diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53ffe490d..6611f24f3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,6 +117,7 @@ jobs: - api - api_client - playout + - shared - worker container: ghcr.io/libretime/libretime-dev:buster @@ -148,6 +149,7 @@ jobs: context: - analyzer - api_client + - shared release: - buster - bullseye diff --git a/shared/.pylintrc b/shared/.pylintrc new file mode 100644 index 000000000..b248ba1b5 --- /dev/null +++ b/shared/.pylintrc @@ -0,0 +1,5 @@ +[MESSAGES CONTROL] +extension-pkg-whitelist = pydantic +disable=missing-module-docstring, + missing-class-docstring, + missing-function-docstring, diff --git a/shared/Makefile b/shared/Makefile new file mode 100644 index 000000000..00f9cd647 --- /dev/null +++ b/shared/Makefile @@ -0,0 +1,16 @@ +all: lint test + +include ../tools/python.mk + +PIP_INSTALL = --editable .[dev] +PYLINT_ARG = libretime_shared tests +MYPY_ARG = --disallow-untyped-calls \ + --disallow-untyped-defs \ + --disallow-incomplete-defs \ + libretime_shared +PYTEST_ARG = --cov=libretime_shared tests + +format: .format +lint: .pylint .mypy +test: .pytest +clean: .clean diff --git a/shared/README.md b/shared/README.md new file mode 100644 index 000000000..bc7f60154 --- /dev/null +++ b/shared/README.md @@ -0,0 +1,77 @@ +# Shared + +The `libretime_shared` package contains reusable functions and classes for the Libretime project. + +## Usage + +This library assumes that: + +- You will use [`Click`](https://github.com/pallets/click) to build a CLI for your app. +- You will use [`Loguru`](https://github.com/delgan/loguru) to log messages from your app. +- You will use [`Pydantic`](https://github.com/samuelcolvin/pydantic/) to validate objects in your app. + +### Configuration + +First define a schema for your configuration in order to validate it. A schema is a class that inherit from `pydantic.BaseModel`. Some existing schemas can be reused such as `libretime_shared.config.RabbitMQ` or `libretime_shared.config.Database`. + +Load your configuration using a subclass of `libretime_shared.config.BaseConfig`. + +```py +from pydantic import BaseModel + +from libretime_shared.config import RabbitMQ, BaseConfig + +class Analyzer(BaseModel): + bpm_enabled: bool = False + bpm_track_max_length: int + +class Config(BaseConfig): + rabbitmq: RabbitMQ + analyzer: Analyzer + +config = Config(filepath="/etc/libretime/config.yml") +``` + +### App + +Create an app class that inherit from `libretime_shared.app.AbstractApp`. + +```py +from libretime_shared.app import AbstractApp + +class LiquidsoapApp(AbstractApp): + name = "liquidsoap" + + def __init__(self, some_arg, **kwargs): + super().__init__(**kwargs) + self.some_arg = some_arg + + def run(self): + ... + + +app = LiquidsoapApp(**kwargs) +app.run() +``` + +### CLI + +Decorate your CLI commands with the shared decorators to add extra flags. + +```py +import click +from libretime_shared.cli import cli_logging_options, cli_config_options + +from .app import App + +@click.group() +def cli(): + pass + +@cli.command() +@cli_config_options +@cli_logging_options +def run(**kwargs): + app = App(**kwargs) + return app.run() +``` diff --git a/shared/libretime_shared/__init__.py b/shared/libretime_shared/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/shared/libretime_shared/app.py b/shared/libretime_shared/app.py new file mode 100644 index 000000000..0e652d2c3 --- /dev/null +++ b/shared/libretime_shared/app.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from os import PathLike +from pathlib import Path +from typing import Optional + +from loguru import logger + +from .logging import LogLevel, setup_logger + + +# pylint: disable=too-few-public-methods +class AbstractApp(ABC): + """ + Abstracts the creation of an application to reduce boilerplate code such + as logging setup. + """ + + log_level: LogLevel + + @property + @abstractmethod + def name(self) -> str: + ... + + def __init__( + self, + *, + verbosity: int, + log_filepath: Optional[PathLike] = None, + ): + if log_filepath is not None: + log_filepath = Path(log_filepath) + + self.log_level = setup_logger(verbosity=verbosity, filepath=log_filepath) + + logger.info(f"Starting {self.name}...") diff --git a/shared/libretime_shared/cli.py b/shared/libretime_shared/cli.py new file mode 100644 index 000000000..9ada00807 --- /dev/null +++ b/shared/libretime_shared/cli.py @@ -0,0 +1,65 @@ +from typing import Callable + +import click + +from .config import DEFAULT_ENV_PREFIX + + +def cli_logging_options(func: Callable) -> Callable: + """ + Decorator function to add logging options to a click application. + + This decorator add the following arguments: + - verbosity: int + - log_filepath: Path + """ + func = click.option( + "-v", + "--verbose", + "verbosity", + envvar=f"{DEFAULT_ENV_PREFIX}_VERBOSITY", + count=True, + default=0, + help="Increase logging verbosity (use -vvv to debug).", + )(func) + + func = click.option( + "-q", + "--quiet", + "verbosity", + is_flag=True, + flag_value=-1, + help="Decrease logging verbosity.", + )(func) + + func = click.option( + "--log-filepath", + "log_filepath", + envvar=f"{DEFAULT_ENV_PREFIX}_LOG_FILEPATH", + type=click.Path(), + help="Path to the logging file.", + default=None, + )(func) + + return func + + +def cli_config_options(func: Callable) -> Callable: + """ + Decorator function to add config file options to a click application. + + This decorator add the following arguments: + - config_filepath: Path + """ + + func = click.option( + "--c", + "--config", + "config_filepath", + envvar=f"{DEFAULT_ENV_PREFIX}_CONFIG_FILEPATH", + type=click.Path(), + help="Path to the config file.", + default=None, + )(func) + + return func diff --git a/shared/libretime_shared/config.py b/shared/libretime_shared/config.py new file mode 100644 index 000000000..bd5e614ec --- /dev/null +++ b/shared/libretime_shared/config.py @@ -0,0 +1,101 @@ +import sys +from os import environ +from pathlib import Path +from typing import Any, Dict, Optional + +from loguru import logger + +# pylint: disable=no-name-in-module +from pydantic import BaseModel, ValidationError +from pydantic.fields import ModelField +from yaml import YAMLError, safe_load + +DEFAULT_ENV_PREFIX = "LIBRETIME" + + +# pylint: disable=too-few-public-methods +class BaseConfig(BaseModel): + """ + Read and validate the configuration from 'filepath' and os environment. + + :param filepath: yaml configuration file to read from + :param env_prefix: prefix for the environment variable names + :returns: configuration class + """ + + def __init__( + self, + *, + env_prefix: str = DEFAULT_ENV_PREFIX, + filepath: Optional[Path] = None, + ) -> None: + file_values = self._load_file_values(filepath) + env_values = self._load_env_values(env_prefix) + + try: + super().__init__( + **{ + **file_values, + **env_values, + }, + ) + except ValidationError as error: + logger.critical(error) + sys.exit(1) + + def _load_env_values(self, env_prefix: str) -> Dict[str, Any]: + return self._get_fields_from_env(env_prefix, self.__fields__) + + def _get_fields_from_env( + self, + env_prefix: str, + fields: Dict[str, ModelField], + ) -> Dict[str, Any]: + result: Dict[str, Any] = {} + + for field in fields.values(): + env_name = (env_prefix + "_" + field.name).upper() + + if field.is_complex(): + result[field.name] = self._get_fields_from_env( + env_name, + field.type_.__fields__, + ) + else: + if env_name in environ: + result[field.name] = environ[env_name] + + return result + + # pylint: disable=no-self-use + def _load_file_values( + self, + filepath: Optional[Path] = None, + ) -> Dict[str, Any]: + if filepath is None: + return {} + + try: + return safe_load(filepath.read_text(encoding="utf-8")) + except YAMLError as error: + logger.critical(error) + sys.exit(1) + + +# pylint: disable=too-few-public-methods +class Database(BaseModel): + host: str = "localhost" + port: int = 5432 + name: str = "libretime" + user: str = "libretime" + password: str = "libretime" + + +# pylint: disable=too-few-public-methods +class RabbitMQ(BaseModel): + host: str = "localhost" + port: int = 5672 + name: str = "libretime" + user: str = "libretime" + password: str = "libretime" + vhost: str = "/libretime" diff --git a/shared/libretime_shared/logging.py b/shared/libretime_shared/logging.py new file mode 100644 index 000000000..634a8ae03 --- /dev/null +++ b/shared/libretime_shared/logging.py @@ -0,0 +1,117 @@ +import sys +from copy import deepcopy +from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple, Optional + +from loguru import logger + +if TYPE_CHECKING: + from loguru import Logger + +logger.remove() + + +class LogLevel(NamedTuple): + name: str + no: int + + def is_debug(self) -> bool: + return self.no <= 10 + + +# See https://loguru.readthedocs.io/en/stable/api/logger.html#levels +ERROR = LogLevel(name="error", no=40) +WARNING = LogLevel(name="warning", no=30) +INFO = LogLevel(name="info", no=20) +DEBUG = LogLevel(name="debug", no=10) +TRACE = LogLevel(name="trace", no=5) + + +def level_from_verbosity(verbosity: int) -> LogLevel: + """ + Find logging level, depending on the verbosity requested. + + -q -1 => ERROR + default 0 => WARNING + -v 1 => INFO + -vv 2 => DEBUG + -vvv 3 => TRACE + + :param verbosity: verbosity (between -1 and 3) of the logger + :returns: log level guessed from the verbosity + """ + if verbosity < 0: + return ERROR + return [WARNING, INFO, DEBUG, TRACE][min(3, verbosity)] + + +def setup_logger( + verbosity: int, + filepath: Optional[Path] = None, + serialize: bool = False, +) -> LogLevel: + """ + Configure the logger and return the computed log level. + + See https://loguru.readthedocs.io/en/stable/overview.html + + :param verbosity: verbosity (between -1 and 3) of the logger + :param filepath: write logs to filepath + :param serialize: generate JSON formatted log records + :returns: log level guessed from the verbosity + """ + level = level_from_verbosity(verbosity) + + handlers = [dict(sink=sys.stderr, level=level.no, serialize=serialize)] + + if filepath is not None: + handlers.append( + dict( + sink=filepath, + enqueue=True, + level=level.no, + serialize=serialize, + rotation="12:00", + retention="7 days", + ) + ) + + logger.configure(handlers=handlers) + + return level + + +_empty_logger = deepcopy(logger) + + +def create_task_logger( + verbosity: int, + filepath: Path, + serialize: bool = False, +) -> "Logger": + """ + Create and configure an independent logger for a task, return the new logger. + + See #creating-independent-loggers-with-separate-set-of-handlers in + https://loguru.readthedocs.io/en/stable/resources/recipes.html + + :returns: new logger + """ + task_logger = deepcopy(_empty_logger) + + level = level_from_verbosity(verbosity) + + task_logger.configure( + handlers=[ + dict( + sink=filepath, + enqueue=True, + level=level.no, + serialize=serialize, + rotation="12:00", + retention="7 days", + ) + ], + ) + + return task_logger diff --git a/shared/libretime_shared/py.typed b/shared/libretime_shared/py.typed new file mode 100644 index 000000000..1242d4327 --- /dev/null +++ b/shared/libretime_shared/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. diff --git a/shared/setup.py b/shared/setup.py new file mode 100644 index 000000000..27c321b75 --- /dev/null +++ b/shared/setup.py @@ -0,0 +1,29 @@ +from os import chdir +from pathlib import Path + +from setuptools import setup + +here = Path(__file__).parent +chdir(here) + +setup( + name="libretime-shared", + version="1.0.0", + description="LibreTime Shared", + url="http://github.com/libretime/libretime", + author="LibreTime Contributors", + license="AGPLv3", + packages=["libretime_shared"], + package_data={"": ["py.typed"]}, + install_requires=[ + "click", + "loguru", + "pydantic", + "pyyaml", + ], + extras_require={ + "dev": [ + "types-pyyaml", + ], + }, +) diff --git a/shared/tests/config_test.py b/shared/tests/config_test.py new file mode 100644 index 000000000..90b251c01 --- /dev/null +++ b/shared/tests/config_test.py @@ -0,0 +1,72 @@ +from os import environ +from unittest import mock + +from pytest import mark, raises + +from libretime_shared.config import BaseConfig, Database + + +# pylint: disable=too-few-public-methods +class FixtureConfig(BaseConfig): + api_key: str + database: Database + + +FIXTURE_CONFIG_RAW = """ +api_key: "f3bf04fc" + +# Comment ! +database: + host: "localhost" + port: 5672 + +ignored: "ignored" +""" + + +def test_base_config(tmpdir): + config_filepath = tmpdir.join("config.yml") + config_filepath.write(FIXTURE_CONFIG_RAW) + + with mock.patch.dict( + environ, + dict( + LIBRETIME_API="invalid", + LIBRETIME_API_KEY="f3bf04fc", + LIBRETIME_DATABASE="invalid", + LIBRETIME_DATABASE_PORT="8888", + WRONGPREFIX_API_KEY="invalid", + ), + ): + config = FixtureConfig(filepath=config_filepath) + + assert config.api_key == "f3bf04fc" + assert config.database.host == "localhost" + assert config.database.port == 8888 + + +FIXTURE_CONFIG_RAW_MISSING = """ +database: + host: "localhost" +""" + +FIXTURE_CONFIG_RAW_INVALID = """ +database + host: "localhost" +""" + + +@mark.parametrize( + "raw,exception", + [ + (FIXTURE_CONFIG_RAW_INVALID, SystemExit), + (FIXTURE_CONFIG_RAW_MISSING, SystemExit), + ], +) +def test_load_config_error(tmpdir, raw, exception): + config_filepath = tmpdir.join("config.yml") + config_filepath.write(raw) + + with raises(exception): + with mock.patch.dict(environ, {}): + FixtureConfig(filepath=config_filepath) diff --git a/shared/tests/logging_test.py b/shared/tests/logging_test.py new file mode 100644 index 000000000..54e08ac90 --- /dev/null +++ b/shared/tests/logging_test.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import pytest +from loguru import logger + +from libretime_shared.logging import ( + create_task_logger, + level_from_verbosity, + setup_logger, +) + + +@pytest.mark.parametrize( + "verbosity,level_name,level_no", + [ + (-100, "error", 40), + (-1, "error", 40), + (0, "warning", 30), + (1, "info", 20), + (2, "debug", 10), + (3, "trace", 5), + (100, "trace", 5), + ], +) +def test_level_from_verbosity(verbosity, level_name, level_no): + level = level_from_verbosity(verbosity) + assert level.name == level_name + assert level.no == level_no + + +def test_setup_logger(tmp_path: Path): + log_filepath = tmp_path / "test.log" + extra_log_filepath = tmp_path / "extra.log" + + setup_logger(1, log_filepath) + + extra_logger = create_task_logger(2, extra_log_filepath, True) + + logger.info("test info") + extra_logger.info("extra info") + logger.debug("test debug") + + extra_logger.complete() + logger.complete() + + assert len(log_filepath.read_text().splitlines()) == 1 + assert len(extra_log_filepath.read_text().splitlines()) == 1