feat: create libretime_shared package (#1349)

* feat: create libretime_shared package

- We don't use pydantic.BaseSettings because of some
  incompatble loading behavior.

* fix: whitelist pydantic in pylintrc

* docs: update to new BaseConfig behavior

* docs: change confusing config filepath

.github/workflows/test.yml

@@ -117,6 +117,7 @@ jobs:
           - api
           - api_client
           - playout
+          - shared
           - worker
 
     container: ghcr.io/libretime/libretime-dev:buster
@@ -148,6 +149,7 @@ jobs:
         context:
           - analyzer
           - api_client
+          - shared
         release:
           - buster
           - bullseye

shared/.pylintrc

@@ -0,0 +1,5 @@
+[MESSAGES CONTROL]
+extension-pkg-whitelist = pydantic
+disable=missing-module-docstring,
+        missing-class-docstring,
+        missing-function-docstring,

shared/Makefile

@@ -0,0 +1,16 @@
+all: lint test
+
+include ../tools/python.mk
+
+PIP_INSTALL = --editable .[dev]
+PYLINT_ARG = libretime_shared tests
+MYPY_ARG =	--disallow-untyped-calls \
+			--disallow-untyped-defs \
+			--disallow-incomplete-defs \
+			libretime_shared
+PYTEST_ARG = --cov=libretime_shared tests
+
+format: .format
+lint: .pylint .mypy
+test: .pytest
+clean: .clean

shared/README.md

@@ -0,0 +1,77 @@
+# Shared
+
+The `libretime_shared` package contains reusable functions and classes for the Libretime project.
+
+## Usage
+
+This library assumes that:
+
+- You will use [`Click`](https://github.com/pallets/click) to build a CLI for your app.
+- You will use [`Loguru`](https://github.com/delgan/loguru) to log messages from your app.
+- You will use [`Pydantic`](https://github.com/samuelcolvin/pydantic/) to validate objects in your app.
+
+### Configuration
+
+First define a schema for your configuration in order to validate it. A schema is a class that inherit from `pydantic.BaseModel`. Some existing schemas can be reused such as `libretime_shared.config.RabbitMQ` or `libretime_shared.config.Database`.
+
+Load your configuration using a subclass of `libretime_shared.config.BaseConfig`.
+
+```py
+from pydantic import BaseModel
+
+from libretime_shared.config import RabbitMQ, BaseConfig
+
+class Analyzer(BaseModel):
+    bpm_enabled: bool = False
+    bpm_track_max_length: int
+
+class Config(BaseConfig):
+    rabbitmq: RabbitMQ
+    analyzer: Analyzer
+
+config = Config(filepath="/etc/libretime/config.yml")
+```
+
+### App
+
+Create an app class that inherit from `libretime_shared.app.AbstractApp`.
+
+```py
+from libretime_shared.app import AbstractApp
+
+class LiquidsoapApp(AbstractApp):
+    name = "liquidsoap"
+
+    def __init__(self, some_arg, **kwargs):
+        super().__init__(**kwargs)
+        self.some_arg = some_arg
+
+    def run(self):
+        ...
+
+
+app = LiquidsoapApp(**kwargs)
+app.run()
+```
+
+### CLI
+
+Decorate your CLI commands with the shared decorators to add extra flags.
+
+```py
+import click
+from libretime_shared.cli import cli_logging_options, cli_config_options
+
+from .app import App
+
+@click.group()
+def cli():
+    pass
+
+@cli.command()
+@cli_config_options
+@cli_logging_options
+def run(**kwargs):
+    app = App(**kwargs)
+    return app.run()
+```

shared/libretime_shared/app.py

@@ -0,0 +1,36 @@
+from abc import ABC, abstractmethod
+from os import PathLike
+from pathlib import Path
+from typing import Optional
+
+from loguru import logger
+
+from .logging import LogLevel, setup_logger
+
+
+# pylint: disable=too-few-public-methods
+class AbstractApp(ABC):
+    """
+    Abstracts the creation of an application to reduce boilerplate code such
+    as logging setup.
+    """
+
+    log_level: LogLevel
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        ...
+
+    def __init__(
+        self,
+        *,
+        verbosity: int,
+        log_filepath: Optional[PathLike] = None,
+    ):
+        if log_filepath is not None:
+            log_filepath = Path(log_filepath)
+
+        self.log_level = setup_logger(verbosity=verbosity, filepath=log_filepath)
+
+        logger.info(f"Starting {self.name}...")

shared/libretime_shared/cli.py

@@ -0,0 +1,65 @@
+from typing import Callable
+
+import click
+
+from .config import DEFAULT_ENV_PREFIX
+
+
+def cli_logging_options(func: Callable) -> Callable:
+    """
+    Decorator function to add logging options to a click application.
+
+    This decorator add the following arguments:
+    - verbosity: int
+    - log_filepath: Path
+    """
+    func = click.option(
+        "-v",
+        "--verbose",
+        "verbosity",
+        envvar=f"{DEFAULT_ENV_PREFIX}_VERBOSITY",
+        count=True,
+        default=0,
+        help="Increase logging verbosity (use -vvv to debug).",
+    )(func)
+
+    func = click.option(
+        "-q",
+        "--quiet",
+        "verbosity",
+        is_flag=True,
+        flag_value=-1,
+        help="Decrease logging verbosity.",
+    )(func)
+
+    func = click.option(
+        "--log-filepath",
+        "log_filepath",
+        envvar=f"{DEFAULT_ENV_PREFIX}_LOG_FILEPATH",
+        type=click.Path(),
+        help="Path to the logging file.",
+        default=None,
+    )(func)
+
+    return func
+
+
+def cli_config_options(func: Callable) -> Callable:
+    """
+    Decorator function to add config file options to a click application.
+
+    This decorator add the following arguments:
+    - config_filepath: Path
+    """
+
+    func = click.option(
+        "--c",
+        "--config",
+        "config_filepath",
+        envvar=f"{DEFAULT_ENV_PREFIX}_CONFIG_FILEPATH",
+        type=click.Path(),
+        help="Path to the config file.",
+        default=None,
+    )(func)
+
+    return func

shared/libretime_shared/config.py

@@ -0,0 +1,101 @@
+import sys
+from os import environ
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from loguru import logger
+
+# pylint: disable=no-name-in-module
+from pydantic import BaseModel, ValidationError
+from pydantic.fields import ModelField
+from yaml import YAMLError, safe_load
+
+DEFAULT_ENV_PREFIX = "LIBRETIME"
+
+
+# pylint: disable=too-few-public-methods
+class BaseConfig(BaseModel):
+    """
+    Read and validate the configuration from 'filepath' and os environment.
+
+    :param filepath: yaml configuration file to read from
+    :param env_prefix: prefix for the environment variable names
+    :returns: configuration class
+    """
+
+    def __init__(
+        self,
+        *,
+        env_prefix: str = DEFAULT_ENV_PREFIX,
+        filepath: Optional[Path] = None,
+    ) -> None:
+        file_values = self._load_file_values(filepath)
+        env_values = self._load_env_values(env_prefix)
+
+        try:
+            super().__init__(
+                **{
+                    **file_values,
+                    **env_values,
+                },
+            )
+        except ValidationError as error:
+            logger.critical(error)
+            sys.exit(1)
+
+    def _load_env_values(self, env_prefix: str) -> Dict[str, Any]:
+        return self._get_fields_from_env(env_prefix, self.__fields__)
+
+    def _get_fields_from_env(
+        self,
+        env_prefix: str,
+        fields: Dict[str, ModelField],
+    ) -> Dict[str, Any]:
+        result: Dict[str, Any] = {}
+
+        for field in fields.values():
+            env_name = (env_prefix + "_" + field.name).upper()
+
+            if field.is_complex():
+                result[field.name] = self._get_fields_from_env(
+                    env_name,
+                    field.type_.__fields__,
+                )
+            else:
+                if env_name in environ:
+                    result[field.name] = environ[env_name]
+
+        return result
+
+    # pylint: disable=no-self-use
+    def _load_file_values(
+        self,
+        filepath: Optional[Path] = None,
+    ) -> Dict[str, Any]:
+        if filepath is None:
+            return {}
+
+        try:
+            return safe_load(filepath.read_text(encoding="utf-8"))
+        except YAMLError as error:
+            logger.critical(error)
+            sys.exit(1)
+
+
+# pylint: disable=too-few-public-methods
+class Database(BaseModel):
+    host: str = "localhost"
+    port: int = 5432
+    name: str = "libretime"
+    user: str = "libretime"
+    password: str = "libretime"
+
+
+# pylint: disable=too-few-public-methods
+class RabbitMQ(BaseModel):
+    host: str = "localhost"
+    port: int = 5672
+    name: str = "libretime"
+    user: str = "libretime"
+    password: str = "libretime"
+    vhost: str = "/libretime"

shared/libretime_shared/logging.py

@@ -0,0 +1,117 @@
+import sys
+from copy import deepcopy
+from pathlib import Path
+from typing import TYPE_CHECKING, NamedTuple, Optional
+
+from loguru import logger
+
+if TYPE_CHECKING:
+    from loguru import Logger
+
+logger.remove()
+
+
+class LogLevel(NamedTuple):
+    name: str
+    no: int
+
+    def is_debug(self) -> bool:
+        return self.no <= 10
+
+
+# See https://loguru.readthedocs.io/en/stable/api/logger.html#levels
+ERROR = LogLevel(name="error", no=40)
+WARNING = LogLevel(name="warning", no=30)
+INFO = LogLevel(name="info", no=20)
+DEBUG = LogLevel(name="debug", no=10)
+TRACE = LogLevel(name="trace", no=5)
+
+
+def level_from_verbosity(verbosity: int) -> LogLevel:
+    """
+    Find logging level, depending on the verbosity requested.
+
+    -q         -1   =>  ERROR
+    default     0   =>  WARNING
+    -v          1   =>  INFO
+    -vv         2   =>  DEBUG
+    -vvv        3   =>  TRACE
+
+    :param verbosity: verbosity (between -1 and 3) of the logger
+    :returns: log level guessed from the verbosity
+    """
+    if verbosity < 0:
+        return ERROR
+    return [WARNING, INFO, DEBUG, TRACE][min(3, verbosity)]
+
+
+def setup_logger(
+    verbosity: int,
+    filepath: Optional[Path] = None,
+    serialize: bool = False,
+) -> LogLevel:
+    """
+    Configure the logger and return the computed log level.
+
+    See https://loguru.readthedocs.io/en/stable/overview.html
+
+    :param verbosity: verbosity (between -1 and 3) of the logger
+    :param filepath: write logs to filepath
+    :param serialize: generate JSON formatted log records
+    :returns: log level guessed from the verbosity
+    """
+    level = level_from_verbosity(verbosity)
+
+    handlers = [dict(sink=sys.stderr, level=level.no, serialize=serialize)]
+
+    if filepath is not None:
+        handlers.append(
+            dict(
+                sink=filepath,
+                enqueue=True,
+                level=level.no,
+                serialize=serialize,
+                rotation="12:00",
+                retention="7 days",
+            )
+        )
+
+    logger.configure(handlers=handlers)
+
+    return level
+
+
+_empty_logger = deepcopy(logger)
+
+
+def create_task_logger(
+    verbosity: int,
+    filepath: Path,
+    serialize: bool = False,
+) -> "Logger":
+    """
+    Create and configure an independent logger for a task, return the new logger.
+
+    See #creating-independent-loggers-with-separate-set-of-handlers in
+    https://loguru.readthedocs.io/en/stable/resources/recipes.html
+
+    :returns: new logger
+    """
+    task_logger = deepcopy(_empty_logger)
+
+    level = level_from_verbosity(verbosity)
+
+    task_logger.configure(
+        handlers=[
+            dict(
+                sink=filepath,
+                enqueue=True,
+                level=level.no,
+                serialize=serialize,
+                rotation="12:00",
+                retention="7 days",
+            )
+        ],
+    )
+
+    return task_logger

shared/libretime_shared/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561.

shared/setup.py

@@ -0,0 +1,29 @@
+from os import chdir
+from pathlib import Path
+
+from setuptools import setup
+
+here = Path(__file__).parent
+chdir(here)
+
+setup(
+    name="libretime-shared",
+    version="1.0.0",
+    description="LibreTime Shared",
+    url="http://github.com/libretime/libretime",
+    author="LibreTime Contributors",
+    license="AGPLv3",
+    packages=["libretime_shared"],
+    package_data={"": ["py.typed"]},
+    install_requires=[
+        "click",
+        "loguru",
+        "pydantic",
+        "pyyaml",
+    ],
+    extras_require={
+        "dev": [
+            "types-pyyaml",
+        ],
+    },
+)

shared/tests/config_test.py

@@ -0,0 +1,72 @@
+from os import environ
+from unittest import mock
+
+from pytest import mark, raises
+
+from libretime_shared.config import BaseConfig, Database
+
+
+# pylint: disable=too-few-public-methods
+class FixtureConfig(BaseConfig):
+    api_key: str
+    database: Database
+
+
+FIXTURE_CONFIG_RAW = """
+api_key: "f3bf04fc"
+
+# Comment !
+database:
+    host: "localhost"
+    port: 5672
+
+ignored: "ignored"
+"""
+
+
+def test_base_config(tmpdir):
+    config_filepath = tmpdir.join("config.yml")
+    config_filepath.write(FIXTURE_CONFIG_RAW)
+
+    with mock.patch.dict(
+        environ,
+        dict(
+            LIBRETIME_API="invalid",
+            LIBRETIME_API_KEY="f3bf04fc",
+            LIBRETIME_DATABASE="invalid",
+            LIBRETIME_DATABASE_PORT="8888",
+            WRONGPREFIX_API_KEY="invalid",
+        ),
+    ):
+        config = FixtureConfig(filepath=config_filepath)
+
+        assert config.api_key == "f3bf04fc"
+        assert config.database.host == "localhost"
+        assert config.database.port == 8888
+
+
+FIXTURE_CONFIG_RAW_MISSING = """
+database:
+    host: "localhost"
+"""
+
+FIXTURE_CONFIG_RAW_INVALID = """
+database
+    host: "localhost"
+"""
+
+
+@mark.parametrize(
+    "raw,exception",
+    [
+        (FIXTURE_CONFIG_RAW_INVALID, SystemExit),
+        (FIXTURE_CONFIG_RAW_MISSING, SystemExit),
+    ],
+)
+def test_load_config_error(tmpdir, raw, exception):
+    config_filepath = tmpdir.join("config.yml")
+    config_filepath.write(raw)
+
+    with raises(exception):
+        with mock.patch.dict(environ, {}):
+            FixtureConfig(filepath=config_filepath)

shared/tests/logging_test.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+import pytest
+from loguru import logger
+
+from libretime_shared.logging import (
+    create_task_logger,
+    level_from_verbosity,
+    setup_logger,
+)
+
+
+@pytest.mark.parametrize(
+    "verbosity,level_name,level_no",
+    [
+        (-100, "error", 40),
+        (-1, "error", 40),
+        (0, "warning", 30),
+        (1, "info", 20),
+        (2, "debug", 10),
+        (3, "trace", 5),
+        (100, "trace", 5),
+    ],
+)
+def test_level_from_verbosity(verbosity, level_name, level_no):
+    level = level_from_verbosity(verbosity)
+    assert level.name == level_name
+    assert level.no == level_no
+
+
+def test_setup_logger(tmp_path: Path):
+    log_filepath = tmp_path / "test.log"
+    extra_log_filepath = tmp_path / "extra.log"
+
+    setup_logger(1, log_filepath)
+
+    extra_logger = create_task_logger(2, extra_log_filepath, True)
+
+    logger.info("test info")
+    extra_logger.info("extra info")
+    logger.debug("test debug")
+
+    extra_logger.complete()
+    logger.complete()
+
+    assert len(log_filepath.read_text().splitlines()) == 1
+    assert len(extra_log_filepath.read_text().splitlines()) == 1