Merge pull request #1246 from jooola/deps_manager

Create dependencies manager

api/packages.ini

@@ -0,0 +1,14 @@
+# This file contains a list of package dependencies.
+[common]
+python3 = buster, bionic
+python3-pip = buster, bionic
+python3-venv = buster, bionic
+
+[django]
+uwsgi = buster, bionic
+uwsgi-plugin-python3 = buster, bionic
+
+[psycopg2]
+# See https://www.psycopg.org/docs/install.html#install-from-source
+python3-dev = buster, bionic
+libpq-dev = buster, bionic

install

@@ -739,16 +739,23 @@ if [ "$ignore_dependencies" = "f" ]; then
   fi
 
   if [ -x /usr/bin/apt-get ]; then
-    verbose "\n * Reading requirements-${dist}-${code}.apt..."
+    verbose "\n * Reading packages.ini files..."
+
+    packages_files=(
+      "${SCRIPT_DIR}/"
+      "${SCRIPT_DIR}/api"
+      "${SCRIPT_DIR}/python_apps/airtime_analyzer"
+      "${SCRIPT_DIR}/python_apps/pypo"
+    )
+
+    set -e
+    package_list=$(
+      "${SCRIPT_DIR}/scripts/packages.py" --format=line "${code}" "${packages_files[@]}" ||
+        (echo "ERROR: could not generate packages list" >&2 && exit 1)
+    )
+    set +e
+
     loudCmd "apt-get -q update"
-    package_list_file="${SCRIPT_DIR}/installer/apt/requirements-${dist}-${code}.apt"
-
-    if [ ! -f "$package_list_file" ]; then
-      echo "ERROR: package file does not exist: $package_list_file" >&2
-      exit 1
-    fi
-
-    package_list="$(grep -vE '^\s*#' "$package_list_file" | tr '\n' ' ')"
     loudCmd "DEBIAN_FRONTEND=noninteractive apt-get -y install $package_list"
     [[ "$in_place" == "t" ]] && loudCmd "DEBIAN_FRONTEND=noninteractive apt-get -y install git"
   else

installer/apt/requirements-debian-buster.apt

@@ -1,69 +0,0 @@
-apache2
-coreutils
-curl
-ecasound
-flac
-git
-gcc
-gir1.2-gtk-3.0
-gstreamer1.0-plugins-bad
-gstreamer1.0-plugins-good
-gstreamer1.0-plugins-ugly
-icecast2
-lame
-libao-ocaml
-libapache2-mod-php7.3
-libcairo2-dev
-libcamomile-ocaml-data
-libfaad2
-libglib2.0-dev
-libgirepository1.0-dev
-libmad-ocaml
-libopus0
-libportaudio2
-libpulse0
-libpq-dev
-libsamplerate0
-libsoundtouch-ocaml
-libtaglib-ocaml
-libvo-aacenc0
-liquidsoap
-lsb-release
-lsb-release
-lsof
-mpg123
-patch
-php7.3
-php7.3-bcmath
-php7.3-curl
-php7.3-dev
-php7.3-gd
-php7.3-mbstring
-php7.3-pgsql
-php-amqplib
-php-apcu
-php-pear
-pkg-config
-postgresql
-postgresql-client
-pwgen
-python3
-python3-dev
-python3-gi
-python3-gi-cairo
-python3-gst-1.0
-python3-pika
-python3-pip
-python3-virtualenv
-python3-cairo
-rabbitmq-server
-silan
-systemd-sysv
-unzip
-uwsgi
-uwsgi-plugin-python3
-vorbisgain
-vorbis-tools
-vorbis-tools
-xmlstarlet
-zip

installer/apt/requirements-ubuntu-bionic.apt

@@ -1,80 +0,0 @@
-apache2
-build-essential
-coreutils
-curl
-ecasound
-flac
-gcc
-gir1.2-gtk-3.0
-gstreamer1.0-plugins-bad
-gstreamer1.0-plugins-good
-gstreamer1.0-plugins-ugly
-icecast2
-lame
-libao-ocaml
-libapache2-mod-php7.2
-libcairo2-dev
-libcamomile-ocaml-data
-libfaad2
-libffi-dev
-libgirepository1.0-dev
-libglib2.0-dev
-libmad-ocaml
-libopus0
-libportaudio2
-libpulse0
-libpq-dev
-libsamplerate0
-libsoundtouch-ocaml
-libssl-dev
-libtaglib-ocaml
-liquidsoap
-liquidsoap-plugin-alsa
-liquidsoap-plugin-ao
-liquidsoap-plugin-faad
-liquidsoap-plugin-flac
-liquidsoap-plugin-icecast
-liquidsoap-plugin-lame
-liquidsoap-plugin-mad
-liquidsoap-plugin-ogg
-liquidsoap-plugin-portaudio
-liquidsoap-plugin-pulseaudio
-liquidsoap-plugin-taglib
-liquidsoap-plugin-voaacenc
-liquidsoap-plugin-vorbis
-lsb-release
-lsof
-mpg123
-patch
-php7.2
-php7.2-curl
-php7.2-gd
-php7.2-pgsql
-php-apcu
-php-bcmath
-php-mbstring
-php-pear
-pkg-config
-postgresql
-postgresql-client
-pwgen
-python3
-python3-dev
-python3-cairo
-python3-dev
-python3-gi
-python3-gi-cairo
-python3-gst-1.0
-python3-pika
-python3-pip
-python3-virtualenv
-rabbitmq-server
-silan
-sysvinit-utils
-unzip
-uwsgi
-uwsgi-plugin-python3
-vorbisgain
-vorbis-tools
-xmlstarlet
-zip

packages.ini

@@ -0,0 +1,39 @@
+# This file contains a list of package dependencies.
+[common]
+icecast2 = buster, bionic
+postgresql = buster, bionic
+postgresql-client = buster, bionic
+rabbitmq-server = buster, bionic
+
+curl = buster, bionic
+
+# See https://github.com/Libretime/libretime/commits/71ce3082c5e0611f77eb423a15b4b36267ebfda4/installer/lib
+unzip = buster, bionic
+zip = buster, bionic
+
+[legacy]
+apache2 = buster, bionic
+php-apcu = buster, bionic
+php-pear = buster, bionic
+# Bionic
+libapache2-mod-php7.2 = bionic
+php-bcmath = bionic
+php-mbstring = bionic
+php7.2 = bionic
+php7.2-curl = bionic
+php7.2-gd = bionic
+php7.2-pgsql = bionic
+# Buster
+libapache2-mod-php7.3 = buster
+php-amqplib = buster
+php7.3 = buster
+php7.3-bcmath = buster
+php7.3-curl = buster
+php7.3-dev = buster
+php7.3-gd = buster
+php7.3-mbstring = buster
+php7.3-pgsql = buster
+
+[installer]
+lsb-release = buster, bionic
+xmlstarlet = buster, bionic

python_apps/airtime_analyzer/packages.ini

@@ -0,0 +1,46 @@
+# This file contains a list of package dependencies.
+[common]
+python3 = buster, bionic
+python3-pip = buster, bionic
+python3-venv = buster, bionic
+
+[liquidsoap]
+# https://github.com/savonet/liquidsoap/blob/main/CHANGES.md
+liquidsoap-plugin-alsa = bionic
+liquidsoap-plugin-ao = bionic
+liquidsoap-plugin-ogg = bionic
+liquidsoap-plugin-portaudio = bionic
+# Already recommended packages in bionic
+# See `apt show liquidsoap`
+; liquidsoap-plugin-faad = bionic
+; liquidsoap-plugin-flac = bionic
+; liquidsoap-plugin-icecast = bionic
+; liquidsoap-plugin-lame = bionic
+; liquidsoap-plugin-mad = bionic
+; liquidsoap-plugin-pulseaudio = bionic
+; liquidsoap-plugin-taglib = bionic
+; liquidsoap-plugin-voaacenc = bionic
+; liquidsoap-plugin-vorbis = bionic
+liquidsoap = buster, bionic
+
+[pika]
+python3-pika = buster, bionic
+
+[rgain3]
+gcc = buster, bionic
+gir1.2-gtk-3.0 = buster, bionic
+gstreamer1.0-plugins-bad = buster, bionic
+gstreamer1.0-plugins-good = buster, bionic
+gstreamer1.0-plugins-ugly = buster, bionic
+libcairo2-dev = buster, bionic
+libgirepository1.0-dev = buster, bionic
+libglib2.0-dev = buster, bionic
+pkg-config = buster, bionic
+python3-cairo = buster, bionic
+python3-dev = buster, bionic
+python3-gi = buster, bionic
+python3-gi-cairo = buster, bionic
+python3-gst-1.0 = buster, bionic
+
+[silan]
+silan = buster, bionic

python_apps/pypo/packages.ini

@@ -0,0 +1,32 @@
+# This file contains a list of package dependencies.
+[common]
+python3 = buster, bionic
+python3-pip = buster, bionic
+python3-venv = buster, bionic
+
+[liquidsoap]
+# https://github.com/savonet/liquidsoap/blob/main/CHANGES.md
+liquidsoap-plugin-alsa = bionic
+liquidsoap-plugin-ao = bionic
+liquidsoap-plugin-ogg = bionic
+liquidsoap-plugin-portaudio = bionic
+# Already recommended packages in bionic
+# See `apt show liquidsoap`
+; liquidsoap-plugin-faad = bionic
+; liquidsoap-plugin-flac = bionic
+; liquidsoap-plugin-icecast = bionic
+; liquidsoap-plugin-lame = bionic
+; liquidsoap-plugin-mad = bionic
+; liquidsoap-plugin-pulseaudio = bionic
+; liquidsoap-plugin-taglib = bionic
+; liquidsoap-plugin-voaacenc = bionic
+; liquidsoap-plugin-vorbis = bionic
+liquidsoap = buster, bionic
+
+[recorder]
+ecasound = buster, bionic
+
+[misc]
+# Used by pypofetch to check if a file is open.
+# TODO: consider using a python library
+lsof = buster, bionic

scripts/packages.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+from argparse import ArgumentParser
+from configparser import ConfigParser
+from os import PathLike
+from pathlib import Path
+from typing import Iterator, Set
+
+DEFAULT_PACKAGES_FILENAME = "packages.ini"
+FORMATS = ("list", "line")
+SYSTEMS = ("buster", "bionic")
+
+SETTINGS_SECTION = "=settings"
+DEVELOPMENT_SECTION = "=development"
+
+
+def load_packages(raw: str, distribution: str, development: bool = False) -> Set[str]:
+    manager = ConfigParser(default_section=SETTINGS_SECTION)
+    manager.read_string(raw)
+
+    packages = set()
+    for section, entries in manager.items():
+        if not development and section == DEVELOPMENT_SECTION:
+            continue
+
+        for package, distributions in entries.items():
+            if distribution in distributions.split(", "):
+                packages.add(package)
+
+    return packages
+
+
+def list_packages_files(paths: str) -> Iterator[Path]:
+    for path in paths:
+        path = Path(path)
+
+        if path.is_dir():
+            path = path / DEFAULT_PACKAGES_FILENAME
+
+        if not path.is_file():
+            raise Exception(f"{path} is not a file!")
+
+        yield path
+
+
+def list_packages(paths: str, distribution: str, development: bool = False) -> Set[str]:
+    packages = set()
+    for package_file in list_packages_files(paths):
+        raw = package_file.read_text()
+        packages.update(load_packages(raw, distribution, development))
+
+    return set(sorted(packages))
+
+
+def run():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-f",
+        "--format",
+        choices=FORMATS,
+        help="print packages list in a specific format.",
+        default="list",
+    ),
+    parser.add_argument(
+        "-d",
+        "--dev",
+        help="include development packages.",
+        action="store_true",
+    )
+    parser.add_argument(
+        "distribution",
+        choices=SYSTEMS,
+        help="list packages for the given distribution.",
+    )
+    parser.add_argument(
+        "path",
+        nargs="+",
+        help="list packages from given files or directories.",
+    )
+    args = parser.parse_args()
+
+    packages = list_packages(args.path, args.distribution, args.dev)
+
+    if args.format == "list":
+        print("\n".join(packages))
+    else:
+        print(" ".join(packages))
+
+
+if __name__ == "__main__":
+    run()

scripts/packages_test.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+from .packages import list_packages, load_packages
+
+package_ini = """
+[common]
+postgresql = buster
+# Some comment
+curl = buster, bionic
+
+[legacy]
+apache2 = bionic
+
+[=development]
+ffmpeg = buster, bionic
+"""
+
+result1 = {"curl", "postgresql"}
+result2 = {"apache2", "curl", "ffmpeg"}
+
+
+def test_load_packages():
+    assert load_packages(package_ini, "buster", False) == result1
+    assert load_packages(package_ini, "bionic", True) == result2
+
+
+def test_list_packages(tmp_path: Path):
+    package_file = tmp_path / "packages.ini"
+    package_file.write_text(package_ini)
+
+    assert list_packages([tmp_path, package_file], "buster", False) == result1
+    assert list_packages([tmp_path, package_file], "bionic", True) == result2