From 18f9c81b746bd4d44b4718414323fe2fe2fc96d1 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 18:48:25 +0200 Subject: [PATCH] Add dependencies manager Used to generate list of packages dependencies for different distributions, and should improve dependencies management by having them in each project. This tools should gather the reason why a dependency is present, so one can remove it when unused. Rename deps script to packages Move installer/packages to scripts and write tests Remove packages-list make target --- api/packages.ini | 9 +++ packages.ini | 54 +++++++++++++ python_apps/airtime_analyzer/packages.ini | 46 +++++++++++ python_apps/pypo/packages.ini | 29 +++++++ scripts/__init__.py | 0 scripts/packages.py | 93 +++++++++++++++++++++++ scripts/packages_test.py | 32 ++++++++ 7 files changed, 263 insertions(+) create mode 100644 api/packages.ini create mode 100644 packages.ini create mode 100644 python_apps/airtime_analyzer/packages.ini create mode 100644 python_apps/pypo/packages.ini create mode 100644 scripts/__init__.py create mode 100755 scripts/packages.py create mode 100644 scripts/packages_test.py diff --git a/api/packages.ini b/api/packages.ini new file mode 100644 index 000000000..def9cd21e --- /dev/null +++ b/api/packages.ini @@ -0,0 +1,9 @@ +# This file contains a list of package dependencies. +[common] +python3 = buster, bionic +python3-pip = buster, bionic +python3-virtualenv = buster, bionic + +[django] +uwsgi = buster, bionic +uwsgi-plugin-python3 = buster, bionic diff --git a/packages.ini b/packages.ini new file mode 100644 index 000000000..24bd455c7 --- /dev/null +++ b/packages.ini @@ -0,0 +1,54 @@ +# This file contains a list of package dependencies. +[common] +icecast2 = buster, bionic +postgresql = buster, bionic +postgresql-client = buster, bionic +rabbitmq-server = buster, bionic + +curl = buster, bionic +unzip = buster, bionic +zip = buster, bionic + +[legacy] +apache2 = buster, bionic +php-apcu = buster, bionic +php-pear = buster, bionic +# Bionic +libapache2-mod-php7.2 = bionic +php-bcmath = bionic +php-mbstring = bionic +php7.2 = bionic +php7.2-curl = bionic +php7.2-gd = bionic +php7.2-pgsql = bionic +# Buster +libapache2-mod-php7.3 = buster +php-amqplib = buster +php7.3 = buster +php7.3-bcmath = buster +php7.3-curl = buster +php7.3-dev = buster +php7.3-gd = buster +php7.3-mbstring = buster +php7.3-pgsql = buster + +[installer] +lsb-release = buster, bionic +xmlstarlet = buster, bionic +; patch = centos + +[unknown] +# TODO: Triage all these package to know whether they are still +# used or if they are installed by other packages. +build-essential = bionic +coreutils = buster, bionic +ecasound = buster, bionic +flac = buster, bionic +lame = buster, bionic +libffi-dev = bionic +libpq-dev = buster, bionic # WHY ? header files for libpq5 (PostgreSQL library) +libssl-dev = bionic +mpg123 = buster, bionic +pwgen = buster, bionic +systemd-sysv = buster +sysvinit-utils = bionic diff --git a/python_apps/airtime_analyzer/packages.ini b/python_apps/airtime_analyzer/packages.ini new file mode 100644 index 000000000..4c06ff3a1 --- /dev/null +++ b/python_apps/airtime_analyzer/packages.ini @@ -0,0 +1,46 @@ +# This file contains a list of package dependencies. +[common] +python3 = buster, bionic +python3-pip = buster, bionic +python3-virtualenv = buster, bionic + +[liquidsoap] +# https://github.com/savonet/liquidsoap/blob/main/CHANGES.md +liquidsoap-plugin-alsa = bionic +liquidsoap-plugin-ao = bionic +liquidsoap-plugin-ogg = bionic +liquidsoap-plugin-portaudio = bionic +# Already recommended packages in bionic +# See `apt show liquidsoap` +; liquidsoap-plugin-faad = bionic +; liquidsoap-plugin-flac = bionic +; liquidsoap-plugin-icecast = bionic +; liquidsoap-plugin-lame = bionic +; liquidsoap-plugin-mad = bionic +; liquidsoap-plugin-pulseaudio = bionic +; liquidsoap-plugin-taglib = bionic +; liquidsoap-plugin-voaacenc = bionic +; liquidsoap-plugin-vorbis = bionic +liquidsoap = buster, bionic + +[pika] +python3-pika = buster, bionic + +[rgain3] +gcc = buster, bionic +gir1.2-gtk-3.0 = buster, bionic +gstreamer1.0-plugins-bad = buster, bionic +gstreamer1.0-plugins-good = buster, bionic +gstreamer1.0-plugins-ugly = buster, bionic +libcairo2-dev = buster, bionic +libgirepository1.0-dev = buster, bionic +libglib2.0-dev = buster, bionic +pkg-config = buster, bionic +python3-cairo = buster, bionic +python3-dev = buster, bionic +python3-gi = buster, bionic +python3-gi-cairo = buster, bionic +python3-gst-1.0 = buster, bionic + +[silan] +silan = buster, bionic diff --git a/python_apps/pypo/packages.ini b/python_apps/pypo/packages.ini new file mode 100644 index 000000000..9280452ac --- /dev/null +++ b/python_apps/pypo/packages.ini @@ -0,0 +1,29 @@ +# This file contains a list of package dependencies. +[common] +python3 = buster, bionic +python3-pip = buster, bionic +python3-virtualenv = buster, bionic + +[liquidsoap] +# https://github.com/savonet/liquidsoap/blob/main/CHANGES.md +liquidsoap-plugin-alsa = bionic +liquidsoap-plugin-ao = bionic +liquidsoap-plugin-ogg = bionic +liquidsoap-plugin-portaudio = bionic +# Already recommended packages in bionic +# See `apt show liquidsoap` +; liquidsoap-plugin-faad = bionic +; liquidsoap-plugin-flac = bionic +; liquidsoap-plugin-icecast = bionic +; liquidsoap-plugin-lame = bionic +; liquidsoap-plugin-mad = bionic +; liquidsoap-plugin-pulseaudio = bionic +; liquidsoap-plugin-taglib = bionic +; liquidsoap-plugin-voaacenc = bionic +; liquidsoap-plugin-vorbis = bionic +liquidsoap = buster, bionic + +[misc] +# Used by pypofetch to check if a file is open. +# TODO: consider using a python library +lsof = buster, bionic diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/packages.py b/scripts/packages.py new file mode 100755 index 000000000..e6a235d99 --- /dev/null +++ b/scripts/packages.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +import json +import sys +from argparse import ArgumentParser +from configparser import ConfigParser +from os import PathLike +from pathlib import Path +from typing import Iterator, Set + +DEFAULT_PACKAGES_FILENAME = "packages.ini" +FORMATS = ("list", "line") +SYSTEMS = ("buster", "bionic") + +SETTINGS_SECTION = "=settings" +DEVELOPMENT_SECTION = "=development" + + +def load_packages(raw: str, distribution: str, development: bool = False) -> Set[str]: + manager = ConfigParser(default_section=SETTINGS_SECTION) + manager.read_string(raw) + + packages = set() + for section, entries in manager.items(): + if not development and section == DEVELOPMENT_SECTION: + continue + + for package, distributions in entries.items(): + if distribution in distributions.split(", "): + packages.add(package) + + return packages + + +def list_packages_files(paths: str) -> Iterator[Path]: + for path in paths: + path = Path(path) + + if path.is_dir(): + path = path / DEFAULT_PACKAGES_FILENAME + + if not path.is_file(): + raise Exception(f"{path} is not a file!") + + yield path + + +def list_packages(paths: str, distribution: str, development: bool = False) -> Set[str]: + packages = set() + for package_file in list_packages_files(paths): + raw = package_file.read_text() + packages.update(load_packages(raw, distribution, development)) + + return set(sorted(packages)) + + +def run(): + parser = ArgumentParser() + parser.add_argument( + "-f", + "--format", + choices=FORMATS, + help="print packages list in a specific format.", + default="list", + ), + parser.add_argument( + "-d", + "--dev", + help="include development packages.", + action="store_true", + ) + parser.add_argument( + "distribution", + choices=SYSTEMS, + help="list packages for the given distribution.", + ) + parser.add_argument( + "path", + nargs="+", + help="list packages from given files or directories.", + ) + args = parser.parse_args() + + packages = list_packages(args.path, args.distribution, args.dev) + + if args.format == "list": + print("\n".join(packages)) + else: + print(" ".join(packages)) + + +if __name__ == "__main__": + run() diff --git a/scripts/packages_test.py b/scripts/packages_test.py new file mode 100644 index 000000000..bdd9edb06 --- /dev/null +++ b/scripts/packages_test.py @@ -0,0 +1,32 @@ +from pathlib import Path + +from .packages import list_packages, load_packages + +package_ini = """ +[common] +postgresql = buster +# Some comment +curl = buster, bionic + +[legacy] +apache2 = bionic + +[=development] +ffmpeg = buster, bionic +""" + +result1 = {"curl", "postgresql"} +result2 = {"apache2", "curl", "ffmpeg"} + + +def test_load_packages(): + assert load_packages(package_ini, "buster", False) == result1 + assert load_packages(package_ini, "bionic", True) == result2 + + +def test_list_packages(tmp_path: Path): + package_file = tmp_path / "packages.ini" + package_file.write_text(package_ini) + + assert list_packages([tmp_path, package_file], "buster", False) == result1 + assert list_packages([tmp_path, package_file], "bionic", True) == result2