From 3e4968244dbff089571e61fd5f7bca9a23895b5e Mon Sep 17 00:00:00 2001 From: jo Date: Fri, 4 Jun 2021 13:24:12 +0200 Subject: [PATCH 01/24] Sort apt requirements files --- installer/apt/requirements-debian-buster.apt | 18 +++++++++--------- installer/apt/requirements-ubuntu-bionic.apt | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/installer/apt/requirements-debian-buster.apt b/installer/apt/requirements-debian-buster.apt index d1b30038c..8275078ec 100644 --- a/installer/apt/requirements-debian-buster.apt +++ b/installer/apt/requirements-debian-buster.apt @@ -3,9 +3,9 @@ coreutils curl ecasound flac -git gcc gir1.2-gtk-3.0 +git gstreamer1.0-plugins-bad gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly @@ -16,13 +16,13 @@ libapache2-mod-php7.3 libcairo2-dev libcamomile-ocaml-data libfaad2 -libglib2.0-dev libgirepository1.0-dev +libglib2.0-dev libmad-ocaml libopus0 libportaudio2 -libpulse0 libpq-dev +libpulse0 libsamplerate0 libsoundtouch-ocaml libtaglib-ocaml @@ -33,6 +33,9 @@ lsb-release lsof mpg123 patch +php-amqplib +php-apcu +php-pear php7.3 php7.3-bcmath php7.3-curl @@ -40,14 +43,12 @@ php7.3-dev php7.3-gd php7.3-mbstring php7.3-pgsql -php-amqplib -php-apcu -php-pear pkg-config postgresql postgresql-client pwgen python3 +python3-cairo python3-dev python3-gi python3-gi-cairo @@ -55,15 +56,14 @@ python3-gst-1.0 python3-pika python3-pip python3-virtualenv -python3-cairo rabbitmq-server silan systemd-sysv unzip uwsgi uwsgi-plugin-python3 +vorbis-tools +vorbis-tools vorbisgain -vorbis-tools -vorbis-tools xmlstarlet zip diff --git a/installer/apt/requirements-ubuntu-bionic.apt b/installer/apt/requirements-ubuntu-bionic.apt index 05ca6b90f..c732d2ee8 100644 --- a/installer/apt/requirements-ubuntu-bionic.apt +++ b/installer/apt/requirements-ubuntu-bionic.apt @@ -22,8 +22,8 @@ libglib2.0-dev libmad-ocaml libopus0 libportaudio2 -libpulse0 libpq-dev +libpulse0 libsamplerate0 libsoundtouch-ocaml libssl-dev @@ -46,22 +46,22 @@ lsb-release lsof mpg123 patch -php7.2 -php7.2-curl -php7.2-gd -php7.2-pgsql php-apcu php-bcmath php-mbstring php-pear +php7.2 +php7.2-curl +php7.2-gd +php7.2-pgsql pkg-config postgresql postgresql-client pwgen python3 -python3-dev python3-cairo python3-dev +python3-dev python3-gi python3-gi-cairo python3-gst-1.0 @@ -74,7 +74,7 @@ sysvinit-utils unzip uwsgi uwsgi-plugin-python3 -vorbisgain vorbis-tools +vorbisgain xmlstarlet zip From 854663e51dd2c3111f15a60961395f8aac13510f Mon Sep 17 00:00:00 2001 From: jo Date: Sun, 6 Jun 2021 03:07:18 +0200 Subject: [PATCH 02/24] Remove duplicate entries --- installer/apt/requirements-debian-buster.apt | 2 -- installer/apt/requirements-ubuntu-bionic.apt | 1 - 2 files changed, 3 deletions(-) diff --git a/installer/apt/requirements-debian-buster.apt b/installer/apt/requirements-debian-buster.apt index 8275078ec..daf7cd847 100644 --- a/installer/apt/requirements-debian-buster.apt +++ b/installer/apt/requirements-debian-buster.apt @@ -29,7 +29,6 @@ libtaglib-ocaml libvo-aacenc0 liquidsoap lsb-release -lsb-release lsof mpg123 patch @@ -63,7 +62,6 @@ unzip uwsgi uwsgi-plugin-python3 vorbis-tools -vorbis-tools vorbisgain xmlstarlet zip diff --git a/installer/apt/requirements-ubuntu-bionic.apt b/installer/apt/requirements-ubuntu-bionic.apt index c732d2ee8..5c315cfd4 100644 --- a/installer/apt/requirements-ubuntu-bionic.apt +++ b/installer/apt/requirements-ubuntu-bionic.apt @@ -61,7 +61,6 @@ pwgen python3 python3-cairo python3-dev -python3-dev python3-gi python3-gi-cairo python3-gst-1.0 From f2b9391a2299c24927b393dd71a2ebd388b232b6 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 17:51:16 +0200 Subject: [PATCH 03/24] Remove unused packages from apt requirements --- installer/apt/requirements-debian-buster.apt | 2 -- installer/apt/requirements-ubuntu-bionic.apt | 1 - 2 files changed, 3 deletions(-) diff --git a/installer/apt/requirements-debian-buster.apt b/installer/apt/requirements-debian-buster.apt index daf7cd847..4143fb843 100644 --- a/installer/apt/requirements-debian-buster.apt +++ b/installer/apt/requirements-debian-buster.apt @@ -5,7 +5,6 @@ ecasound flac gcc gir1.2-gtk-3.0 -git gstreamer1.0-plugins-bad gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly @@ -31,7 +30,6 @@ liquidsoap lsb-release lsof mpg123 -patch php-amqplib php-apcu php-pear diff --git a/installer/apt/requirements-ubuntu-bionic.apt b/installer/apt/requirements-ubuntu-bionic.apt index 5c315cfd4..e95691d68 100644 --- a/installer/apt/requirements-ubuntu-bionic.apt +++ b/installer/apt/requirements-ubuntu-bionic.apt @@ -45,7 +45,6 @@ liquidsoap-plugin-vorbis lsb-release lsof mpg123 -patch php-apcu php-bcmath php-mbstring From 30611f1d807970dde4a8d0a181099296de9e9709 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 18:39:06 +0200 Subject: [PATCH 04/24] Remove already recommended packages by liquidsoap Many packages are already in the recommends list of the liquidsoap package or from the plugin packages (only bionic), so we don't need to install them manually. --- installer/apt/requirements-debian-buster.apt | 13 ------------ installer/apt/requirements-ubuntu-bionic.apt | 22 -------------------- 2 files changed, 35 deletions(-) diff --git a/installer/apt/requirements-debian-buster.apt b/installer/apt/requirements-debian-buster.apt index 4143fb843..1676c1861 100644 --- a/installer/apt/requirements-debian-buster.apt +++ b/installer/apt/requirements-debian-buster.apt @@ -10,22 +10,11 @@ gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 lame -libao-ocaml libapache2-mod-php7.3 libcairo2-dev -libcamomile-ocaml-data -libfaad2 libgirepository1.0-dev libglib2.0-dev -libmad-ocaml -libopus0 -libportaudio2 libpq-dev -libpulse0 -libsamplerate0 -libsoundtouch-ocaml -libtaglib-ocaml -libvo-aacenc0 liquidsoap lsb-release lsof @@ -59,7 +48,5 @@ systemd-sysv unzip uwsgi uwsgi-plugin-python3 -vorbis-tools -vorbisgain xmlstarlet zip diff --git a/installer/apt/requirements-ubuntu-bionic.apt b/installer/apt/requirements-ubuntu-bionic.apt index e95691d68..209fd9dd3 100644 --- a/installer/apt/requirements-ubuntu-bionic.apt +++ b/installer/apt/requirements-ubuntu-bionic.apt @@ -11,37 +11,17 @@ gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 lame -libao-ocaml libapache2-mod-php7.2 libcairo2-dev -libcamomile-ocaml-data -libfaad2 libffi-dev libgirepository1.0-dev libglib2.0-dev -libmad-ocaml -libopus0 -libportaudio2 -libpq-dev -libpulse0 -libsamplerate0 -libsoundtouch-ocaml libssl-dev -libtaglib-ocaml liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao -liquidsoap-plugin-faad -liquidsoap-plugin-flac -liquidsoap-plugin-icecast -liquidsoap-plugin-lame -liquidsoap-plugin-mad liquidsoap-plugin-ogg liquidsoap-plugin-portaudio -liquidsoap-plugin-pulseaudio -liquidsoap-plugin-taglib -liquidsoap-plugin-voaacenc -liquidsoap-plugin-vorbis lsb-release lsof mpg123 @@ -72,7 +52,5 @@ sysvinit-utils unzip uwsgi uwsgi-plugin-python3 -vorbis-tools -vorbisgain xmlstarlet zip From 18f9c81b746bd4d44b4718414323fe2fe2fc96d1 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 18:48:25 +0200 Subject: [PATCH 05/24] Add dependencies manager Used to generate list of packages dependencies for different distributions, and should improve dependencies management by having them in each project. This tools should gather the reason why a dependency is present, so one can remove it when unused. Rename deps script to packages Move installer/packages to scripts and write tests Remove packages-list make target --- api/packages.ini | 9 +++ packages.ini | 54 +++++++++++++ python_apps/airtime_analyzer/packages.ini | 46 +++++++++++ python_apps/pypo/packages.ini | 29 +++++++ scripts/__init__.py | 0 scripts/packages.py | 93 +++++++++++++++++++++++ scripts/packages_test.py | 32 ++++++++ 7 files changed, 263 insertions(+) create mode 100644 api/packages.ini create mode 100644 packages.ini create mode 100644 python_apps/airtime_analyzer/packages.ini create mode 100644 python_apps/pypo/packages.ini create mode 100644 scripts/__init__.py create mode 100755 scripts/packages.py create mode 100644 scripts/packages_test.py diff --git a/api/packages.ini b/api/packages.ini new file mode 100644 index 000000000..def9cd21e --- /dev/null +++ b/api/packages.ini @@ -0,0 +1,9 @@ +# This file contains a list of package dependencies. +[common] +python3 = buster, bionic +python3-pip = buster, bionic +python3-virtualenv = buster, bionic + +[django] +uwsgi = buster, bionic +uwsgi-plugin-python3 = buster, bionic diff --git a/packages.ini b/packages.ini new file mode 100644 index 000000000..24bd455c7 --- /dev/null +++ b/packages.ini @@ -0,0 +1,54 @@ +# This file contains a list of package dependencies. +[common] +icecast2 = buster, bionic +postgresql = buster, bionic +postgresql-client = buster, bionic +rabbitmq-server = buster, bionic + +curl = buster, bionic +unzip = buster, bionic +zip = buster, bionic + +[legacy] +apache2 = buster, bionic +php-apcu = buster, bionic +php-pear = buster, bionic +# Bionic +libapache2-mod-php7.2 = bionic +php-bcmath = bionic +php-mbstring = bionic +php7.2 = bionic +php7.2-curl = bionic +php7.2-gd = bionic +php7.2-pgsql = bionic +# Buster +libapache2-mod-php7.3 = buster +php-amqplib = buster +php7.3 = buster +php7.3-bcmath = buster +php7.3-curl = buster +php7.3-dev = buster +php7.3-gd = buster +php7.3-mbstring = buster +php7.3-pgsql = buster + +[installer] +lsb-release = buster, bionic +xmlstarlet = buster, bionic +; patch = centos + +[unknown] +# TODO: Triage all these package to know whether they are still +# used or if they are installed by other packages. +build-essential = bionic +coreutils = buster, bionic +ecasound = buster, bionic +flac = buster, bionic +lame = buster, bionic +libffi-dev = bionic +libpq-dev = buster, bionic # WHY ? header files for libpq5 (PostgreSQL library) +libssl-dev = bionic +mpg123 = buster, bionic +pwgen = buster, bionic +systemd-sysv = buster +sysvinit-utils = bionic diff --git a/python_apps/airtime_analyzer/packages.ini b/python_apps/airtime_analyzer/packages.ini new file mode 100644 index 000000000..4c06ff3a1 --- /dev/null +++ b/python_apps/airtime_analyzer/packages.ini @@ -0,0 +1,46 @@ +# This file contains a list of package dependencies. +[common] +python3 = buster, bionic +python3-pip = buster, bionic +python3-virtualenv = buster, bionic + +[liquidsoap] +# https://github.com/savonet/liquidsoap/blob/main/CHANGES.md +liquidsoap-plugin-alsa = bionic +liquidsoap-plugin-ao = bionic +liquidsoap-plugin-ogg = bionic +liquidsoap-plugin-portaudio = bionic +# Already recommended packages in bionic +# See `apt show liquidsoap` +; liquidsoap-plugin-faad = bionic +; liquidsoap-plugin-flac = bionic +; liquidsoap-plugin-icecast = bionic +; liquidsoap-plugin-lame = bionic +; liquidsoap-plugin-mad = bionic +; liquidsoap-plugin-pulseaudio = bionic +; liquidsoap-plugin-taglib = bionic +; liquidsoap-plugin-voaacenc = bionic +; liquidsoap-plugin-vorbis = bionic +liquidsoap = buster, bionic + +[pika] +python3-pika = buster, bionic + +[rgain3] +gcc = buster, bionic +gir1.2-gtk-3.0 = buster, bionic +gstreamer1.0-plugins-bad = buster, bionic +gstreamer1.0-plugins-good = buster, bionic +gstreamer1.0-plugins-ugly = buster, bionic +libcairo2-dev = buster, bionic +libgirepository1.0-dev = buster, bionic +libglib2.0-dev = buster, bionic +pkg-config = buster, bionic +python3-cairo = buster, bionic +python3-dev = buster, bionic +python3-gi = buster, bionic +python3-gi-cairo = buster, bionic +python3-gst-1.0 = buster, bionic + +[silan] +silan = buster, bionic diff --git a/python_apps/pypo/packages.ini b/python_apps/pypo/packages.ini new file mode 100644 index 000000000..9280452ac --- /dev/null +++ b/python_apps/pypo/packages.ini @@ -0,0 +1,29 @@ +# This file contains a list of package dependencies. +[common] +python3 = buster, bionic +python3-pip = buster, bionic +python3-virtualenv = buster, bionic + +[liquidsoap] +# https://github.com/savonet/liquidsoap/blob/main/CHANGES.md +liquidsoap-plugin-alsa = bionic +liquidsoap-plugin-ao = bionic +liquidsoap-plugin-ogg = bionic +liquidsoap-plugin-portaudio = bionic +# Already recommended packages in bionic +# See `apt show liquidsoap` +; liquidsoap-plugin-faad = bionic +; liquidsoap-plugin-flac = bionic +; liquidsoap-plugin-icecast = bionic +; liquidsoap-plugin-lame = bionic +; liquidsoap-plugin-mad = bionic +; liquidsoap-plugin-pulseaudio = bionic +; liquidsoap-plugin-taglib = bionic +; liquidsoap-plugin-voaacenc = bionic +; liquidsoap-plugin-vorbis = bionic +liquidsoap = buster, bionic + +[misc] +# Used by pypofetch to check if a file is open. +# TODO: consider using a python library +lsof = buster, bionic diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/packages.py b/scripts/packages.py new file mode 100755 index 000000000..e6a235d99 --- /dev/null +++ b/scripts/packages.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +import json +import sys +from argparse import ArgumentParser +from configparser import ConfigParser +from os import PathLike +from pathlib import Path +from typing import Iterator, Set + +DEFAULT_PACKAGES_FILENAME = "packages.ini" +FORMATS = ("list", "line") +SYSTEMS = ("buster", "bionic") + +SETTINGS_SECTION = "=settings" +DEVELOPMENT_SECTION = "=development" + + +def load_packages(raw: str, distribution: str, development: bool = False) -> Set[str]: + manager = ConfigParser(default_section=SETTINGS_SECTION) + manager.read_string(raw) + + packages = set() + for section, entries in manager.items(): + if not development and section == DEVELOPMENT_SECTION: + continue + + for package, distributions in entries.items(): + if distribution in distributions.split(", "): + packages.add(package) + + return packages + + +def list_packages_files(paths: str) -> Iterator[Path]: + for path in paths: + path = Path(path) + + if path.is_dir(): + path = path / DEFAULT_PACKAGES_FILENAME + + if not path.is_file(): + raise Exception(f"{path} is not a file!") + + yield path + + +def list_packages(paths: str, distribution: str, development: bool = False) -> Set[str]: + packages = set() + for package_file in list_packages_files(paths): + raw = package_file.read_text() + packages.update(load_packages(raw, distribution, development)) + + return set(sorted(packages)) + + +def run(): + parser = ArgumentParser() + parser.add_argument( + "-f", + "--format", + choices=FORMATS, + help="print packages list in a specific format.", + default="list", + ), + parser.add_argument( + "-d", + "--dev", + help="include development packages.", + action="store_true", + ) + parser.add_argument( + "distribution", + choices=SYSTEMS, + help="list packages for the given distribution.", + ) + parser.add_argument( + "path", + nargs="+", + help="list packages from given files or directories.", + ) + args = parser.parse_args() + + packages = list_packages(args.path, args.distribution, args.dev) + + if args.format == "list": + print("\n".join(packages)) + else: + print(" ".join(packages)) + + +if __name__ == "__main__": + run() diff --git a/scripts/packages_test.py b/scripts/packages_test.py new file mode 100644 index 000000000..bdd9edb06 --- /dev/null +++ b/scripts/packages_test.py @@ -0,0 +1,32 @@ +from pathlib import Path + +from .packages import list_packages, load_packages + +package_ini = """ +[common] +postgresql = buster +# Some comment +curl = buster, bionic + +[legacy] +apache2 = bionic + +[=development] +ffmpeg = buster, bionic +""" + +result1 = {"curl", "postgresql"} +result2 = {"apache2", "curl", "ffmpeg"} + + +def test_load_packages(): + assert load_packages(package_ini, "buster", False) == result1 + assert load_packages(package_ini, "bionic", True) == result2 + + +def test_list_packages(tmp_path: Path): + package_file = tmp_path / "packages.ini" + package_file.write_text(package_ini) + + assert list_packages([tmp_path, package_file], "buster", False) == result1 + assert list_packages([tmp_path, package_file], "bionic", True) == result2 From bfd8e8aecf465be15250b1cbc0f0d716725a8d6b Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 18:53:38 +0200 Subject: [PATCH 06/24] ecasound is used by the show recorder in pypo --- packages.ini | 1 - python_apps/pypo/packages.ini | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages.ini b/packages.ini index 24bd455c7..c4e3a9b01 100644 --- a/packages.ini +++ b/packages.ini @@ -42,7 +42,6 @@ xmlstarlet = buster, bionic # used or if they are installed by other packages. build-essential = bionic coreutils = buster, bionic -ecasound = buster, bionic flac = buster, bionic lame = buster, bionic libffi-dev = bionic diff --git a/python_apps/pypo/packages.ini b/python_apps/pypo/packages.ini index 9280452ac..fc82348c5 100644 --- a/python_apps/pypo/packages.ini +++ b/python_apps/pypo/packages.ini @@ -23,6 +23,9 @@ liquidsoap-plugin-portaudio = bionic ; liquidsoap-plugin-vorbis = bionic liquidsoap = buster, bionic +[recorder] +ecasound = buster, bionic + [misc] # Used by pypofetch to check if a file is open. # TODO: consider using a python library From 1cfd1cd0137b95562043ae8f9018d0efe78696bc Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 18:56:27 +0200 Subject: [PATCH 07/24] Add comment about lame --- packages.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.ini b/packages.ini index c4e3a9b01..8788f6f73 100644 --- a/packages.ini +++ b/packages.ini @@ -43,7 +43,7 @@ xmlstarlet = buster, bionic build-essential = bionic coreutils = buster, bionic flac = buster, bionic -lame = buster, bionic +lame = buster, bionic # Maybe liquidsoap-plugin-lame, but it uses a library not the lame binary libffi-dev = bionic libpq-dev = buster, bionic # WHY ? header files for libpq5 (PostgreSQL library) libssl-dev = bionic From 5b5505d42aa94e639edf41e0784176694cc0edcc Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 18:57:56 +0200 Subject: [PATCH 08/24] Add comment about build-essential --- packages.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.ini b/packages.ini index 8788f6f73..f939ee4a3 100644 --- a/packages.ini +++ b/packages.ini @@ -40,7 +40,7 @@ xmlstarlet = buster, bionic [unknown] # TODO: Triage all these package to know whether they are still # used or if they are installed by other packages. -build-essential = bionic +build-essential = bionic # Maybe related to libffi, libpq and libssl below. coreutils = buster, bionic flac = buster, bionic lame = buster, bionic # Maybe liquidsoap-plugin-lame, but it uses a library not the lame binary From 60c5c5fbb2484f3466f9425c30f582495a28cbf8 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 18:59:49 +0200 Subject: [PATCH 09/24] Move icecast from project to pypo packages --- packages.ini | 1 - python_apps/pypo/packages.ini | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages.ini b/packages.ini index f939ee4a3..8f35f2aa5 100644 --- a/packages.ini +++ b/packages.ini @@ -1,6 +1,5 @@ # This file contains a list of package dependencies. [common] -icecast2 = buster, bionic postgresql = buster, bionic postgresql-client = buster, bionic rabbitmq-server = buster, bionic diff --git a/python_apps/pypo/packages.ini b/python_apps/pypo/packages.ini index fc82348c5..fdd20b361 100644 --- a/python_apps/pypo/packages.ini +++ b/python_apps/pypo/packages.ini @@ -4,6 +4,9 @@ python3 = buster, bionic python3-pip = buster, bionic python3-virtualenv = buster, bionic +[icecast] +icecast2 = buster, bionic + [liquidsoap] # https://github.com/savonet/liquidsoap/blob/main/CHANGES.md liquidsoap-plugin-alsa = bionic From 62938c3a83b03f8358eccc2c9f9235bb97a765d6 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 19:06:00 +0200 Subject: [PATCH 10/24] Remove unused patch packages for centos --- packages.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/packages.ini b/packages.ini index 8f35f2aa5..b00e09e3d 100644 --- a/packages.ini +++ b/packages.ini @@ -34,7 +34,6 @@ php7.3-pgsql = buster [installer] lsb-release = buster, bionic xmlstarlet = buster, bionic -; patch = centos [unknown] # TODO: Triage all these package to know whether they are still From 01141606fd0fd851c3af4fd773f973c190cabdb3 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 19:28:14 +0200 Subject: [PATCH 11/24] libpq-dev is required for psycopg2 in production This is a package for the API. --- api/packages.ini | 5 +++++ packages.ini | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/packages.ini b/api/packages.ini index def9cd21e..467cf0f53 100644 --- a/api/packages.ini +++ b/api/packages.ini @@ -7,3 +7,8 @@ python3-virtualenv = buster, bionic [django] uwsgi = buster, bionic uwsgi-plugin-python3 = buster, bionic + +[psycopg2] +# See https://www.psycopg.org/docs/install.html#install-from-source +python3-dev = buster, bionic +libpq-dev = buster, bionic diff --git a/packages.ini b/packages.ini index b00e09e3d..f9ee081da 100644 --- a/packages.ini +++ b/packages.ini @@ -43,7 +43,6 @@ coreutils = buster, bionic flac = buster, bionic lame = buster, bionic # Maybe liquidsoap-plugin-lame, but it uses a library not the lame binary libffi-dev = bionic -libpq-dev = buster, bionic # WHY ? header files for libpq5 (PostgreSQL library) libssl-dev = bionic mpg123 = buster, bionic pwgen = buster, bionic From b1cffc7b7dd3979d5773134e63ca139aa7f9ea00 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 19:28:39 +0200 Subject: [PATCH 12/24] Add comment for zip/unzip packages --- packages.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages.ini b/packages.ini index f9ee081da..1dbda7043 100644 --- a/packages.ini +++ b/packages.ini @@ -5,6 +5,8 @@ postgresql-client = buster, bionic rabbitmq-server = buster, bionic curl = buster, bionic + +# See https://github.com/Libretime/libretime/commits/71ce3082c5e0611f77eb423a15b4b36267ebfda4/installer/lib unzip = buster, bionic zip = buster, bionic From 154ee7cc22cef6075efb57e188143a1cb401be00 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 19:29:36 +0200 Subject: [PATCH 13/24] Remove packages used for a workaroung on ubuntu 14 See https://github.com/LibreTime/libretime/commit/2ae2377ee29b1d4c28aab904f625605779a1442c#diff-0991ed0aaae32d2b0c9723cd9127f95bc19d176a5fd668d9b331f487136318fe --- packages.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages.ini b/packages.ini index 1dbda7043..93257dd2a 100644 --- a/packages.ini +++ b/packages.ini @@ -40,12 +40,9 @@ xmlstarlet = buster, bionic [unknown] # TODO: Triage all these package to know whether they are still # used or if they are installed by other packages. -build-essential = bionic # Maybe related to libffi, libpq and libssl below. coreutils = buster, bionic flac = buster, bionic lame = buster, bionic # Maybe liquidsoap-plugin-lame, but it uses a library not the lame binary -libffi-dev = bionic -libssl-dev = bionic mpg123 = buster, bionic pwgen = buster, bionic systemd-sysv = buster From 34e836c746e87e32446222d182f9d7ffc2d85f7c Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 19:48:45 +0200 Subject: [PATCH 14/24] pwgen package is not used anymore See https://github.com/LibreTime/libretime/commit/989f040603994dd60aa53766c8a6204e9560aed0 --- packages.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.ini b/packages.ini index 93257dd2a..1cae83de3 100644 --- a/packages.ini +++ b/packages.ini @@ -44,6 +44,6 @@ coreutils = buster, bionic flac = buster, bionic lame = buster, bionic # Maybe liquidsoap-plugin-lame, but it uses a library not the lame binary mpg123 = buster, bionic -pwgen = buster, bionic + systemd-sysv = buster sysvinit-utils = bionic From 7080aa4758a57e3581eae332f1b104dd412c0e2f Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 19:49:07 +0200 Subject: [PATCH 15/24] Add comment for sysv related packages --- packages.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/packages.ini b/packages.ini index 1cae83de3..502defdcc 100644 --- a/packages.ini +++ b/packages.ini @@ -45,5 +45,6 @@ flac = buster, bionic lame = buster, bionic # Maybe liquidsoap-plugin-lame, but it uses a library not the lame binary mpg123 = buster, bionic +# See https://github.com/LibreTime/libretime/commit/bbcd4fbe2ee05cdb2e3a62ad2b7549157a1cd950#diff-0991ed0aaae32d2b0c9723cd9127f95bc19d176a5fd668d9b331f487136318fe systemd-sysv = buster sysvinit-utils = bionic From 06547efec7f72319e873bdf243dda9ca4ee90fb9 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 20:18:27 +0200 Subject: [PATCH 16/24] Remove binaries not used by liquidsoap anymore --- packages.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages.ini b/packages.ini index 502defdcc..a42439389 100644 --- a/packages.ini +++ b/packages.ini @@ -41,9 +41,6 @@ xmlstarlet = buster, bionic # TODO: Triage all these package to know whether they are still # used or if they are installed by other packages. coreutils = buster, bionic -flac = buster, bionic -lame = buster, bionic # Maybe liquidsoap-plugin-lame, but it uses a library not the lame binary -mpg123 = buster, bionic # See https://github.com/LibreTime/libretime/commit/bbcd4fbe2ee05cdb2e3a62ad2b7549157a1cd950#diff-0991ed0aaae32d2b0c9723cd9127f95bc19d176a5fd668d9b331f487136318fe systemd-sysv = buster From ee4cf4268d3a87c19f37b211795c2052ecfe6a2d Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 20:19:31 +0200 Subject: [PATCH 17/24] Remove coreutils as most system ship it --- packages.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages.ini b/packages.ini index a42439389..e0e67b41f 100644 --- a/packages.ini +++ b/packages.ini @@ -38,10 +38,6 @@ lsb-release = buster, bionic xmlstarlet = buster, bionic [unknown] -# TODO: Triage all these package to know whether they are still -# used or if they are installed by other packages. -coreutils = buster, bionic - # See https://github.com/LibreTime/libretime/commit/bbcd4fbe2ee05cdb2e3a62ad2b7549157a1cd950#diff-0991ed0aaae32d2b0c9723cd9127f95bc19d176a5fd668d9b331f487136318fe systemd-sysv = buster sysvinit-utils = bionic From f5288109e5e56ae268974b4ccbab40357615a5cc Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 20:24:44 +0200 Subject: [PATCH 18/24] Generate installer/apt/ files Generate installer/apt/ files with our dependencies manager. --- installer/apt/requirements-debian-buster.apt | 5 ----- installer/apt/requirements-ubuntu-bionic.apt | 9 +-------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/installer/apt/requirements-debian-buster.apt b/installer/apt/requirements-debian-buster.apt index 1676c1861..d83f8053b 100644 --- a/installer/apt/requirements-debian-buster.apt +++ b/installer/apt/requirements-debian-buster.apt @@ -1,15 +1,12 @@ apache2 -coreutils curl ecasound -flac gcc gir1.2-gtk-3.0 gstreamer1.0-plugins-bad gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 -lame libapache2-mod-php7.3 libcairo2-dev libgirepository1.0-dev @@ -18,7 +15,6 @@ libpq-dev liquidsoap lsb-release lsof -mpg123 php-amqplib php-apcu php-pear @@ -32,7 +28,6 @@ php7.3-pgsql pkg-config postgresql postgresql-client -pwgen python3 python3-cairo python3-dev diff --git a/installer/apt/requirements-ubuntu-bionic.apt b/installer/apt/requirements-ubuntu-bionic.apt index 209fd9dd3..90b5a682b 100644 --- a/installer/apt/requirements-ubuntu-bionic.apt +++ b/installer/apt/requirements-ubuntu-bionic.apt @@ -1,22 +1,17 @@ apache2 -build-essential -coreutils curl ecasound -flac gcc gir1.2-gtk-3.0 gstreamer1.0-plugins-bad gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly icecast2 -lame libapache2-mod-php7.2 libcairo2-dev -libffi-dev libgirepository1.0-dev libglib2.0-dev -libssl-dev +libpq-dev liquidsoap liquidsoap-plugin-alsa liquidsoap-plugin-ao @@ -24,7 +19,6 @@ liquidsoap-plugin-ogg liquidsoap-plugin-portaudio lsb-release lsof -mpg123 php-apcu php-bcmath php-mbstring @@ -36,7 +30,6 @@ php7.2-pgsql pkg-config postgresql postgresql-client -pwgen python3 python3-cairo python3-dev From 3b546a05c649aa95974c93483b2c20cfe50c3e87 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 23:26:53 +0200 Subject: [PATCH 19/24] Remove install/apt folder --- installer/apt/requirements-debian-buster.apt | 47 ------------------- installer/apt/requirements-ubuntu-bionic.apt | 49 -------------------- 2 files changed, 96 deletions(-) delete mode 100644 installer/apt/requirements-debian-buster.apt delete mode 100644 installer/apt/requirements-ubuntu-bionic.apt diff --git a/installer/apt/requirements-debian-buster.apt b/installer/apt/requirements-debian-buster.apt deleted file mode 100644 index d83f8053b..000000000 --- a/installer/apt/requirements-debian-buster.apt +++ /dev/null @@ -1,47 +0,0 @@ -apache2 -curl -ecasound -gcc -gir1.2-gtk-3.0 -gstreamer1.0-plugins-bad -gstreamer1.0-plugins-good -gstreamer1.0-plugins-ugly -icecast2 -libapache2-mod-php7.3 -libcairo2-dev -libgirepository1.0-dev -libglib2.0-dev -libpq-dev -liquidsoap -lsb-release -lsof -php-amqplib -php-apcu -php-pear -php7.3 -php7.3-bcmath -php7.3-curl -php7.3-dev -php7.3-gd -php7.3-mbstring -php7.3-pgsql -pkg-config -postgresql -postgresql-client -python3 -python3-cairo -python3-dev -python3-gi -python3-gi-cairo -python3-gst-1.0 -python3-pika -python3-pip -python3-virtualenv -rabbitmq-server -silan -systemd-sysv -unzip -uwsgi -uwsgi-plugin-python3 -xmlstarlet -zip diff --git a/installer/apt/requirements-ubuntu-bionic.apt b/installer/apt/requirements-ubuntu-bionic.apt deleted file mode 100644 index 90b5a682b..000000000 --- a/installer/apt/requirements-ubuntu-bionic.apt +++ /dev/null @@ -1,49 +0,0 @@ -apache2 -curl -ecasound -gcc -gir1.2-gtk-3.0 -gstreamer1.0-plugins-bad -gstreamer1.0-plugins-good -gstreamer1.0-plugins-ugly -icecast2 -libapache2-mod-php7.2 -libcairo2-dev -libgirepository1.0-dev -libglib2.0-dev -libpq-dev -liquidsoap -liquidsoap-plugin-alsa -liquidsoap-plugin-ao -liquidsoap-plugin-ogg -liquidsoap-plugin-portaudio -lsb-release -lsof -php-apcu -php-bcmath -php-mbstring -php-pear -php7.2 -php7.2-curl -php7.2-gd -php7.2-pgsql -pkg-config -postgresql -postgresql-client -python3 -python3-cairo -python3-dev -python3-gi -python3-gi-cairo -python3-gst-1.0 -python3-pika -python3-pip -python3-virtualenv -rabbitmq-server -silan -sysvinit-utils -unzip -uwsgi -uwsgi-plugin-python3 -xmlstarlet -zip From b5c9a6b2c7428a94db148a72f705fd8b5128add9 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 17 Aug 2021 23:11:04 +0200 Subject: [PATCH 20/24] Use packages manager in install script --- install | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/install b/install index b1c7fa9db..2447669e0 100755 --- a/install +++ b/install @@ -739,16 +739,23 @@ if [ "$ignore_dependencies" = "f" ]; then fi if [ -x /usr/bin/apt-get ]; then - verbose "\n * Reading requirements-${dist}-${code}.apt..." + verbose "\n * Reading packages.ini files..." + + packages_files=( + "${SCRIPT_DIR}/" + "${SCRIPT_DIR}/api" + "${SCRIPT_DIR}/python_apps/airtime_analyzer" + "${SCRIPT_DIR}/python_apps/pypo" + ) + + set -e + package_list=$( + "${SCRIPT_DIR}/installer/packages.py" --format=line "${code}" "${packages_files[@]}" || + (echo "ERROR: could not generate packages list" >&2 && exit 1) + ) + set +e + loudCmd "apt-get -q update" - package_list_file="${SCRIPT_DIR}/installer/apt/requirements-${dist}-${code}.apt" - - if [ ! -f "$package_list_file" ]; then - echo "ERROR: package file does not exist: $package_list_file" >&2 - exit 1 - fi - - package_list="$(grep -vE '^\s*#' "$package_list_file" | tr '\n' ' ')" loudCmd "DEBIAN_FRONTEND=noninteractive apt-get -y install $package_list" [[ "$in_place" == "t" ]] && loudCmd "DEBIAN_FRONTEND=noninteractive apt-get -y install git" else From 42acf124a01df2701658d636fb57c08246ff6ad2 Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 31 Aug 2021 11:40:12 +0200 Subject: [PATCH 21/24] Revert "Move icecast from project to pypo packages" This reverts commit a6cca16e579aecd4540df228c9e288d4f4f3534a. --- packages.ini | 1 + python_apps/pypo/packages.ini | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages.ini b/packages.ini index e0e67b41f..f48f46a04 100644 --- a/packages.ini +++ b/packages.ini @@ -1,5 +1,6 @@ # This file contains a list of package dependencies. [common] +icecast2 = buster, bionic postgresql = buster, bionic postgresql-client = buster, bionic rabbitmq-server = buster, bionic diff --git a/python_apps/pypo/packages.ini b/python_apps/pypo/packages.ini index fdd20b361..fc82348c5 100644 --- a/python_apps/pypo/packages.ini +++ b/python_apps/pypo/packages.ini @@ -4,9 +4,6 @@ python3 = buster, bionic python3-pip = buster, bionic python3-virtualenv = buster, bionic -[icecast] -icecast2 = buster, bionic - [liquidsoap] # https://github.com/savonet/liquidsoap/blob/main/CHANGES.md liquidsoap-plugin-alsa = bionic From fb3ee2d27ae6ae30364ce9e8d01c8dfcf3d05ffe Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 31 Aug 2021 15:06:27 +0200 Subject: [PATCH 22/24] Install venv instead of virtualenv --- api/packages.ini | 2 +- python_apps/airtime_analyzer/packages.ini | 2 +- python_apps/pypo/packages.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/packages.ini b/api/packages.ini index 467cf0f53..ebfdf9585 100644 --- a/api/packages.ini +++ b/api/packages.ini @@ -2,7 +2,7 @@ [common] python3 = buster, bionic python3-pip = buster, bionic -python3-virtualenv = buster, bionic +python3-venv = buster, bionic [django] uwsgi = buster, bionic diff --git a/python_apps/airtime_analyzer/packages.ini b/python_apps/airtime_analyzer/packages.ini index 4c06ff3a1..681dac421 100644 --- a/python_apps/airtime_analyzer/packages.ini +++ b/python_apps/airtime_analyzer/packages.ini @@ -2,7 +2,7 @@ [common] python3 = buster, bionic python3-pip = buster, bionic -python3-virtualenv = buster, bionic +python3-venv = buster, bionic [liquidsoap] # https://github.com/savonet/liquidsoap/blob/main/CHANGES.md diff --git a/python_apps/pypo/packages.ini b/python_apps/pypo/packages.ini index fc82348c5..0f7aafad7 100644 --- a/python_apps/pypo/packages.ini +++ b/python_apps/pypo/packages.ini @@ -2,7 +2,7 @@ [common] python3 = buster, bionic python3-pip = buster, bionic -python3-virtualenv = buster, bionic +python3-venv = buster, bionic [liquidsoap] # https://github.com/savonet/liquidsoap/blob/main/CHANGES.md From 4bd76f096ca731f7b492ddc0d1453dc7f88c5161 Mon Sep 17 00:00:00 2001 From: jo Date: Wed, 1 Sep 2021 15:33:48 +0200 Subject: [PATCH 23/24] Fix packages path in installer --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 2447669e0..f578b297f 100755 --- a/install +++ b/install @@ -750,7 +750,7 @@ if [ "$ignore_dependencies" = "f" ]; then set -e package_list=$( - "${SCRIPT_DIR}/installer/packages.py" --format=line "${code}" "${packages_files[@]}" || + "${SCRIPT_DIR}/scripts/packages.py" --format=line "${code}" "${packages_files[@]}" || (echo "ERROR: could not generate packages list" >&2 && exit 1) ) set +e From fa35fc5b0e15a7c46810aadfa8c27111ef954b99 Mon Sep 17 00:00:00 2001 From: jo Date: Wed, 1 Sep 2021 15:51:40 +0200 Subject: [PATCH 24/24] Remove sysv related packages --- packages.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages.ini b/packages.ini index f48f46a04..e1ecfc7d1 100644 --- a/packages.ini +++ b/packages.ini @@ -37,8 +37,3 @@ php7.3-pgsql = buster [installer] lsb-release = buster, bionic xmlstarlet = buster, bionic - -[unknown] -# See https://github.com/LibreTime/libretime/commit/bbcd4fbe2ee05cdb2e3a62ad2b7549157a1cd950#diff-0991ed0aaae32d2b0c9723cd9127f95bc19d176a5fd668d9b331f487136318fe -systemd-sysv = buster -sysvinit-utils = bionic