chore: rewrite extract-requirements to python

Using python ast to guess requirements in the setup.py file.

.pre-commit-config.yaml

@@ -23,7 +23,6 @@ repos:
         args: [--fix=lf]
       - id: trailing-whitespace
 
-      - id: requirements-txt-fixer
       - id: name-tests-test
         # TODO: Remove once the django api uses pytest
         exclude: ^(api.*)$
@@ -66,7 +65,7 @@ repos:
       - id: requirements.txt
         name: requirements.txt
         description: Generate requirements.txt
-        entry: tools/extract-requirements.sh dev
+        entry: tools/extract_requirements.py dev
         pass_filenames: false
         language: script
         files: setup.py$

analyzer/requirements.txt

@@ -1,5 +1,5 @@
-# This file is auto-generated by tools/extract-requirements.sh.
 # Please do not edit this file, edit the setup.py file!
+# This file is auto-generated by tools/extract_requirements.py.
 mutagen>=1.45.1
 pika>=1.0.0
 requests>=2.7.0

api-client/requirements.txt

@@ -1,4 +1,4 @@
-# This file is auto-generated by tools/extract-requirements.sh.
 # Please do not edit this file, edit the setup.py file!
+# This file is auto-generated by tools/extract_requirements.py.
 python-dateutil>=2.7.0
 requests

api/requirements.txt

@@ -1,8 +1,8 @@
-# This file is auto-generated by tools/extract-requirements.sh.
 # Please do not edit this file, edit the setup.py file!
-django<4.0,>=3.2.12
+# This file is auto-generated by tools/extract_requirements.py.
-django-filter<22.0,>=2.4.0
+django-filter>=2.4.0,<22.0
-djangorestframework<3.14,>=3.12.1
+django>=3.2.12,<4.0
-drf-spectacular<0.23,>=0.22.1
+djangorestframework>=3.12.1,<3.14
-psycopg2<2.10,>=2.8.6
+drf-spectacular>=0.22.1,<0.23
-requests<2.29,>=2.25.1
+psycopg2>=2.8.6,<2.10
+requests>=2.25.1,<2.29

playout/requirements.txt

@@ -1,5 +1,5 @@
-# This file is auto-generated by tools/extract-requirements.sh.
 # Please do not edit this file, edit the setup.py file!
+# This file is auto-generated by tools/extract_requirements.py.
 amqplib
 defusedxml
 kombu

shared/requirements.txt

@@ -1,5 +1,5 @@
-# This file is auto-generated by tools/extract-requirements.sh.
 # Please do not edit this file, edit the setup.py file!
+# This file is auto-generated by tools/extract_requirements.py.
 click~=8.0.4
 loguru==0.6.0
 pydantic

tools/extract-requirements.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-# Extract the dependencies from the setup.py files
-# and save the result to requirements.txt.
-#
-# You can filter any extra require by adding the name as argument.
-#
-# Examples:
-#   tools/extract-requirements.sh
-#   tools/extract-requirements.sh dev
-
-set -u
-
-error() {
-  echo >&2 "error: $*"
-  exit 1
-}
-
-command -v python3 > /dev/null || error "python3 command not found!"
-command -v sed > /dev/null || error "sed command not found!"
-
-for setup_path in */setup.py; do
-  path="$(dirname "$setup_path")"
-
-  # Build egg
-  python3 "$setup_path" egg_info > /dev/null 2>&1 || true
-  egg_path="$(echo "$path"/*.egg-info)"
-
-  # Remove entire extra section from require file
-  for arg in "$@"; do
-    sed --in-place "/^\[$arg\]/,/^\[/d" -- "$egg_path/requires.txt"
-  done
-
-  # Generate requirements.txt
-  cat << EOF > "$path/requirements.txt"
-# This file is auto-generated by tools/extract-requirements.sh.
-# Please do not edit this file, edit the setup.py file!
-EOF
-
-  cat -- "$egg_path/requires.txt" |
-    sed '/^$/d' |
-    LC_ALL=en_US.UTF-8 sort >> "$path/requirements.txt"
-done

tools/extract_requirements.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+# Extract the dependencies from the setup.py files
+# and save the result to requirements.txt.
+#
+# You can filter any extra require by adding the name as argument.
+#
+# Examples:
+#   tools/extract_requirements.py
+#   tools/extract_requirements.py dev
+
+
+import ast
+from glob import glob
+from pathlib import Path
+from sys import argv
+
+
+class RemoveJoinedStr(ast.NodeTransformer):
+    def visit_JoinedStr(self, _node):  # pylint: disable=invalid-name
+        pass
+
+
+for setup in glob("*/setup.py"):
+    setup_path = Path(setup)
+    requirements_path = setup_path.parent / "requirements.txt"
+
+    lines = [
+        "# Please do not edit this file, edit the setup.py file!",
+        "# This file is auto-generated by tools/extract_requirements.py.",
+    ]
+
+    requires = []
+
+    for node in ast.walk(ast.parse(setup_path.read_text(encoding="utf-8"))):
+        if (
+            isinstance(node, ast.Expr)
+            and isinstance(node.value, ast.Call)
+            and isinstance(node.value.func, ast.Name)
+            and node.value.func.id == "setup"
+        ):
+
+            for keyword in node.value.keywords:
+                if keyword.arg == "install_requires":
+                    requires.extend(ast.literal_eval(keyword.value))
+
+                if keyword.arg == "extras_require":
+                    extras = ast.literal_eval(RemoveJoinedStr().visit(keyword.value))
+
+                    for key, values in extras.items():
+                        if key in argv:
+                            continue
+                        requires.extend(values)
+
+    lines.extend(sorted(requires))
+
+    requirements_path.write_text("\n".join(lines) + "\n", encoding="utf-8")

worker/requirements.txt

@@ -1,5 +1,5 @@
-# This file is auto-generated by tools/extract-requirements.sh.
 # Please do not edit this file, edit the setup.py file!
+# This file is auto-generated by tools/extract_requirements.py.
 celery==4.4.7
 kombu==4.6.11
 mutagen>=1.31.0