Merge pull request #1339 from jooola/feat/ci_start_linting

Start linting 🎉 !

.github/annotations/pylint.json

@@ -0,0 +1,32 @@
+{
+  "problemMatcher": [
+    {
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+): (E\\d+): \\033\\[[\\d;]+m([^\\033]+).*$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "code": 4,
+          "message": 5
+        }
+      ],
+      "owner": "pylint-error"
+    },
+    {
+      "severity": "warning",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+): ([A-DF-Z]\\d+): \\033\\[[\\d;]+m([^\\033]+).*$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "code": 4,
+          "message": 5
+        }
+      ],
+      "owner": "pylint-warning"
+    }
+  ]
+}

.github/workflows/test.yml

@@ -91,13 +91,42 @@ jobs:
           make test
         working-directory: airtime_mvc
 
+  # Start lint the code without failing the entire workflow, should be merged
+  # into 'test' when the entire matrix succeeds.
+  lint:
+    runs-on: ubuntu-latest
+    container: ghcr.io/libretime/libretime-dev:buster
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Add annotations matchers
+        run: |
+          echo "::add-matcher::.github/annotations/pylint.json"
+
+      - name: Lint
+        run: |
+          make -C api lint
+          make -C python_apps/airtime_analyzer lint
+          make -C python_apps/airtime-celery lint
+          make -C python_apps/api_clients lint
+          make -C python_apps/pypo lint
+
   test:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        release: [bionic, buster]
         context: [python_apps/airtime_analyzer, python_apps/api_clients]
+        release: [bionic, buster]
 
     container: ghcr.io/libretime/libretime-dev:${{ matrix.release }}
     defaults:
@@ -110,20 +139,10 @@ jobs:
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py', '**/requirements-dev.txt') }}
+          key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-${{ matrix.context }}
 
-      - name: Install dependencies
-        run: |
-          python3 -m venv venv && source venv/bin/activate
-          pip install --upgrade pip setuptools wheel
-          pip install -r requirements-dev.txt
-          pip install -e .
-        working-directory: ${{ matrix.context }}
-
       - name: Test
-        run: |
+        run: make test
-          source venv/bin/activate
-          make test
         working-directory: ${{ matrix.context }}

.pre-commit-config.yaml

@@ -46,11 +46,6 @@ repos:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
 
-  # - repo: https://github.com/pre-commit/mirrors-pylint
-  #   rev: v3.0.0a3
-  #   hooks:
-  #     - id: pylint
-
   - repo: https://github.com/codespell-project/codespell
     rev: v2.1.0
     hooks:

api/Makefile

@@ -0,0 +1,10 @@
+all: lint
+
+include ../tools/python.mk
+
+PIP_INSTALL := --editable .[dev]
+PYLINT_ARG := libretimeapi
+MYPY_ARG := libretimeapi
+
+lint: .pylint .mypy
+clean: .clean

api/setup.py

@@ -20,13 +20,23 @@ setup(
     packages=find_packages(),
     include_package_data=True,
     scripts=["bin/libretime-api"],
+    python_requires=">=3.6",
     install_requires=[
         "coreapi",
-        "Django~=3.0",
+        "django~=3.0",
         "djangorestframework",
         "django-url-filter",
         "markdown",
         "model_bakery",
-        "psycopg2",
     ],
+    extras_require={
+        "prod": [
+            "psycopg2",
+        ],
+        "dev": [
+            "psycopg2-binary",
+            "mypy",
+            "pylint",
+        ],
+    },
 )

python_apps/airtime-celery/Makefile

@@ -0,0 +1,10 @@
+all: lint
+
+include ../../tools/python.mk
+
+PIP_INSTALL := --editable .[dev]
+PYLINT_ARG := airtime-celery
+MYPY_ARG := airtime-celery
+
+lint: .pylint .mypy
+clean: .clean

python_apps/airtime-celery/setup.py

@@ -18,10 +18,18 @@ setup(
     },
     license="MIT",
     packages=["airtime-celery"],
+    python_requires=">=3.6",
     install_requires=[
         "celery==4.4.7",
         "kombu==4.6.10",
         "configobj",
     ],
+    extras_require={
+        "prod": [],
+        "dev": [
+            "mypy",
+            "pylint",
+        ],
+    },
     zip_safe=False,
 )

python_apps/airtime_analyzer/Makefile

@@ -1,19 +1,16 @@
-.PHONY: lint test
+all: lint test
 
-SHELL := bash
+include ../../tools/python.mk
-CPU_CORES := $(shell nproc)
 
-MODULE_APP := airtime_analyzer
+PIP_INSTALL := --editable .[dev]
-MODULE_TESTS := tests
+PYLINT_ARG := airtime_analyzer tests
+MYPY_ARG := airtime_analyzer tests
+PYTEST_ARG := --cov=airtime_analyzer tests
 
-lint:
+lint: .pylint .mypy
-	pylint ${MODULE_APP}
-	pylint ${MODULE_TESTS}
 
 fixtures:
 	bash tests/fixtures/generate.sh
 
-test: fixtures
+test: fixtures .pytest
-	pytest -n ${CPU_CORES} --color=yes -v --cov=${MODULE_APP} ${MODULE_TESTS}
+clean: .clean
-
-all: lint test

python_apps/airtime_analyzer/requirements-dev.txt

@@ -1,5 +0,0 @@
-distro
-pylint
-pytest
-pytest-cov
-pytest-xdist

python_apps/airtime_analyzer/setup.py

@@ -23,6 +23,7 @@ setup(
             "libretime-analyzer=airtime_analyzer.cli:main",
         ]
     },
+    python_requires=">=3.6",
     install_requires=[
         "mutagen>=1.31.0",
         "pika>=1.0.0",
@@ -33,5 +34,16 @@ setup(
         # If this version is changed, it needs changing in the install script too
         "pycairo==1.19.1",
     ],
+    extras_require={
+        "prod": [],
+        "dev": [
+            "distro",
+            "mypy",
+            "pylint",
+            "pytest",
+            "pytest-cov",
+            "pytest-xdist",
+        ],
+    },
     zip_safe=False,
 )

python_apps/api_clients/Makefile

@@ -1,16 +1,12 @@
-.PHONY: lint test
-
-SHELL := bash
-CPU_CORES := $(shell nproc)
-
-MODULE_APP := api_clients
-MODULE_TESTS := tests
-
-lint:
-	pylint ${MODULE_APP}
-	pylint ${MODULE_TESTS}
-
-test:
-	pytest -n ${CPU_CORES} --color=yes -v --cov=${MODULE_APP} ${MODULE_TESTS}
-
 all: lint test
+
+include ../../tools/python.mk
+
+PIP_INSTALL := --editable .[dev]
+PYLINT_ARG := api_clients tests
+MYPY_ARG := api_clients tests
+PYTEST_ARG := --cov=api_clients tests
+
+lint: .pylint .mypy
+test: .pytest
+clean: .clean

python_apps/api_clients/requirements-dev.txt

@@ -1,4 +0,0 @@
-pylint
-pytest
-pytest-cov
-pytest-xdist

python_apps/api_clients/setup.py

@@ -18,10 +18,21 @@ setup(
     },
     license="AGPLv3",
     packages=["api_clients"],
+    python_requires=">=3.6",
     install_requires=[
         "configobj",
         "python-dateutil>=2.7.0",
         "requests",
     ],
+    extras_require={
+        "prod": [],
+        "dev": [
+            "mypy",
+            "pylint",
+            "pytest",
+            "pytest-cov",
+            "pytest-xdist",
+        ],
+    },
     zip_safe=False,
 )

python_apps/pypo/Makefile

@@ -0,0 +1,10 @@
+all: lint
+
+include ../../tools/python.mk
+
+PIP_INSTALL := --editable .[dev]
+PYLINT_ARG := liquidsoap pypo
+MYPY_ARG := liquidsoap pypo
+
+lint: .pylint .mypy
+clean: .clean

python_apps/pypo/setup.py

@@ -27,6 +27,7 @@ setup(
         "bin/airtime-liquidsoap",
         "bin/pyponotify",
     ],
+    python_requires=">=3.6",
     install_requires=[
         "amqplib",
         "configobj",
@@ -37,5 +38,12 @@ setup(
         "pytz",
         "requests",
     ],
+    extras_require={
+        "prod": [],
+        "dev": [
+            "mypy",
+            "pylint",
+        ],
+    },
     zip_safe=False,
 )

tools/Makefile

@@ -1,23 +1,12 @@
-.PHONY: lint test
-.ONESHELL:
-
-SHELL := bash
-CPU_CORES := $(shell nproc)
-
 all: lint test
 
-venv:
+include python.mk
-	python3 -m venv venv
-	source venv/bin/activate
-	pip install -r requirements-dev.txt
 
-lint: venv
+PIP_INSTALL := -r requirements-dev.txt
-	source venv/bin/activate
+PYLINT_ARG := tools
-	pylint tools
+MYPY_ARG := .
+PYTEST_ARG := .
 
-test: venv
+lint: .pylint .mypy
-	source venv/bin/activate
+test: .pytest
-	pytest -n ${CPU_CORES} --color=yes -v .
+clean: .clean
-
-clean:
-	rm -Rf venv

tools/python.mk

@@ -0,0 +1,35 @@
+.ONESHELL:
+
+SHELL := bash
+CPU_CORES := $(shell nproc)
+
+# PIP_INSTALL := --editable .[dev]
+# PYLINT_ARG :=
+# MYPY_ARG :=
+# PYTEST_ARG :=
+
+VENV := venv
+$(VENV):
+	python3 -m venv $(VENV)
+	source $(VENV)/bin/activate
+	pip install --upgrade pip setuptools wheel
+	pip install $(PIP_INSTALL)
+
+.PHONY: .pylint
+.pylint: $(VENV)
+	source $(VENV)/bin/activate
+	pylint --output-format=colorized $(PYLINT_ARG) || true
+
+.PHONY: .mypy
+.mypy: $(VENV)
+	source $(VENV)/bin/activate
+	mypy $(MYPY_ARG) || true
+
+.PHONY: .pytest
+.pytest: $(VENV)
+	source venv/bin/activate
+	pytest -n $(CPU_CORES) --color=yes -v $(PYTEST_ARG)
+
+.PHONY: .clean
+.clean:
+	rm -Rf $(VENV)

tools/requirements-dev.txt

@@ -1,3 +1,4 @@
+mypy
 pylint
 pytest
 pytest-xdist