diff --git a/.github/annotations/pylint.json b/.github/annotations/pylint.json new file mode 100644 index 000000000..649e1d09b --- /dev/null +++ b/.github/annotations/pylint.json @@ -0,0 +1,32 @@ +{ + "problemMatcher": [ + { + "severity": "error", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): (E\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-error" + }, + { + "severity": "warning", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): ([A-DF-Z]\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-warning" + } + ] +} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a3f98baf0..c511ee8c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,13 +91,42 @@ jobs: make test working-directory: airtime_mvc + # Start lint the code without failing the entire workflow, should be merged + # into 'test' when the entire matrix succeeds. + lint: + runs-on: ubuntu-latest + container: ghcr.io/libretime/libretime-dev:buster + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Add annotations matchers + run: | + echo "::add-matcher::.github/annotations/pylint.json" + + - name: Lint + run: | + make -C api lint + make -C python_apps/airtime_analyzer lint + make -C python_apps/airtime-celery lint + make -C python_apps/api_clients lint + make -C python_apps/pypo lint + test: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - release: [bionic, buster] context: [python_apps/airtime_analyzer, python_apps/api_clients] + release: [bionic, buster] container: ghcr.io/libretime/libretime-dev:${{ matrix.release }} defaults: @@ -110,20 +139,10 @@ jobs: - uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py', '**/requirements-dev.txt') }} + key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip-${{ matrix.context }} - - name: Install dependencies - run: | - python3 -m venv venv && source venv/bin/activate - pip install --upgrade pip setuptools wheel - pip install -r requirements-dev.txt - pip install -e . - working-directory: ${{ matrix.context }} - - name: Test - run: | - source venv/bin/activate - make test + run: make test working-directory: ${{ matrix.context }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fd657dc45..39836899e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,11 +46,6 @@ repos: - id: isort args: ["--profile", "black", "--filter-files"] - # - repo: https://github.com/pre-commit/mirrors-pylint - # rev: v3.0.0a3 - # hooks: - # - id: pylint - - repo: https://github.com/codespell-project/codespell rev: v2.1.0 hooks: diff --git a/api/Makefile b/api/Makefile new file mode 100644 index 000000000..60e3bf54a --- /dev/null +++ b/api/Makefile @@ -0,0 +1,10 @@ +all: lint + +include ../tools/python.mk + +PIP_INSTALL := --editable .[dev] +PYLINT_ARG := libretimeapi +MYPY_ARG := libretimeapi + +lint: .pylint .mypy +clean: .clean diff --git a/api/setup.py b/api/setup.py index 724901cbe..5f4b77c4b 100644 --- a/api/setup.py +++ b/api/setup.py @@ -20,13 +20,23 @@ setup( packages=find_packages(), include_package_data=True, scripts=["bin/libretime-api"], + python_requires=">=3.6", install_requires=[ "coreapi", - "Django~=3.0", + "django~=3.0", "djangorestframework", "django-url-filter", "markdown", "model_bakery", - "psycopg2", ], + extras_require={ + "prod": [ + "psycopg2", + ], + "dev": [ + "psycopg2-binary", + "mypy", + "pylint", + ], + }, ) diff --git a/python_apps/airtime-celery/Makefile b/python_apps/airtime-celery/Makefile new file mode 100644 index 000000000..b74c3b0e1 --- /dev/null +++ b/python_apps/airtime-celery/Makefile @@ -0,0 +1,10 @@ +all: lint + +include ../../tools/python.mk + +PIP_INSTALL := --editable .[dev] +PYLINT_ARG := airtime-celery +MYPY_ARG := airtime-celery + +lint: .pylint .mypy +clean: .clean diff --git a/python_apps/airtime-celery/setup.py b/python_apps/airtime-celery/setup.py index ac0e3ced4..4a9a81741 100644 --- a/python_apps/airtime-celery/setup.py +++ b/python_apps/airtime-celery/setup.py @@ -18,10 +18,18 @@ setup( }, license="MIT", packages=["airtime-celery"], + python_requires=">=3.6", install_requires=[ "celery==4.4.7", "kombu==4.6.10", "configobj", ], + extras_require={ + "prod": [], + "dev": [ + "mypy", + "pylint", + ], + }, zip_safe=False, ) diff --git a/python_apps/airtime_analyzer/Makefile b/python_apps/airtime_analyzer/Makefile index 9d4260e07..d0c69d657 100644 --- a/python_apps/airtime_analyzer/Makefile +++ b/python_apps/airtime_analyzer/Makefile @@ -1,19 +1,16 @@ -.PHONY: lint test +all: lint test -SHELL := bash -CPU_CORES := $(shell nproc) +include ../../tools/python.mk -MODULE_APP := airtime_analyzer -MODULE_TESTS := tests +PIP_INSTALL := --editable .[dev] +PYLINT_ARG := airtime_analyzer tests +MYPY_ARG := airtime_analyzer tests +PYTEST_ARG := --cov=airtime_analyzer tests -lint: - pylint ${MODULE_APP} - pylint ${MODULE_TESTS} +lint: .pylint .mypy fixtures: bash tests/fixtures/generate.sh -test: fixtures - pytest -n ${CPU_CORES} --color=yes -v --cov=${MODULE_APP} ${MODULE_TESTS} - -all: lint test +test: fixtures .pytest +clean: .clean diff --git a/python_apps/airtime_analyzer/requirements-dev.txt b/python_apps/airtime_analyzer/requirements-dev.txt deleted file mode 100644 index e8b5dfdfc..000000000 --- a/python_apps/airtime_analyzer/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -distro -pylint -pytest -pytest-cov -pytest-xdist diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index 01d899f7c..82b44e0e4 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -23,6 +23,7 @@ setup( "libretime-analyzer=airtime_analyzer.cli:main", ] }, + python_requires=">=3.6", install_requires=[ "mutagen>=1.31.0", "pika>=1.0.0", @@ -33,5 +34,16 @@ setup( # If this version is changed, it needs changing in the install script too "pycairo==1.19.1", ], + extras_require={ + "prod": [], + "dev": [ + "distro", + "mypy", + "pylint", + "pytest", + "pytest-cov", + "pytest-xdist", + ], + }, zip_safe=False, ) diff --git a/python_apps/api_clients/Makefile b/python_apps/api_clients/Makefile index 267fb7986..28d7d7224 100644 --- a/python_apps/api_clients/Makefile +++ b/python_apps/api_clients/Makefile @@ -1,16 +1,12 @@ -.PHONY: lint test - -SHELL := bash -CPU_CORES := $(shell nproc) - -MODULE_APP := api_clients -MODULE_TESTS := tests - -lint: - pylint ${MODULE_APP} - pylint ${MODULE_TESTS} - -test: - pytest -n ${CPU_CORES} --color=yes -v --cov=${MODULE_APP} ${MODULE_TESTS} - all: lint test + +include ../../tools/python.mk + +PIP_INSTALL := --editable .[dev] +PYLINT_ARG := api_clients tests +MYPY_ARG := api_clients tests +PYTEST_ARG := --cov=api_clients tests + +lint: .pylint .mypy +test: .pytest +clean: .clean diff --git a/python_apps/api_clients/requirements-dev.txt b/python_apps/api_clients/requirements-dev.txt deleted file mode 100644 index 2b9579643..000000000 --- a/python_apps/api_clients/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -pylint -pytest -pytest-cov -pytest-xdist diff --git a/python_apps/api_clients/setup.py b/python_apps/api_clients/setup.py index af6f5354f..adb6efd3f 100644 --- a/python_apps/api_clients/setup.py +++ b/python_apps/api_clients/setup.py @@ -18,10 +18,21 @@ setup( }, license="AGPLv3", packages=["api_clients"], + python_requires=">=3.6", install_requires=[ "configobj", "python-dateutil>=2.7.0", "requests", ], + extras_require={ + "prod": [], + "dev": [ + "mypy", + "pylint", + "pytest", + "pytest-cov", + "pytest-xdist", + ], + }, zip_safe=False, ) diff --git a/python_apps/pypo/Makefile b/python_apps/pypo/Makefile new file mode 100644 index 000000000..8f4eefc4c --- /dev/null +++ b/python_apps/pypo/Makefile @@ -0,0 +1,10 @@ +all: lint + +include ../../tools/python.mk + +PIP_INSTALL := --editable .[dev] +PYLINT_ARG := liquidsoap pypo +MYPY_ARG := liquidsoap pypo + +lint: .pylint .mypy +clean: .clean diff --git a/python_apps/pypo/setup.py b/python_apps/pypo/setup.py index ddb3b1037..ebf0c268c 100644 --- a/python_apps/pypo/setup.py +++ b/python_apps/pypo/setup.py @@ -27,6 +27,7 @@ setup( "bin/airtime-liquidsoap", "bin/pyponotify", ], + python_requires=">=3.6", install_requires=[ "amqplib", "configobj", @@ -37,5 +38,12 @@ setup( "pytz", "requests", ], + extras_require={ + "prod": [], + "dev": [ + "mypy", + "pylint", + ], + }, zip_safe=False, ) diff --git a/tools/Makefile b/tools/Makefile index 511ba08af..ad8adad3f 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,23 +1,12 @@ -.PHONY: lint test -.ONESHELL: - -SHELL := bash -CPU_CORES := $(shell nproc) - all: lint test -venv: - python3 -m venv venv - source venv/bin/activate - pip install -r requirements-dev.txt +include python.mk -lint: venv - source venv/bin/activate - pylint tools +PIP_INSTALL := -r requirements-dev.txt +PYLINT_ARG := tools +MYPY_ARG := . +PYTEST_ARG := . -test: venv - source venv/bin/activate - pytest -n ${CPU_CORES} --color=yes -v . - -clean: - rm -Rf venv +lint: .pylint .mypy +test: .pytest +clean: .clean diff --git a/tools/python.mk b/tools/python.mk new file mode 100644 index 000000000..814cbcf92 --- /dev/null +++ b/tools/python.mk @@ -0,0 +1,35 @@ +.ONESHELL: + +SHELL := bash +CPU_CORES := $(shell nproc) + +# PIP_INSTALL := --editable .[dev] +# PYLINT_ARG := +# MYPY_ARG := +# PYTEST_ARG := + +VENV := venv +$(VENV): + python3 -m venv $(VENV) + source $(VENV)/bin/activate + pip install --upgrade pip setuptools wheel + pip install $(PIP_INSTALL) + +.PHONY: .pylint +.pylint: $(VENV) + source $(VENV)/bin/activate + pylint --output-format=colorized $(PYLINT_ARG) || true + +.PHONY: .mypy +.mypy: $(VENV) + source $(VENV)/bin/activate + mypy $(MYPY_ARG) || true + +.PHONY: .pytest +.pytest: $(VENV) + source venv/bin/activate + pytest -n $(CPU_CORES) --color=yes -v $(PYTEST_ARG) + +.PHONY: .clean +.clean: + rm -Rf $(VENV) diff --git a/tools/requirements-dev.txt b/tools/requirements-dev.txt index 430110048..cda4c0a74 100644 --- a/tools/requirements-dev.txt +++ b/tools/requirements-dev.txt @@ -1,3 +1,4 @@ +mypy pylint pytest pytest-xdist