From e320ce204ed7ca94095cc4825bd322fc06bfaf56 Mon Sep 17 00:00:00 2001 From: Jonas L Date: Mon, 25 Apr 2022 15:27:10 +0200 Subject: [PATCH] ci: reduce usage (#1804) * ci: move analyzer jobs in a dedicated workflow * ci: move worker jobs in a dedicated workflow * ci: move legacy jobs in a dedicated workflow * ci: move shared jobs in a dedicated workflow * ci: move api-client jobs to dedicated workflow * ci: remove unused test job * ci: move api jobs in a dedicated workflow * ci: move playout ci jobs in a dedicated workflow * ci: remove unused test job * ci: move test-with-database in api workflow * ci: run playout on api-client changes * ci: rename tools workflow to dev-tools * ci: rename generic tests workflow to project * ci: rename lint-pr workflow to pr * ci: update python tests concurrency * ci: update python caching keys * ci: update website/docs trigger condition * ci: update legacy trigger condition * ci: readd cancel in progress * ci :update api cache key --- .github/workflows/_python.yml | 75 ++++++ .github/workflows/analyzer.yml | 22 ++ .github/workflows/api-client.yml | 22 ++ .github/workflows/api.yml | 68 ++++++ .../workflows/{tools.yml => dev-tools.yml} | 4 +- .github/workflows/docs.yml | 7 +- .github/workflows/legacy.yml | 77 ++++++ .github/workflows/playout.yml | 25 ++ .github/workflows/{lint-pr.yml => pr.yml} | 2 +- .github/workflows/project.yml | 48 ++++ .github/workflows/shared.yml | 20 ++ .github/workflows/test.yml | 220 ------------------ .github/workflows/website.yml | 4 +- .github/workflows/worker.yml | 21 ++ 14 files changed, 389 insertions(+), 226 deletions(-) create mode 100644 .github/workflows/_python.yml create mode 100644 .github/workflows/analyzer.yml create mode 100644 .github/workflows/api-client.yml create mode 100644 .github/workflows/api.yml rename .github/workflows/{tools.yml => dev-tools.yml} (97%) create mode 100644 .github/workflows/legacy.yml create mode 100644 .github/workflows/playout.yml rename .github/workflows/{lint-pr.yml => pr.yml} (98%) create mode 100644 .github/workflows/project.yml create mode 100644 .github/workflows/shared.yml delete mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/worker.yml diff --git a/.github/workflows/_python.yml b/.github/workflows/_python.yml new file mode 100644 index 000000000..2b8c68416 --- /dev/null +++ b/.github/workflows/_python.yml @@ -0,0 +1,75 @@ +on: + workflow_call: + inputs: + context: + required: true + type: string + + lint: + required: false + default: true + type: boolean + + test: + required: false + default: true + type: boolean + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + if: inputs.lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ inputs.context }}-${{ hashFiles(format('{0}/{1}', inputs.context, '**/setup.py')) }} + restore-keys: | + ${{ runner.os }}-pip-${{ inputs.context }} + + - name: Add annotations matchers + run: | + echo "::add-matcher::.github/annotations/pylint.json" + + - name: Lint + run: make lint + working-directory: ${{ inputs.context }} + + test: + if: inputs.test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + release: + - buster + - bullseye + - bionic + - focal + - jammy + + container: ghcr.io/libretime/libretime-dev:${{ matrix.release }} + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ inputs.context }}-${{ hashFiles(format('{0}/{1}', inputs.context, '**/setup.py')) }} + restore-keys: | + ${{ runner.os }}-pip-${{ inputs.context }} + + - name: Test + run: make test + working-directory: ${{ inputs.context }} diff --git a/.github/workflows/analyzer.yml b/.github/workflows/analyzer.yml new file mode 100644 index 000000000..6f41b4b59 --- /dev/null +++ b/.github/workflows/analyzer.yml @@ -0,0 +1,22 @@ +name: Analyzer + +on: + push: + branches: [main] + paths: + - .github/workflows/** + - analyzer/** + - shared/** + + pull_request: + branches: [main] + paths: + - .github/workflows/** + - analyzer/** + - shared/** + +jobs: + python: + uses: ./.github/workflows/_python.yml + with: + context: analyzer diff --git a/.github/workflows/api-client.yml b/.github/workflows/api-client.yml new file mode 100644 index 000000000..ad72fc514 --- /dev/null +++ b/.github/workflows/api-client.yml @@ -0,0 +1,22 @@ +name: API Client + +on: + push: + branches: [main] + paths: + - .github/workflows/** + - api-client/** + - shared/** + + pull_request: + branches: [main] + paths: + - .github/workflows/** + - api-client/** + - shared/** + +jobs: + python: + uses: ./.github/workflows/_python.yml + with: + context: api-client diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml new file mode 100644 index 000000000..b3f3f77c3 --- /dev/null +++ b/.github/workflows/api.yml @@ -0,0 +1,68 @@ +name: API + +on: + push: + branches: [main] + paths: + - .github/workflows/** + - api/** + - shared/** + + pull_request: + branches: [main] + paths: + - .github/workflows/** + - api/** + - shared/** + +jobs: + python: + uses: ./.github/workflows/_python.yml + with: + context: api + test: false + + test-with-database: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + release: + - buster + - bullseye + - bionic + - focal + - jammy + + services: + postgres: + image: postgres + env: + POSTGRES_USER: libretime + POSTGRES_PASSWORD: libretime + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + container: ghcr.io/libretime/libretime-dev:${{ matrix.release }} + defaults: + run: + shell: bash + env: + LIBRETIME_DATABASE_HOST: postgres + + steps: + - uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-api-${{ hashFiles('api/**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip-api + + - name: Test + run: make test + working-directory: api diff --git a/.github/workflows/tools.yml b/.github/workflows/dev-tools.yml similarity index 97% rename from .github/workflows/tools.yml rename to .github/workflows/dev-tools.yml index 69745dd33..a336cebd5 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/dev-tools.yml @@ -1,4 +1,4 @@ -name: Tools +name: Dev Tools on: schedule: @@ -6,8 +6,8 @@ on: push: branches: [main] paths: + - ".github/workflows/dev-tools.yml" - "**/packages.ini" - - ".github/workflows/tools.yml" jobs: docker-dev-image: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a7ffed840..f10596396 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,10 +5,15 @@ on: branches: [main] paths: - .github/workflows/docs.yml - - website/** - docs/** + - website/** + pull_request: branches: [main] + paths: + - .github/workflows/docs.yml + - docs/** + - website/** jobs: lint: diff --git a/.github/workflows/legacy.yml b/.github/workflows/legacy.yml new file mode 100644 index 000000000..a9b03fa3a --- /dev/null +++ b/.github/workflows/legacy.yml @@ -0,0 +1,77 @@ +name: Legacy + +on: + push: + branches: [main] + paths: + - .github/workflows/** + - api/** + - legacy/** + + pull_request: + branches: [main] + paths: + - .github/workflows/** + - api/** + - legacy/** + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: shivammathur/setup-php@v2 + with: + php-version: 7.4 + - name: Lint + run: make lint + working-directory: legacy + + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - php-version: "7.2" # Bionic + - php-version: "7.3" # Buster + - php-version: "7.4" # Bullseye, Focal + + env: + ENVIRONMENT: testing + steps: + - uses: actions/checkout@v3 + + - name: Setup PostgreSQL + run: | + sudo systemctl start postgresql.service + pg_isready + sudo -u postgres psql -c 'CREATE DATABASE libretime;' + sudo -u postgres psql -c "CREATE USER libretime WITH PASSWORD 'libretime';" + sudo -u postgres psql -c 'GRANT CONNECT ON DATABASE libretime TO libretime;' + sudo -u postgres psql -c 'ALTER USER libretime CREATEDB;' + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-version }} + + - name: Get Composer Cache Directory + id: composer-cache + run: | + echo "::set-output name=dir::$(composer config cache-files-dir)" + + - uses: actions/cache@v3 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Run tests + run: make test + working-directory: legacy diff --git a/.github/workflows/playout.yml b/.github/workflows/playout.yml new file mode 100644 index 000000000..47578708f --- /dev/null +++ b/.github/workflows/playout.yml @@ -0,0 +1,25 @@ +name: Playout + +on: + push: + branches: [main] + paths: + - .github/workflows/** + - playout/** + - api-client/** + - shared/** + + pull_request: + branches: [main] + paths: + - .github/workflows/** + - playout/** + - api-client/** + - shared/** + +jobs: + python: + uses: ./.github/workflows/_python.yml + with: + context: playout + test: false diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/pr.yml similarity index 98% rename from .github/workflows/lint-pr.yml rename to .github/workflows/pr.yml index ebef30840..d18ea485c 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/pr.yml @@ -1,4 +1,4 @@ -name: Lint PR +name: PR on: pull_request_target: diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml new file mode 100644 index 000000000..6aed0dd70 --- /dev/null +++ b/.github/workflows/project.yml @@ -0,0 +1,48 @@ +name: Project + +on: + push: + branches: [main] + pull_request: + types: [opened, reopened, synchronize, edited] + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v2.0.3 + + check-shell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - run: | + python -m venv venv && source venv/bin/activate + pip install gh-release-install + + sudo venv/bin/gh-release-install \ + koalaman/shellcheck \ + shellcheck-{tag}.linux.x86_64.tar.xz --extract shellcheck-{tag}/shellcheck \ + /usr/bin/shellcheck + + sudo venv/bin/gh-release-install \ + mvdan/sh \ + shfmt_{tag}_linux_amd64 \ + /usr/bin/shfmt + + - run: SEVERITY=warning make shell-check + + test-tools: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - run: make all + working-directory: tools diff --git a/.github/workflows/shared.yml b/.github/workflows/shared.yml new file mode 100644 index 000000000..32223aef7 --- /dev/null +++ b/.github/workflows/shared.yml @@ -0,0 +1,20 @@ +name: Shared + +on: + push: + branches: [main] + paths: + - .github/workflows/** + - shared/** + + pull_request: + branches: [main] + paths: + - .github/workflows/** + - shared/** + +jobs: + python: + uses: ./.github/workflows/_python.yml + with: + context: shared diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 9bb488c8a..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,220 +0,0 @@ -name: Tests - -on: - push: - branches: [main] - pull_request: - types: [opened, reopened, synchronize, edited] - branches: [main] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v2.0.3 - - check-shell: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - run: | - python -m venv venv && source venv/bin/activate - pip install gh-release-install - - sudo venv/bin/gh-release-install \ - koalaman/shellcheck \ - shellcheck-{tag}.linux.x86_64.tar.xz --extract shellcheck-{tag}/shellcheck \ - /usr/bin/shellcheck - - sudo venv/bin/gh-release-install \ - mvdan/sh \ - shfmt_{tag}_linux_amd64 \ - /usr/bin/shfmt - - - run: SEVERITY=warning make shell-check - - test-tools: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - run: make all - working-directory: tools - - lint-legacy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: shivammathur/setup-php@v2 - with: - php-version: 7.4 - - name: Lint - run: make lint - working-directory: legacy - - test-legacy: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - php-version: "7.2" # Bionic - - php-version: "7.3" # Buster - - php-version: "7.4" # Bullseye, Focal - - env: - ENVIRONMENT: testing - steps: - - uses: actions/checkout@v3 - - - name: Setup PostgreSQL - run: | - sudo systemctl start postgresql.service - pg_isready - sudo -u postgres psql -c 'CREATE DATABASE libretime;' - sudo -u postgres psql -c "CREATE USER libretime WITH PASSWORD 'libretime';" - sudo -u postgres psql -c 'GRANT CONNECT ON DATABASE libretime TO libretime;' - sudo -u postgres psql -c 'ALTER USER libretime CREATEDB;' - - - name: Setup PHP - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-version }} - - - name: Get Composer Cache Directory - id: composer-cache - run: | - echo "::set-output name=dir::$(composer config cache-files-dir)" - - - uses: actions/cache@v3 - with: - path: ${{ steps.composer-cache.outputs.dir }} - key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} - restore-keys: | - ${{ runner.os }}-composer- - - - name: Run tests - run: make test - working-directory: legacy - - # Start lint the code without failing the entire workflow, should be merged - # into 'test' when the entire matrix succeeds. - lint: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - context: - - analyzer - - api - - api-client - - playout - - shared - - worker - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Add annotations matchers - run: | - echo "::add-matcher::.github/annotations/pylint.json" - - - name: Lint - run: make lint - working-directory: ${{ matrix.context }} - - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - context: - - analyzer - - api-client - - shared - release: - - buster - - bullseye - - bionic - - focal - - jammy - - container: ghcr.io/libretime/libretime-dev:${{ matrix.release }} - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.context }} - - - name: Test - run: make test - working-directory: ${{ matrix.context }} - - test-with-database: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - context: - - api - release: - - buster - - bullseye - - bionic - - focal - - jammy - - services: - postgres: - image: postgres - env: - POSTGRES_USER: libretime - POSTGRES_PASSWORD: libretime - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - container: ghcr.io/libretime/libretime-dev:${{ matrix.release }} - defaults: - run: - shell: bash - env: - LIBRETIME_DATABASE_HOST: postgres - - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.context }} - - - name: Test - run: make test - working-directory: ${{ matrix.context }} diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 09b8cac19..2be313312 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -5,15 +5,15 @@ on: branches: [main] paths: - .github/workflows/website.yml - - website/** - docs/** + - website/** pull_request: branches: [main] paths: - .github/workflows/website.yml - - website/** - docs/** + - website/** jobs: deploy: diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml new file mode 100644 index 000000000..866bcef0b --- /dev/null +++ b/.github/workflows/worker.yml @@ -0,0 +1,21 @@ +name: Worker + +on: + push: + branches: [main] + paths: + - .github/workflows/** + - worker/** + + pull_request: + branches: [main] + paths: + - .github/workflows/** + - worker/** + +jobs: + python: + uses: ./.github/workflows/_python.yml + with: + context: worker + test: false