From 4a3ce313d18ea3d29e15bbd563b6821f155cf4ad Mon Sep 17 00:00:00 2001 From: jo Date: Tue, 31 Aug 2021 16:13:48 +0200 Subject: [PATCH] Enhance CI workflow - Explode tests in multiple jobs. - Cache apt dependencies in a shared docker image. - Run tests on different distribution (debian/ubuntu, and maybe centos). --- .github/scripts/install-bionic.sh | 30 --------- .github/scripts/python-pkg-install.sh | 10 --- .github/scripts/python-pkg-test.sh | 21 ------ .github/workflows/test.yml | 93 +++++++++++++++++++-------- .github/workflows/tools.yml | 77 ++++++++++++++++++++++ 5 files changed, 142 insertions(+), 89 deletions(-) delete mode 100755 .github/scripts/install-bionic.sh delete mode 100755 .github/scripts/python-pkg-install.sh delete mode 100755 .github/scripts/python-pkg-test.sh create mode 100644 .github/workflows/tools.yml diff --git a/.github/scripts/install-bionic.sh b/.github/scripts/install-bionic.sh deleted file mode 100755 index a39de7135..000000000 --- a/.github/scripts/install-bionic.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Adding repos and packages -add-apt-repository -y ppa:libretime/libretime -apt-get -q update -apt-get install -y gstreamer1.0-plugins-base \ - gstreamer1.0-plugins-good \ - gstreamer1.0-plugins-bad \ - gstreamer1.0-plugins-ugly \ - libgirepository1.0-dev \ - liquidsoap \ - liquidsoap-plugin-faad \ - liquidsoap-plugin-lame \ - liquidsoap-plugin-mad \ - liquidsoap-plugin-vorbis \ - python3-gst-1.0 \ - silan \ - ffmpeg \ - gcc \ - gir1.2-gtk-3.0 \ - python3-setuptools \ - python3-gi \ - python3-gi-cairo \ - python-cairo \ - pkg-config \ - libcairo2-dev - -# Making log directory for PHP tests -mkdir -p "$LIBRETIME_LOG_DIR" -chown runner:runner "$LIBRETIME_LOG_DIR" diff --git a/.github/scripts/python-pkg-install.sh b/.github/scripts/python-pkg-install.sh deleted file mode 100755 index 063b2df46..000000000 --- a/.github/scripts/python-pkg-install.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -echo "::group::Install Python apps" -for app in python_apps/*; do - if [[ -f "$app/requirements-dev.txt" ]]; then - pip3 install -r "$app/requirements-dev.txt" - fi - pip3 install -e "$app" -done -echo "::endgroup::" diff --git a/.github/scripts/python-pkg-test.sh b/.github/scripts/python-pkg-test.sh deleted file mode 100755 index 96f7cf18c..000000000 --- a/.github/scripts/python-pkg-test.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -failed="false" - -echo "::group::Airtime Analyzer" -if ! make -C python_apps/airtime_analyzer test; then - failed="true" -fi -echo "::endgroup::" - -echo "::group::API Client" -if ! make -C python_apps/api_clients test; then - failed="true" -fi -echo "::endgroup::" - -if [[ $failed == "true" ]]; then - echo "Python tests failed" - exit 1 -fi -echo "Python tests passed!" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f534545a..5bdeb93c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,21 +1,11 @@ -name: Python and PHP Tests +name: Tests + on: push: - paths-ignore: - - "docs/**" + branches: [master] pull_request: - types: - [ - opened, - ready_for_review, - review_requested, - edited, - reopened, - synchronize, - ] - paths-ignore: - - "docs/**" - workflow_dispatch: + types: [opened, reopened, synchronize, edited] + branches: [master] jobs: pre-commit: @@ -46,14 +36,12 @@ jobs: - run: SEVERITY=warning make shell-check - test: + test-legacy: strategy: matrix: include: - runs-on: ubuntu-18.04 - python-version: "3.6" php-version: "7.2" - prerequisites-script: install-bionic.sh runs-on: ${{ matrix.runs-on }} env: @@ -61,9 +49,7 @@ jobs: LIBRETIME_LOG_DIR: /tmp/log/libretime steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - name: Setup PostgreSQL run: | sudo systemctl start postgresql.service @@ -72,18 +58,69 @@ jobs: sudo -u postgres psql -c "CREATE USER libretime WITH PASSWORD 'libretime';" sudo -u postgres psql -c 'GRANT CONNECT ON DATABASE libretime TO libretime;' sudo -u postgres psql -c 'ALTER USER libretime CREATEDB;' - - name: Setup PHP with specific version + + - name: Setup PHP uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php-version }} - - name: Install prerequisites - run: sudo -E ./.github/scripts/${{ matrix.prerequisites-script }} - - name: Run Python tests + + - name: Get Composer Cache Directory + id: composer-cache run: | - sudo ./.github/scripts/python-pkg-install.sh - ./.github/scripts/python-pkg-test.sh - - name: Run PHP tests + echo "::set-output name=dir::$(composer config cache-files-dir)" + + - uses: actions/cache@v2 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Install dependencies run: | composer install --no-progress --dev + + - name: Run PHP tests + run: | + sudo mkdir -p "$LIBRETIME_LOG_DIR" + sudo chown runner:runner "$LIBRETIME_LOG_DIR" + cd airtime_mvc/tests php ../../vendor/bin/phpunit + + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + release: [bionic, buster] + context: [python_apps/airtime_analyzer, python_apps/api_clients] + + container: ghcr.io/${{ github.repository_owner }}/libretime-dev:${{ matrix.release }} + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v2 + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py', '**/requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.context }} + + - name: Install dependencies + run: | + python3 -m venv venv && source venv/bin/activate + pip install --upgrade pip setuptools wheel + pip install -r requirements-dev.txt + pip install -e . + working-directory: ${{ matrix.context }} + + - name: Test + run: | + source venv/bin/activate + make test + working-directory: ${{ matrix.context }} diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml new file mode 100644 index 000000000..a032f4cd5 --- /dev/null +++ b/.github/workflows/tools.yml @@ -0,0 +1,77 @@ +name: Tools + +on: + schedule: + - cron: "0 3 * * 0" + push: + branches: [master] + paths: + - "**/packages.ini" + - ".github/workflows/tools.yml" + +jobs: + docker-dev-image: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - distribution: ubuntu + release: bionic + - distribution: debian + release: buster + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate packages list + run: | + scripts/packages.py -f line ${{ matrix.release }} \ + python_apps/airtime_analyzer \ + python_apps/pypo \ + > packages.list + + - name: Generate Dockerfile + run: | + cat <> Dockerfile + FROM ${{ matrix.distribution }}:${{ matrix.release }} + + COPY packages.list packages.list + EOF + + [[ "${{ matrix.release }}" == "bionic" ]] && \ + cat <> Dockerfile + RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:libretime/libretime + EOF + + cat <> Dockerfile + RUN apt-get update && \ + apt-get -y install \ + python3 \ + python3-pip \ + $(cat packages.list) + + ARG USER=docker + ARG UID=1000 + ARG GID=1000 + + RUN useradd -m ${USER} --uid=${UID} + USER ${UID}:${GID} + WORKDIR /home/${USER} + EOF + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ghcr.io/${{ github.repository_owner }}/libretime-dev:${{ matrix.release }}