Merge pull request #1315 from jooola/feat/rework_ci

Rework CI pipeline

.github/scripts/install-bionic.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-# Adding repos and packages
-add-apt-repository -y ppa:libretime/libretime
-apt-get -q update
-apt-get install -y gstreamer1.0-plugins-base \
-  gstreamer1.0-plugins-good \
-  gstreamer1.0-plugins-bad \
-  gstreamer1.0-plugins-ugly \
-  libgirepository1.0-dev \
-  liquidsoap \
-  liquidsoap-plugin-faad \
-  liquidsoap-plugin-lame \
-  liquidsoap-plugin-mad \
-  liquidsoap-plugin-vorbis \
-  python3-gst-1.0 \
-  silan \
-  ffmpeg \
-  gcc \
-  gir1.2-gtk-3.0 \
-  python3-setuptools \
-  python3-gi \
-  python3-gi-cairo \
-  python-cairo \
-  pkg-config \
-  libcairo2-dev
-
-# Making log directory for PHP tests
-mkdir -p "$LIBRETIME_LOG_DIR"
-chown runner:runner "$LIBRETIME_LOG_DIR"

.github/scripts/python-pkg-install.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-echo "::group::Install Python apps"
-for app in python_apps/*; do
-  if [[ -f "$app/requirements-dev.txt" ]]; then
-    pip3 install -r "$app/requirements-dev.txt"
-  fi
-  pip3 install -e "$app"
-done
-echo "::endgroup::"

.github/scripts/python-pkg-test.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-failed="false"
-
-echo "::group::Airtime Analyzer"
-if ! make -C python_apps/airtime_analyzer test; then
-  failed="true"
-fi
-echo "::endgroup::"
-
-echo "::group::API Client"
-if ! make -C python_apps/api_clients test; then
-  failed="true"
-fi
-echo "::endgroup::"
-
-if [[ $failed == "true" ]]; then
-  echo "Python tests failed"
-  exit 1
-fi
-echo "Python tests passed!"

.github/workflows/test.yml

@@ -1,21 +1,11 @@
-name: Python and PHP Tests
+name: Tests
+
 on:
   push:
-    paths-ignore:
-      - "docs/**"
+    branches: [master]
   pull_request:
-    types:
-      [
-        opened,
-        ready_for_review,
-        review_requested,
-        edited,
-        reopened,
-        synchronize,
-      ]
-    paths-ignore:
-      - "docs/**"
-  workflow_dispatch:
+    types: [opened, reopened, synchronize, edited]
+    branches: [master]
 
 jobs:
   pre-commit:
@@ -46,14 +36,12 @@ jobs:
 
       - run: SEVERITY=warning make shell-check
 
-  test:
+  test-legacy:
     strategy:
       matrix:
         include:
           - runs-on: ubuntu-18.04
-            python-version: "3.6"
             php-version: "7.2"
-            prerequisites-script: install-bionic.sh
 
     runs-on: ${{ matrix.runs-on }}
     env:
@@ -61,9 +49,7 @@ jobs:
       LIBRETIME_LOG_DIR: /tmp/log/libretime
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
+
       - name: Setup PostgreSQL
         run: |
           sudo systemctl start postgresql.service
@@ -72,18 +58,69 @@ jobs:
           sudo -u postgres psql -c "CREATE USER libretime WITH PASSWORD 'libretime';"
           sudo -u postgres psql -c 'GRANT CONNECT ON DATABASE libretime TO libretime;'
           sudo -u postgres psql -c 'ALTER USER libretime CREATEDB;'
-      - name: Setup PHP with specific version
+
+      - name: Setup PHP
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ matrix.php-version }}
-      - name: Install prerequisites
-        run: sudo -E ./.github/scripts/${{ matrix.prerequisites-script }}
-      - name: Run Python tests
+
+      - name: Get Composer Cache Directory
+        id: composer-cache
         run: |
-          sudo ./.github/scripts/python-pkg-install.sh
-          ./.github/scripts/python-pkg-test.sh
-      - name: Run PHP tests
+          echo "::set-output name=dir::$(composer config cache-files-dir)"
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-
+
+      - name: Install dependencies
         run: |
           composer install --no-progress --dev
+
+      - name: Run PHP tests
+        run: |
+          sudo mkdir -p "$LIBRETIME_LOG_DIR"
+          sudo chown runner:runner "$LIBRETIME_LOG_DIR"
+
           cd airtime_mvc/tests
           php ../../vendor/bin/phpunit
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        release: [bionic, buster]
+        context: [python_apps/airtime_analyzer, python_apps/api_clients]
+
+    container: ghcr.io/${{ github.repository_owner }}/libretime-dev:${{ matrix.release }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.context }}-${{ hashFiles('**/setup.py', '**/requirements-dev.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ matrix.context }}
+
+      - name: Install dependencies
+        run: |
+          python3 -m venv venv && source venv/bin/activate
+          pip install --upgrade pip setuptools wheel
+          pip install -r requirements-dev.txt
+          pip install -e .
+        working-directory: ${{ matrix.context }}
+
+      - name: Test
+        run: |
+          source venv/bin/activate
+          make test
+        working-directory: ${{ matrix.context }}

.github/workflows/tools.yml

@@ -0,0 +1,77 @@
+name: Tools
+
+on:
+  schedule:
+    - cron: "0 3 * * 0"
+  push:
+    branches: [master]
+    paths:
+      - "**/packages.ini"
+      - ".github/workflows/tools.yml"
+
+jobs:
+  docker-dev-image:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - distribution: ubuntu
+            release: bionic
+          - distribution: debian
+            release: buster
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate packages list
+        run: |
+          scripts/packages.py --dev --format line ${{ matrix.release }} \
+            python_apps/airtime_analyzer \
+            python_apps/pypo \
+            > packages.list
+
+      - name: Generate Dockerfile
+        run: |
+          cat <<EOF >> Dockerfile
+          FROM ${{ matrix.distribution }}:${{ matrix.release }}
+
+          COPY packages.list packages.list
+          EOF
+
+          [[ "${{ matrix.release }}" == "bionic" ]] && \
+          cat <<EOF >> Dockerfile
+          RUN apt-get update && \
+              apt-get install -y software-properties-common && \
+              add-apt-repository -y ppa:libretime/libretime
+          EOF
+
+          cat <<EOF >> Dockerfile
+          RUN apt-get update && \
+              apt-get -y install \
+              python3 \
+              python3-pip \
+              $(cat packages.list)
+
+          ARG USER=docker
+          ARG UID=1000
+          ARG GID=1000
+
+          RUN useradd -m ${USER} --uid=${UID}
+          USER ${UID}:${GID}
+          WORKDIR /home/${USER}
+          EOF
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/libretime-dev:${{ matrix.release }}

python_apps/airtime_analyzer/packages.ini

@@ -44,3 +44,7 @@ python3-gst-1.0 = buster, bionic
 
 [silan]
 silan = buster, bionic
+
+[=development]
+# Generate fixtures
+ffmpeg = buster, bionic