usegalaxy-eu/infrastructure-playbook (opens in new tab)

---
name: CI
'on':
  pull_request:
  push:
    branches:
      - master

defaults:
  run:
    working-directory: 'infrastructure-playbook'

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v2
        with:
          path: 'infrastructure-playbook'

      - name: Set up Python 3.
        uses: actions/setup-python@v2
        with:
          python-version: '3.x'

      - name: Install test dependencies.
        run: pip3 install yamllint

      - name: Lint code.
        run: |
          yamllint .

---
name: Python formatting

"on": pull_request

jobs:
  PEP8:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          cache: 'pip'
          cache-dependency-path: '.github/requirements-python-lint.txt'

      - id: pip_install
        run: pip install -r '.github/requirements-python-lint.txt'

      - name: isort
        run: isort . --check --diff
        if: (success() || failure()) && steps.pip_install.conclusion == 'success'

      - name: Black
        uses: psf/black@stable
        with:
          version: "~=23.0"
          options: "--check --diff"
          src: "."
        if: (success() || failure()) && steps.pip_install.conclusion == 'success'

      - name: Flake8
        run: flake8 .
        if: (success() || failure()) && steps.pip_install.conclusion == 'success'

---
name: Total Perspective Vortex

"on":
  pull_request:
    paths:
      - "files/galaxy/tpv/**"
  push:
    branches:
      - master
    paths:
      - "files/galaxy/tpv/**"

jobs:
  lint:
    name: Total Perspective Vortex linter
    runs-on: ubuntu-latest
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v5
        with:
          path: "infrastructure-playbook"

      - name: Workaround Ansible vault password not being available to GitHub.
        working-directory: "infrastructure-playbook"
        run: |
          rm -f group_vars/htcondor/vault.yml
          rm -f group_vars/htcondor-secondary/vault.yml
          rm -f group_vars/all/ssh-keys_vault.yml

      - name: Update git submodules.
        working-directory: "infrastructure-playbook"
        run: |
          git submodule update --init --recursive --remote --checkout

      - name: Set up Python 3.
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - name: Install uv.
        uses: astral-sh/setup-uv@v7

      # Install Ansible.
      - name: Install Ansible.
        working-directory: "infrastructure-playbook"
        run: |
          # Install an Ansible version compatible with the version of
          # ansible-core specified in requirements.txt for the
          # infrastructure-playbook repo.
          ANSIBLE_CORE_REQ=$(perl -pe 's/\\\n/ /' requirements.txt | grep ansible-core)
          uv pip install --system ansible "$ANSIBLE_CORE_REQ"

      # Total Perspective Vortex needs the Galaxy logic, which should be
      # installed automatically when running `pip3 install
      # total-perspective-vortex[cli]` (the `galaxy-app` package).
      # However:
      # - `galaxy-app` package on PyPI is outdated (see issue #15999 on
      #   the Galaxy repo: https://github.com/galaxyproject/galaxy/issues/15999)
      # - Ideally the version of Galaxy should exactly match the one running on
      #   usegalaxy.eu.
      # Therefore, we clone Galaxy and add it to the PYTHONPATH.
      - name: Get Galaxy repo and commit id.
        id: commits-galaxy
        working-directory: "infrastructure-playbook"
        run: |
          # Get the Galaxy repository URL and commit from Ansible variables.
          export TMP_FILE=`mktemp`
          openssl rand -base64 24 > .vault_password
          ansible localhost --connection local \
            --inventory hosts --module-name copy \
            --args "content={{hostvars['sn09.galaxyproject.eu']}} dest=${TMP_FILE}" \
            > /dev/null
          export GALAXY_COMMIT_ID=$(cat ${TMP_FILE} | jq -r .galaxy_commit_id)
          export GALAXY_REPO=$(cat ${TMP_FILE} | jq -r .galaxy_repo)
          rm ${TMP_FILE}
          echo "commit=$GALAXY_COMMIT_ID" >> $GITHUB_OUTPUT
          echo "repo=$GALAXY_REPO" >> $GITHUB_OUTPUT
      - name: Cache Galaxy
        id: cache-galaxy
        uses: actions/cache@v4
        with:
          path: galaxy
          key: galaxy-${{ steps.commits-galaxy.outputs.repo }}-${{ steps.commits-galaxy.outputs.commit }}
      - name: Clone Galaxy
        if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
        run: |
          git clone ${{ steps.commits-galaxy.outputs.repo }} galaxy
      - name: Checkout production Galaxy commit
        if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
        working-directory: galaxy
        run: |
          git checkout "${{ steps.commits-galaxy.outputs.commit }}"
      - name: Install Galaxy requirements.
        working-directory: "galaxy"
        run: uv pip install --system -r requirements.txt

      # Install the Total Perspective Vortex version that should be running on
      # usegalaxy.eu
      - name: Install Total Perspective Vortex.
        working-directory: "galaxy"
        run: |
          TPV_REQ=$(perl -pe 's/\\\n/ /' lib/galaxy/dependencies/conditional-requirements.txt | grep total-perspective-vortex)
          uv pip install --system --upgrade "$TPV_REQ"

      - name: Install port of Ansible filters for Jinja (required for the next step).
        run: |
          uv pip install --system jinja2-ansible-filters

      - name: Create mounts vars file.
        working-directory: "infrastructure-playbook/mounts"
        run: |
          make dest/all.yml

      - name: Create a playbook to template the TPV files.
        shell: python -u {0}
        working-directory: "infrastructure-playbook"
        run: |
          import glob
          import importlib.util
          import sys

          # import tpv.py
          spec = importlib.util.spec_from_file_location('tpv_ci', '.github/workflows/tpv.py')
          module = importlib.util.module_from_spec(spec)
          spec.loader.exec_module(module)
          make_playbook = module.make_playbook

          tpv_path = "files/galaxy/tpv/"
          templates = tuple(
              (file, '.'.join(file.split('.')[:-1]))  # remove j2 extension
              for file in glob.glob(f"{tpv_path}/*.yaml.j2") + glob.glob(f"{tpv_path}/*.yml.j2")
          )

          playbook = make_playbook('sn09.yml', templates=templates)
          with open('../playbook_path', 'w') as file:
            file.write(str(playbook))

      - name: Render TPV configuration files.
        run: |
          PLAYBOOK="$(cat playbook_path)"
          BASENAME="$(basename $PLAYBOOK)"
          DIRNAME="$(dirname $PLAYBOOK)"
          cd "$DIRNAME"

          shopt -s nullglob

          ansible-playbook --connection=local --diff "$BASENAME"

      - name: Run Total Perspective Vortex linter.
        run: |
          export PYTHONPATH=$(realpath ./galaxy/lib)

          PLAYBOOK="$(cat playbook_path)"
          BASENAME="$(basename $PLAYBOOK)"
          DIRNAME="$(dirname $PLAYBOOK)"
          cd "$DIRNAME"

          shopt -s nullglob

          for file in files/galaxy/tpv/*.{yml,yaml}; do
            echo Running TPV linter on "$file"...
            tpv lint $file || exit 1
          done

  dry-run:
    name: Total Perspective Vortex dry-run
    runs-on: ubuntu-latest
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v5
        with:
          fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
          path: "infrastructure-playbook"

      - name: Workaround Ansible vault password not being available to GitHub.
        working-directory: "infrastructure-playbook"
        run: |
          rm -f group_vars/htcondor/vault.yml
          rm -f group_vars/htcondor-secondary/vault.yml
          rm -f group_vars/all/ssh-keys_vault.yml

      - name: Update git submodules.
        working-directory: "infrastructure-playbook"
        run: |
          git submodule update --init --recursive --remote --checkout

      - name: Set up Python 3.
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - name: Install uv.
        uses: astral-sh/setup-uv@v7

      # Install Ansible.
      - name: Install Ansible.
        working-directory: "infrastructure-playbook"
        run: |
          # Install an Ansible version compatible with the version of
          # ansible-core specified in requirements.txt for the
          # infrastructure-playbook repo.
          ANSIBLE_CORE_REQ=$(perl -pe 's/\\\n/ /' requirements.txt | grep ansible-core)
          uv pip install --system ansible "$ANSIBLE_CORE_REQ"

      # Total Perspective Vortex needs the Galaxy logic, which should be
      # installed automatically when running `pip3 install
      # total-perspective-vortex[cli]` (the `galaxy-app` package).
      # However:
      # - `galaxy-app` package on PyPI is outdated (see issue #15999 on
      #   the Galaxy repo: https://github.com/galaxyproject/galaxy/issues/15999)
      # - Ideally the version of Galaxy should exactly match the one running on
      #   usegalaxy.eu.
      # Therefore, we clone Galaxy and add it to the PYTHONPATH.
      - name: Get Galaxy repo and commit id.
        id: commits-galaxy
        working-directory: "infrastructure-playbook"
        run: |
          # Get the Galaxy repository URL and commit from Ansible variables.
          export TMP_FILE=`mktemp`
          openssl rand -base64 24 > .vault_password
          ansible localhost --connection local \
            --inventory hosts --module-name copy \
            --args "content={{hostvars['sn09.galaxyproject.eu']}} dest=${TMP_FILE}" \
            > /dev/null
          export GALAXY_COMMIT_ID=$(cat ${TMP_FILE} | jq -r .galaxy_commit_id)
          export GALAXY_REPO=$(cat ${TMP_FILE} | jq -r .galaxy_repo)
          rm ${TMP_FILE}
          echo "commit=$GALAXY_COMMIT_ID" >> $GITHUB_OUTPUT
          echo "repo=$GALAXY_REPO" >> $GITHUB_OUTPUT
      - name: Cache Galaxy
        id: cache-galaxy
        uses: actions/cache@v4
        with:
          path: galaxy
          key: galaxy-${{ steps.commits-galaxy.outputs.repo }}-${{ steps.commits-galaxy.outputs.commit }}
      - name: Clone Galaxy
        if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
        run: |
          git clone ${{ steps.commits-galaxy.outputs.repo }} galaxy
      - name: Checkout production Galaxy commit
        if: ${{ steps.cache-galaxy.outputs.cache-hit != 'true' }}
        working-directory: galaxy
        run: |
          git checkout "${{ steps.commits-galaxy.outputs.commit }}"
      - name: Install Galaxy requirements.
        working-directory: "galaxy"
        run: uv pip install --system -r requirements.txt

      # Install the Total Perspective Vortex version that should be running on
      # usegalaxy.eu
      - name: Install Total Perspective Vortex.
        working-directory: "galaxy"
        run: |
          TPV_REQ=$(perl -pe 's/\\\n/ /' lib/galaxy/dependencies/conditional-requirements.txt | grep total-perspective-vortex)
          uv pip install --system --upgrade "$TPV_REQ"

      - name: Install port of Ansible filters for Jinja.
        run: |
          uv pip install --system jinja2-ansible-filters

      - name: Create mounts vars file.
        working-directory: "infrastructure-playbook/mounts"
        run: |
          make dest/all.yml

      - name: Get commit ids before/after push or pull request.
        id: commits-infrastructure-playbook
        working-directory: "infrastructure-playbook"
        run: |
          set -Eeo pipefail
          if ${{ github.event_name == 'pull_request' }}; then
            echo "before=$(git rev-parse HEAD^1)" >> $GITHUB_OUTPUT
            echo "after=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
          else
            echo "before=${{ github.event.before }}" >> $GITHUB_OUTPUT
            echo "after=${{ github.event.after }}" >> $GITHUB_OUTPUT
          fi

      - name: Create playbooks to template the TPV files.
        id: playbooks
        shell: python -u {0}
        working-directory: "infrastructure-playbook"
        run: |
          import glob
          import importlib.util
          import os
          import subprocess
          import sys

          # import make_playbook from tpv.py
          spec = importlib.util.spec_from_file_location('tpv_ci', '.github/workflows/tpv.py')
          module = importlib.util.module_from_spec(spec)
          spec.loader.exec_module(module)
          make_playbook = module.make_playbook

          tpv_path = 'files/galaxy/tpv/'
          templates = tuple(
              # TPV configuration files
              (file, '.'.join(file.split('.')[:-1]))  # remove j2 extension
              for file in glob.glob(f'{tpv_path}/*.yaml.j2') + glob.glob(f'{tpv_path}/*.yml.j2')
          ) + (
              # job_conf.yml
              ('templates/galaxy/config/job_conf.yml.j2', 'templates/galaxy/config/job_conf.yml'),
          )

          # template files based on the new version
          playbook = make_playbook('sn09.yml', templates=templates)
          with open(os.environ["GITHUB_OUTPUT"], "a") as file:
              file.write(f"new={playbook}\n")

          # template files based on the old version
          subprocess.run(
            ["git", "checkout", "${{ steps.commits-infrastructure-playbook.outputs.before }}"]
          )
          playbook = make_playbook('sn09.yml', templates=templates)
          with open(os.environ["GITHUB_OUTPUT"], "a") as file:
              file.write(f"old={playbook}\n")

      - name: Render TPV configuration files.
        run: |
          shopt -s nullglob

          PLAYBOOK="${{ steps.playbooks.outputs.old }}"
          BASENAME="$(basename $PLAYBOOK)"
          DIRNAME="$(dirname $PLAYBOOK)"
          cd "$DIRNAME"
          ansible-playbook --connection=local --diff "$BASENAME"

          PLAYBOOK="${{ steps.playbooks.outputs.new }}"
          BASENAME="$(basename $PLAYBOOK)"
          DIRNAME="$(dirname $PLAYBOOK)"
          cd "$DIRNAME"
          ansible-playbook --connection=local --diff "$BASENAME"

      - name: Change paths of TPV configuration files in job_conf.yml
        shell: python -u {0}
        run: |
          import os
          from pathlib import Path
          from urllib.parse import urlparse

          import yaml

          repository = Path("${{ steps.playbooks.outputs.new }}").parent
          job_conf_path = Path('templates/galaxy/config/job_conf.yml')
          tpv_path = Path('files/galaxy/tpv/')

          job_conf = yaml.safe_load(open(repository / job_conf_path, 'r'))
          tpv_config_files = job_conf['execution']['environments']['tpv_dispatcher']['tpv_config_files']
          for i, file in enumerate(tpv_config_files):
              if urlparse(file).scheme in {'file', ''}:
                  tpv_config_files[i] = str(repository / tpv_path / Path(file).name)
          job_conf['execution']['environments']['tpv_dispatcher']['tpv_config_files'] = tpv_config_files

          yaml.dump(job_conf, open(repository / job_conf_path, 'w'))

      - name: Detect tools that have changed.
        id: tools-changed
        shell: python -u {0}
        run: |
          import glob
          import hashlib
          import os
          from pathlib import Path

          import yaml

          playbooks = {
            'old': Path("${{ steps.playbooks.outputs.old }}"),
            'new': Path("${{ steps.playbooks.outputs.new }}"),
          }
          os.chdir(playbooks['new'].parent)

          tpv_path = Path('files/galaxy/tpv/')

          changed = set()
          for file in (Path(path) for path in glob.glob(f"{tpv_path}/*.yml") + glob.glob(f"{tpv_path}/*.yaml")):
              comparison = {}
              for key, playbook in playbooks.items():
                  comparison[key] = yaml.safe_load(open(playbooks[key].parent / file, 'r')).get("tools", {})
                  comparison[key] = {
                    key: hashlib.sha256(yaml.dump(value, sort_keys=True).encode('utf-8')).hexdigest()
                    for key, value in comparison[key].items() if not value.get("abstract", False) # ignore abstract tools
                  }
              changed |= set(comparison['new']) - set(comparison['old'])
              changed |= {key for key in set(comparison['new']) & set(comparison['old']) if comparison['new'][key] != comparison['old'][key]}

          with open(os.environ["GITHUB_OUTPUT"], 'a') as file:
            file.write(f"changed={' '.join(changed)}\n")

      - name: Run Total Perspective Vortex dry-run.
        env:
          TOOLS: ${{ steps.tools-changed.outputs.changed }}
          PLAYBOOK: ${{ steps.playbooks.outputs.new }}
        run: |
          set -Eeo pipefail
          shopt -s nullglob

          export PYTHONPATH=$(realpath ./galaxy/lib)

          BASENAME="$(basename $PLAYBOOK)"
          DIRNAME="$(dirname $PLAYBOOK)"
          cd "$DIRNAME"

          IFS=" " read -a tools <<< "$TOOLS"
          for tool in "${tools[@]}"; do
              echo Running TPV dry-run for "$tool..."
              tpv dry-run --job-conf templates/galaxy/config/job_conf.yml --tool "$tool"
          done