galaxyproject/planemo-ci-action (opens in new tab)

on: [push, pull_request]

name: 'Shellcheck'

jobs:
  shellcheck:
    name: Shellcheck
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Run ShellCheck
      uses: ludeeus/action-shellcheck@master

on: [push, pull_request]

name: 'Test action for tools'

env:
  GALAXY_FORK: galaxyproject
  GALAXY_BRANCH: master
  MAX_CHUNKS: 4

jobs:
  setup-pr-tools:
    name: Setup as in PR for tools
    runs-on: ubuntu-latest
    outputs:
      galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
      repository-list: ${{ steps.discover.outputs.repository-list }}
      tool-list: ${{ steps.discover.outputs.tool-list }}
      chunk-count: ${{ steps.discover.outputs.chunk-count }}
      chunk-list: ${{ steps.discover.outputs.chunk-list }}
    strategy:
      matrix:
        python-version: [3.11]
    steps:
    - name: Print github context properties
      run: |
        echo 'event: ${{ github.event_name }}'
        echo 'sha: ${{ github.sha }}'
        echo 'ref: ${{ github.ref }}'
        echo 'head_ref: ${{ github.head_ref }}'
        echo 'base_ref: ${{ github.base_ref }}'
        echo 'event.before: ${{ github.event.before }}'
        echo 'event.after: ${{ github.event.after }}'
    - name: Determine latest commit in the Galaxy repo
      id: get-galaxy-sha
      run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
    - name: Cache .planemo
      uses: actions/cache@v3
      id: cache-planemo
      with:
        path: ~/.planemo
        key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
    # Install the `wheel` package so that when installing other packages which
    # are not available as wheels, pip will build a wheel for them, which can be cached.
    - name: Install wheel
      run: pip install wheel
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Introduce a change on test tool1
      run: |
        git config --global user.name "Arthur Dent"
        git config --global user.email "a.dent@galaxyproject.org"
        echo "" >> test/tools/tool1/tool1.xml
        git commit -m bump test/tools/tool1/tool1.xml
    - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
      uses: ./
      id: discover
      with:
        create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }}
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        max-chunks: ${{ env.MAX_CHUNKS }}
        python-version: ${{ matrix.python-version }}
    - name: Check commit range
      run: if ! grep "\.\." <<<$(echo ${{ steps.discover.outputs.commit-range }}); then echo "wrong commit range"; exit 1; fi
    - name: Check content of repository list
      run: |
        if ! grep -q "tool1" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool1 must be in repo list"; exit 1; fi
        if grep -q "tool2" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool2 must not be in repo list"; exit 1; fi
    - name: Check content of tool list
      run: |
        if ! grep -q "tool1.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool1.xml must be in tool list"; exit 1; fi
        if grep -q "tool2.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool2.xml must not be in tool list"; exit 1; fi
    - name: Check number of chunks
      run: if [ "${{ steps.discover.outputs.chunk-count }}" != "1" ]; then echo "wrong chunk-count"; exit 1; fi

  setup-ci-tools:
    name: Setup as in CI for tools
    runs-on: ubuntu-latest
    outputs:
      galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
      fork: ${{ steps.get-fork-branch.outputs.fork }}
      branch: ${{ steps.get-fork-branch.outputs.branch }}
      repository-list: ${{ steps.discover.outputs.repository-list }}
      chunk-count: ${{ steps.discover.outputs.chunk-count }}
      chunk-list: ${{ steps.discover.outputs.chunk-list }}
    strategy:
      matrix:
        python-version: [3.11]
    steps:
    - name: Determine latest commit in the Galaxy repo
      id: get-galaxy-sha
      run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
    # Install the `wheel` package so that when installing other packages which
    # are not available as wheels, pip will build a wheel for them, which can be cached.
    - name: Install wheel
      run: pip install wheel
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - name: Artificially exclude workflows (because ci_find_repos discovers tool and workflow repos)
      run: echo 'test/workflows/' > .tt_skip
    - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
      uses: ./
      id: discover
      env:
        GITHUB_EVENT_NAME: schedule
      with:
        create-cache: true
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        max-chunks: ${{ env.MAX_CHUNKS }}
        python-version: ${{ matrix.python-version }}
        github-event-name-override: 'schedule'
    - name: Check for empty commit range
      run: if [ -n "${{ steps.discover.outputs.commit-range }}" ]; then exit 1; fi
    # note: ci_find_repos will find all tools and workflows
    - name: Check that all tools an workflows are in the repository list
      run: |
        if ! grep -q "tool1" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool1 must be in the repo list"; exit 1; fi
        if ! grep -q "tool2" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool2 must be in the repo list"; exit 1; fi
        if grep -q "example3" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example3 must not be in the repo list"; exit 1; fi
        if grep -q "example4" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example4 must not be in the repo list"; exit 1; fi
    # note: ci_find_tools will find onlytools
    - name: Check that all tools are in the tool list
      run: |
        if ! grep -q "tool1.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool1.xml must be in the repo list"; exit 1; fi
        if ! grep -q "tool2.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool2.xml must be in the repo list"; exit 1; fi
    # chunk-count will only consider tools/workflows depending on the workflow input to the action
    - name: Check that the number of chunks is four, i.e. the number of tools
      run: if [ "${{ steps.discover.outputs.chunk-count }}" != "4" ]; then exit 1; fi

  lint:
    name: Test linting of tools
    needs: [setup-ci-tools]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.11]
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
    - name: Install wheel
      run: pip install wheel
    - name: Planemo lint tools
      uses: ./
      with:
        mode: lint
        report-level: all
        fail-level: warn
        repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
        tool-list: ${{ needs.setup-ci-tools.outputs.tool-list }}
        additional-planemo-options: --report_level all
      continue-on-error: true
    - name: check if all test tools were linted
      run: |
        set -x
        # check if the ToolIDValid linter is called (no of tools - 1 ) times, i.e. is skipped once
        if [ "$(grep -c ToolIDValid lint_report.txt)" != "3" ]; then
          echo "expecting exactly 2 tests for ToolIDValid"; exit 1; 
        fi
        grep tool1 lint_report.txt
        grep tool2 lint_report.txt
        grep fail lint_report.txt
        grep "ERROR: Error 'HTTPConnectionPool" lint_report.txt
        grep "TestsNoValid" lint_report.txt
  test-tools:
    name: Test testing of tools
    needs: [setup-ci-tools]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        chunk: ${{ fromJson(needs.setup-ci-tools.outputs.chunk-list) }}
        python-version: [3.11]
    services:
      postgres:
        image: postgres:11
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        ports:
          - 5432:5432
    steps:
    # checkout the repository
    # and use it as the current working directory
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
    - name: Get number of CPU cores
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores
    - name: Planemo test tools
      uses: ./
      id: test-tools
      with:
        mode: test
        repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        chunk: ${{ matrix.chunk }}
        chunk-count: ${{ needs.setup-ci-tools.outputs.chunk-count }}
        additional-planemo-options: --simultaneous_uploads --check_uploads_ok
        galaxy-slots: ${{ steps.cpu-cores.outputs.count }}
    - uses: actions/upload-artifact@v4
      with:
        name: 'Tool test output ${{ matrix.chunk }}'
        path: upload

  combine_outputs:
    name: Test combining 'chunked' test results
    needs: [setup-ci-tools, test-tools]
    strategy:
      matrix:
        python-version: [3.11]
    # This job runs on Linux
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/download-artifact@v4
      with:
        path: artifacts
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
    - name: Combine outputs
      uses: ./
      id: combine
      with:
        mode: combine
        html-report: true
    - name: Check statistics
      run: |
        if ! grep -q "3\s\+success" <<<$(echo "${{ steps.combine.outputs.statistics }}"); then
          echo "wrong statistics: expected 3 success";
          echo "${{ steps.combine.outputs.statistics }}";
          exit 1;
        fi
        if ! grep -q "1\s\+failure" <<<$(echo "${{ steps.combine.outputs.statistics }}"); then
          echo "wrong statistics: expected 1 failure";
          echo "${{ steps.combine.outputs.statistics }}";
          exit 1;
        fi
    - uses: actions/upload-artifact@v4
      with:
        name: 'All tool test results'
        path: upload
    - name: Check outputs
      uses: ./
      continue-on-error: true
      id: check
      with:
        mode: check
    - name: Check on failures
      if: steps.check.outcome != 'failure'
      run: |
        echo "outcome  ${{ steps.check.outcome }}" 
        exit 1

name: Keep the versions up-to-date

on:
  release:
    types: [published, edited]

jobs:
  actions-tagger:
    runs-on: ubuntu-latest
    steps:
      - uses: Actions-R-Us/actions-tagger@v2
        env:
          GITHUB_TOKEN: "${{ github.token }}"
        with:
          publish_latest_tag: true

on: [push, pull_request]

name: 'Test action for workflows'

env:
  GALAXY_FORK: galaxyproject
  GALAXY_BRANCH: master
  MAX_CHUNKS: 4

jobs:

  setup-pr-workflows:
    name: Setup as in PR for workflows
    runs-on: ubuntu-latest
    outputs:
      galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
      repository-list: ${{ steps.discover.outputs.repository-list }}
      tool-list: ${{ steps.discover.outputs.tool-list }}
      chunk-count: ${{ steps.discover.outputs.chunk-count }}
      chunk-list: ${{ steps.discover.outputs.chunk-list }}
    strategy:
      matrix:
        python-version: [3.11]
    steps:
    - name: Print github context properties
      run: |
        echo 'event: ${{ github.event_name }}'
        echo 'sha: ${{ github.sha }}'
        echo 'ref: ${{ github.ref }}'
        echo 'head_ref: ${{ github.head_ref }}'
        echo 'base_ref: ${{ github.base_ref }}'
        echo 'event.before: ${{ github.event.before }}'
        echo 'event.after: ${{ github.event.after }}'
    - name: Determine latest commit in the Galaxy repo
      id: get-galaxy-sha
      run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
    - name: Cache .planemo
      uses: actions/cache@v3
      id: cache-planemo
      with:
        path: ~/.planemo
        key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
    # Install the `wheel` package so that when installing other packages which
    # are not available as wheels, pip will build a wheel for them, which can be cached.
    - name: Install wheel
      run: pip install wheel
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Introduce a change on a test workflow
      run: |
        git config --global user.name "Arthur Dent"
        git config --global user.email "a.dent@galaxyproject.org"
        echo "" >> test/workflows/example3/tutorial.ga
        git commit -m bump test/workflows/example3/tutorial.ga
    - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
      uses: ./
      id: discover
      with:
        workflows: true
        create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }}
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        max-chunks: ${{ env.MAX_CHUNKS }}
        python-version: ${{ matrix.python-version }}
    - name: Check commit range
      run: if ! grep "\.\." <<<$(echo ${{ steps.discover.outputs.commit-range }}); then echo "wrong commit range"; exit 1; fi
    - name: Check content of repository list
      run: |
        if ! grep -q "example3" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example3 must be in repo list"; exit 1; fi
        if grep -q "example4" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example4 must not be in repo list"; exit 1; fi
    - name: Check content of tool list
      run: if [ -n "${{ steps.discover.outputs.tool-list }}" ]; then echo "tool list must be empty"; exit 1; fi
    - name: Check number of chunks
      run: if [ "${{ steps.discover.outputs.chunk-count }}" != "1" ]; then echo "wrong chunk-count"; fi
  
  setup-ci-workflows:
    name: Setup as in CI for workflows
    runs-on: ubuntu-latest
    outputs:
      galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
      fork: ${{ steps.get-fork-branch.outputs.fork }}
      branch: ${{ steps.get-fork-branch.outputs.branch }}
      repository-list: ${{ steps.discover.outputs.repository-list }}
      chunk-count: ${{ steps.discover.outputs.chunk-count }}
      chunk-list: ${{ steps.discover.outputs.chunk-list }}
    strategy:
      matrix:
        python-version: [3.11]
    steps:
    - name: Determine latest commit in the Galaxy repo
      id: get-galaxy-sha
      run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
    # Install the `wheel` package so that when installing other packages which
    # are not available as wheels, pip will build a wheel for them, which can be cached.
    - name: Install wheel
      run: pip install wheel
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - name: Artificially exclude tools (because ci_find_repos discovers tool and workflow repos)
      run: echo 'test/tools/' > .tt_skip
    - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
      uses: ./
      id: discover
      env:
        GITHUB_EVENT_NAME: schedule
      with:
        workflows: true
        create-cache: true
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        max-chunks: ${{ env.MAX_CHUNKS }}
        python-version: ${{ matrix.python-version }}
        github-event-name-override: 'schedule'
    - name: Check for empty commit range
      run: if [ -n "${{ steps.discover.outputs.commit-range }}" ]; then exit 1; fi
    # note: ci_find_repos will find all tools and workflows
    - name: Check that all tools an workflows are in the repository list
      run: |
        if grep -q "tool1" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool1 must not be in the repo list"; exit 1; fi
        if grep -q "tool2" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool2 must not be in the repo list"; exit 1; fi
        if ! grep -q "example3" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example3 must be in the repo list"; exit 1; fi
        if ! grep -q "example4" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example4 must be in the repo list"; exit 1; fi
    # note: ci_find_tools will find only tools, i.e. it should be empty here
    - name: Check that all tools are in the tool list
      run: if [ -n "${{ steps.discover.outputs.tool-list }}" ]; then exit 1; fi
    # chunk-count will only consider tools/workflows depending on the workflow input to the action
    - name: Check that the number of chunks is two, i.e. the number of tools
      run: if [ "${{ steps.discover.outputs.chunk-count }}" != "2" ]; then exit 1; fi

  lint:
    name: Test linting of workflows
    needs: [setup-ci-workflows]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.11]
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-workflows.outputs.galaxy-head-sha }}
    - name: Install wheel
      run: pip install wheel
    - name: Planemo lint workflows
      uses: ./
      with:
        mode: lint
        workflows: true
        repository-list: "${{ needs.setup-ci-workflows.outputs.repository-list }}"
        tool-list: "${{ needs.setup-ci-workflows.outputs.tool-list }}"

  test-workflows:
    name: Test testing of workflows
    needs: [setup-ci-workflows]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        chunk: ${{ fromJson(needs.setup-ci-workflows.outputs.chunk-list) }}
        python-version: [3.11]
    services:
      postgres:
        image: postgres:11
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        ports:
          - 5432:5432
    steps:
    # checkout the repository
    # and use it as the current working directory
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-workflows.outputs.galaxy-head-sha }}
    - name: Planemo test workflows
      uses: ./
      id: test-workflows
      with:
        mode: test
        workflows: true
        setup-cvmfs: true
        repository-list: ${{ needs.setup-ci-workflows.outputs.repository-list }}
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        chunk: ${{ matrix.chunk }}
        chunk-count: ${{ needs.setup-ci-workflows.outputs.chunk-count }}
    - name: 2nd run test workflows
      uses: ./
      id: test-workflows-2
      with:
        mode: test
        workflows: true
        setup-cvmfs: false
        repository-list: ${{ needs.setup-ci-workflows.outputs.repository-list }}
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        chunk: ${{ matrix.chunk }}
        chunk-count: ${{ needs.setup-ci-workflows.outputs.chunk-count }}
    - uses: actions/upload-artifact@v4
      with:
        name: 'Workflow test output ${{ matrix.chunk }}'
        path: upload

  combine_outputs:
    name: Test combining 'chunked' test results
    needs: [setup-ci-workflows, test-workflows]
    strategy:
      matrix:
        python-version: [3.11]
    # This job runs on Linux
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/download-artifact@v4
      with:
        path: artifacts
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
    - name: Combine outputs
      uses: ./
      id: combine
      with:
        mode: combine
        html-report: true
    - name: Check statistics
      run: if ! grep -q "4\s\+success" <<<$(echo ${{ steps.combine.outputs.statistics }}); then echo "wrong statistics"; exit 1; fi
    - uses: actions/upload-artifact@v4
      with:
        name: 'All tool test results'
        path: upload
    - name: Check outputs
      uses: ./
      id: check
      with:
        mode: check