galaxyproject/pulsar (opens in new tab)

name: Deploy
on: [push, pull_request]
jobs:
  build_packages:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - uses: actions/setup-python@v6
        with:
          python-version: '3.14'
          cache: pip
      - name: Install requirements
        run: |
          python3 -m pip install build twine
      - name: Create and check sdist and wheel packages
        run: make dist-all
      - uses: actions/upload-artifact@v7
        with:
          name: packages
          path: dist/
  pypi-publish:
    needs: [build_packages]
    name: Upload release to PyPI
    runs-on: ubuntu-latest
    permissions:
      id-token: write
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && github.repository_owner == 'galaxyproject'
    steps:
      - uses: actions/download-artifact@v8
        with:
          name: packages
          path: dist
      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1

name: Framework tests
on: [push, pull_request]
env:
  GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
jobs:
  test:
    name: Test
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.10']
        galaxy-branch: ['dev', 'master']
        metadata-strategy: ['directory']
        include:
          - python-version: '3.10'
            galaxy-branch: 'dev'
            metadata-strategy: 'extended'
    services:
      postgres:
        image: postgres:13
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v6
        with:
          path: 'pulsar'
          persist-credentials: false
      - name: Checkout tools repo
        uses: actions/checkout@v6
        with:
          repository: galaxyproject/galaxy
          ref: ${{ matrix.galaxy-branch }}
          path: galaxy
          persist-credentials: false
      - uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: Build Pulsar wheel and replace Galaxy's requirements.
        run: python tools/replace_galaxy_requirements_for_ci.py
        working-directory: 'pulsar'
      - name: Run tests
        run: ./run_tests.sh --framework
        working-directory: 'galaxy'
        env:
          GALAXY_TEST_JOB_CONFIG_FILE: ../pulsar/test_data/test_job_conf.yaml
          GALAXY_CONFIG_OVERRIDE_METADATA_STRATEGY: ${{ matrix.metadata-strategy }}
      - uses: actions/upload-artifact@v7
        with:
          name: Framework test results (${{ matrix.galaxy-branch }} ${{ matrix.metadata-strategy  }} ${{ matrix.python-version }})
          path: 'galaxy/run_framework_tests.html'

name: pulsar
on: [push, pull_request]
jobs:
  lint:
    name: Lint
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.7']
        tox-env: [lint, docs]
        include:
          - python-version: '3.14'
            tox-env: lint
    steps:
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
    - name: Install tox
      run: pip install tox
    - name: Run lint
      run: tox -e ${{ matrix.tox-env }}
  mypy:
    name: MyPy
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        tox-env: [mypy]
        python-version: ['3.14']
    steps:
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
    - name: Install tox
      run: pip install tox
    - name: Setup pycurl
      run: sudo apt update; sudo apt install -y libxml2-dev libxslt1-dev libcurl4-openssl-dev openssh-server
    - name: Run tox
      run: tox -e ${{ matrix.tox-env }}
  test:
    name: Run Tests
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        tox-env: [test-ci, test-unit]
        python-version: ['3.7', '3.11', '3.12', '3.13', '3.14']
        include:
        - tox-env: install_wheel
          python-version: '3.10'
    services:
      job-files:
        image: galaxy/simple-job-files:latest
        env:
          JOB_FILES_ROOT_DIRECTORY: /tmp
        ports:
          - '8000:8000'
        volumes:
          - '/tmp:/tmp:rw'
    steps:
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
    - name: Install tox
      run: pip install tox
    - name: Run setup
      run: .ci/setup_tests.sh
    - name: Run tests
      run: . local_env.sh; tox -e ${{ matrix.tox-env }}
      env:
        PULSAR_TEST_KEY: "test_data/testkey"
        PULSAR_TEST_EXTERNAL_JOB_FILES_URL: "http://localhost:8000"
        PULSAR_TEST_EXTERNAL_JOB_FILES_DIRECTORY: "/tmp"
        TEST_WEBAPP_POST_SHUTDOWN_SLEEP: 1
  resilience:
    name: Resilience Suite
    runs-on: ubuntu-22.04
    timeout-minutes: 45
    steps:
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: actions/setup-python@v6
      with:
        python-version: '3.11'
        cache: pip
    - name: Install test runner deps
      run: pip install pytest pytest-timeout requests
    - name: Build images
      run: docker compose -f test/resilience/docker-compose.yml build
    - name: Bring up the docker-compose stack
      run: docker compose -f test/resilience/docker-compose.yml up -d
    - name: Wait for stack
      run: |
        # Toxiproxy admin API is the readiness signal — when it answers,
        # rabbitmq, relay, valkey and mock-galaxy have all reached a usable
        # state because they're toxiproxy's depends_on chain.
        for i in $(seq 1 60); do
          if curl -sf http://localhost:8474/version >/dev/null; then
            echo "stack ready"; exit 0;
          fi
          sleep 2
        done
        echo "stack did not become ready" >&2
        docker compose -f test/resilience/docker-compose.yml ps
        docker compose -f test/resilience/docker-compose.yml logs
        exit 1
    - name: Run resilience suite
      # Fail-fast (-x) for the first CI run on this job: if a scenario
      # fails we want diagnostics quickly, not 45 minutes of cascade. The
      # captured docker logs from the next step are scoped to whatever
      # was happening when the first failure hit. Drop -x once the job
      # is stable.
      run: pytest test/resilience -v -x --keep-stack
      env:
        PYTHONUNBUFFERED: "1"
    - name: Show stack logs on failure
      if: failure()
      run: |
        echo "=== ps ==="
        docker compose -f test/resilience/docker-compose.yml ps
        echo "=== pulsar ==="
        docker compose -f test/resilience/docker-compose.yml logs --no-color --timestamps pulsar | tail -800
        echo "=== mock-galaxy ==="
        docker compose -f test/resilience/docker-compose.yml logs --no-color --timestamps mock-galaxy | grep -vE "GET /_recorder/events" | tail -400
        echo "=== rabbitmq ==="
        docker compose -f test/resilience/docker-compose.yml logs --tail=100 rabbitmq
        echo "=== relay ==="
        docker compose -f test/resilience/docker-compose.yml logs --no-color --timestamps relay | tail -400
        echo "=== toxiproxy ==="
        docker compose -f test/resilience/docker-compose.yml logs --no-color --timestamps toxiproxy | tail -200
    - name: Tear down
      if: always()
      run: docker compose -f test/resilience/docker-compose.yml down -v
  ## Can't get the following tests to work - they fall over on the containerized job not being able to
  ## connect to the host running the Pulsar server for file transfers in stage out/in.
  # tes-test:
  #   name: Run Tests
  #   runs-on: ubuntu-20.04
  #   strategy:
  #     matrix:
  #       include:
  #       - tox-env: test-funnel
  #         python-version: 3.7
  #   steps:
  #   - uses: actions/checkout@v6
  #     with:
  #       persist-credentials: false
  #   - uses: actions/setup-python@v6
  #     with:
  #       python-version: ${{ matrix.python-version }}
  #       cache: pip
  #   - name: Set up Go 1.x
  #     uses: actions/setup-go@v2
  #     with:
  #       go-version: ^1.13
  #   - name: Pre-fetch required docker containers
  #     run: docker pull 'galaxy/pulsar-pod-staging:0.15.0.0' && docker pull 'conda/miniconda3'
  #   - name: Install tox
  #     run: pip install tox
  #   - name: Run tests
  #     run: PULSAR_TEST_INFRASTRUCTURE_HOST="_PLATFORM_AUTO_" FUNNEL_SERVER_TARGET=DEPLOY PULSAR_TES_SERVER_TARGET=http://localhost:8000/ tox -e ${{ matrix.tox-env }}