From fb40a5c0d1578d96903704953832d41618a4bf34 Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Wed, 24 Nov 2021 18:14:41 +0100 Subject: [PATCH] Partial CI workflows: take 2 (#60294) --- .github/workflows/ci.yaml | 353 ++++++++++++++++++-------------------- 1 file changed, 167 insertions(+), 186 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c9a9ac4eff8..4a35f7b8e35 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,6 +20,106 @@ concurrency: cancel-in-progress: true jobs: + changes: + name: Determine what has changed + outputs: + # In case of issues with the partial run, use the following line instead: + # test_full_suite: 'true' + test_full_suite: ${{ steps.info.outputs.test_full_suite }} + core: ${{ steps.core.outputs.changes }} + integrations: ${{ steps.integrations.outputs.changes }} + integrations_glob: ${{ steps.info.outputs.integrations_glob }} + tests: ${{ steps.info.outputs.tests }} + tests_glob: ${{ steps.info.outputs.tests_glob }} + test_groups: ${{ steps.info.outputs.test_groups }} + test_group_count: ${{ steps.info.outputs.test_group_count }} + runs-on: ubuntu-latest + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.4.0 + - name: Filter for core changes + uses: dorny/paths-filter@v2.10.2 + id: core + with: + filters: .core_files.yaml + - name: Create a list of integrations to filter for changes + run: | + integrations=$(ls -Ad ./homeassistant/components/[!_]* | xargs -n 1 basename) + touch .integration_paths.yaml + for integration in $integrations; do + echo "${integration}: [homeassistant/components/${integration}/*, tests/components/${integration}/*]" \ + >> .integration_paths.yaml; + done + echo "Result:" + cat .integration_paths.yaml + - name: Filter for integration changes + uses: dorny/paths-filter@v2.10.2 + id: integrations + with: + filters: .integration_paths.yaml + - name: Collect additional information + id: info + run: | + # Defaults + integrations_glob="" + test_full_suite="true" + test_groups="[1, 2, 3, 4, 5, 6]" + test_group_count=6 + tests="[]" + tests_glob="" + + if [[ "${{ steps.integrations.outputs.changes }}" != "[]" ]]; + then + # Create a file glob for the integrations + integrations_glob=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '. | join(",")') + [[ "${integrations_glob}" == *","* ]] && integrations_glob="{${integrations_glob}}" + + # Create list of testable integrations + possible_integrations=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '.[]') + tests=$( + for integration in ${possible_integrations}; + do + if [[ -d "tests/components/${integration}" ]]; then + echo -n "\"${integration}\","; + fi; + done + ) + + [[ ! -z "${tests}" ]] && tests="${tests::-1}" + tests="[${tests}]" + test_groups="${tests}" + # Test group count should be 1, we don't split partial tests + test_group_count=1 + + # Create a file glob for the integrations tests + tests_glob=$(echo "${tests}" | jq -cSr '. | join(",")') + [[ "${tests_glob}" == *","* ]] && tests_glob="{${tests_glob}}" + + test_full_suite="false" + fi + + # We need to run the full suite on certain branches + if [[ "${{ github.ref }}" == "refs/heads/dev" ]] \ + || [[ "${{ github.ref }}" == "refs/heads/master" ]] \ + || [[ "${{ github.ref }}" == "refs/heads/rc" ]]; + then + test_full_suite="true" + fi + + # Output & sent to GitHub Actions + echo "test_full_suite: ${test_full_suite}" + echo "::set-output name=test_full_suite::${test_full_suite}" + echo "integrations_glob: ${integrations_glob}" + echo "::set-output name=integrations_glob::${integrations_glob}" + echo "test_group_count: ${test_group_count}" + echo "::set-output name=test_group_count::${test_group_count}" + echo "test_groups: ${test_groups}" + echo "::set-output name=test_groups::${test_groups}" + echo "tests: ${tests}" + echo "::set-output name=tests::${tests}" + echo "tests_glob: ${tests_glob}" + echo "::set-output name=tests_glob::${tests_glob}" + # Separate job to pre-populate the base dependency cache # This prevent upcoming jobs to do the same individually prepare-base: @@ -85,7 +185,9 @@ jobs: lint-bandit: name: Check bandit runs-on: ubuntu-latest - needs: prepare-base + needs: + - changes + - prepare-base steps: - name: Check out code from GitHub uses: actions/checkout@v2.4.0 @@ -117,15 +219,23 @@ jobs: run: | echo "Failed to restore pre-commit environment from cache" exit 1 - - name: Run bandit + - name: Run bandit (fully) + if: needs.changes.outputs.test_full_suite == 'true' run: | . venv/bin/activate pre-commit run --hook-stage manual bandit --all-files --show-diff-on-failure + - name: Run bandit (partially) + if: needs.changes.outputs.test_full_suite == 'false' + run: | + . venv/bin/activate + pre-commit run --hook-stage manual bandit --files {homeassistant,tests}/components/${{ needs.changes.outputs.integrations_glob }}/* --show-diff-on-failure lint-black: name: Check black runs-on: ubuntu-latest - needs: prepare-base + needs: + - changes + - prepare-base steps: - name: Check out code from GitHub uses: actions/checkout@v2.4.0 @@ -157,15 +267,23 @@ jobs: run: | echo "Failed to restore pre-commit environment from cache" exit 1 - - name: Run black + - name: Run black (fully) + if: needs.changes.outputs.test_full_suite == 'true' run: | . venv/bin/activate pre-commit run --hook-stage manual black --all-files --show-diff-on-failure + - name: Run black (partially) + if: needs.changes.outputs.test_full_suite == 'false' + run: | + . venv/bin/activate + pre-commit run --hook-stage manual black --files {homeassistant,tests}/components/${{ needs.changes.outputs.integrations_glob }}/* --show-diff-on-failure lint-flake8: name: Check flake8 runs-on: ubuntu-latest - needs: prepare-base + needs: + - changes + - prepare-base steps: - name: Check out code from GitHub uses: actions/checkout@v2.4.0 @@ -200,10 +318,16 @@ jobs: - name: Register flake8 problem matcher run: | echo "::add-matcher::.github/workflows/matchers/flake8.json" - - name: Run flake8 + - name: Run flake8 (fully) + if: needs.changes.outputs.test_full_suite == 'true' run: | . venv/bin/activate pre-commit run --hook-stage manual flake8 --all-files + - name: Run flake8 (partially) + if: needs.changes.outputs.test_full_suite == 'false' + run: | + . venv/bin/activate + pre-commit run --hook-stage manual flake8 --files {homeassistant,tests}/components/${{ needs.changes.outputs.integrations_glob }}/* lint-isort: name: Check isort @@ -436,7 +560,9 @@ jobs: pylint: name: Check pylint runs-on: ubuntu-latest - needs: prepare-tests + needs: + - changes + - prepare-tests strategy: matrix: python-version: [3.8] @@ -459,15 +585,23 @@ jobs: - name: Register pylint problem matcher run: | echo "::add-matcher::.github/workflows/matchers/pylint.json" - - name: Run pylint + - name: Run pylint (fully) + if: needs.changes.outputs.test_full_suite == 'true' run: | . venv/bin/activate pylint homeassistant + - name: Run pylint (partially) + if: needs.changes.outputs.test_full_suite == 'false' + run: | + . venv/bin/activate + pylint homeassistant/components/${{ needs.changes.outputs.integrations_glob }} mypy: name: Check mypy runs-on: ubuntu-latest - needs: prepare-tests + needs: + - changes + - prepare-tests strategy: matrix: python-version: [3.8] @@ -490,62 +624,20 @@ jobs: - name: Register mypy problem matcher run: | echo "::add-matcher::.github/workflows/matchers/mypy.json" - - name: Run mypy + - name: Run mypy (fully) + if: needs.changes.outputs.test_full_suite == 'true' run: | . venv/bin/activate mypy homeassistant - - changes: - name: Determine what has changed - outputs: - # core: ${{ steps.core.outputs.any }} - # Temporary disable - core: 'true' - integrations: ${{ steps.integrations.outputs.changes }} - tests: ${{ steps.tests.outputs.integrations }} - runs-on: ubuntu-latest - steps: - - name: Check out code from GitHub - uses: actions/checkout@v2.4.0 - - name: Filter for core changes - uses: dorny/paths-filter@v2.10.2 - id: core - with: - filters: .core_files.yaml - - name: Create a list of integrations to filter for changes - id: integration-filters + - name: Run mypy (partially) + if: needs.changes.outputs.test_full_suite == 'false' run: | - integrations=$(ls -Ad ./homeassistant/components/[!_]* | xargs -n 1 basename) - touch .integration_paths.yaml - for integration in $integrations; do - echo "${integration}: [homeassistant/components/${integration}/*, tests/components/${integration}/*]" \ - >> .integration_paths.yaml; - done - echo "Result:" - cat .integration_paths.yaml - - name: Filter for integration changes - uses: dorny/paths-filter@v2.10.2 - id: integrations - with: - filters: .integration_paths.yaml - - name: Determine integration tests to run - if: ${{ steps.integrations.outputs.changes }} - id: tests - run: | - possible_integrations=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '. | join(" ")') - integrations=$(for integration in $possible_integrations; do [[ -d "tests/components/${integration}" ]] && echo -n "${integration},"; done) - integrations="${integrations::-1}" + . venv/bin/activate + mypy homeassistant/components/${{ needs.changes.outputs.integrations_glob }} - # If more than one, add brackets to it - if [[ "${integrations}" == *","* ]]; then - integrations="{${integrations}}" - fi - - echo "::set-output name=integrations::${integrations}" - - pytest-full: - if: ${{ needs.changes.outputs.core == 'true' }} + pytest: runs-on: ubuntu-latest + if: needs.changes.outputs.test_full_suite == 'true' || needs.changes.outputs.tests_glob needs: - changes - gen-requirements-all @@ -559,10 +651,10 @@ jobs: strategy: fail-fast: false matrix: - group: [1, 2, 3, 4, 5, 6] + group: ${{ fromJson(needs.changes.outputs.test_groups) }} python-version: [3.8, 3.9] name: >- - Run tests Python ${{ matrix.python-version }} (group ${{ matrix.group }}) + Run tests Python ${{ matrix.python-version }} (${{ matrix.group }}) container: homeassistant/ci-azure:${{ matrix.python-version }} steps: - name: Check out code from GitHub @@ -589,7 +681,8 @@ jobs: # However this plugin is fairly new and doesn't run correctly # on a non-GitHub environment. pip install pytest-github-actions-annotate-failures==0.1.3 - - name: Run pytest + - name: Run pytest (fully) + if: needs.changes.outputs.test_full_suite == 'true' run: | . venv/bin/activate python3 -X dev -m pytest \ @@ -598,61 +691,15 @@ jobs: --durations=10 \ -n auto \ --dist=loadfile \ - --test-group-count 6 \ + --test-group-count ${{ needs.changes.outputs.test_group_count }} \ --test-group=${{ matrix.group }} \ --cov homeassistant \ --cov-report= \ -o console_output_style=count \ -p no:sugar \ tests - - name: Upload coverage artifact - uses: actions/upload-artifact@v2.2.4 - with: - name: coverage-${{ matrix.python-version }}-group${{ matrix.group }} - path: .coverage - - name: Check dirty - run: | - ./script/check_dirty - - pytest-partial: - if: ${{ needs.changes.outputs.core == 'false' }} - runs-on: ubuntu-latest - needs: - - changes - - prepare-tests - strategy: - fail-fast: false - matrix: - python-version: [3.8, 3.9] - name: >- - Run partial tests Python ${{ matrix.python-version }} - container: homeassistant/ci-azure:${{ matrix.python-version }} - steps: - - name: Check out code from GitHub - uses: actions/checkout@v2.4.0 - - name: Restore full Python ${{ matrix.python-version }} virtual environment - id: cache-venv - uses: actions/cache@v2.1.7 - with: - path: venv - key: ${{ runner.os }}-${{ matrix.python-version }}-${{ - needs.prepare-tests.outputs.python-key }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Register Python problem matcher - run: | - echo "::add-matcher::.github/workflows/matchers/python.json" - - name: Install Pytest Annotation plugin - run: | - . venv/bin/activate - # Ideally this should be part of our dependencies - # However this plugin is fairly new and doesn't run correctly - # on a non-GitHub environment. - pip install pytest-github-actions-annotate-failures==0.1.3 - - name: Run pytest + - name: Run pytest (partially) + if: needs.changes.outputs.test_full_suite == 'false' run: | . venv/bin/activate python3 -X dev -m pytest \ @@ -661,85 +708,19 @@ jobs: --durations=10 \ -n auto \ --dist=loadfile \ - --cov homeassistant \ + --cov homeassistant.components.${{ matrix.group }} \ --cov-report= \ -o console_output_style=count \ -p no:sugar \ - tests/components/${{ needs.changes.outputs.tests }} - - name: Upload coverage artifact - uses: actions/upload-artifact@v2.2.4 + tests/components/${{ matrix.group }} + - name: Upload coverage to Codecov (full coverage) + if: needs.changes.outputs.test_full_suite == 'true' + uses: codecov/codecov-action@v2.1.0 + - name: Upload coverage to Codecov (partial coverage) + if: needs.changes.outputs.test_full_suite == 'false' + uses: codecov/codecov-action@v2.1.0 with: - name: coverage-${{ matrix.python-version }} - path: .coverage + flags: ${{ matrix.group }} - name: Check dirty run: | ./script/check_dirty - - coverage-full: - name: Process test coverage - runs-on: ubuntu-latest - needs: ["prepare-tests", "pytest-full"] - strategy: - matrix: - python-version: [3.8] - container: homeassistant/ci-azure:${{ matrix.python-version }} - steps: - - name: Check out code from GitHub - uses: actions/checkout@v2.4.0 - - name: Restore full Python ${{ matrix.python-version }} virtual environment - id: cache-venv - uses: actions/cache@v2.1.7 - with: - path: venv - key: ${{ runner.os }}-${{ matrix.python-version }}-${{ - needs.prepare-tests.outputs.python-key }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Download all coverage artifacts - uses: actions/download-artifact@v2 - - name: Combine coverage results - run: | - . venv/bin/activate - coverage combine coverage*/.coverage* - coverage report --fail-under=94 - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2.1.0 - - coverage-partial: - name: Process partial test coverage - runs-on: ubuntu-latest - needs: ["prepare-tests", "pytest-partial"] - strategy: - matrix: - python-version: [3.8] - container: homeassistant/ci-azure:${{ matrix.python-version }} - steps: - - name: Check out code from GitHub - uses: actions/checkout@v2.4.0 - - name: Restore full Python ${{ matrix.python-version }} virtual environment - id: cache-venv - uses: actions/cache@v2.1.7 - with: - path: venv - key: ${{ runner.os }}-${{ matrix.python-version }}-${{ - needs.prepare-tests.outputs.python-key }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Download all coverage artifacts - uses: actions/download-artifact@v2 - - name: Combine coverage results - run: | - . venv/bin/activate - coverage combine coverage*/.coverage* - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2.1.0 - with: - flags: partial