Partial CI workflows: take 2 (#60294)

.github/workflows/ci.yaml

@@ -20,6 +20,106 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  changes:
+    name: Determine what has changed
+    outputs:
+      # In case of issues with the partial run, use the following line instead:
+      # test_full_suite: 'true'
+      test_full_suite: ${{ steps.info.outputs.test_full_suite }}
+      core: ${{ steps.core.outputs.changes }}
+      integrations: ${{ steps.integrations.outputs.changes }}
+      integrations_glob: ${{ steps.info.outputs.integrations_glob }}
+      tests: ${{ steps.info.outputs.tests }}
+      tests_glob: ${{ steps.info.outputs.tests_glob }}
+      test_groups: ${{ steps.info.outputs.test_groups }}
+      test_group_count: ${{ steps.info.outputs.test_group_count }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v2.4.0
+      - name: Filter for core changes
+        uses: dorny/paths-filter@v2.10.2
+        id: core
+        with:
+          filters: .core_files.yaml
+      - name: Create a list of integrations to filter for changes
+        run: |
+          integrations=$(ls -Ad ./homeassistant/components/[!_]*  | xargs -n 1 basename)
+          touch .integration_paths.yaml
+          for integration in $integrations; do
+            echo "${integration}: [homeassistant/components/${integration}/*, tests/components/${integration}/*]" \
+              >> .integration_paths.yaml;
+          done
+          echo "Result:"
+          cat .integration_paths.yaml
+      - name: Filter for integration changes
+        uses: dorny/paths-filter@v2.10.2
+        id: integrations
+        with:
+          filters: .integration_paths.yaml
+      - name: Collect additional information
+        id: info
+        run: |
+          # Defaults
+          integrations_glob=""
+          test_full_suite="true"
+          test_groups="[1, 2, 3, 4, 5, 6]"
+          test_group_count=6
+          tests="[]"
+          tests_glob=""
+
+          if [[ "${{ steps.integrations.outputs.changes }}" != "[]" ]];
+          then
+            # Create a file glob for the integrations
+            integrations_glob=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '. | join(",")')
+            [[ "${integrations_glob}" == *","* ]] && integrations_glob="{${integrations_glob}}"
+
+            # Create list of testable integrations
+            possible_integrations=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '.[]')
+            tests=$(
+              for integration in ${possible_integrations};
+              do
+                if [[ -d "tests/components/${integration}" ]]; then
+                  echo -n "\"${integration}\",";
+                fi;
+              done
+            )
+
+            [[ ! -z "${tests}" ]] && tests="${tests::-1}"
+            tests="[${tests}]"
+            test_groups="${tests}"
+            # Test group count should be 1, we don't split partial tests
+            test_group_count=1
+
+            # Create a file glob for the integrations tests
+            tests_glob=$(echo "${tests}" | jq -cSr '. | join(",")')
+            [[ "${tests_glob}" == *","* ]] && tests_glob="{${tests_glob}}"
+
+            test_full_suite="false"
+          fi
+
+          # We need to run the full suite on certain branches
+          if [[ "${{ github.ref }}" == "refs/heads/dev" ]] \
+            || [[ "${{ github.ref }}" == "refs/heads/master" ]] \
+            || [[ "${{ github.ref }}" == "refs/heads/rc" ]];
+          then
+            test_full_suite="true"
+          fi
+
+          # Output & sent to GitHub Actions
+          echo "test_full_suite: ${test_full_suite}"
+          echo "::set-output name=test_full_suite::${test_full_suite}"
+          echo "integrations_glob: ${integrations_glob}"
+          echo "::set-output name=integrations_glob::${integrations_glob}"
+          echo "test_group_count: ${test_group_count}"
+          echo "::set-output name=test_group_count::${test_group_count}"
+          echo "test_groups: ${test_groups}"
+          echo "::set-output name=test_groups::${test_groups}"
+          echo "tests: ${tests}"
+          echo "::set-output name=tests::${tests}"
+          echo "tests_glob: ${tests_glob}"
+          echo "::set-output name=tests_glob::${tests_glob}"
+
   # Separate job to pre-populate the base dependency cache
   # This prevent upcoming jobs to do the same individually
   prepare-base:
@@ -85,7 +185,9 @@ jobs:
   lint-bandit:
     name: Check bandit
     runs-on: ubuntu-latest
-    needs: prepare-base
+    needs:
+      - changes
+      - prepare-base
     steps:
       - name: Check out code from GitHub
         uses: actions/checkout@v2.4.0
@@ -117,15 +219,23 @@ jobs:
         run: |
           echo "Failed to restore pre-commit environment from cache"
           exit 1
-      - name: Run bandit
+      - name: Run bandit (fully)
+        if: needs.changes.outputs.test_full_suite == 'true'
         run: |
           . venv/bin/activate
           pre-commit run --hook-stage manual bandit --all-files --show-diff-on-failure
+      - name: Run bandit (partially)
+        if: needs.changes.outputs.test_full_suite == 'false'
+        run: |
+          . venv/bin/activate
+          pre-commit run --hook-stage manual bandit --files {homeassistant,tests}/components/${{ needs.changes.outputs.integrations_glob }}/* --show-diff-on-failure
 
   lint-black:
     name: Check black
     runs-on: ubuntu-latest
-    needs: prepare-base
+    needs:
+      - changes
+      - prepare-base
     steps:
       - name: Check out code from GitHub
         uses: actions/checkout@v2.4.0
@@ -157,15 +267,23 @@ jobs:
         run: |
           echo "Failed to restore pre-commit environment from cache"
           exit 1
-      - name: Run black
+      - name: Run black (fully)
+        if: needs.changes.outputs.test_full_suite == 'true'
         run: |
           . venv/bin/activate
           pre-commit run --hook-stage manual black --all-files --show-diff-on-failure
+      - name: Run black (partially)
+        if: needs.changes.outputs.test_full_suite == 'false'
+        run: |
+          . venv/bin/activate
+          pre-commit run --hook-stage manual black --files {homeassistant,tests}/components/${{ needs.changes.outputs.integrations_glob }}/* --show-diff-on-failure
 
   lint-flake8:
     name: Check flake8
     runs-on: ubuntu-latest
-    needs: prepare-base
+    needs:
+      - changes
+      - prepare-base
     steps:
       - name: Check out code from GitHub
         uses: actions/checkout@v2.4.0
@@ -200,10 +318,16 @@ jobs:
       - name: Register flake8 problem matcher
         run: |
           echo "::add-matcher::.github/workflows/matchers/flake8.json"
-      - name: Run flake8
+      - name: Run flake8 (fully)
+        if: needs.changes.outputs.test_full_suite == 'true'
         run: |
           . venv/bin/activate
           pre-commit run --hook-stage manual flake8 --all-files
+      - name: Run flake8 (partially)
+        if: needs.changes.outputs.test_full_suite == 'false'
+        run: |
+          . venv/bin/activate
+          pre-commit run --hook-stage manual flake8 --files {homeassistant,tests}/components/${{ needs.changes.outputs.integrations_glob }}/*
 
   lint-isort:
     name: Check isort
@@ -436,7 +560,9 @@ jobs:
   pylint:
     name: Check pylint
     runs-on: ubuntu-latest
-    needs: prepare-tests
+    needs:
+      - changes
+      - prepare-tests
     strategy:
       matrix:
         python-version: [3.8]
@@ -459,15 +585,23 @@ jobs:
       - name: Register pylint problem matcher
         run: |
           echo "::add-matcher::.github/workflows/matchers/pylint.json"
-      - name: Run pylint
+      - name: Run pylint (fully)
+        if: needs.changes.outputs.test_full_suite == 'true'
         run: |
           . venv/bin/activate
           pylint homeassistant
+      - name: Run pylint (partially)
+        if: needs.changes.outputs.test_full_suite == 'false'
+        run: |
+          . venv/bin/activate
+          pylint homeassistant/components/${{ needs.changes.outputs.integrations_glob }}
 
   mypy:
     name: Check mypy
     runs-on: ubuntu-latest
-    needs: prepare-tests
+    needs:
+      - changes
+      - prepare-tests
     strategy:
       matrix:
         python-version: [3.8]
@@ -490,62 +624,20 @@ jobs:
       - name: Register mypy problem matcher
         run: |
           echo "::add-matcher::.github/workflows/matchers/mypy.json"
-      - name: Run mypy
+      - name: Run mypy (fully)
+        if: needs.changes.outputs.test_full_suite == 'true'
         run: |
           . venv/bin/activate
           mypy homeassistant
-
+      - name: Run mypy (partially)
-  changes:
+        if: needs.changes.outputs.test_full_suite == 'false'
-    name: Determine what has changed
-    outputs:
-      # core: ${{ steps.core.outputs.any }}
-      # Temporary disable
-      core: 'true'
-      integrations: ${{ steps.integrations.outputs.changes }}
-      tests: ${{ steps.tests.outputs.integrations }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code from GitHub
-        uses: actions/checkout@v2.4.0
-      - name: Filter for core changes
-        uses: dorny/paths-filter@v2.10.2
-        id: core
-        with:
-          filters: .core_files.yaml
-      - name: Create a list of integrations to filter for changes
-        id: integration-filters
         run: |
-          integrations=$(ls -Ad ./homeassistant/components/[!_]*  | xargs -n 1 basename)
+          . venv/bin/activate
-          touch .integration_paths.yaml
+          mypy homeassistant/components/${{ needs.changes.outputs.integrations_glob }}
-          for integration in $integrations; do
-            echo "${integration}: [homeassistant/components/${integration}/*, tests/components/${integration}/*]" \
-              >> .integration_paths.yaml;
-          done
-          echo "Result:"
-          cat .integration_paths.yaml
-      - name: Filter for integration changes
-        uses: dorny/paths-filter@v2.10.2
-        id: integrations
-        with:
-          filters: .integration_paths.yaml
-      - name: Determine integration tests to run
-        if: ${{ steps.integrations.outputs.changes }}
-        id: tests
-        run: |
-          possible_integrations=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '. | join(" ")')
-          integrations=$(for integration in $possible_integrations; do [[ -d "tests/components/${integration}" ]] && echo -n "${integration},"; done)
-          integrations="${integrations::-1}"
 
-          # If more than one, add brackets to it
+  pytest:
-          if [[ "${integrations}" == *","* ]]; then
-            integrations="{${integrations}}"
-          fi
-
-          echo "::set-output name=integrations::${integrations}"
-
-  pytest-full:
-    if: ${{ needs.changes.outputs.core == 'true' }}
     runs-on: ubuntu-latest
+    if: needs.changes.outputs.test_full_suite == 'true' || needs.changes.outputs.tests_glob
     needs:
       - changes
       - gen-requirements-all
@@ -559,10 +651,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        group: [1, 2, 3, 4, 5, 6]
+        group: ${{ fromJson(needs.changes.outputs.test_groups) }}
         python-version: [3.8, 3.9]
     name: >-
-      Run tests Python ${{ matrix.python-version }} (group ${{ matrix.group }})
+      Run tests Python ${{ matrix.python-version }} (${{ matrix.group }})
     container: homeassistant/ci-azure:${{ matrix.python-version }}
     steps:
       - name: Check out code from GitHub
@@ -589,7 +681,8 @@ jobs:
           # However this plugin is fairly new and doesn't run correctly
           # on a non-GitHub environment.
           pip install pytest-github-actions-annotate-failures==0.1.3
-      - name: Run pytest
+      - name: Run pytest (fully)
+        if: needs.changes.outputs.test_full_suite == 'true'
         run: |
           . venv/bin/activate
           python3 -X dev -m pytest \
@@ -598,61 +691,15 @@ jobs:
             --durations=10 \
             -n auto \
             --dist=loadfile \
-            --test-group-count 6 \
+            --test-group-count ${{ needs.changes.outputs.test_group_count }} \
             --test-group=${{ matrix.group }} \
             --cov homeassistant \
             --cov-report= \
             -o console_output_style=count \
             -p no:sugar \
             tests
-      - name: Upload coverage artifact
+      - name: Run pytest (partially)
-        uses: actions/upload-artifact@v2.2.4
+        if: needs.changes.outputs.test_full_suite == 'false'
-        with:
-          name: coverage-${{ matrix.python-version }}-group${{ matrix.group }}
-          path: .coverage
-      - name: Check dirty
-        run: |
-          ./script/check_dirty
-
-  pytest-partial:
-    if: ${{ needs.changes.outputs.core == 'false' }}
-    runs-on: ubuntu-latest
-    needs:
-      - changes
-      - prepare-tests
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [3.8, 3.9]
-    name: >-
-      Run partial tests Python ${{ matrix.python-version }}
-    container: homeassistant/ci-azure:${{ matrix.python-version }}
-    steps:
-      - name: Check out code from GitHub
-        uses: actions/checkout@v2.4.0
-      - name: Restore full Python ${{ matrix.python-version }} virtual environment
-        id: cache-venv
-        uses: actions/cache@v2.1.7
-        with:
-          path: venv
-          key: ${{ runner.os }}-${{ matrix.python-version }}-${{
-            needs.prepare-tests.outputs.python-key }}
-      - name: Fail job if Python cache restore failed
-        if: steps.cache-venv.outputs.cache-hit != 'true'
-        run: |
-          echo "Failed to restore Python virtual environment from cache"
-          exit 1
-      - name: Register Python problem matcher
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Install Pytest Annotation plugin
-        run: |
-          . venv/bin/activate
-          # Ideally this should be part of our dependencies
-          # However this plugin is fairly new and doesn't run correctly
-          # on a non-GitHub environment.
-          pip install pytest-github-actions-annotate-failures==0.1.3
-      - name: Run pytest
         run: |
           . venv/bin/activate
           python3 -X dev -m pytest \
@@ -661,85 +708,19 @@ jobs:
             --durations=10 \
             -n auto \
             --dist=loadfile \
-            --cov homeassistant \
+            --cov homeassistant.components.${{ matrix.group }} \
             --cov-report= \
             -o console_output_style=count \
             -p no:sugar \
-            tests/components/${{ needs.changes.outputs.tests }}
+            tests/components/${{ matrix.group }}
-      - name: Upload coverage artifact
+      - name: Upload coverage to Codecov (full coverage)
-        uses: actions/upload-artifact@v2.2.4
+        if: needs.changes.outputs.test_full_suite == 'true'
+        uses: codecov/codecov-action@v2.1.0
+      - name: Upload coverage to Codecov (partial coverage)
+        if: needs.changes.outputs.test_full_suite == 'false'
+        uses: codecov/codecov-action@v2.1.0
         with:
-          name: coverage-${{ matrix.python-version }}
+          flags: ${{ matrix.group }}
-          path: .coverage
       - name: Check dirty
         run: |
           ./script/check_dirty
-
-  coverage-full:
-    name: Process test coverage
-    runs-on: ubuntu-latest
-    needs: ["prepare-tests", "pytest-full"]
-    strategy:
-      matrix:
-        python-version: [3.8]
-    container: homeassistant/ci-azure:${{ matrix.python-version }}
-    steps:
-      - name: Check out code from GitHub
-        uses: actions/checkout@v2.4.0
-      - name: Restore full Python ${{ matrix.python-version }} virtual environment
-        id: cache-venv
-        uses: actions/cache@v2.1.7
-        with:
-          path: venv
-          key: ${{ runner.os }}-${{ matrix.python-version }}-${{
-            needs.prepare-tests.outputs.python-key }}
-      - name: Fail job if Python cache restore failed
-        if: steps.cache-venv.outputs.cache-hit != 'true'
-        run: |
-          echo "Failed to restore Python virtual environment from cache"
-          exit 1
-      - name: Download all coverage artifacts
-        uses: actions/download-artifact@v2
-      - name: Combine coverage results
-        run: |
-          . venv/bin/activate
-          coverage combine coverage*/.coverage*
-          coverage report --fail-under=94
-          coverage xml
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2.1.0
-
-  coverage-partial:
-    name: Process partial test coverage
-    runs-on: ubuntu-latest
-    needs: ["prepare-tests", "pytest-partial"]
-    strategy:
-      matrix:
-        python-version: [3.8]
-    container: homeassistant/ci-azure:${{ matrix.python-version }}
-    steps:
-      - name: Check out code from GitHub
-        uses: actions/checkout@v2.4.0
-      - name: Restore full Python ${{ matrix.python-version }} virtual environment
-        id: cache-venv
-        uses: actions/cache@v2.1.7
-        with:
-          path: venv
-          key: ${{ runner.os }}-${{ matrix.python-version }}-${{
-            needs.prepare-tests.outputs.python-key }}
-      - name: Fail job if Python cache restore failed
-        if: steps.cache-venv.outputs.cache-hit != 'true'
-        run: |
-          echo "Failed to restore Python virtual environment from cache"
-          exit 1
-      - name: Download all coverage artifacts
-        uses: actions/download-artifact@v2
-      - name: Combine coverage results
-        run: |
-          . venv/bin/activate
-          coverage combine coverage*/.coverage*
-          coverage xml
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2.1.0
-        with:
-          flags: partial