Run partial test suite in CI if core untouched (#60230)

Co-authored-by: J. Nick Koston <nick@koston.org>
Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

.core_files.yaml

@@ -0,0 +1,120 @@
+# Defines a list of files that are part of main core of Home Assistant.
+# Changes to these files/filters define how our CI test suite is ran.
+core: &core
+  - homeassistant/*.py
+  - homeassistant/auth/**
+  - homeassistant/helpers/*
+  - homeassistant/package_constraints.txt
+  - homeassistant/util/*
+  - pyproject.yaml
+  - requirements.txt
+  - setup.cfg
+
+# Our base platforms, that are used by other integrations
+base_platforms: &base_platforms
+  - homeassistant/components/air_quality/*
+  - homeassistant/components/alarm_control_panel/*
+  - homeassistant/components/binary_sensor/*
+  - homeassistant/components/button/*
+  - homeassistant/components/calendar/*
+  - homeassistant/components/camera/*
+  - homeassistant/components/climate/*
+  - homeassistant/components/cover/*
+  - homeassistant/components/device_tracker/*
+  - homeassistant/components/fan/*
+  - homeassistant/components/geo_location/*
+  - homeassistant/components/humidifier/*
+  - homeassistant/components/image_processing/*
+  - homeassistant/components/light/*
+  - homeassistant/components/lock/*
+  - homeassistant/components/media_player/*
+  - homeassistant/components/notify/*
+  - homeassistant/components/number/*
+  - homeassistant/components/remote/*
+  - homeassistant/components/scene/*
+  - homeassistant/components/select/*
+  - homeassistant/components/sensor/*
+  - homeassistant/components/siren/*
+  - homeassistant/components/stt/*
+  - homeassistant/components/switch/*
+  - homeassistant/components/tts/*
+  - homeassistant/components/vacuum/*
+  - homeassistant/components/water_heater/*
+  - homeassistant/components/weather/*
+
+# Extra components that trigger the full suite
+components: &components
+  - homeassistant/components/alert/*
+  - homeassistant/components/alexa/*
+  - homeassistant/components/auth/*
+  - homeassistant/components/automation/*
+  - homeassistant/components/cloud/*
+  - homeassistant/components/config/*
+  - homeassistant/components/configurator/*
+  - homeassistant/components/conversation/*
+  - homeassistant/components/demo/*
+  - homeassistant/components/device_automation/*
+  - homeassistant/components/dhcp/*
+  - homeassistant/components/discovery/*
+  - homeassistant/components/energy/*
+  - homeassistant/components/ffmpeg/*
+  - homeassistant/components/frontend/*
+  - homeassistant/components/google_assistant/*
+  - homeassistant/components/group/*
+  - homeassistant/components/hassio/*
+  - homeassistant/components/homeassistant/**
+  - homeassistant/components/image/*
+  - homeassistant/components/input_boolean/*
+  - homeassistant/components/input_datetime/*
+  - homeassistant/components/input_number/*
+  - homeassistant/components/input_select/*
+  - homeassistant/components/input_text/*
+  - homeassistant/components/logbook/*
+  - homeassistant/components/logger/*
+  - homeassistant/components/lovelace/*
+  - homeassistant/components/media_source/*
+  - homeassistant/components/mqtt/*
+  - homeassistant/components/network/*
+  - homeassistant/components/onboarding/*
+  - homeassistant/components/otp/*
+  - homeassistant/components/persistent_notification/*
+  - homeassistant/components/person/*
+  - homeassistant/components/recorder/*
+  - homeassistant/components/safe_mode/*
+  - homeassistant/components/script/*
+  - homeassistant/components/shopping_list/*
+  - homeassistant/components/ssdp/*
+  - homeassistant/components/stream/*
+  - homeassistant/components/sun/*
+  - homeassistant/components/system_health/*
+  - homeassistant/components/tag/*
+  - homeassistant/components/template/*
+  - homeassistant/components/timer/*
+  - homeassistant/components/usb/*
+  - homeassistant/components/webhook/*
+  - homeassistant/components/websocket_api/*
+  - homeassistant/components/zeroconf/*
+  - homeassistant/components/zone/*
+
+# Testing related files that affect the whole test/linting suite
+tests: &tests
+  - codecov.yaml
+  - requirements_test_pre_commit.txt
+  - requirements_test.txt
+  - tests/common.py
+  - tests/conftest.py
+  - tests/ignore_uncaught_exceptions.py
+  - tests/mock/*
+  - tests/test_util/*
+  - tests/testing_config/**
+
+other: &other
+  - .github/workflows/*
+  - homeassistant/scripts/**
+
+any:
+  - *base_platforms
+  - *components
+  - *core
+  - *other
+  - *tests

.github/workflows/ci.yaml

@@ -651,9 +651,57 @@ jobs:
           . venv/bin/activate
           mypy homeassistant
 
-  pytest:
+  changes:
+    name: Determine what has changed
+    outputs:
+      core: ${{ steps.core.outputs.any }}
+      integrations: ${{ steps.integrations.outputs.changes }}
+      tests: ${{ steps.tests.outputs.integrations }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v2.4.0
+      - name: Filter for core changes
+        uses: dorny/paths-filter@v2.10.2
+        id: core
+        with:
+          filters: .core_files.yaml
+      - name: Create a list of integrations to filter for changes
+        id: integration-filters
+        run: |
+          integrations=$(ls -Ad ./homeassistant/components/[!_]*  | xargs -n 1 basename)
+          touch .integration_paths.yaml
+          for integration in $integrations; do
+            echo "${integration}: [homeassistant/components/${integration}/*, tests/components/${integration}/*]" \
+              >> .integration_paths.yaml;
+          done
+          echo "Result:"
+          cat .integration_paths.yaml
+      - name: Filter for integration changes
+        uses: dorny/paths-filter@v2.10.2
+        id: integrations
+        with:
+          filters: .integration_paths.yaml
+      - name: Determine integration tests to run
+        if: ${{ steps.integrations.outputs.changes }}
+        id: tests
+        run: |
+          possible_integrations=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '. | join(" ")')
+          integrations=$(for integration in $possible_integrations; do [[ -d "tests/components/${integration}" ]] && echo -n "${integration},"; done)
+          integrations="${integrations::-1}"
+
+          # If more than one, add brackets to it
+          if [[ "${integrations}" == *","* ]]; then
+            integrations="{${integrations}}"
+          fi
+
+          echo "::set-output name=integrations::${integrations}"
+
+  pytest-full:
+    if: ${{ needs.changes.outputs.core == 'true' }}
     runs-on: ubuntu-latest
     needs:
+      - changes
       - gen-requirements-all
       - hassfest
       - lint-bandit
@@ -725,10 +773,83 @@ jobs:
         run: |
           ./script/check_dirty
 
-  coverage:
+  pytest-partial:
+    if: ${{ needs.changes.outputs.core == 'false' }}
+    runs-on: ubuntu-latest
+    needs:
+      - changes
+      - gen-requirements-all
+      - hassfest
+      - lint-bandit
+      - lint-black
+      - lint-codespell
+      - lint-dockerfile
+      - lint-executable-shebangs
+      - lint-isort
+      - lint-json
+      - lint-pyupgrade
+      - lint-yaml
+      - mypy
+      - prepare-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8, 3.9]
+    name: >-
+      Run partial tests Python ${{ matrix.python-version }}
+    container: homeassistant/ci-azure:${{ matrix.python-version }}
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v2.4.0
+      - name: Restore full Python ${{ matrix.python-version }} virtual environment
+        id: cache-venv
+        uses: actions/cache@v2.1.7
+        with:
+          path: venv
+          key: ${{ runner.os }}-${{ matrix.python-version }}-${{
+            needs.prepare-tests.outputs.python-key }}
+      - name: Fail job if Python cache restore failed
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore Python virtual environment from cache"
+          exit 1
+      - name: Register Python problem matcher
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - name: Install Pytest Annotation plugin
+        run: |
+          . venv/bin/activate
+          # Ideally this should be part of our dependencies
+          # However this plugin is fairly new and doesn't run correctly
+          # on a non-GitHub environment.
+          pip install pytest-github-actions-annotate-failures==0.1.3
+      - name: Run pytest
+        run: |
+          . venv/bin/activate
+          python3 -X dev -m pytest \
+            -qq \
+            --timeout=9 \
+            --durations=10 \
+            -n auto \
+            --dist=loadfile \
+            --cov homeassistant \
+            --cov-report= \
+            -o console_output_style=count \
+            -p no:sugar \
+            tests/components/${{ needs.changes.outputs.tests }}
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v2.2.4
+        with:
+          name: coverage-${{ matrix.python-version }}
+          path: .coverage
+      - name: Check dirty
+        run: |
+          ./script/check_dirty
+
+  coverage-full:
     name: Process test coverage
     runs-on: ubuntu-latest
-    needs: ["prepare-tests", "pytest"]
+    needs: ["prepare-tests", "pytest-full"]
     strategy:
       matrix:
         python-version: [3.8]
@@ -758,3 +879,36 @@ jobs:
           coverage xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2.1.0
+
+  coverage-partial:
+    name: Process partial test coverage
+    runs-on: ubuntu-latest
+    needs: ["prepare-tests", "pytest-partial"]
+    strategy:
+      matrix:
+        python-version: [3.8]
+    container: homeassistant/ci-azure:${{ matrix.python-version }}
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v2.4.0
+      - name: Restore full Python ${{ matrix.python-version }} virtual environment
+        id: cache-venv
+        uses: actions/cache@v2.1.7
+        with:
+          path: venv
+          key: ${{ runner.os }}-${{ matrix.python-version }}-${{
+            needs.prepare-tests.outputs.python-key }}
+      - name: Fail job if Python cache restore failed
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore Python virtual environment from cache"
+          exit 1
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@v2
+      - name: Combine coverage results
+        run: |
+          . venv/bin/activate
+          coverage combine coverage*/.coverage*
+          coverage xml
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2.1.0

codecov.yml

@@ -7,3 +7,9 @@ coverage:
         target: 90
         threshold: 0.09
 comment: false
+
+# To make partial tests possible,
+# we need to carry forward.
+flag_management:
+  default_rules:
+    carryforward: true