From 6b59e305cb6a867a253f9e9102118d8d498e62ad Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Tue, 23 Nov 2021 23:57:45 +0100 Subject: [PATCH] Run partial test suite in CI if core untouched (#60230) Co-authored-by: J. Nick Koston Co-authored-by: Martin Hjelmare --- .core_files.yaml | 120 ++++++++++++++++++++++++++++ .github/workflows/ci.yaml | 160 +++++++++++++++++++++++++++++++++++++- codecov.yml | 6 ++ 3 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 .core_files.yaml diff --git a/.core_files.yaml b/.core_files.yaml new file mode 100644 index 00000000000..27daff11f35 --- /dev/null +++ b/.core_files.yaml @@ -0,0 +1,120 @@ +# Defines a list of files that are part of main core of Home Assistant. +# Changes to these files/filters define how our CI test suite is ran. +core: &core + - homeassistant/*.py + - homeassistant/auth/** + - homeassistant/helpers/* + - homeassistant/package_constraints.txt + - homeassistant/util/* + - pyproject.yaml + - requirements.txt + - setup.cfg + +# Our base platforms, that are used by other integrations +base_platforms: &base_platforms + - homeassistant/components/air_quality/* + - homeassistant/components/alarm_control_panel/* + - homeassistant/components/binary_sensor/* + - homeassistant/components/button/* + - homeassistant/components/calendar/* + - homeassistant/components/camera/* + - homeassistant/components/climate/* + - homeassistant/components/cover/* + - homeassistant/components/device_tracker/* + - homeassistant/components/fan/* + - homeassistant/components/geo_location/* + - homeassistant/components/humidifier/* + - homeassistant/components/image_processing/* + - homeassistant/components/light/* + - homeassistant/components/lock/* + - homeassistant/components/media_player/* + - homeassistant/components/notify/* + - homeassistant/components/number/* + - homeassistant/components/remote/* + - homeassistant/components/scene/* + - homeassistant/components/select/* + - homeassistant/components/sensor/* + - homeassistant/components/siren/* + - homeassistant/components/stt/* + - homeassistant/components/switch/* + - homeassistant/components/tts/* + - homeassistant/components/vacuum/* + - homeassistant/components/water_heater/* + - homeassistant/components/weather/* + +# Extra components that trigger the full suite +components: &components + - homeassistant/components/alert/* + - homeassistant/components/alexa/* + - homeassistant/components/auth/* + - homeassistant/components/automation/* + - homeassistant/components/cloud/* + - homeassistant/components/config/* + - homeassistant/components/configurator/* + - homeassistant/components/conversation/* + - homeassistant/components/demo/* + - homeassistant/components/device_automation/* + - homeassistant/components/dhcp/* + - homeassistant/components/discovery/* + - homeassistant/components/energy/* + - homeassistant/components/ffmpeg/* + - homeassistant/components/frontend/* + - homeassistant/components/google_assistant/* + - homeassistant/components/group/* + - homeassistant/components/hassio/* + - homeassistant/components/homeassistant/** + - homeassistant/components/image/* + - homeassistant/components/input_boolean/* + - homeassistant/components/input_datetime/* + - homeassistant/components/input_number/* + - homeassistant/components/input_select/* + - homeassistant/components/input_text/* + - homeassistant/components/logbook/* + - homeassistant/components/logger/* + - homeassistant/components/lovelace/* + - homeassistant/components/media_source/* + - homeassistant/components/mqtt/* + - homeassistant/components/network/* + - homeassistant/components/onboarding/* + - homeassistant/components/otp/* + - homeassistant/components/persistent_notification/* + - homeassistant/components/person/* + - homeassistant/components/recorder/* + - homeassistant/components/safe_mode/* + - homeassistant/components/script/* + - homeassistant/components/shopping_list/* + - homeassistant/components/ssdp/* + - homeassistant/components/stream/* + - homeassistant/components/sun/* + - homeassistant/components/system_health/* + - homeassistant/components/tag/* + - homeassistant/components/template/* + - homeassistant/components/timer/* + - homeassistant/components/usb/* + - homeassistant/components/webhook/* + - homeassistant/components/websocket_api/* + - homeassistant/components/zeroconf/* + - homeassistant/components/zone/* + +# Testing related files that affect the whole test/linting suite +tests: &tests + - codecov.yaml + - requirements_test_pre_commit.txt + - requirements_test.txt + - tests/common.py + - tests/conftest.py + - tests/ignore_uncaught_exceptions.py + - tests/mock/* + - tests/test_util/* + - tests/testing_config/** + +other: &other + - .github/workflows/* + - homeassistant/scripts/** + +any: + - *base_platforms + - *components + - *core + - *other + - *tests diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b02dd8a63f..29e0a7951e6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -651,9 +651,57 @@ jobs: . venv/bin/activate mypy homeassistant - pytest: + changes: + name: Determine what has changed + outputs: + core: ${{ steps.core.outputs.any }} + integrations: ${{ steps.integrations.outputs.changes }} + tests: ${{ steps.tests.outputs.integrations }} + runs-on: ubuntu-latest + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.4.0 + - name: Filter for core changes + uses: dorny/paths-filter@v2.10.2 + id: core + with: + filters: .core_files.yaml + - name: Create a list of integrations to filter for changes + id: integration-filters + run: | + integrations=$(ls -Ad ./homeassistant/components/[!_]* | xargs -n 1 basename) + touch .integration_paths.yaml + for integration in $integrations; do + echo "${integration}: [homeassistant/components/${integration}/*, tests/components/${integration}/*]" \ + >> .integration_paths.yaml; + done + echo "Result:" + cat .integration_paths.yaml + - name: Filter for integration changes + uses: dorny/paths-filter@v2.10.2 + id: integrations + with: + filters: .integration_paths.yaml + - name: Determine integration tests to run + if: ${{ steps.integrations.outputs.changes }} + id: tests + run: | + possible_integrations=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '. | join(" ")') + integrations=$(for integration in $possible_integrations; do [[ -d "tests/components/${integration}" ]] && echo -n "${integration},"; done) + integrations="${integrations::-1}" + + # If more than one, add brackets to it + if [[ "${integrations}" == *","* ]]; then + integrations="{${integrations}}" + fi + + echo "::set-output name=integrations::${integrations}" + + pytest-full: + if: ${{ needs.changes.outputs.core == 'true' }} runs-on: ubuntu-latest needs: + - changes - gen-requirements-all - hassfest - lint-bandit @@ -725,10 +773,83 @@ jobs: run: | ./script/check_dirty - coverage: + pytest-partial: + if: ${{ needs.changes.outputs.core == 'false' }} + runs-on: ubuntu-latest + needs: + - changes + - gen-requirements-all + - hassfest + - lint-bandit + - lint-black + - lint-codespell + - lint-dockerfile + - lint-executable-shebangs + - lint-isort + - lint-json + - lint-pyupgrade + - lint-yaml + - mypy + - prepare-tests + strategy: + fail-fast: false + matrix: + python-version: [3.8, 3.9] + name: >- + Run partial tests Python ${{ matrix.python-version }} + container: homeassistant/ci-azure:${{ matrix.python-version }} + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.4.0 + - name: Restore full Python ${{ matrix.python-version }} virtual environment + id: cache-venv + uses: actions/cache@v2.1.7 + with: + path: venv + key: ${{ runner.os }}-${{ matrix.python-version }}-${{ + needs.prepare-tests.outputs.python-key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python virtual environment from cache" + exit 1 + - name: Register Python problem matcher + run: | + echo "::add-matcher::.github/workflows/matchers/python.json" + - name: Install Pytest Annotation plugin + run: | + . venv/bin/activate + # Ideally this should be part of our dependencies + # However this plugin is fairly new and doesn't run correctly + # on a non-GitHub environment. + pip install pytest-github-actions-annotate-failures==0.1.3 + - name: Run pytest + run: | + . venv/bin/activate + python3 -X dev -m pytest \ + -qq \ + --timeout=9 \ + --durations=10 \ + -n auto \ + --dist=loadfile \ + --cov homeassistant \ + --cov-report= \ + -o console_output_style=count \ + -p no:sugar \ + tests/components/${{ needs.changes.outputs.tests }} + - name: Upload coverage artifact + uses: actions/upload-artifact@v2.2.4 + with: + name: coverage-${{ matrix.python-version }} + path: .coverage + - name: Check dirty + run: | + ./script/check_dirty + + coverage-full: name: Process test coverage runs-on: ubuntu-latest - needs: ["prepare-tests", "pytest"] + needs: ["prepare-tests", "pytest-full"] strategy: matrix: python-version: [3.8] @@ -758,3 +879,36 @@ jobs: coverage xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v2.1.0 + + coverage-partial: + name: Process partial test coverage + runs-on: ubuntu-latest + needs: ["prepare-tests", "pytest-partial"] + strategy: + matrix: + python-version: [3.8] + container: homeassistant/ci-azure:${{ matrix.python-version }} + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.4.0 + - name: Restore full Python ${{ matrix.python-version }} virtual environment + id: cache-venv + uses: actions/cache@v2.1.7 + with: + path: venv + key: ${{ runner.os }}-${{ matrix.python-version }}-${{ + needs.prepare-tests.outputs.python-key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python virtual environment from cache" + exit 1 + - name: Download all coverage artifacts + uses: actions/download-artifact@v2 + - name: Combine coverage results + run: | + . venv/bin/activate + coverage combine coverage*/.coverage* + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2.1.0 diff --git a/codecov.yml b/codecov.yml index 7a9eea730d8..5c0750e659f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -7,3 +7,9 @@ coverage: target: 90 threshold: 0.09 comment: false + +# To make partial tests possible, +# we need to carry forward. +flag_management: + default_rules: + carryforward: true