From bf41a971a27e895acaf0b6daa9e3307b60fce000 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Tue, 24 Jan 2023 13:15:16 +0200 Subject: [PATCH] Introduce ruff (eventually replacing autoflake, pyupgrade, flake8) (#86224) --- .github/workflows/ci.yaml | 57 +++++++++++++++++++++++++++- .github/workflows/matchers/ruff.json | 30 +++++++++++++++ .pre-commit-config.yaml | 9 +++++ .vscode/tasks.json | 14 +++++++ pyproject.toml | 44 +++++++++++++++++++++ requirements_test.txt | 2 +- requirements_test_pre_commit.txt | 1 + script/lint | 4 ++ script/lint_and_test.py | 35 ++++++++++++----- 9 files changed, 185 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/matchers/ruff.json diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ccfef6244be..ab83359267c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -324,7 +324,62 @@ jobs: . venv/bin/activate shopt -s globstar pre-commit run --hook-stage manual flake8 --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*} - + lint-ruff: + name: Check ruff + runs-on: ubuntu-latest + needs: + - info + - pre-commit + steps: + - name: Check out code from GitHub + uses: actions/checkout@v3.3.0 + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + uses: actions/setup-python@v4.5.0 + id: python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + check-latest: true + - name: Restore base Python virtual environment + id: cache-venv + uses: actions/cache/restore@v3.2.3 + with: + path: venv + key: >- + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ + needs.info.outputs.pre-commit_cache_key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python virtual environment from cache" + exit 1 + - name: Restore pre-commit environment from cache + id: cache-precommit + uses: actions/cache/restore@v3.2.3 + with: + path: ${{ env.PRE_COMMIT_CACHE }} + key: >- + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + needs.info.outputs.pre-commit_cache_key }} + - name: Fail job if pre-commit cache restore failed + if: steps.cache-precommit.outputs.cache-hit != 'true' + run: | + echo "Failed to restore pre-commit environment from cache" + exit 1 + - name: Register ruff problem matcher + run: | + echo "::add-matcher::.github/workflows/matchers/ruff.json" + - name: Run ruff (fully) + if: needs.info.outputs.test_full_suite == 'true' + run: | + . venv/bin/activate + pre-commit run --hook-stage manual ruff --all-files + - name: Run ruff (partially) + if: needs.info.outputs.test_full_suite == 'false' + shell: bash + run: | + . venv/bin/activate + shopt -s globstar + pre-commit run --hook-stage manual ruff --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*} lint-isort: name: Check isort runs-on: ubuntu-20.04 diff --git a/.github/workflows/matchers/ruff.json b/.github/workflows/matchers/ruff.json new file mode 100644 index 00000000000..d189a3656a5 --- /dev/null +++ b/.github/workflows/matchers/ruff.json @@ -0,0 +1,30 @@ +{ + "problemMatcher": [ + { + "owner": "ruff-error", + "severity": "error", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s([EF]\\d{3}\\s.*)$", + "file": 1, + "line": 2, + "column": 3, + "message": 4 + } + ] + }, + { + "owner": "ruff-warning", + "severity": "warning", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s([CDNW]\\d{3}\\s.*)$", + "file": 1, + "line": 2, + "column": 3, + "message": 4 + } + ] + } + ] +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bc71b75912..ce1f4e8d635 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,16 @@ repos: + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.230 + hooks: + - id: ruff + args: + - --fix - repo: https://github.com/asottile/pyupgrade rev: v3.3.1 hooks: - id: pyupgrade args: [--py310-plus] + stages: [manual] - repo: https://github.com/PyCQA/autoflake rev: v2.0.0 hooks: @@ -11,6 +18,7 @@ repos: args: - --in-place - --remove-all-unused-imports + stages: [manual] - repo: https://github.com/psf/black rev: 22.12.0 hooks: @@ -41,6 +49,7 @@ repos: - flake8-noqa==1.3.0 - mccabe==0.7.0 exclude: docs/source/conf.py + stages: [manual] - repo: https://github.com/PyCQA/bandit rev: 1.7.4 hooks: diff --git a/.vscode/tasks.json b/.vscode/tasks.json index d71571d2594..4b62a16042d 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -41,6 +41,20 @@ }, "problemMatcher": [] }, + { + "label": "Ruff", + "type": "shell", + "command": "pre-commit run ruff --all-files", + "group": { + "kind": "test", + "isDefault": true + }, + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": [] + }, { "label": "Pylint", "type": "shell", diff --git a/pyproject.toml b/pyproject.toml index e2ec3ddc12f..df98a7bec1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,3 +237,47 @@ norecursedirs = [ log_format = "%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(name)s:%(filename)s:%(lineno)s %(message)s" log_date_format = "%Y-%m-%d %H:%M:%S" asyncio_mode = "auto" + +[tool.ruff] +target-version = "py310" +exclude = [] + +ignore = [ + "D202", # No blank lines allowed after function docstring + "D203", # 1 blank line required before class docstring + "D212", # Multi-line docstring summary should start at the first line + "D213", # Multi-line docstring summary should start at the second line + "D401", # TODO: Enable when https://github.com/charliermarsh/ruff/pull/2071 is released + "D404", # First word of the docstring should not be This + "D406", # Section name should end with a newline + "D407", # Section name underlining + "D411", # Missing blank line before section + "D418", # Function decorated with `@overload` shouldn't contain a docstring + "E501", # line too long + "E713", # Test for membership should be 'not in' + "E731", # do not assign a lambda expression, use a def + "UP024", # Replace aliased errors with `OSError` +] +select = [ + "C", # complexity + "D", # docstrings + "E", # pycodestyle + "F", # pyflakes/autoflake + "W", # pycodestyle + "UP", # pyupgrade + "PGH004", # Use specific rule codes when using noqa +] + +[tool.ruff.per-file-ignores] + +# TODO: these files have functions that are too complex, but flake8's and ruff's +# complexity (and/or nested-function) handling differs; trying to add a noqa doesn't work +# because the flake8-noqa plugin then disagrees on whether there should be a C901 noqa +# on that line. So, for now, we just ignore C901s on these files as far as ruff is concerned. + +"homeassistant/components/light/__init__.py" = ["C901"] +"homeassistant/components/mqtt/discovery.py" = ["C901"] +"homeassistant/components/websocket_api/http.py" = ["C901"] + +[tool.ruff.mccabe] +max-complexity = 25 diff --git a/requirements_test.txt b/requirements_test.txt index 183efcd3e8c..a238e8aea58 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,4 @@ -# linters such as flake8 and pylint should be pinned, as new releases +# linters such as pylint should be pinned, as new releases # make new things fail. Manually update these pins when pulling in a # new version diff --git a/requirements_test_pre_commit.txt b/requirements_test_pre_commit.txt index 8644ae23a16..0ac9fbb12c7 100644 --- a/requirements_test_pre_commit.txt +++ b/requirements_test_pre_commit.txt @@ -14,4 +14,5 @@ pycodestyle==2.10.0 pydocstyle==6.2.3 pyflakes==3.0.1 pyupgrade==3.3.1 +ruff==0.0.230 yamllint==1.28.0 diff --git a/script/lint b/script/lint index 378c8c68d39..450733cecfd 100755 --- a/script/lint +++ b/script/lint @@ -16,6 +16,10 @@ echo "================" echo "LINT with flake8" echo "================" pre-commit run flake8 --files $files +echo "==============" +echo "LINT with ruff" +echo "==============" +pre-commit run ruff --files $files echo "================" echo "LINT with pylint" echo "================" diff --git a/script/lint_and_test.py b/script/lint_and_test.py index d7b6bc19e23..03765701530 100755 --- a/script/lint_and_test.py +++ b/script/lint_and_test.py @@ -6,6 +6,7 @@ This is NOT a full CI/linting replacement, only a quick check during development """ import asyncio from collections import namedtuple +import itertools import os import re import shlex @@ -115,9 +116,9 @@ async def pylint(files): return res -async def flake8(files): - """Exec flake8.""" - _, log = await async_exec("pre-commit", "run", "flake8", "--files", *files) +async def _ruff_or_flake8(tool, files): + """Exec ruff or flake8.""" + _, log = await async_exec("pre-commit", "run", tool, "--files", *files) res = [] for line in log.splitlines(): line = line.split(":") @@ -128,17 +129,33 @@ async def flake8(files): return res +async def flake8(files): + """Exec flake8.""" + return await _ruff_or_flake8("flake8", files) + + +async def ruff(files): + """Exec ruff.""" + return await _ruff_or_flake8("ruff", files) + + async def lint(files): """Perform lint.""" files = [file for file in files if os.path.isfile(file)] - fres, pres = await asyncio.gather(flake8(files), pylint(files)) - - res = fres + pres - res.sort(key=lambda item: item.file) + res = sorted( + itertools.chain( + *await asyncio.gather( + flake8(files), + pylint(files), + ruff(files), + ) + ), + key=lambda item: item.file, + ) if res: - print("Pylint & Flake8 errors:") + print("Lint errors:") else: - printc(PASS, "Pylint and Flake8 passed") + printc(PASS, "Lint passed") lint_ok = True for err in res: