Introduce ruff (eventually replacing autoflake, pyupgrade, flake8) (#86224)

.github/workflows/ci.yaml

@@ -324,7 +324,62 @@ jobs:
           . venv/bin/activate
           shopt -s globstar
           pre-commit run --hook-stage manual flake8 --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*}
-
+  lint-ruff:
+    name: Check ruff
+    runs-on: ubuntu-latest
+    needs:
+      - info
+      - pre-commit
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v3.3.0
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        uses: actions/setup-python@v4.5.0
+        id: python
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+          check-latest: true
+      - name: Restore base Python virtual environment
+        id: cache-venv
+        uses: actions/cache/restore@v3.2.3
+        with:
+          path: venv
+          key: >-
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{
+              needs.info.outputs.pre-commit_cache_key }}
+      - name: Fail job if Python cache restore failed
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore Python virtual environment from cache"
+          exit 1
+      - name: Restore pre-commit environment from cache
+        id: cache-precommit
+        uses: actions/cache/restore@v3.2.3
+        with:
+          path: ${{ env.PRE_COMMIT_CACHE }}
+          key: >-
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+              needs.info.outputs.pre-commit_cache_key }}
+      - name: Fail job if pre-commit cache restore failed
+        if: steps.cache-precommit.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore pre-commit environment from cache"
+          exit 1
+      - name: Register ruff problem matcher
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/ruff.json"
+      - name: Run ruff (fully)
+        if: needs.info.outputs.test_full_suite == 'true'
+        run: |
+          . venv/bin/activate
+          pre-commit run --hook-stage manual ruff --all-files
+      - name: Run ruff (partially)
+        if: needs.info.outputs.test_full_suite == 'false'
+        shell: bash
+        run: |
+          . venv/bin/activate
+          shopt -s globstar
+          pre-commit run --hook-stage manual ruff --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*}
   lint-isort:
     name: Check isort
     runs-on: ubuntu-20.04

.github/workflows/matchers/ruff.json

@@ -0,0 +1,30 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "ruff-error",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^(.*):(\\d+):(\\d+):\\s([EF]\\d{3}\\s.*)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4
+        }
+      ]
+    },
+    {
+      "owner": "ruff-warning",
+      "severity": "warning",
+      "pattern": [
+        {
+          "regexp": "^(.*):(\\d+):(\\d+):\\s([CDNW]\\d{3}\\s.*)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4
+        }
+      ]
+    }
+  ]
+}

.pre-commit-config.yaml

@@ -1,9 +1,16 @@
 repos:
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.230
+    hooks:
+      - id: ruff
+        args:
+          - --fix
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.3.1
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
+        stages: [manual]
   - repo: https://github.com/PyCQA/autoflake
     rev: v2.0.0
     hooks:
@@ -11,6 +18,7 @@ repos:
         args:
           - --in-place
           - --remove-all-unused-imports
+        stages: [manual]
   - repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
@@ -41,6 +49,7 @@ repos:
           - flake8-noqa==1.3.0
           - mccabe==0.7.0
         exclude: docs/source/conf.py
+        stages: [manual]
   - repo: https://github.com/PyCQA/bandit
     rev: 1.7.4
     hooks:

.vscode/tasks.json

@@ -41,6 +41,20 @@
       },
       "problemMatcher": []
     },
+    {
+      "label": "Ruff",
+      "type": "shell",
+      "command": "pre-commit run ruff --all-files",
+      "group": {
+        "kind": "test",
+        "isDefault": true
+      },
+      "presentation": {
+        "reveal": "always",
+        "panel": "new"
+      },
+      "problemMatcher": []
+    },
     {
       "label": "Pylint",
       "type": "shell",

pyproject.toml

@@ -237,3 +237,47 @@ norecursedirs = [
 log_format = "%(asctime)s.%(msecs)03d %(levelname)-8s %(threadName)s %(name)s:%(filename)s:%(lineno)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 asyncio_mode = "auto"
+
+[tool.ruff]
+target-version = "py310"
+exclude = []
+
+ignore = [
+    "D202",  # No blank lines allowed after function docstring
+    "D203",  # 1 blank line required before class docstring
+    "D212",  # Multi-line docstring summary should start at the first line
+    "D213",  # Multi-line docstring summary should start at the second line
+    "D401",  # TODO: Enable when https://github.com/charliermarsh/ruff/pull/2071 is released
+    "D404",  # First word of the docstring should not be This
+    "D406",  # Section name should end with a newline
+    "D407",  # Section name underlining
+    "D411",  # Missing blank line before section
+    "D418",  # Function decorated with `@overload` shouldn't contain a docstring
+    "E501",  # line too long
+    "E713",  # Test for membership should be 'not in'
+    "E731",  # do not assign a lambda expression, use a def
+    "UP024", # Replace aliased errors with `OSError`
+]
+select = [
+    "C",  # complexity
+    "D",  # docstrings
+    "E",  # pycodestyle
+    "F",  # pyflakes/autoflake
+    "W",  # pycodestyle
+    "UP",  # pyupgrade
+    "PGH004",  # Use specific rule codes when using noqa
+]
+
+[tool.ruff.per-file-ignores]
+
+# TODO: these files have functions that are too complex, but flake8's and ruff's
+#       complexity (and/or nested-function) handling differs; trying to add a noqa doesn't work
+#       because the flake8-noqa plugin then disagrees on whether there should be a C901 noqa
+#       on that line.  So, for now, we just ignore C901s on these files as far as ruff is concerned.
+
+"homeassistant/components/light/__init__.py" = ["C901"]
+"homeassistant/components/mqtt/discovery.py" = ["C901"]
+"homeassistant/components/websocket_api/http.py" = ["C901"]
+
+[tool.ruff.mccabe]
+max-complexity = 25

requirements_test.txt

@@ -1,4 +1,4 @@
-# linters such as flake8 and pylint should be pinned, as new releases
+# linters such as pylint should be pinned, as new releases
 # make new things fail. Manually update these pins when pulling in a
 # new version
 

requirements_test_pre_commit.txt

@@ -14,4 +14,5 @@ pycodestyle==2.10.0
 pydocstyle==6.2.3
 pyflakes==3.0.1
 pyupgrade==3.3.1
+ruff==0.0.230
 yamllint==1.28.0

script/lint

@@ -16,6 +16,10 @@ echo "================"
 echo "LINT with flake8"
 echo "================"
 pre-commit run flake8 --files $files
+echo "=============="
+echo "LINT with ruff"
+echo "=============="
+pre-commit run ruff --files $files
 echo "================"
 echo "LINT with pylint"
 echo "================"

script/lint_and_test.py

@@ -6,6 +6,7 @@ This is NOT a full CI/linting replacement, only a quick check during development
 """
 import asyncio
 from collections import namedtuple
+import itertools
 import os
 import re
 import shlex
@@ -115,9 +116,9 @@ async def pylint(files):
     return res
 
 
-async def flake8(files):
-    """Exec flake8."""
-    _, log = await async_exec("pre-commit", "run", "flake8", "--files", *files)
+async def _ruff_or_flake8(tool, files):
+    """Exec ruff or flake8."""
+    _, log = await async_exec("pre-commit", "run", tool, "--files", *files)
     res = []
     for line in log.splitlines():
         line = line.split(":")
@@ -128,17 +129,33 @@ async def flake8(files):
     return res
 
 
+async def flake8(files):
+    """Exec flake8."""
+    return await _ruff_or_flake8("flake8", files)
+
+
+async def ruff(files):
+    """Exec ruff."""
+    return await _ruff_or_flake8("ruff", files)
+
+
 async def lint(files):
     """Perform lint."""
     files = [file for file in files if os.path.isfile(file)]
-    fres, pres = await asyncio.gather(flake8(files), pylint(files))
-
-    res = fres + pres
-    res.sort(key=lambda item: item.file)
+    res = sorted(
+        itertools.chain(
+            *await asyncio.gather(
+                flake8(files),
+                pylint(files),
+                ruff(files),
+            )
+        ),
+        key=lambda item: item.file,
+    )
     if res:
-        print("Pylint & Flake8 errors:")
+        print("Lint errors:")
     else:
-        printc(PASS, "Pylint and Flake8 passed")
+        printc(PASS, "Lint passed")
 
     lint_ok = True
     for err in res: