diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 581a36be953..0dc8f34570c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,7 +97,8 @@ jobs: hashFiles('requirements_test.txt', 'requirements_test_pre_commit.txt') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_all.txt') }}-${{ - hashFiles('homeassistant/package_constraints.txt') }}" >> $GITHUB_OUTPUT + hashFiles('homeassistant/package_constraints.txt') }}-${{ + hashFiles('script/gen_requirements_all.py') }}" >> $GITHUB_OUTPUT - name: Generate partial pre-commit restore key id: generate_pre-commit_cache_key run: >- @@ -497,8 +498,9 @@ jobs: python --version pip install "$(grep '^uv' < requirements_test.txt)" uv pip install -U "pip>=21.3.1" setuptools wheel - uv pip install -r requirements_all.txt - uv pip install "$(grep 'python-gammu' < requirements_all.txt | sed -e 's|# python-gammu|python-gammu|g')" + uv pip install -r requirements.txt + python -m script.gen_requirements_all ci + uv pip install -r requirements_all_pytest.txt uv pip install -r requirements_test.txt uv pip install -e . --config-settings editable_mode=compat diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 24033a92fd5..6618eb9963b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -14,6 +14,10 @@ on: - "homeassistant/package_constraints.txt" - "requirements_all.txt" - "requirements.txt" + - "script/gen_requirements_all.py" + +env: + DEFAULT_PYTHON: "3.12" concurrency: group: ${{ github.workflow }}-${{ github.ref_name}} @@ -30,6 +34,21 @@ jobs: - name: Checkout the repository uses: actions/checkout@v4.1.3 + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + id: python + uses: actions/setup-python@v5.1.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + check-latest: true + + - name: Create Python virtual environment + run: | + python -m venv venv + . venv/bin/activate + python --version + pip install "$(grep '^uv' < requirements_test.txt)" + uv pip install -r requirements.txt + - name: Get information id: info uses: home-assistant/actions/helpers/info@master @@ -76,6 +95,17 @@ jobs: path: ./requirements_diff.txt overwrite: true + - name: Generate requirements + run: | + . venv/bin/activate + python -m script.gen_requirements_all ci + + - name: Upload requirements_all_wheels + uses: actions/upload-artifact@v4.3.1 + with: + name: requirements_all_wheels + path: ./requirements_all_wheels_*.txt + core: name: Build Core wheels ${{ matrix.abi }} for ${{ matrix.arch }} (musllinux_1_2) if: github.repository_owner == 'home-assistant' @@ -138,30 +168,10 @@ jobs: with: name: requirements_diff - - name: (Un)comment packages - run: | - requirement_files="requirements_all.txt requirements_diff.txt" - for requirement_file in ${requirement_files}; do - sed -i "s|# pyuserinput|pyuserinput|g" ${requirement_file} - sed -i "s|# evdev|evdev|g" ${requirement_file} - sed -i "s|# pycups|pycups|g" ${requirement_file} - sed -i "s|# decora-wifi|decora-wifi|g" ${requirement_file} - sed -i "s|# python-gammu|python-gammu|g" ${requirement_file} - - # Some packages are not buildable on armhf anymore - if [ "${{ matrix.arch }}" = "armhf" ]; then - - # Pandas has issues building on armhf, it is expected they - # will drop the platform in the near future (they consider it - # "flimsy" on 386). The following packages depend on pandas, - # so we comment them out. - sed -i "s|env-canada|# env-canada|g" ${requirement_file} - sed -i "s|noaa-coops|# noaa-coops|g" ${requirement_file} - sed -i "s|pyezviz|# pyezviz|g" ${requirement_file} - sed -i "s|pykrakenapi|# pykrakenapi|g" ${requirement_file} - fi - - done + - name: Download requirements_all_wheels + uses: actions/download-artifact@v4.1.4 + with: + name: requirements_all_wheels - name: Split requirements all run: | @@ -169,7 +179,7 @@ jobs: # This is to prevent the build from running out of memory when # resolving packages on 32-bits systems (like armhf, armv7). - split -l $(expr $(expr $(cat requirements_all.txt | wc -l) + 1) / 3) requirements_all.txt requirements_all.txt + split -l $(expr $(expr $(cat requirements_all.txt | wc -l) + 1) / 3) requirements_all_wheels_${{ matrix.arch }}.txt requirements_all.txt - name: Create requirements for cython<3 run: | diff --git a/script/gen_requirements_all.py b/script/gen_requirements_all.py index 7fc0907e756..a5db9997d9d 100755 --- a/script/gen_requirements_all.py +++ b/script/gen_requirements_all.py @@ -17,7 +17,10 @@ from typing import Any from homeassistant.util.yaml.loader import load_yaml from script.hassfest.model import Integration -COMMENT_REQUIREMENTS = ( +# Requirements which can't be installed on all systems because they rely on additional +# system packages. Requirements listed in EXCLUDED_REQUIREMENTS_ALL will be commented-out +# in requirements_all.txt and requirements_test_all.txt. +EXCLUDED_REQUIREMENTS_ALL = { "atenpdu", # depends on pysnmp which is not maintained at this time "avea", # depends on bluepy "avion", @@ -36,10 +39,39 @@ COMMENT_REQUIREMENTS = ( "pyuserinput", "tensorflow", "tf-models-official", -) +} -COMMENT_REQUIREMENTS_NORMALIZED = { - commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS +# Requirements excluded by EXCLUDED_REQUIREMENTS_ALL which should be included when +# building integration wheels for all architectures. +INCLUDED_REQUIREMENTS_WHEELS = { + "decora-wifi", + "evdev", + "pycups", + "python-gammu", + "pyuserinput", +} + + +# Requirements to exclude or include when running github actions. +# Requirements listed in "exclude" will be commented-out in +# requirements_all_{action}.txt +# Requirements listed in "include" must be listed in EXCLUDED_REQUIREMENTS_CI, and +# will be included in requirements_all_{action}.txt + +OVERRIDDEN_REQUIREMENTS_ACTIONS = { + "pytest": {"exclude": set(), "include": {"python-gammu"}}, + "wheels_aarch64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS}, + # Pandas has issues building on armhf, it is expected they + # will drop the platform in the near future (they consider it + # "flimsy" on 386). The following packages depend on pandas, + # so we comment them out. + "wheels_armhf": { + "exclude": {"env-canada", "noaa-coops", "pyezviz", "pykrakenapi"}, + "include": INCLUDED_REQUIREMENTS_WHEELS, + }, + "wheels_armv7": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS}, + "wheels_amd64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS}, + "wheels_i386": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS}, } IGNORE_PIN = ("colorlog>2.1,<3", "urllib3") @@ -254,6 +286,12 @@ def gather_recursive_requirements( return reqs +def _normalize_package_name(package_name: str) -> str: + """Normalize a package name.""" + # pipdeptree needs lowercase and dash instead of underscore or period as separator + return package_name.lower().replace("_", "-").replace(".", "-") + + def normalize_package_name(requirement: str) -> str: """Return a normalized package name from a requirement string.""" # This function is also used in hassfest. @@ -262,12 +300,24 @@ def normalize_package_name(requirement: str) -> str: return "" # pipdeptree needs lowercase and dash instead of underscore or period as separator - return match.group(1).lower().replace("_", "-").replace(".", "-") + return _normalize_package_name(match.group(1)) def comment_requirement(req: str) -> bool: """Comment out requirement. Some don't install on all systems.""" - return normalize_package_name(req) in COMMENT_REQUIREMENTS_NORMALIZED + return normalize_package_name(req) in EXCLUDED_REQUIREMENTS_ALL + + +def process_action_requirement(req: str, action: str) -> str: + """Process requirement for a specific github action.""" + normalized_package_name = normalize_package_name(req) + if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["exclude"]: + return f"# {req}" + if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["include"]: + return req + if normalized_package_name in EXCLUDED_REQUIREMENTS_ALL: + return f"# {req}" + return req def gather_modules() -> dict[str, list[str]] | None: @@ -353,6 +403,16 @@ def generate_requirements_list(reqs: dict[str, list[str]]) -> str: return "".join(output) +def generate_action_requirements_list(reqs: dict[str, list[str]], action: str) -> str: + """Generate a pip file based on requirements.""" + output = [] + for pkg, requirements in sorted(reqs.items(), key=itemgetter(0)): + output.extend(f"\n# {req}" for req in sorted(requirements)) + processed_pkg = process_action_requirement(pkg, action) + output.append(f"\n{processed_pkg}\n") + return "".join(output) + + def requirements_output() -> str: """Generate output for requirements.""" output = [ @@ -379,6 +439,18 @@ def requirements_all_output(reqs: dict[str, list[str]]) -> str: return "".join(output) +def requirements_all_action_output(reqs: dict[str, list[str]], action: str) -> str: + """Generate output for requirements_all_{action}.""" + output = [ + f"# Home Assistant Core, full dependency set for {action}\n", + GENERATED_MESSAGE, + "-r requirements.txt\n", + ] + output.append(generate_action_requirements_list(reqs, action)) + + return "".join(output) + + def requirements_test_all_output(reqs: dict[str, list[str]]) -> str: """Generate output for test_requirements.""" output = [ @@ -459,7 +531,7 @@ def diff_file(filename: str, content: str) -> list[str]: ) -def main(validate: bool) -> int: +def main(validate: bool, ci: bool) -> int: """Run the script.""" if not os.path.isfile("requirements_all.txt"): print("Run this from HA root dir") @@ -472,17 +544,28 @@ def main(validate: bool) -> int: reqs_file = requirements_output() reqs_all_file = requirements_all_output(data) + reqs_all_action_files = { + action: requirements_all_action_output(data, action) + for action in OVERRIDDEN_REQUIREMENTS_ACTIONS + } reqs_test_all_file = requirements_test_all_output(data) + # Always calling requirements_pre_commit_output is intentional to ensure + # the code is called by the pre-commit hooks. reqs_pre_commit_file = requirements_pre_commit_output() constraints = gather_constraints() - files = ( + files = [ ("requirements.txt", reqs_file), ("requirements_all.txt", reqs_all_file), ("requirements_test_pre_commit.txt", reqs_pre_commit_file), ("requirements_test_all.txt", reqs_test_all_file), ("homeassistant/package_constraints.txt", constraints), - ) + ] + if ci: + files.extend( + (f"requirements_all_{action}.txt", reqs_all_file) + for action, reqs_all_file in reqs_all_action_files.items() + ) if validate: errors = [] @@ -511,4 +594,5 @@ def main(validate: bool) -> int: if __name__ == "__main__": _VAL = sys.argv[-1] == "validate" - sys.exit(main(_VAL)) + _CI = sys.argv[-1] == "ci" + sys.exit(main(_VAL, _CI)) diff --git a/script/hassfest/requirements.py b/script/hassfest/requirements.py index ee63bf07f90..2c4ed47b158 100644 --- a/script/hassfest/requirements.py +++ b/script/hassfest/requirements.py @@ -15,13 +15,13 @@ from awesomeversion import AwesomeVersion, AwesomeVersionStrategy from tqdm import tqdm import homeassistant.util.package as pkg_util -from script.gen_requirements_all import COMMENT_REQUIREMENTS, normalize_package_name +from script.gen_requirements_all import ( + EXCLUDED_REQUIREMENTS_ALL, + normalize_package_name, +) from .model import Config, Integration -IGNORE_PACKAGES = { - commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS -} PACKAGE_REGEX = re.compile( r"^(?:--.+\s)?([-_,\.\w\d\[\]]+)(==|>=|<=|~=|!=|<|>|===)*(.*)$" ) @@ -116,7 +116,7 @@ def validate_requirements(integration: Integration) -> None: f"Failed to normalize package name from requirement {req}", ) return - if package in IGNORE_PACKAGES: + if package in EXCLUDED_REQUIREMENTS_ALL: continue integration_requirements.add(req) integration_packages.add(package) diff --git a/tests/script/__init__.py b/tests/script/__init__.py new file mode 100644 index 00000000000..209299782c9 --- /dev/null +++ b/tests/script/__init__.py @@ -0,0 +1 @@ +"""Tests for scripts.""" diff --git a/tests/script/test_gen_requirements_all.py b/tests/script/test_gen_requirements_all.py new file mode 100644 index 00000000000..793b3de63c5 --- /dev/null +++ b/tests/script/test_gen_requirements_all.py @@ -0,0 +1,25 @@ +"""Tests for the gen_requirements_all script.""" + +from script import gen_requirements_all + + +def test_overrides_normalized() -> None: + """Test override lists are using normalized package names.""" + for req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL: + assert req == gen_requirements_all._normalize_package_name(req) + for req in gen_requirements_all.INCLUDED_REQUIREMENTS_WHEELS: + assert req == gen_requirements_all._normalize_package_name(req) + for overrides in gen_requirements_all.OVERRIDDEN_REQUIREMENTS_ACTIONS.values(): + for req in overrides["exclude"]: + assert req == gen_requirements_all._normalize_package_name(req) + for req in overrides["include"]: + assert req == gen_requirements_all._normalize_package_name(req) + + +def test_include_overrides_subsets() -> None: + """Test packages in include override lists are present in the exclude list.""" + for req in gen_requirements_all.INCLUDED_REQUIREMENTS_WHEELS: + assert req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL + for overrides in gen_requirements_all.OVERRIDDEN_REQUIREMENTS_ACTIONS.values(): + for req in overrides["include"]: + assert req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL