From bd2be0a763791a196833c656f64390ccbe59f244 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Fri, 30 Aug 2024 13:09:10 +0200 Subject: [PATCH] Optimize hassfest image (#124855) * Optimize hassfest docker image * Adjust CI * Use dynamic uv version * Remove workaround --- .github/workflows/builder.yml | 10 +- script/hassfest/docker.py | 134 +++++++++++++++--- script/hassfest/docker/Dockerfile | 35 +++-- .../hassfest/docker/Dockerfile.dockerignore | 8 ++ script/hassfest/docker/entrypoint.sh | 22 +-- 5 files changed, 163 insertions(+), 46 deletions(-) create mode 100644 script/hassfest/docker/Dockerfile.dockerignore diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index d206f8fe8c8..65ad0e240bc 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -491,7 +491,7 @@ jobs: packages: write attestations: write id-token: write - needs: ["init", "build_base"] + needs: ["init"] if: github.repository_owner == 'home-assistant' env: HASSFEST_IMAGE_NAME: ghcr.io/home-assistant/hassfest @@ -510,8 +510,8 @@ jobs: - name: Build Docker image uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 with: - context: ./script/hassfest/docker - build-args: BASE_IMAGE=ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }} + context: . # So action will not pull the repository again + file: ./script/hassfest/docker/Dockerfile load: true tags: ${{ env.HASSFEST_IMAGE_TAG }} @@ -523,8 +523,8 @@ jobs: id: push uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 with: - context: ./script/hassfest/docker - build-args: BASE_IMAGE=ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }} + context: . # So action will not pull the repository again + file: ./script/hassfest/docker/Dockerfile push: true tags: ${{ env.HASSFEST_IMAGE_TAG }},${{ env.HASSFEST_IMAGE_NAME }}:latest diff --git a/script/hassfest/docker.py b/script/hassfest/docker.py index e38a238be7d..6e39a5c350b 100644 --- a/script/hassfest/docker.py +++ b/script/hassfest/docker.py @@ -1,7 +1,12 @@ """Generate and validate the dockerfile.""" +from dataclasses import dataclass +from pathlib import Path + from homeassistant import core +from homeassistant.const import Platform from homeassistant.util import executor, thread +from script.gen_requirements_all import gather_recursive_requirements from .model import Config, Integration from .requirements import PACKAGE_REGEX, PIP_VERSION_RANGE_SEPARATOR @@ -20,7 +25,7 @@ ENV \ ARG QEMU_CPU # Install uv -RUN pip3 install uv=={uv_version} +RUN pip3 install uv=={uv} WORKDIR /usr/src @@ -61,30 +66,105 @@ COPY rootfs / WORKDIR /config """ +_HASSFEST_TEMPLATE = r"""# Automatically generated by hassfest. +# +# To update, run python3 -m script.hassfest -p docker +FROM python:alpine3.20 -def _get_uv_version() -> str: - with open("requirements_test.txt") as fp: +ENV \ + UV_SYSTEM_PYTHON=true \ + UV_EXTRA_INDEX_URL="https://wheels.home-assistant.io/musllinux-index/" + +SHELL ["/bin/sh", "-o", "pipefail", "-c"] +ENTRYPOINT ["/usr/src/homeassistant/script/hassfest/docker/entrypoint.sh"] +WORKDIR "/github/workspace" + +# Install uv +COPY --from=ghcr.io/astral-sh/uv:{uv} /uv /bin/uv + +COPY . /usr/src/homeassistant + +RUN \ + # Required for PyTurboJPEG + apk add --no-cache libturbojpeg \ + && cd /usr/src/homeassistant \ + && uv pip install \ + --no-build \ + --no-cache \ + -c homeassistant/package_constraints.txt \ + -r requirements.txt \ + stdlib-list==0.10.0 pipdeptree=={pipdeptree} tqdm=={tqdm} ruff=={ruff} \ + {required_components_packages} + +LABEL "name"="hassfest" +LABEL "maintainer"="Home Assistant " + +LABEL "com.github.actions.name"="hassfest" +LABEL "com.github.actions.description"="Run hassfest to validate standalone integration repositories" +LABEL "com.github.actions.icon"="terminal" +LABEL "com.github.actions.color"="gray-dark" +""" + + +def _get_package_versions(file: str, packages: set[str]) -> dict[str, str]: + package_versions: dict[str, str] = {} + with open(file, encoding="UTF-8") as fp: for _, line in enumerate(fp): + if package_versions.keys() == packages: + return package_versions + if match := PACKAGE_REGEX.match(line): pkg, sep, version = match.groups() - if pkg != "uv": + if pkg not in packages: continue if sep != "==" or not version: raise RuntimeError( - 'Requirement uv need to be pinned "uv==".' + f'Requirement {pkg} need to be pinned "{pkg}==".' ) for part in version.split(";", 1)[0].split(","): version_part = PIP_VERSION_RANGE_SEPARATOR.match(part) if version_part: - return version_part.group(2) + package_versions[pkg] = version_part.group(2) + break - raise RuntimeError("Invalid uv requirement in requirements_test.txt") + if package_versions.keys() == packages: + return package_versions + + raise RuntimeError("At least one package was not found in the requirements file.") -def _generate_dockerfile() -> str: +@dataclass +class File: + """File.""" + + content: str + path: Path + + +def _generate_hassfest_dockerimage( + config: Config, timeout: int, package_versions: dict[str, str] +) -> File: + packages = set() + already_checked_domains = set() + for platform in Platform: + packages.update( + gather_recursive_requirements(platform.value, already_checked_domains) + ) + + return File( + _HASSFEST_TEMPLATE.format( + timeout=timeout, + required_components_packages=" ".join(sorted(packages)), + **package_versions, + ), + config.root / "script/hassfest/docker/Dockerfile", + ) + + +def _generate_files(config: Config) -> list[File]: timeout = ( core.STOPPING_STAGE_SHUTDOWN_TIMEOUT + core.STOP_STAGE_SHUTDOWN_TIMEOUT @@ -93,27 +173,39 @@ def _generate_dockerfile() -> str: + executor.EXECUTOR_SHUTDOWN_TIMEOUT + thread.THREADING_SHUTDOWN_TIMEOUT + 10 + ) * 1000 + + package_versions = _get_package_versions( + "requirements_test.txt", {"pipdeptree", "tqdm", "uv"} ) - return DOCKERFILE_TEMPLATE.format( - timeout=timeout * 1000, uv_version=_get_uv_version() + package_versions |= _get_package_versions( + "requirements_test_pre_commit.txt", {"ruff"} ) + return [ + File( + DOCKERFILE_TEMPLATE.format(timeout=timeout, **package_versions), + config.root / "Dockerfile", + ), + _generate_hassfest_dockerimage(config, timeout, package_versions), + ] + def validate(integrations: dict[str, Integration], config: Config) -> None: """Validate dockerfile.""" - dockerfile_content = _generate_dockerfile() - config.cache["dockerfile"] = dockerfile_content + docker_files = _generate_files(config) + config.cache["docker"] = docker_files - dockerfile_path = config.root / "Dockerfile" - if dockerfile_path.read_text() != dockerfile_content: - config.add_error( - "docker", - "File Dockerfile is not up to date. Run python3 -m script.hassfest", - fixable=True, - ) + for file in docker_files: + if file.content != file.path.read_text(): + config.add_error( + "docker", + f"File {file.path} is not up to date. Run python3 -m script.hassfest", + fixable=True, + ) def generate(integrations: dict[str, Integration], config: Config) -> None: """Generate dockerfile.""" - dockerfile_path = config.root / "Dockerfile" - dockerfile_path.write_text(config.cache["dockerfile"]) + for file in _generate_files(config): + file.path.write_text(file.content) diff --git a/script/hassfest/docker/Dockerfile b/script/hassfest/docker/Dockerfile index 8921d92307e..4fc60c0c621 100644 --- a/script/hassfest/docker/Dockerfile +++ b/script/hassfest/docker/Dockerfile @@ -1,17 +1,32 @@ -ARG BASE_IMAGE=ghcr.io/home-assistant/home-assistant:beta -FROM $BASE_IMAGE +# Automatically generated by hassfest. +# +# To update, run python3 -m script.hassfest -p docker +FROM python:alpine3.20 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV \ + UV_SYSTEM_PYTHON=true \ + UV_EXTRA_INDEX_URL="https://wheels.home-assistant.io/musllinux-index/" -COPY entrypoint.sh /entrypoint.sh +SHELL ["/bin/sh", "-o", "pipefail", "-c"] +ENTRYPOINT ["/usr/src/homeassistant/script/hassfest/docker/entrypoint.sh"] +WORKDIR "/github/workspace" + +# Install uv +COPY --from=ghcr.io/astral-sh/uv:0.2.27 /uv /bin/uv + +COPY . /usr/src/homeassistant RUN \ - uv pip install stdlib-list==0.10.0 \ - $(grep -e "^pipdeptree" -e "^tqdm" /usr/src/homeassistant/requirements_test.txt) \ - $(grep -e "^ruff" /usr/src/homeassistant/requirements_test_pre_commit.txt) - -WORKDIR "/github/workspace" -ENTRYPOINT ["/entrypoint.sh"] + # Required for PyTurboJPEG + apk add --no-cache libturbojpeg \ + && cd /usr/src/homeassistant \ + && uv pip install \ + --no-build \ + --no-cache \ + -c homeassistant/package_constraints.txt \ + -r requirements.txt \ + stdlib-list==0.10.0 pipdeptree==2.23.1 tqdm==4.66.4 ruff==0.6.2 \ + PyTurboJPEG==1.7.5 ha-ffmpeg==3.2.0 hassil==1.7.4 home-assistant-intents==2024.8.29 mutagen==1.47.0 LABEL "name"="hassfest" LABEL "maintainer"="Home Assistant " diff --git a/script/hassfest/docker/Dockerfile.dockerignore b/script/hassfest/docker/Dockerfile.dockerignore new file mode 100644 index 00000000000..75ed4f0e5d3 --- /dev/null +++ b/script/hassfest/docker/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +# Ignore everything except the specified files +* + +!homeassistant/ +!requirements.txt +!script/ +script/hassfest/docker/ +!script/hassfest/docker/entrypoint.sh diff --git a/script/hassfest/docker/entrypoint.sh b/script/hassfest/docker/entrypoint.sh index 33330f63161..7b75eb186d2 100755 --- a/script/hassfest/docker/entrypoint.sh +++ b/script/hassfest/docker/entrypoint.sh @@ -1,16 +1,18 @@ -#!/usr/bin/env bashio -declare -a integrations -declare integration_path +#!/bin/sh -shopt -s globstar nullglob -for manifest in **/manifest.json; do +integrations="" +integration_path="" + +# Enable recursive globbing using find +for manifest in $(find . -name "manifest.json"); do manifest_path=$(realpath "${manifest}") - integrations+=(--integration-path "${manifest_path%/*}") + integrations="$integrations --integration-path ${manifest_path%/*}" done -if [[ ${#integrations[@]} -eq 0 ]]; then - bashio::exit.nok "No integrations found!" +if [ -z "$integrations" ]; then + echo "Error: No integrations found!" + exit 1 fi -cd /usr/src/homeassistant -exec python3 -m script.hassfest --action validate "${integrations[@]}" "$@" \ No newline at end of file +cd /usr/src/homeassistant || exit 1 +exec python3 -m script.hassfest --action validate $integrations "$@"