diff --git a/supervisor/addons/build.py b/supervisor/addons/build.py index 675d1af58..ee9962d95 100644 --- a/supervisor/addons/build.py +++ b/supervisor/addons/build.py @@ -15,6 +15,7 @@ from ..const import ( ATTR_SQUASH, FILE_SUFFIX_CONFIGURATION, META_ADDON, + SOCKET_DOCKER, ) from ..coresys import CoreSys, CoreSysAttributes from ..docker.interface import MAP_ARCH @@ -121,39 +122,64 @@ class AddonBuild(FileConfiguration, CoreSysAttributes): except HassioArchNotFound: return False - def get_docker_args(self, version: AwesomeVersion, image: str | None = None): - """Create a dict with Docker build arguments. + def get_docker_args( + self, version: AwesomeVersion, image_tag: str + ) -> dict[str, Any]: + """Create a dict with Docker run args.""" + dockerfile_path = self.get_dockerfile().relative_to(self.addon.path_location) - Must be run in executor. - """ - args: dict[str, Any] = { - "path": str(self.addon.path_location), - "tag": f"{image or self.addon.image}:{version!s}", - "dockerfile": str(self.get_dockerfile()), - "pull": True, - "forcerm": not self.sys_dev, - "squash": self.squash, - "platform": MAP_ARCH[self.arch], - "labels": { - "io.hass.version": version, - "io.hass.arch": self.arch, - "io.hass.type": META_ADDON, - "io.hass.name": self._fix_label("name"), - "io.hass.description": self._fix_label("description"), - **self.additional_labels, - }, - "buildargs": { - "BUILD_FROM": self.base_image, - "BUILD_VERSION": version, - "BUILD_ARCH": self.sys_arch.default, - **self.additional_args, - }, + build_cmd = [ + "docker", + "buildx", + "build", + ".", + "--tag", + image_tag, + "--file", + str(dockerfile_path), + "--platform", + MAP_ARCH[self.arch], + "--pull", + ] + + labels = { + "io.hass.version": version, + "io.hass.arch": self.arch, + "io.hass.type": META_ADDON, + "io.hass.name": self._fix_label("name"), + "io.hass.description": self._fix_label("description"), + **self.additional_labels, } if self.addon.url: - args["labels"]["io.hass.url"] = self.addon.url + labels["io.hass.url"] = self.addon.url - return args + for key, value in labels.items(): + build_cmd.extend(["--label", f"{key}={value}"]) + + build_args = { + "BUILD_FROM": self.base_image, + "BUILD_VERSION": version, + "BUILD_ARCH": self.sys_arch.default, + **self.additional_args, + } + + for key, value in build_args.items(): + build_cmd.extend(["--build-arg", f"{key}={value}"]) + + # The addon path will be mounted from the host system + addon_extern_path = self.sys_config.local_to_extern_path( + self.addon.path_location + ) + + return { + "command": build_cmd, + "volumes": { + SOCKET_DOCKER: {"bind": "/var/run/docker.sock", "mode": "rw"}, + addon_extern_path: {"bind": "/addon", "mode": "ro"}, + }, + "working_dir": "/addon", + } def _fix_label(self, label_name: str) -> str: """Remove characters they are not supported.""" diff --git a/supervisor/docker/addon.py b/supervisor/docker/addon.py index 815be5979..422925a15 100644 --- a/supervisor/docker/addon.py +++ b/supervisor/docker/addon.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, cast from attr import evolve from awesomeversion import AwesomeVersion import docker +import docker.errors from docker.types import Mount import requests @@ -43,6 +44,7 @@ from ..jobs.decorator import Job from ..resolution.const import CGROUP_V2_VERSION, ContextType, IssueType, SuggestionType from ..utils.sentry import async_capture_exception from .const import ( + ADDON_BUILDER_IMAGE, ENV_TIME, ENV_TOKEN, ENV_TOKEN_OLD, @@ -673,10 +675,41 @@ class DockerAddon(DockerInterface): _LOGGER.info("Starting build for %s:%s", self.image, version) def build_image(): - return self.sys_docker.images.build( - use_config_proxy=False, **build_env.get_docker_args(version, image) + if build_env.squash: + _LOGGER.warning( + "Ignoring squash build option for %s as Docker BuildKit does not support it.", + self.addon.slug, + ) + + addon_image_tag = f"{image or self.addon.image}:{version!s}" + + docker_version = self.sys_docker.info.version + builder_version_tag = f"{docker_version.major}.{docker_version.minor}.{docker_version.micro}-cli" + + builder_name = f"addon_builder_{self.addon.slug}" + + # Remove dangling builder container if it exists by any chance + # E.g. because of an abrupt host shutdown/reboot during a build + with suppress(docker.errors.NotFound): + self.sys_docker.containers.get(builder_name).remove(force=True, v=True) + + result = self.sys_docker.run_command( + ADDON_BUILDER_IMAGE, + version=builder_version_tag, + name=builder_name, + **build_env.get_docker_args(version, addon_image_tag), ) + logs = result.output.decode("utf-8") + + if result.exit_code != 0: + error_message = f"Docker build failed for {addon_image_tag} (exit code {result.exit_code}). Build output:\n{logs}" + raise docker.errors.DockerException(error_message) + + addon_image = self.sys_docker.images.get(addon_image_tag) + + return addon_image, logs + try: docker_image, log = await self.sys_run_in_executor(build_image) @@ -687,15 +720,6 @@ class DockerAddon(DockerInterface): except (docker.errors.DockerException, requests.RequestException) as err: _LOGGER.error("Can't build %s:%s: %s", self.image, version, err) - if hasattr(err, "build_log"): - log = "\n".join( - [ - x["stream"] - for x in err.build_log # pylint: disable=no-member - if isinstance(x, dict) and "stream" in x - ] - ) - _LOGGER.error("Build log: \n%s", log) raise DockerError() from err _LOGGER.info("Build %s:%s done", self.image, version) diff --git a/supervisor/docker/const.py b/supervisor/docker/const.py index 02feed247..bde3752e4 100644 --- a/supervisor/docker/const.py +++ b/supervisor/docker/const.py @@ -107,3 +107,6 @@ PATH_BACKUP = PurePath("/backup") PATH_SHARE = PurePath("/share") PATH_MEDIA = PurePath("/media") PATH_CLOUD_BACKUP = PurePath("/cloud_backup") + +# https://hub.docker.com/_/docker +ADDON_BUILDER_IMAGE = "docker.io/library/docker" diff --git a/supervisor/docker/manager.py b/supervisor/docker/manager.py index 0c00b4c5b..e350f19d6 100644 --- a/supervisor/docker/manager.py +++ b/supervisor/docker/manager.py @@ -295,7 +295,7 @@ class DockerAPI: self, image: str, version: str = "latest", - command: str | None = None, + command: str | list[str] | None = None, **kwargs: Any, ) -> CommandReturn: """Create a temporary container and run command. diff --git a/supervisor/resolution/evaluations/container.py b/supervisor/resolution/evaluations/container.py index 308e17e54..83136fb53 100644 --- a/supervisor/resolution/evaluations/container.py +++ b/supervisor/resolution/evaluations/container.py @@ -5,6 +5,8 @@ import logging from docker.errors import DockerException from requests import RequestException +from supervisor.docker.const import ADDON_BUILDER_IMAGE + from ...const import CoreState from ...coresys import CoreSys from ..const import ( @@ -63,6 +65,7 @@ class EvaluateContainer(EvaluateBase): self.sys_supervisor.image or self.sys_supervisor.default_image, *(plugin.image for plugin in self.sys_plugins.all_plugins if plugin.image), *(addon.image for addon in self.sys_addons.installed if addon.image), + ADDON_BUILDER_IMAGE, } async def evaluate(self) -> bool: diff --git a/tests/addons/test_addon.py b/tests/addons/test_addon.py index 258eccf05..1d06bc1aa 100644 --- a/tests/addons/test_addon.py +++ b/tests/addons/test_addon.py @@ -18,6 +18,7 @@ from supervisor.const import AddonBoot, AddonState, BusEvent from supervisor.coresys import CoreSys from supervisor.docker.addon import DockerAddon from supervisor.docker.const import ContainerState +from supervisor.docker.manager import CommandReturn from supervisor.docker.monitor import DockerContainerStateEvent from supervisor.exceptions import AddonsError, AddonsJobError, AudioUpdateError from supervisor.hardware.helper import HwHelper @@ -27,7 +28,7 @@ from supervisor.utils.dt import utcnow from .test_manager import BOOT_FAIL_ISSUE, BOOT_FAIL_SUGGESTIONS -from tests.common import get_fixture_path +from tests.common import get_fixture_path, is_in_list from tests.const import TEST_ADDON_SLUG @@ -840,10 +841,25 @@ async def test_addon_loads_wrong_image( install_addon_ssh.persist["image"] = "local/aarch64-addon-ssh" assert install_addon_ssh.image == "local/aarch64-addon-ssh" - with patch("pathlib.Path.is_file", return_value=True): + with ( + patch("pathlib.Path.is_file", return_value=True), + patch.object( + coresys.docker, + "run_command", + new=PropertyMock(return_value=CommandReturn(0, b"Build successful")), + ) as mock_run_command, + patch.object( + type(coresys.config), + "local_to_extern_path", + return_value="/addon/path/on/host", + ), + ): await install_addon_ssh.load() - container.remove.assert_called_once_with(force=True, v=True) + container.remove.assert_called_with(force=True, v=True) + # one for removing the addon, one for removing the addon builder + assert coresys.docker.images.remove.call_count == 2 + assert coresys.docker.images.remove.call_args_list[0].kwargs == { "image": "local/aarch64-addon-ssh:latest", "force": True, @@ -852,12 +868,18 @@ async def test_addon_loads_wrong_image( "image": "local/aarch64-addon-ssh:9.2.1", "force": True, } - coresys.docker.images.build.assert_called_once() - assert ( - coresys.docker.images.build.call_args.kwargs["tag"] - == "local/amd64-addon-ssh:9.2.1" + mock_run_command.assert_called_once() + assert mock_run_command.call_args.args[0] == "docker.io/library/docker" + assert mock_run_command.call_args.kwargs["version"] == "1.0.0-cli" + command = mock_run_command.call_args.kwargs["command"] + assert is_in_list( + ["--platform", "linux/amd64"], + command, + ) + assert is_in_list( + ["--tag", "local/amd64-addon-ssh:9.2.1"], + command, ) - assert coresys.docker.images.build.call_args.kwargs["platform"] == "linux/amd64" assert install_addon_ssh.image == "local/amd64-addon-ssh" coresys.addons.data.save_data.assert_called_once() @@ -871,15 +893,33 @@ async def test_addon_loads_missing_image( """Test addon corrects a missing image on load.""" coresys.docker.images.get.side_effect = ImageNotFound("missing") - with patch("pathlib.Path.is_file", return_value=True): + with ( + patch("pathlib.Path.is_file", return_value=True), + patch.object( + coresys.docker, + "run_command", + new=PropertyMock(return_value=CommandReturn(0, b"Build successful")), + ) as mock_run_command, + patch.object( + type(coresys.config), + "local_to_extern_path", + return_value="/addon/path/on/host", + ), + ): await install_addon_ssh.load() - coresys.docker.images.build.assert_called_once() - assert ( - coresys.docker.images.build.call_args.kwargs["tag"] - == "local/amd64-addon-ssh:9.2.1" + mock_run_command.assert_called_once() + assert mock_run_command.call_args.args[0] == "docker.io/library/docker" + assert mock_run_command.call_args.kwargs["version"] == "1.0.0-cli" + command = mock_run_command.call_args.kwargs["command"] + assert is_in_list( + ["--platform", "linux/amd64"], + command, + ) + assert is_in_list( + ["--tag", "local/amd64-addon-ssh:9.2.1"], + command, ) - assert coresys.docker.images.build.call_args.kwargs["platform"] == "linux/amd64" assert install_addon_ssh.image == "local/amd64-addon-ssh" @@ -900,7 +940,14 @@ async def test_addon_load_succeeds_with_docker_errors( # Image build failure coresys.docker.images.build.side_effect = DockerException() caplog.clear() - with patch("pathlib.Path.is_file", return_value=True): + with ( + patch("pathlib.Path.is_file", return_value=True), + patch.object( + type(coresys.config), + "local_to_extern_path", + return_value="/addon/path/on/host", + ), + ): await install_addon_ssh.load() assert "Can't build local/amd64-addon-ssh:9.2.1" in caplog.text diff --git a/tests/addons/test_build.py b/tests/addons/test_build.py index 800f4e21e..243fbef97 100644 --- a/tests/addons/test_build.py +++ b/tests/addons/test_build.py @@ -8,10 +8,13 @@ from supervisor.addons.addon import Addon from supervisor.addons.build import AddonBuild from supervisor.coresys import CoreSys +from tests.common import is_in_list + async def test_platform_set(coresys: CoreSys, install_addon_ssh: Addon): - """Test platform set in docker args.""" + """Test platform set in container build args.""" build = await AddonBuild(coresys, install_addon_ssh).load_config() + with ( patch.object( type(coresys.arch), "supported", new=PropertyMock(return_value=["amd64"]) @@ -19,17 +22,23 @@ async def test_platform_set(coresys: CoreSys, install_addon_ssh: Addon): patch.object( type(coresys.arch), "default", new=PropertyMock(return_value="amd64") ), + patch.object( + type(coresys.config), + "local_to_extern_path", + return_value="/addon/path/on/host", + ), ): args = await coresys.run_in_executor( - build.get_docker_args, AwesomeVersion("latest") + build.get_docker_args, AwesomeVersion("latest"), "test-image:latest" ) - assert args["platform"] == "linux/amd64" + assert is_in_list(["--platform", "linux/amd64"], args["command"]) async def test_dockerfile_evaluation(coresys: CoreSys, install_addon_ssh: Addon): - """Test platform set in docker args.""" + """Test dockerfile path in container build args.""" build = await AddonBuild(coresys, install_addon_ssh).load_config() + with ( patch.object( type(coresys.arch), "supported", new=PropertyMock(return_value=["amd64"]) @@ -37,12 +46,17 @@ async def test_dockerfile_evaluation(coresys: CoreSys, install_addon_ssh: Addon) patch.object( type(coresys.arch), "default", new=PropertyMock(return_value="amd64") ), + patch.object( + type(coresys.config), + "local_to_extern_path", + return_value="/addon/path/on/host", + ), ): args = await coresys.run_in_executor( - build.get_docker_args, AwesomeVersion("latest") + build.get_docker_args, AwesomeVersion("latest"), "test-image:latest" ) - assert args["dockerfile"].endswith("fixtures/addons/local/ssh/Dockerfile") + assert is_in_list(["--file", "Dockerfile"], args["command"]) assert str(await coresys.run_in_executor(build.get_dockerfile)).endswith( "fixtures/addons/local/ssh/Dockerfile" ) @@ -50,8 +64,9 @@ async def test_dockerfile_evaluation(coresys: CoreSys, install_addon_ssh: Addon) async def test_dockerfile_evaluation_arch(coresys: CoreSys, install_addon_ssh: Addon): - """Test platform set in docker args.""" + """Test dockerfile arch evaluation in container build args.""" build = await AddonBuild(coresys, install_addon_ssh).load_config() + with ( patch.object( type(coresys.arch), "supported", new=PropertyMock(return_value=["aarch64"]) @@ -59,12 +74,17 @@ async def test_dockerfile_evaluation_arch(coresys: CoreSys, install_addon_ssh: A patch.object( type(coresys.arch), "default", new=PropertyMock(return_value="aarch64") ), + patch.object( + type(coresys.config), + "local_to_extern_path", + return_value="/addon/path/on/host", + ), ): args = await coresys.run_in_executor( - build.get_docker_args, AwesomeVersion("latest") + build.get_docker_args, AwesomeVersion("latest"), "test-image:latest" ) - assert args["dockerfile"].endswith("fixtures/addons/local/ssh/Dockerfile.aarch64") + assert is_in_list(["--file", "Dockerfile.aarch64"], args["command"]) assert str(await coresys.run_in_executor(build.get_dockerfile)).endswith( "fixtures/addons/local/ssh/Dockerfile.aarch64" ) diff --git a/tests/api/test_addons.py b/tests/api/test_addons.py index c04b0129b..64e87bf1a 100644 --- a/tests/api/test_addons.py +++ b/tests/api/test_addons.py @@ -14,6 +14,7 @@ from supervisor.const import AddonState from supervisor.coresys import CoreSys from supervisor.docker.addon import DockerAddon from supervisor.docker.const import ContainerState +from supervisor.docker.manager import CommandReturn from supervisor.docker.monitor import DockerContainerStateEvent from supervisor.exceptions import HassioError from supervisor.store.repository import Repository @@ -239,6 +240,19 @@ async def test_api_addon_rebuild_healthcheck( patch.object(Addon, "need_build", new=PropertyMock(return_value=True)), patch.object(CpuArch, "supported", new=PropertyMock(return_value=["amd64"])), patch.object(DockerAddon, "run", new=container_events_task), + patch.object( + coresys.docker, + "run_command", + new=PropertyMock(return_value=CommandReturn(0, b"Build successful")), + ), + patch.object( + DockerAddon, "healthcheck", new=PropertyMock(return_value={"exists": True}) + ), + patch.object( + type(coresys.config), + "local_to_extern_path", + return_value="/addon/path/on/host", + ), ): resp = await api_client.post("/addons/local_ssh/rebuild") diff --git a/tests/common.py b/tests/common.py index 5ce105193..944f4348c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -105,6 +105,20 @@ def reset_last_call(func, group: str | None = None) -> None: get_job_decorator(func).set_last_call(datetime.min, group) +def is_in_list(a: list, b: list): + """Check if all elements in list a are in list b in order. + + Taken from https://stackoverflow.com/a/69175987/12156188. + """ + + for c in a: + if c in b: + b = b[b.index(c) :] + else: + return False + return True + + class MockResponse: """Mock response for aiohttp requests.""" diff --git a/tests/conftest.py b/tests/conftest.py index f55948bb2..e9fd01deb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -131,7 +131,7 @@ async def docker() -> DockerAPI: docker_obj.info.logging = "journald" docker_obj.info.storage = "overlay2" - docker_obj.info.version = "1.0.0" + docker_obj.info.version = AwesomeVersion("1.0.0") yield docker_obj