From e577d8acb24886f39920b32ac417ca3777e42c16 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Fri, 19 Jul 2019 11:49:00 +0200 Subject: [PATCH 1/9] Bump version 171 --- hassio/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hassio/const.py b/hassio/const.py index 24eb2677a..69219c7ba 100644 --- a/hassio/const.py +++ b/hassio/const.py @@ -3,7 +3,7 @@ from pathlib import Path from ipaddress import ip_network -HASSIO_VERSION = "170" +HASSIO_VERSION = "171" URL_HASSIO_ADDONS = "https://github.com/home-assistant/hassio-addons" URL_HASSIO_VERSION = "https://version.home-assistant.io/{channel}.json" From 0e7e95ba200405e1d85c6249377d64113d2fcffc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2019 14:18:11 +0200 Subject: [PATCH 2/9] Bump gitpython from 2.1.11 to 2.1.12 (#1171) Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 2.1.11 to 2.1.12. - [Release notes](https://github.com/gitpython-developers/GitPython/releases) - [Changelog](https://github.com/gitpython-developers/GitPython/blob/master/CHANGES) - [Commits](https://github.com/gitpython-developers/GitPython/compare/2.1.11...2.1.12) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bdfa609c3..6a572bef2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ colorlog==4.0.2 cpe==1.2.1 cryptography==2.7 docker==4.0.2 -gitpython==2.1.11 +gitpython==2.1.12 pytz==2019.1 pyudev==0.21.0 uvloop==0.12.2 From 7ae430e7a86f36b5a0b5f8299d3e367ba5b3a4e5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2019 14:53:54 +0200 Subject: [PATCH 3/9] Bump gitpython from 2.1.12 to 2.1.13 (#1178) Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 2.1.12 to 2.1.13. - [Release notes](https://github.com/gitpython-developers/GitPython/releases) - [Changelog](https://github.com/gitpython-developers/GitPython/blob/master/CHANGES) - [Commits](https://github.com/gitpython-developers/GitPython/compare/2.1.12...2.1.13) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a572bef2..012d19738 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ colorlog==4.0.2 cpe==1.2.1 cryptography==2.7 docker==4.0.2 -gitpython==2.1.12 +gitpython==2.1.13 pytz==2019.1 pyudev==0.21.0 uvloop==0.12.2 From 814b504fa96c3e638fa52f22ce836a81545d57b1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2019 17:01:28 +0200 Subject: [PATCH 4/9] Bump ptvsd from 4.2.10 to 4.3.0 (#1179) Bumps [ptvsd](https://github.com/Microsoft/ptvsd) from 4.2.10 to 4.3.0. - [Release notes](https://github.com/Microsoft/ptvsd/releases) - [Commits](https://github.com/Microsoft/ptvsd/compare/v4.2.10...v4.3.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 012d19738..c43ed210a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ pytz==2019.1 pyudev==0.21.0 uvloop==0.12.2 voluptuous==0.11.5 -ptvsd==4.2.10 +ptvsd==4.3.0 From b7c07a255531b8f877d692953a69d6314e04be28 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2019 10:32:04 +0200 Subject: [PATCH 5/9] Bump pytz from 2019.1 to 2019.2 (#1184) Bumps [pytz](https://github.com/stub42/pytz) from 2019.1 to 2019.2. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2019.1...release_2019.2) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c43ed210a..4a96eb776 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ cpe==1.2.1 cryptography==2.7 docker==4.0.2 gitpython==2.1.13 -pytz==2019.1 +pytz==2019.2 pyudev==0.21.0 uvloop==0.12.2 voluptuous==0.11.5 From 882586b246e53cc922cecfcc703ed87368881db3 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Tue, 6 Aug 2019 09:24:22 +0200 Subject: [PATCH 6/9] Fix time adjustments on latest boot (#1187) * Fix time adjustments on latest boot * Fix spell --- hassio/core.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hassio/core.py b/hassio/core.py index cd3d08f41..db676d91b 100644 --- a/hassio/core.py +++ b/hassio/core.py @@ -116,8 +116,7 @@ class HassIO(CoreSysAttributes): await self.sys_addons.boot(STARTUP_APPLICATION) # store new last boot - self.sys_config.last_boot = self.sys_hardware.last_boot - self.sys_config.save_data() + self._update_last_boot() finally: # Add core tasks into scheduler @@ -134,6 +133,9 @@ class HassIO(CoreSysAttributes): # don't process scheduler anymore self.sys_scheduler.suspend = True + # store new last boot / prevent time adjustments + self._update_last_boot() + # process async stop tasks try: with async_timeout.timeout(10): @@ -162,3 +164,8 @@ class HassIO(CoreSysAttributes): await self.sys_addons.shutdown(STARTUP_SERVICES) await self.sys_addons.shutdown(STARTUP_SYSTEM) await self.sys_addons.shutdown(STARTUP_INITIALIZE) + + def _update_last_boot(self): + """Update last boot time.""" + self.sys_config.last_boot = self.sys_hardware.last_boot + self.sys_config.save_data() From 778bc46848ee14fcf4bb0ee600ed1c7153422464 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 7 Aug 2019 09:51:27 +0200 Subject: [PATCH 7/9] Don't relay on latest with HA/Addons (#1175) * Don't relay on latest with HA/Addons * Fix latest on install * Revert some options * Fix attach * migrate to new version handling * Fix thread * Fix is running * Allow wait * debug code * Fix debug value * Fix list * Fix regex * Some better log output * Fix logic * Improve cleanup handling * Fix bug * Cleanup old code * Improve version handling * Fix the way to attach --- .devcontainer/devcontainer.json | 9 ++++- azure-pipelines.yml | 2 +- hassio/__main__.py | 3 +- hassio/addons/__init__.py | 12 +++++- hassio/addons/addon.py | 2 +- hassio/bootstrap.py | 3 +- hassio/core.py | 7 ++-- hassio/docker/__init__.py | 20 ++++++---- hassio/docker/addon.py | 10 +++-- hassio/docker/hassos_cli.py | 4 +- hassio/docker/homeassistant.py | 49 +++++++++++++++++++++---- hassio/docker/interface.py | 56 ++++++++++++++-------------- hassio/docker/supervisor.py | 2 +- hassio/hassos.py | 8 +++- hassio/homeassistant.py | 65 +++++++++++++++++++++++---------- hassio/misc/dns.py | 2 +- hassio/supervisor.py | 4 +- hassio/validate.py | 2 + 18 files changed, 175 insertions(+), 85 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6979f8569..929334718 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,8 @@ "context": "..", "dockerFile": "Dockerfile", "runArgs": [ - "-e", "GIT_EDTIOR='code --wait'" + "-e", + "GIT_EDTIOR='code --wait'" ], "extensions": [ "ms-python.python" @@ -14,9 +15,13 @@ "python.linting.pylintEnabled": true, "python.linting.enabled": true, "python.formatting.provider": "black", + "python.formatting.blackArgs": [ + "--target--version", + "py37" + ], "editor.formatOnPaste": false, "editor.formatOnSave": true, "editor.formatOnType": true, "files.trimTrailingWhitespace": true } -} +} \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ababa205d..5a1eb8e35 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,7 +52,7 @@ stages: versionSpec: '3.7' - script: pip install black displayName: 'Install black' - - script: black --check hassio tests + - script: black --target-version py37 --check hassio tests displayName: 'Run Black' - job: 'JQ' pool: diff --git a/hassio/__main__.py b/hassio/__main__.py index 1962a6d86..baed3dda8 100644 --- a/hassio/__main__.py +++ b/hassio/__main__.py @@ -38,9 +38,10 @@ if __name__ == "__main__": _LOGGER.info("Initialize Hass.io setup") coresys = loop.run_until_complete(bootstrap.initialize_coresys()) + loop.run_until_complete(coresys.core.connect()) - bootstrap.migrate_system_env(coresys) bootstrap.supervisor_debugger(coresys) + bootstrap.migrate_system_env(coresys) _LOGGER.info("Setup HassIO") loop.run_until_complete(coresys.core.setup()) diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index f1b8bf499..84e64a28f 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -130,6 +130,7 @@ class AddonManager(CoreSysAttributes): raise AddonsError() from None else: self.local[slug] = addon + _LOGGER.info("Add-on '%s' successfully installed", slug) async def uninstall(self, slug: str) -> None: """Remove an add-on.""" @@ -159,6 +160,8 @@ class AddonManager(CoreSysAttributes): self.data.uninstall(addon) self.local.pop(slug) + _LOGGER.info("Add-on '%s' successfully removed", slug) + async def update(self, slug: str) -> None: """Update add-on.""" if slug not in self.local: @@ -184,9 +187,15 @@ class AddonManager(CoreSysAttributes): last_state = await addon.state() try: await addon.instance.update(store.version, store.image) + + # Cleanup + with suppress(DockerAPIError): + await addon.instance.cleanup() except DockerAPIError: raise AddonsError() from None - self.data.update(store) + else: + self.data.update(store) + _LOGGER.info("Add-on '%s' successfully updated", slug) # Setup/Fix AppArmor profile await addon.install_apparmor() @@ -224,6 +233,7 @@ class AddonManager(CoreSysAttributes): raise AddonsError() from None else: self.data.update(store) + _LOGGER.info("Add-on '%s' successfully rebuilded", slug) # restore state if last_state == STATE_STARTED: diff --git a/hassio/addons/addon.py b/hassio/addons/addon.py index 69061db75..ec0e8cb7f 100644 --- a/hassio/addons/addon.py +++ b/hassio/addons/addon.py @@ -75,7 +75,7 @@ class Addon(AddonModel): async def load(self) -> None: """Async initialize of object.""" with suppress(DockerAPIError): - await self.instance.attach() + await self.instance.attach(tag=self.version) @property def ip_address(self) -> IPv4Address: diff --git a/hassio/bootstrap.py b/hassio/bootstrap.py index f6c07118c..05ab934bb 100644 --- a/hassio/bootstrap.py +++ b/hassio/bootstrap.py @@ -218,7 +218,7 @@ def reg_signal(loop): def supervisor_debugger(coresys: CoreSys) -> None: """Setup debugger if needed.""" - if not coresys.config.debug or not coresys.dev: + if not coresys.config.debug: return import ptvsd @@ -226,4 +226,5 @@ def supervisor_debugger(coresys: CoreSys) -> None: ptvsd.enable_attach(address=("0.0.0.0", 33333), redirect_output=True) if coresys.config.debug_block: + _LOGGER.info("Wait until debugger is attached") ptvsd.wait_for_attach() diff --git a/hassio/core.py b/hassio/core.py index db676d91b..ce8e4c0e0 100644 --- a/hassio/core.py +++ b/hassio/core.py @@ -24,11 +24,12 @@ class HassIO(CoreSysAttributes): """Initialize Hass.io object.""" self.coresys = coresys - async def setup(self): - """Setup HassIO orchestration.""" - # Load Supervisor + async def connect(self): + """Connect Supervisor container.""" await self.sys_supervisor.load() + async def setup(self): + """Setup HassIO orchestration.""" # Load DBus await self.sys_dbus.load() diff --git a/hassio/docker/__init__.py b/hassio/docker/__init__.py index af01df582..13d893291 100644 --- a/hassio/docker/__init__.py +++ b/hassio/docker/__init__.py @@ -50,15 +50,15 @@ class DockerAPI: return self.docker.api def run( - self, image: str, **kwargs: Dict[str, Any] + self, image: str, version: str = "latest", **kwargs: Dict[str, Any] ) -> docker.models.containers.Container: """"Create a Docker container and run it. Need run inside executor. """ - name = kwargs.get("name", image) - network_mode = kwargs.get("network_mode") - hostname = kwargs.get("hostname") + name: str = kwargs.get("name", image) + network_mode: str = kwargs.get("network_mode") + hostname: str = kwargs.get("hostname") # Setup network kwargs["dns_search"] = ["."] @@ -71,7 +71,7 @@ class DockerAPI: # Create container try: container = self.docker.containers.create( - image, use_config_proxy=False, **kwargs + f"{image}:{version}", use_config_proxy=False, **kwargs ) except docker.errors.DockerException as err: _LOGGER.error("Can't create container from %s: %s", name, err) @@ -102,7 +102,11 @@ class DockerAPI: return container def run_command( - self, image: str, command: Optional[str] = None, **kwargs: Dict[str, Any] + self, + image: str, + version: str = "latest", + command: Optional[str] = None, + **kwargs: Dict[str, Any], ) -> CommandReturn: """Create a temporary container and run command. @@ -114,11 +118,11 @@ class DockerAPI: _LOGGER.info("Run command '%s' on %s", command, image) try: container = self.docker.containers.run( - image, + f"{image}:{version}", command=command, network=self.network.name, use_config_proxy=False, - **kwargs + **kwargs, ) # wait until command is done diff --git a/hassio/docker/addon.py b/hassio/docker/addon.py index cd812c378..df7b4c799 100644 --- a/hassio/docker/addon.py +++ b/hassio/docker/addon.py @@ -327,6 +327,7 @@ class DockerAddon(DockerInterface): # Create & Run container docker_container = self.sys_docker.run( self.image, + version=self.addon.version, name=self.name, hostname=self.hostname, detach=True, @@ -346,10 +347,12 @@ class DockerAddon(DockerInterface): tmpfs=self.tmpfs, ) - _LOGGER.info("Start Docker add-on %s with version %s", self.image, self.version) self._meta = docker_container.attrs + _LOGGER.info("Start Docker add-on %s with version %s", self.image, self.version) - def _install(self, tag: str, image: Optional[str] = None) -> None: + def _install( + self, tag: str, image: Optional[str] = None, latest: bool = False + ) -> None: """Pull Docker image or build it. Need run inside executor. @@ -357,7 +360,7 @@ class DockerAddon(DockerInterface): if self.addon.need_build: self._build(tag) else: - super()._install(tag, image) + super()._install(tag, image, latest) def _build(self, tag: str) -> None: """Build a Docker container. @@ -373,7 +376,6 @@ class DockerAddon(DockerInterface): ) _LOGGER.debug("Build %s:%s done: %s", self.image, tag, log) - image.tag(self.image, tag="latest") # Update meta data self._meta = image.attrs diff --git a/hassio/docker/hassos_cli.py b/hassio/docker/hassos_cli.py index 97a0c0a5f..e9413ea3f 100644 --- a/hassio/docker/hassos_cli.py +++ b/hassio/docker/hassos_cli.py @@ -21,12 +21,12 @@ class DockerHassOSCli(DockerInterface, CoreSysAttributes): """Don't need stop.""" return True - def _attach(self): + def _attach(self, tag: str): """Attach to running Docker container. Need run inside executor. """ try: - image = self.sys_docker.images.get(self.image) + image = self.sys_docker.images.get(f"{self.image}:{tag}") except docker.errors.DockerException: _LOGGER.warning("Can't find a HassOS CLI %s", self.image) diff --git a/hassio/docker/homeassistant.py b/hassio/docker/homeassistant.py index f9367cfa2..6e80dacc8 100644 --- a/hassio/docker/homeassistant.py +++ b/hassio/docker/homeassistant.py @@ -1,8 +1,10 @@ """Init file for Hass.io Docker object.""" +from distutils.version import StrictVersion from contextlib import suppress from ipaddress import IPv4Address import logging -from typing import Awaitable +import re +from typing import Awaitable, List, Optional import docker @@ -13,30 +15,31 @@ from .interface import CommandReturn, DockerInterface _LOGGER = logging.getLogger(__name__) HASS_DOCKER_NAME = "homeassistant" +RE_VERSION = re.compile(r"(?P\d+\.\d+\.\d+(?:b\d+|d\d+)?)") class DockerHomeAssistant(DockerInterface): """Docker Hass.io wrapper for Home Assistant.""" @property - def machine(self): + def machine(self) -> Optional[str]: """Return machine of Home Assistant Docker image.""" if self._meta and LABEL_MACHINE in self._meta["Config"]["Labels"]: return self._meta["Config"]["Labels"][LABEL_MACHINE] return None @property - def image(self): + def image(self) -> str: """Return name of Docker image.""" return self.sys_homeassistant.image @property - def name(self): + def name(self) -> str: """Return name of Docker container.""" return HASS_DOCKER_NAME @property - def timeout(self) -> str: + def timeout(self) -> int: """Return timeout for Docker actions.""" return 60 @@ -60,6 +63,7 @@ class DockerHomeAssistant(DockerInterface): # Create & Run container docker_container = self.sys_docker.run( self.image, + version=self.sys_homeassistant.version, name=self.name, hostname=self.name, detach=True, @@ -84,8 +88,8 @@ class DockerHomeAssistant(DockerInterface): }, ) - _LOGGER.info("Start homeassistant %s with version %s", self.image, self.version) self._meta = docker_container.attrs + _LOGGER.info("Start homeassistant %s with version %s", self.image, self.version) def _execute_command(self, command: str) -> CommandReturn: """Create a temporary container and run command. @@ -94,7 +98,8 @@ class DockerHomeAssistant(DockerInterface): """ return self.sys_docker.run_command( self.image, - command, + version=self.sys_homeassistant.version, + command=command, privileged=True, init=True, detach=True, @@ -134,3 +139,33 @@ class DockerHomeAssistant(DockerInterface): return False return True + + def get_latest_version(self) -> Awaitable[str]: + """Return latest version of local Home Asssistant image.""" + return self.sys_run_in_executor(self._get_latest_version) + + def _get_latest_version(self) -> str: + """Return latest version of local Home Asssistant image. + + Need run inside executor. + """ + available_version: List[str] = [] + try: + for image in self.sys_docker.images.list(self.image): + for tag in image.tags: + match = RE_VERSION.search(tag) + if not match: + continue + available_version.append(match.group("version")) + + assert available_version + + except (docker.errors.DockerException, AssertionError): + _LOGGER.warning("No local HA version found") + raise DockerAPIError() + else: + _LOGGER.debug("Found HA versions: %s", available_version) + + # Sort version and return latest version + available_version.sort(key=StrictVersion, reverse=True) + return available_version[0] diff --git a/hassio/docker/interface.py b/hassio/docker/interface.py index 2d65bdfa9..1b0593778 100644 --- a/hassio/docker/interface.py +++ b/hassio/docker/interface.py @@ -68,11 +68,13 @@ class DockerInterface(CoreSysAttributes): return self.lock.locked() @process_lock - def install(self, tag: str, image: Optional[str] = None): + def install(self, tag: str, image: Optional[str] = None, latest: bool = False): """Pull docker image.""" - return self.sys_run_in_executor(self._install, tag, image) + return self.sys_run_in_executor(self._install, tag, image, latest) - def _install(self, tag: str, image: Optional[str] = None) -> None: + def _install( + self, tag: str, image: Optional[str] = None, latest: bool = False + ) -> None: """Pull Docker image. Need run inside executor. @@ -80,12 +82,12 @@ class DockerInterface(CoreSysAttributes): image = image or self.image image = image.partition(":")[0] # remove potential tag + _LOGGER.info("Pull image %s tag %s.", image, tag) try: - _LOGGER.info("Pull image %s tag %s.", image, tag) docker_image = self.sys_docker.images.pull(f"{image}:{tag}") - - _LOGGER.info("Tag image %s with version %s as latest", image, tag) - docker_image.tag(image, tag="latest") + if latest: + _LOGGER.info("Tag image %s with version %s as latest", image, tag) + docker_image.tag(image, tag="latest") except docker.errors.APIError as err: _LOGGER.error("Can't install %s:%s -> %s.", image, tag, err) raise DockerAPIError() from None @@ -123,7 +125,6 @@ class DockerInterface(CoreSysAttributes): """ try: docker_container = self.sys_docker.containers.get(self.name) - docker_image = self.sys_docker.images.get(self.image) except docker.errors.DockerException: return False @@ -131,28 +132,24 @@ class DockerInterface(CoreSysAttributes): if docker_container.status != "running": return False - # we run on an old image, stop and start it - if docker_container.image.id != docker_image.id: - return False - return True @process_lock - def attach(self): + def attach(self, tag: str): """Attach to running Docker container.""" - return self.sys_run_in_executor(self._attach) + return self.sys_run_in_executor(self._attach, tag) - def _attach(self) -> None: + def _attach(self, tag: str) -> None: """Attach to running docker container. Need run inside executor. """ - try: - if self.image: - self._meta = self.sys_docker.images.get(self.image).attrs + with suppress(docker.errors.DockerException): self._meta = self.sys_docker.containers.get(self.name).attrs - except docker.errors.DockerException: - pass + + with suppress(docker.errors.DockerException): + if not self._meta and self.image: + self._meta = self.sys_docker.images.get(f"{self.image}:{tag}").attrs # Successfull? if not self._meta: @@ -250,11 +247,15 @@ class DockerInterface(CoreSysAttributes): self._meta = None @process_lock - def update(self, tag: str, image: Optional[str] = None) -> Awaitable[None]: + def update( + self, tag: str, image: Optional[str] = None, latest: bool = False + ) -> Awaitable[None]: """Update a Docker image.""" return self.sys_run_in_executor(self._update, tag, image) - def _update(self, tag: str, image: Optional[str] = None) -> None: + def _update( + self, tag: str, image: Optional[str] = None, latest: bool = False + ) -> None: """Update a docker image. Need run inside executor. @@ -266,14 +267,11 @@ class DockerInterface(CoreSysAttributes): ) # Update docker image - self._install(tag, image) + self._install(tag, image, latest) # Stop container & cleanup with suppress(DockerAPIError): - try: - self._stop() - finally: - self._cleanup() + self._stop() def logs(self) -> Awaitable[bytes]: """Return Docker logs of container. @@ -308,13 +306,13 @@ class DockerInterface(CoreSysAttributes): Need run inside executor. """ try: - latest = self.sys_docker.images.get(self.image) + origin = self.sys_docker.images.get(f"{self.image}:{self.version}") except docker.errors.DockerException: _LOGGER.warning("Can't find %s for cleanup", self.image) raise DockerAPIError() from None for image in self.sys_docker.images.list(name=self.image): - if latest.id == image.id: + if origin.id == image.id: continue with suppress(docker.errors.DockerException): diff --git a/hassio/docker/supervisor.py b/hassio/docker/supervisor.py index 2d239ca71..4fbe92d86 100644 --- a/hassio/docker/supervisor.py +++ b/hassio/docker/supervisor.py @@ -25,7 +25,7 @@ class DockerSupervisor(DockerInterface, CoreSysAttributes): """Return IP address of this container.""" return self.sys_docker.network.supervisor - def _attach(self) -> None: + def _attach(self, tag: str) -> None: """Attach to running docker container. Need run inside executor. diff --git a/hassio/hassos.py b/hassio/hassos.py index a6f8e3ec4..2e0617e61 100644 --- a/hassio/hassos.py +++ b/hassio/hassos.py @@ -130,7 +130,7 @@ class HassOS(CoreSysAttributes): _LOGGER.info("Detect HassOS %s on host system", self.version) with suppress(DockerAPIError): - await self.instance.attach() + await self.instance.attach(tag="latest") def config_sync(self) -> Awaitable[None]: """Trigger a host config reload from usb. @@ -187,7 +187,11 @@ class HassOS(CoreSysAttributes): return try: - await self.instance.update(version) + await self.instance.update(version, latest=True) + + # Cleanup + with suppress(DockerAPIError): + await self.instance.cleanup() except DockerAPIError: _LOGGER.error("HassOS CLI update fails") raise HassOSUpdateError() from None diff --git a/hassio/homeassistant.py b/hassio/homeassistant.py index 0e831c907..253896582 100644 --- a/hassio/homeassistant.py +++ b/hassio/homeassistant.py @@ -26,6 +26,7 @@ from .const import ( ATTR_REFRESH_TOKEN, ATTR_SSL, ATTR_UUID, + ATTR_VERSION, ATTR_WAIT_BOOT, ATTR_WATCHDOG, FILE_HASSIO_HOMEASSISTANT, @@ -41,7 +42,7 @@ from .exceptions import ( HomeAssistantError, HomeAssistantUpdateError, ) -from .utils import convert_to_ascii, process_lock, check_port +from .utils import check_port, convert_to_ascii, process_lock from .utils.json import JsonConfig from .validate import SCHEMA_HASS_CONFIG @@ -76,7 +77,15 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): async def load(self) -> None: """Prepare Home Assistant object.""" with suppress(DockerAPIError): - await self.instance.attach() + # Evaluate Version if we lost this information + if not self.version: + if await self.instance.is_running(): + self.version = self.instance.version + else: + self.version = await self.instance.get_latest_version() + self.save_data() + + await self.instance.attach(tag=self.version) return _LOGGER.info("No Home Assistant Docker image %s found.", self.image) @@ -159,11 +168,6 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): """Set time to wait for Home Assistant startup.""" self._data[ATTR_WAIT_BOOT] = value - @property - def version(self) -> str: - """Return version of running Home Assistant.""" - return self.instance.version - @property def latest_version(self) -> str: """Return last available version of Home Assistant.""" @@ -199,6 +203,16 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): """Return True if a custom image is used.""" return all(attr in self._data for attr in (ATTR_IMAGE, ATTR_LAST_VERSION)) + @property + def version(self) -> Optional[str]: + """Return version of local version.""" + return self._data.get(ATTR_VERSION) + + @version.setter + def version(self, value: str) -> None: + """Set installed version.""" + self._data[ATTR_VERSION] = value + @property def boot(self) -> bool: """Return True if Home Assistant boot is enabled.""" @@ -234,11 +248,16 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): """Install a landing page.""" _LOGGER.info("Setup HomeAssistant landingpage") while True: - with suppress(DockerAPIError): + try: await self.instance.install("landingpage") - return - _LOGGER.warning("Fails install landingpage, retry after 30sec") - await asyncio.sleep(30) + except DockerAPIError: + _LOGGER.warning("Fails install landingpage, retry after 30sec") + await asyncio.sleep(30) + else: + break + + self.version = self.instance.version + self.save_data() @process_lock async def install(self) -> None: @@ -257,21 +276,23 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): _LOGGER.warning("Error on install Home Assistant. Retry in 30sec") await asyncio.sleep(30) - # finishing _LOGGER.info("Home Assistant docker now installed") + self.version = self.instance.version + self.save_data() + + # finishing try: - if not self.boot: - return _LOGGER.info("Start Home Assistant") await self._start() except HomeAssistantError: _LOGGER.error("Can't start Home Assistant!") - finally: - with suppress(DockerAPIError): - await self.instance.cleanup() + + # Cleanup + with suppress(DockerAPIError): + await self.instance.cleanup() @process_lock - async def update(self, version=None) -> None: + async def update(self, version: Optional[str] = None) -> None: """Update HomeAssistant version.""" version = version or self.latest_version rollback = self.version if not self.error_state else None @@ -283,7 +304,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): return # process an update - async def _update(to_version): + async def _update(to_version: str) -> None: """Run Home Assistant update.""" _LOGGER.info("Update Home Assistant to version %s", to_version) try: @@ -291,10 +312,16 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): except DockerAPIError: _LOGGER.warning("Update Home Assistant image fails") raise HomeAssistantUpdateError() from None + else: + self.version = self.instance.version if running: await self._start() + _LOGGER.info("Successful run Home Assistant %s", to_version) + self.save_data() + with suppress(DockerAPIError): + await self.instance.cleanup() # Update Home Assistant with suppress(HomeAssistantError): diff --git a/hassio/misc/dns.py b/hassio/misc/dns.py index d87fa0b55..564db926c 100644 --- a/hassio/misc/dns.py +++ b/hassio/misc/dns.py @@ -24,7 +24,7 @@ class DNSForward: *shlex.split(COMMAND), stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL + stderr=asyncio.subprocess.DEVNULL, ) except OSError as err: _LOGGER.error("Can't start DNS forwarding: %s", err) diff --git a/hassio/supervisor.py b/hassio/supervisor.py index dbb15fd30..fb63bcc5b 100644 --- a/hassio/supervisor.py +++ b/hassio/supervisor.py @@ -34,7 +34,7 @@ class Supervisor(CoreSysAttributes): async def load(self) -> None: """Prepare Home Assistant object.""" try: - await self.instance.attach() + await self.instance.attach(tag="latest") except DockerAPIError: _LOGGER.fatal("Can't setup Supervisor Docker container!") @@ -109,7 +109,7 @@ class Supervisor(CoreSysAttributes): _LOGGER.info("Update Supervisor to version %s", version) try: - await self.instance.install(version) + await self.instance.install(version, latest=True) except DockerAPIError: _LOGGER.error("Update of Hass.io fails!") raise SupervisorUpdateError() from None diff --git a/hassio/validate.py b/hassio/validate.py index 4df3b06a7..037acdcf3 100644 --- a/hassio/validate.py +++ b/hassio/validate.py @@ -27,6 +27,7 @@ from .const import ( ATTR_SSL, ATTR_TIMEZONE, ATTR_UUID, + ATTR_VERSION, ATTR_WAIT_BOOT, ATTR_WATCHDOG, CHANNEL_BETA, @@ -82,6 +83,7 @@ DOCKER_PORTS_DESCRIPTION = vol.Schema( SCHEMA_HASS_CONFIG = vol.Schema( { vol.Optional(ATTR_UUID, default=lambda: uuid.uuid4().hex): UUID_MATCH, + vol.Optional(ATTR_VERSION): vol.Maybe(vol.Coerce(str)), vol.Optional(ATTR_ACCESS_TOKEN): TOKEN, vol.Optional(ATTR_BOOT, default=True): vol.Boolean(), vol.Inclusive(ATTR_IMAGE, "custom_hass"): DOCKER_IMAGE, From 2fc5e3b7d994d24405405c0db4f644c4abfd9ec7 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 7 Aug 2019 17:26:32 +0200 Subject: [PATCH 8/9] Repair / fixup docker overlayfs issues (#1170) * Add a repair modus * Add repair to add-ons * repair to cli * Add API call * fix sync call * Clean all images * Fix repair * Fix supervisor * Add new function to core * fix tagging * better style * use retag * new retag function * Fix lint * Fix import export --- API.md | 4 ++++ hassio/addons/__init__.py | 35 +++++++++++++++++++++++++++++++++++ hassio/addons/addon.py | 2 +- hassio/api/__init__.py | 1 + hassio/api/supervisor.py | 5 +++++ hassio/core.py | 15 +++++++++++++++ hassio/docker/__init__.py | 31 +++++++++++++++++++++++++++++++ hassio/docker/addon.py | 13 ++++++------- hassio/docker/interface.py | 11 ++++------- hassio/docker/supervisor.py | 19 +++++++++++++++++++ hassio/hassos.py | 11 +++++++++++ hassio/homeassistant.py | 11 +++++++++++ hassio/supervisor.py | 15 +++++++++++++-- 13 files changed, 156 insertions(+), 17 deletions(-) diff --git a/API.md b/API.md index 48c2cc5f5..6d5e76f38 100644 --- a/API.md +++ b/API.md @@ -112,6 +112,10 @@ Output is the raw docker log. } ``` +- GET `/supervisor/repair` + +Repair overlayfs issue and restore lost images + ### Snapshot - GET `/snapshots` diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index 84e64a28f..e522da543 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -256,3 +256,38 @@ class AddonManager(CoreSysAttributes): _LOGGER.info("Detect new Add-on after restore %s", slug) self.local[slug] = addon + + async def repair(self) -> None: + """Repair local add-ons.""" + needs_repair: List[Addon] = [] + + # Evaluate Add-ons to repair + for addon in self.installed: + if await addon.instance.exists(): + continue + needs_repair.append(addon) + + _LOGGER.info("Found %d add-ons to repair", len(needs_repair)) + if not needs_repair: + return + + for addon in needs_repair: + _LOGGER.info("Start repair for add-on: %s", addon.slug) + + with suppress(DockerAPIError, KeyError): + # Need pull a image again + if not addon.need_build: + await addon.instance.install(addon.version, addon.image) + continue + + # Need local lookup + elif addon.need_build and not addon.is_detached: + store = self.store[addon.slug] + # If this add-on is available for rebuild + if addon.version == store.version: + await addon.instance.install(addon.version, addon.image) + continue + + _LOGGER.error("Can't repair %s", addon.slug) + with suppress(AddonsError): + await self.uninstall(addon.slug) diff --git a/hassio/addons/addon.py b/hassio/addons/addon.py index ec0e8cb7f..d694bd916 100644 --- a/hassio/addons/addon.py +++ b/hassio/addons/addon.py @@ -618,7 +618,7 @@ class Addon(AddonModel): image_file = Path(temp, "image.tar") if image_file.is_file(): with suppress(DockerAPIError): - await self.instance.import_image(image_file, version) + await self.instance.import_image(image_file) else: with suppress(DockerAPIError): await self.instance.install(version, restore_image) diff --git a/hassio/api/__init__.py b/hassio/api/__init__.py index 0d36fd530..aebad8bba 100644 --- a/hassio/api/__init__.py +++ b/hassio/api/__init__.py @@ -130,6 +130,7 @@ class RestAPI(CoreSysAttributes): web.post("/supervisor/update", api_supervisor.update), web.post("/supervisor/reload", api_supervisor.reload), web.post("/supervisor/options", api_supervisor.options), + web.post("/supervisor/repair", api_supervisor.repair), ] ) diff --git a/hassio/api/supervisor.py b/hassio/api/supervisor.py index 46b4cb3fb..fafe40d63 100644 --- a/hassio/api/supervisor.py +++ b/hassio/api/supervisor.py @@ -161,6 +161,11 @@ class APISupervisor(CoreSysAttributes): """Reload add-ons, configuration, etc.""" return asyncio.shield(self.sys_updater.reload()) + @api_process + def repair(self, request: web.Request) -> Awaitable[None]: + """Try to repair the local setup / overlayfs.""" + return asyncio.shield(self.sys_core.repair()) + @api_process_raw(CONTENT_TYPE_BINARY) def logs(self, request: web.Request) -> Awaitable[bytes]: """Return supervisor Docker logs.""" diff --git a/hassio/core.py b/hassio/core.py index ce8e4c0e0..a49b97db8 100644 --- a/hassio/core.py +++ b/hassio/core.py @@ -170,3 +170,18 @@ class HassIO(CoreSysAttributes): """Update last boot time.""" self.sys_config.last_boot = self.sys_hardware.last_boot self.sys_config.save_data() + + async def repair(self): + """Repair system integrity.""" + await self.sys_run_in_executor(self.sys_docker.repair) + + # Restore core functionality + await self.sys_addons.repair() + await self.sys_homeassistant.repair() + + # Fix HassOS specific + if self.sys_hassos.available: + await self.sys_hassos.repair_cli() + + # Tag version for latest + await self.sys_supervisor.repair() diff --git a/hassio/docker/__init__.py b/hassio/docker/__init__.py index 13d893291..0b86ef359 100644 --- a/hassio/docker/__init__.py +++ b/hassio/docker/__init__.py @@ -139,3 +139,34 @@ class DockerAPI: container.remove(force=True) return CommandReturn(result.get("StatusCode"), output) + + def repair(self) -> None: + """Repair local docker overlayfs2 issues.""" + + _LOGGER.info("Prune stale containers") + try: + output = self.docker.api.prune_containers() + _LOGGER.debug("Containers prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for containers prune: %s", err) + + _LOGGER.info("Prune stale images") + try: + output = self.docker.api.prune_images(filters={"dangling": False}) + _LOGGER.debug("Images prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for images prune: %s", err) + + _LOGGER.info("Prune stale builds") + try: + output = self.docker.api.prune_builds() + _LOGGER.debug("Builds prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for builds prune: %s", err) + + _LOGGER.info("Prune stale volumes") + try: + output = self.docker.api.prune_builds() + _LOGGER.debug("Volumes prune: %s", output) + except docker.errors.APIError as err: + _LOGGER.warning("Error for volumes prune: %s", err) diff --git a/hassio/docker/addon.py b/hassio/docker/addon.py index df7b4c799..51d1dda85 100644 --- a/hassio/docker/addon.py +++ b/hassio/docker/addon.py @@ -397,7 +397,7 @@ class DockerAddon(DockerInterface): Need run inside executor. """ try: - image = self.sys_docker.api.get_image(self.image) + image = self.sys_docker.api.get_image(f"{self.image}:{self.version}") except docker.errors.DockerException as err: _LOGGER.error("Can't fetch image %s: %s", self.image, err) raise DockerAPIError() from None @@ -414,11 +414,11 @@ class DockerAddon(DockerInterface): _LOGGER.info("Export image %s done", self.image) @process_lock - def import_image(self, tar_file: Path, tag: str) -> Awaitable[None]: + def import_image(self, tar_file: Path) -> Awaitable[None]: """Import a tar file as image.""" - return self.sys_run_in_executor(self._import_image, tar_file, tag) + return self.sys_run_in_executor(self._import_image, tar_file) - def _import_image(self, tar_file: Path, tag: str) -> None: + def _import_image(self, tar_file: Path) -> None: """Import a tar file as image. Need run inside executor. @@ -427,14 +427,13 @@ class DockerAddon(DockerInterface): with tar_file.open("rb") as read_tar: self.sys_docker.api.load_image(read_tar, quiet=True) - docker_image = self.sys_docker.images.get(self.image) - docker_image.tag(self.image, tag=tag) + docker_image = self.sys_docker.images.get(f"{self.image}:{self.version}") except (docker.errors.DockerException, OSError) as err: _LOGGER.error("Can't import image %s: %s", self.image, err) raise DockerAPIError() from None - _LOGGER.info("Import image %s and tag %s", tar_file, tag) self._meta = docker_image.attrs + _LOGGER.info("Import image %s and version %s", tar_file, self.version) with suppress(DockerAPIError): self._cleanup() diff --git a/hassio/docker/interface.py b/hassio/docker/interface.py index 1b0593778..a59cbfeab 100644 --- a/hassio/docker/interface.py +++ b/hassio/docker/interface.py @@ -103,13 +103,10 @@ class DockerInterface(CoreSysAttributes): Need run inside executor. """ - try: - docker_image = self.sys_docker.images.get(self.image) - assert f"{self.image}:{self.version}" in docker_image.tags - except (docker.errors.DockerException, AssertionError): - return False - - return True + with suppress(docker.errors.DockerException): + self.sys_docker.images.get(f"{self.image}:{self.version}") + return True + return False def is_running(self) -> Awaitable[bool]: """Return True if Docker is running. diff --git a/hassio/docker/supervisor.py b/hassio/docker/supervisor.py index 4fbe92d86..6104d3a76 100644 --- a/hassio/docker/supervisor.py +++ b/hassio/docker/supervisor.py @@ -2,6 +2,7 @@ from ipaddress import IPv4Address import logging import os +from typing import Awaitable import docker @@ -49,3 +50,21 @@ class DockerSupervisor(DockerInterface, CoreSysAttributes): self.sys_docker.network.attach_container( docker_container, alias=["hassio"], ipv4=self.sys_docker.network.supervisor ) + + def retag(self) -> Awaitable[None]: + """Retag latest image to version.""" + return self.sys_run_in_executor(self._retag) + + def _retag(self) -> None: + """Retag latest image to version. + + Need run inside executor. + """ + try: + docker_container = self.sys_docker.containers.get(self.name) + + docker_container.image.tag(self.image, tag=self.version) + docker_container.image.tag(self.image, tag="latest") + except docker.errors.DockerException as err: + _LOGGER.error("Can't retag supervisor version: %s", err) + raise DockerAPIError() from None diff --git a/hassio/hassos.py b/hassio/hassos.py index 2e0617e61..0237ce942 100644 --- a/hassio/hassos.py +++ b/hassio/hassos.py @@ -195,3 +195,14 @@ class HassOS(CoreSysAttributes): except DockerAPIError: _LOGGER.error("HassOS CLI update fails") raise HassOSUpdateError() from None + + async def repair_cli(self) -> None: + """Repair CLI container.""" + if await self.instance.exists(): + return + + _LOGGER.info("Repair HassOS CLI %s", self.version_cli) + try: + await self.instance.install(self.version_cli, latest=True) + except DockerAPIError: + _LOGGER.error("Repairing of HassOS CLI fails") diff --git a/hassio/homeassistant.py b/hassio/homeassistant.py index 253896582..c4702b3a1 100644 --- a/hassio/homeassistant.py +++ b/hassio/homeassistant.py @@ -597,3 +597,14 @@ class HomeAssistant(JsonConfig, CoreSysAttributes): self._error_state = True raise HomeAssistantError() + + async def repair(self): + """Repair local Home Assistant data.""" + if await self.instance.exists(): + return + + _LOGGER.info("Repair Home Assistant %s", self.version) + try: + await self.instance.install(self.version) + except DockerAPIError: + _LOGGER.error("Repairing of Home Assistant fails") diff --git a/hassio/supervisor.py b/hassio/supervisor.py index fb63bcc5b..644dc30f8 100644 --- a/hassio/supervisor.py +++ b/hassio/supervisor.py @@ -9,7 +9,7 @@ from typing import Awaitable, Optional import aiohttp -from .const import URL_HASSIO_APPARMOR +from .const import URL_HASSIO_APPARMOR, HASSIO_VERSION from .coresys import CoreSys, CoreSysAttributes from .docker.stats import DockerStats from .docker.supervisor import DockerSupervisor @@ -54,7 +54,7 @@ class Supervisor(CoreSysAttributes): @property def version(self) -> str: """Return version of running Home Assistant.""" - return self.instance.version + return HASSIO_VERSION @property def latest_version(self) -> str: @@ -136,3 +136,14 @@ class Supervisor(CoreSysAttributes): return await self.instance.stats() except DockerAPIError: raise SupervisorError() from None + + async def repair(self): + """Repair local Supervisor data.""" + if await self.instance.exists(): + return + + _LOGGER.info("Repair Supervisor %s", self.version) + try: + await self.instance.retag() + except DockerAPIError: + _LOGGER.error("Repairing of Supervisor fails") From e5f0d80d9689d22fcd3b2b81e3dafbb27c562121 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Wed, 7 Aug 2019 18:03:56 +0200 Subject: [PATCH 9/9] Start API server before he beform a self update (#1189) --- hassio/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hassio/core.py b/hassio/core.py index a49b97db8..335dd9d80 100644 --- a/hassio/core.py +++ b/hassio/core.py @@ -74,6 +74,8 @@ class HassIO(CoreSysAttributes): async def start(self): """Start Hass.io orchestration.""" + await self.sys_api.start() + # on release channel, try update itself if self.sys_supervisor.need_update: try: @@ -87,9 +89,6 @@ class HassIO(CoreSysAttributes): "future version of Home Assistant!" ) - # start api - await self.sys_api.start() - # start addon mark as initialize await self.sys_addons.boot(STARTUP_INITIALIZE)