Don't relay on latest with HA/Addons (#1175)

* Don't relay on latest with HA/Addons

* Fix latest on install

* Revert some options

* Fix attach

* migrate to new version handling

* Fix thread

* Fix is running

* Allow wait

* debug code

* Fix debug value

* Fix list

* Fix regex

* Some better log output

* Fix logic

* Improve cleanup handling

* Fix bug

* Cleanup old code

* Improve version handling

* Fix the way to attach

.devcontainer/devcontainer.json

@@ -4,7 +4,8 @@
 	"context": "..",
 	"dockerFile": "Dockerfile",
 	"runArgs": [
-		"-e", "GIT_EDTIOR='code --wait'"
+		"-e",
+		"GIT_EDTIOR='code --wait'"
 	],
 	"extensions": [
 		"ms-python.python"
@@ -14,9 +15,13 @@
 		"python.linting.pylintEnabled": true,
 		"python.linting.enabled": true,
 		"python.formatting.provider": "black",
+		"python.formatting.blackArgs": [
+			"--target--version",
+			"py37"
+		],
 		"editor.formatOnPaste": false,
 		"editor.formatOnSave": true,
 		"editor.formatOnType": true,
 		"files.trimTrailingWhitespace": true
 	}
-}
+}

azure-pipelines.yml

@@ -52,7 +52,7 @@ stages:
         versionSpec: '3.7'
     - script: pip install black
       displayName: 'Install black'
-    - script: black --check hassio tests
+    - script: black --target-version py37 --check hassio tests
       displayName: 'Run Black'
   - job: 'JQ'
     pool:

hassio/__main__.py

@@ -38,9 +38,10 @@ if __name__ == "__main__":
 
     _LOGGER.info("Initialize Hass.io setup")
     coresys = loop.run_until_complete(bootstrap.initialize_coresys())
+    loop.run_until_complete(coresys.core.connect())
 
-    bootstrap.migrate_system_env(coresys)
     bootstrap.supervisor_debugger(coresys)
+    bootstrap.migrate_system_env(coresys)
 
     _LOGGER.info("Setup HassIO")
     loop.run_until_complete(coresys.core.setup())

hassio/addons/__init__.py

@@ -130,6 +130,7 @@ class AddonManager(CoreSysAttributes):
             raise AddonsError() from None
         else:
             self.local[slug] = addon
+            _LOGGER.info("Add-on '%s' successfully installed", slug)
 
     async def uninstall(self, slug: str) -> None:
         """Remove an add-on."""
@@ -159,6 +160,8 @@ class AddonManager(CoreSysAttributes):
         self.data.uninstall(addon)
         self.local.pop(slug)
 
+        _LOGGER.info("Add-on '%s' successfully removed", slug)
+
     async def update(self, slug: str) -> None:
         """Update add-on."""
         if slug not in self.local:
@@ -184,9 +187,15 @@ class AddonManager(CoreSysAttributes):
         last_state = await addon.state()
         try:
             await addon.instance.update(store.version, store.image)
+
+            # Cleanup
+            with suppress(DockerAPIError):
+                await addon.instance.cleanup()
         except DockerAPIError:
             raise AddonsError() from None
-        self.data.update(store)
+        else:
+            self.data.update(store)
+            _LOGGER.info("Add-on '%s' successfully updated", slug)
 
         # Setup/Fix AppArmor profile
         await addon.install_apparmor()
@@ -224,6 +233,7 @@ class AddonManager(CoreSysAttributes):
             raise AddonsError() from None
         else:
             self.data.update(store)
+            _LOGGER.info("Add-on '%s' successfully rebuilded", slug)
 
         # restore state
         if last_state == STATE_STARTED:

hassio/addons/addon.py

@@ -75,7 +75,7 @@ class Addon(AddonModel):
     async def load(self) -> None:
         """Async initialize of object."""
         with suppress(DockerAPIError):
-            await self.instance.attach()
+            await self.instance.attach(tag=self.version)
 
     @property
     def ip_address(self) -> IPv4Address:

hassio/bootstrap.py

@@ -218,7 +218,7 @@ def reg_signal(loop):
 
 def supervisor_debugger(coresys: CoreSys) -> None:
     """Setup debugger if needed."""
-    if not coresys.config.debug or not coresys.dev:
+    if not coresys.config.debug:
         return
     import ptvsd
 
@@ -226,4 +226,5 @@ def supervisor_debugger(coresys: CoreSys) -> None:
 
     ptvsd.enable_attach(address=("0.0.0.0", 33333), redirect_output=True)
     if coresys.config.debug_block:
+        _LOGGER.info("Wait until debugger is attached")
         ptvsd.wait_for_attach()

hassio/core.py

@@ -24,11 +24,12 @@ class HassIO(CoreSysAttributes):
         """Initialize Hass.io object."""
         self.coresys = coresys
 
-    async def setup(self):
+    async def connect(self):
-        """Setup HassIO orchestration."""
+        """Connect Supervisor container."""
-        # Load Supervisor
         await self.sys_supervisor.load()
 
+    async def setup(self):
+        """Setup HassIO orchestration."""
         # Load DBus
         await self.sys_dbus.load()
 

hassio/docker/__init__.py

@@ -50,15 +50,15 @@ class DockerAPI:
         return self.docker.api
 
     def run(
-        self, image: str, **kwargs: Dict[str, Any]
+        self, image: str, version: str = "latest", **kwargs: Dict[str, Any]
     ) -> docker.models.containers.Container:
         """"Create a Docker container and run it.
 
         Need run inside executor.
         """
-        name = kwargs.get("name", image)
+        name: str = kwargs.get("name", image)
-        network_mode = kwargs.get("network_mode")
+        network_mode: str = kwargs.get("network_mode")
-        hostname = kwargs.get("hostname")
+        hostname: str = kwargs.get("hostname")
 
         # Setup network
         kwargs["dns_search"] = ["."]
@@ -71,7 +71,7 @@ class DockerAPI:
         # Create container
         try:
             container = self.docker.containers.create(
-                image, use_config_proxy=False, **kwargs
+                f"{image}:{version}", use_config_proxy=False, **kwargs
             )
         except docker.errors.DockerException as err:
             _LOGGER.error("Can't create container from %s: %s", name, err)
@@ -102,7 +102,11 @@ class DockerAPI:
         return container
 
     def run_command(
-        self, image: str, command: Optional[str] = None, **kwargs: Dict[str, Any]
+        self,
+        image: str,
+        version: str = "latest",
+        command: Optional[str] = None,
+        **kwargs: Dict[str, Any],
     ) -> CommandReturn:
         """Create a temporary container and run command.
 
@@ -114,11 +118,11 @@ class DockerAPI:
         _LOGGER.info("Run command '%s' on %s", command, image)
         try:
             container = self.docker.containers.run(
-                image,
+                f"{image}:{version}",
                 command=command,
                 network=self.network.name,
                 use_config_proxy=False,
-                **kwargs
+                **kwargs,
             )
 
             # wait until command is done

hassio/docker/addon.py

@@ -327,6 +327,7 @@ class DockerAddon(DockerInterface):
         # Create & Run container
         docker_container = self.sys_docker.run(
             self.image,
+            version=self.addon.version,
             name=self.name,
             hostname=self.hostname,
             detach=True,
@@ -346,10 +347,12 @@ class DockerAddon(DockerInterface):
             tmpfs=self.tmpfs,
         )
 
-        _LOGGER.info("Start Docker add-on %s with version %s", self.image, self.version)
         self._meta = docker_container.attrs
+        _LOGGER.info("Start Docker add-on %s with version %s", self.image, self.version)
 
-    def _install(self, tag: str, image: Optional[str] = None) -> None:
+    def _install(
+        self, tag: str, image: Optional[str] = None, latest: bool = False
+    ) -> None:
         """Pull Docker image or build it.
 
         Need run inside executor.
@@ -357,7 +360,7 @@ class DockerAddon(DockerInterface):
         if self.addon.need_build:
             self._build(tag)
         else:
-            super()._install(tag, image)
+            super()._install(tag, image, latest)
 
     def _build(self, tag: str) -> None:
         """Build a Docker container.
@@ -373,7 +376,6 @@ class DockerAddon(DockerInterface):
             )
 
             _LOGGER.debug("Build %s:%s done: %s", self.image, tag, log)
-            image.tag(self.image, tag="latest")
 
             # Update meta data
             self._meta = image.attrs

hassio/docker/hassos_cli.py

@@ -21,12 +21,12 @@ class DockerHassOSCli(DockerInterface, CoreSysAttributes):
         """Don't need stop."""
         return True
 
-    def _attach(self):
+    def _attach(self, tag: str):
         """Attach to running Docker container.
         Need run inside executor.
         """
         try:
-            image = self.sys_docker.images.get(self.image)
+            image = self.sys_docker.images.get(f"{self.image}:{tag}")
 
         except docker.errors.DockerException:
             _LOGGER.warning("Can't find a HassOS CLI %s", self.image)

hassio/docker/homeassistant.py

@@ -1,8 +1,10 @@
 """Init file for Hass.io Docker object."""
+from distutils.version import StrictVersion
 from contextlib import suppress
 from ipaddress import IPv4Address
 import logging
-from typing import Awaitable
+import re
+from typing import Awaitable, List, Optional
 
 import docker
 
@@ -13,30 +15,31 @@ from .interface import CommandReturn, DockerInterface
 _LOGGER = logging.getLogger(__name__)
 
 HASS_DOCKER_NAME = "homeassistant"
+RE_VERSION = re.compile(r"(?P<version>\d+\.\d+\.\d+(?:b\d+|d\d+)?)")
 
 
 class DockerHomeAssistant(DockerInterface):
     """Docker Hass.io wrapper for Home Assistant."""
 
     @property
-    def machine(self):
+    def machine(self) -> Optional[str]:
         """Return machine of Home Assistant Docker image."""
         if self._meta and LABEL_MACHINE in self._meta["Config"]["Labels"]:
             return self._meta["Config"]["Labels"][LABEL_MACHINE]
         return None
 
     @property
-    def image(self):
+    def image(self) -> str:
         """Return name of Docker image."""
         return self.sys_homeassistant.image
 
     @property
-    def name(self):
+    def name(self) -> str:
         """Return name of Docker container."""
         return HASS_DOCKER_NAME
 
     @property
-    def timeout(self) -> str:
+    def timeout(self) -> int:
         """Return timeout for Docker actions."""
         return 60
 
@@ -60,6 +63,7 @@ class DockerHomeAssistant(DockerInterface):
         # Create & Run container
         docker_container = self.sys_docker.run(
             self.image,
+            version=self.sys_homeassistant.version,
             name=self.name,
             hostname=self.name,
             detach=True,
@@ -84,8 +88,8 @@ class DockerHomeAssistant(DockerInterface):
             },
         )
 
-        _LOGGER.info("Start homeassistant %s with version %s", self.image, self.version)
         self._meta = docker_container.attrs
+        _LOGGER.info("Start homeassistant %s with version %s", self.image, self.version)
 
     def _execute_command(self, command: str) -> CommandReturn:
         """Create a temporary container and run command.
@@ -94,7 +98,8 @@ class DockerHomeAssistant(DockerInterface):
         """
         return self.sys_docker.run_command(
             self.image,
-            command,
+            version=self.sys_homeassistant.version,
+            command=command,
             privileged=True,
             init=True,
             detach=True,
@@ -134,3 +139,33 @@ class DockerHomeAssistant(DockerInterface):
             return False
 
         return True
+
+    def get_latest_version(self) -> Awaitable[str]:
+        """Return latest version of local Home Asssistant image."""
+        return self.sys_run_in_executor(self._get_latest_version)
+
+    def _get_latest_version(self) -> str:
+        """Return latest version of local Home Asssistant image.
+
+        Need run inside executor.
+        """
+        available_version: List[str] = []
+        try:
+            for image in self.sys_docker.images.list(self.image):
+                for tag in image.tags:
+                    match = RE_VERSION.search(tag)
+                    if not match:
+                        continue
+                    available_version.append(match.group("version"))
+
+            assert available_version
+
+        except (docker.errors.DockerException, AssertionError):
+            _LOGGER.warning("No local HA version found")
+            raise DockerAPIError()
+        else:
+            _LOGGER.debug("Found HA versions: %s", available_version)
+
+        # Sort version and return latest version
+        available_version.sort(key=StrictVersion, reverse=True)
+        return available_version[0]

hassio/docker/interface.py

@@ -68,11 +68,13 @@ class DockerInterface(CoreSysAttributes):
         return self.lock.locked()
 
     @process_lock
-    def install(self, tag: str, image: Optional[str] = None):
+    def install(self, tag: str, image: Optional[str] = None, latest: bool = False):
         """Pull docker image."""
-        return self.sys_run_in_executor(self._install, tag, image)
+        return self.sys_run_in_executor(self._install, tag, image, latest)
 
-    def _install(self, tag: str, image: Optional[str] = None) -> None:
+    def _install(
+        self, tag: str, image: Optional[str] = None, latest: bool = False
+    ) -> None:
         """Pull Docker image.
 
         Need run inside executor.
@@ -80,12 +82,12 @@ class DockerInterface(CoreSysAttributes):
         image = image or self.image
         image = image.partition(":")[0]  # remove potential tag
 
+        _LOGGER.info("Pull image %s tag %s.", image, tag)
         try:
-            _LOGGER.info("Pull image %s tag %s.", image, tag)
             docker_image = self.sys_docker.images.pull(f"{image}:{tag}")
-
+            if latest:
-            _LOGGER.info("Tag image %s with version %s as latest", image, tag)
+                _LOGGER.info("Tag image %s with version %s as latest", image, tag)
-            docker_image.tag(image, tag="latest")
+                docker_image.tag(image, tag="latest")
         except docker.errors.APIError as err:
             _LOGGER.error("Can't install %s:%s -> %s.", image, tag, err)
             raise DockerAPIError() from None
@@ -123,7 +125,6 @@ class DockerInterface(CoreSysAttributes):
         """
         try:
             docker_container = self.sys_docker.containers.get(self.name)
-            docker_image = self.sys_docker.images.get(self.image)
         except docker.errors.DockerException:
             return False
 
@@ -131,28 +132,24 @@ class DockerInterface(CoreSysAttributes):
         if docker_container.status != "running":
             return False
 
-        # we run on an old image, stop and start it
-        if docker_container.image.id != docker_image.id:
-            return False
-
         return True
 
     @process_lock
-    def attach(self):
+    def attach(self, tag: str):
         """Attach to running Docker container."""
-        return self.sys_run_in_executor(self._attach)
+        return self.sys_run_in_executor(self._attach, tag)
 
-    def _attach(self) -> None:
+    def _attach(self, tag: str) -> None:
         """Attach to running docker container.
 
         Need run inside executor.
         """
-        try:
+        with suppress(docker.errors.DockerException):
-            if self.image:
-                self._meta = self.sys_docker.images.get(self.image).attrs
             self._meta = self.sys_docker.containers.get(self.name).attrs
-        except docker.errors.DockerException:
+
-            pass
+        with suppress(docker.errors.DockerException):
+            if not self._meta and self.image:
+                self._meta = self.sys_docker.images.get(f"{self.image}:{tag}").attrs
 
         # Successfull?
         if not self._meta:
@@ -250,11 +247,15 @@ class DockerInterface(CoreSysAttributes):
         self._meta = None
 
     @process_lock
-    def update(self, tag: str, image: Optional[str] = None) -> Awaitable[None]:
+    def update(
+        self, tag: str, image: Optional[str] = None, latest: bool = False
+    ) -> Awaitable[None]:
         """Update a Docker image."""
         return self.sys_run_in_executor(self._update, tag, image)
 
-    def _update(self, tag: str, image: Optional[str] = None) -> None:
+    def _update(
+        self, tag: str, image: Optional[str] = None, latest: bool = False
+    ) -> None:
         """Update a docker image.
 
         Need run inside executor.
@@ -266,14 +267,11 @@ class DockerInterface(CoreSysAttributes):
         )
 
         # Update docker image
-        self._install(tag, image)
+        self._install(tag, image, latest)
 
         # Stop container & cleanup
         with suppress(DockerAPIError):
-            try:
+            self._stop()
-                self._stop()
-            finally:
-                self._cleanup()
 
     def logs(self) -> Awaitable[bytes]:
         """Return Docker logs of container.
@@ -308,13 +306,13 @@ class DockerInterface(CoreSysAttributes):
         Need run inside executor.
         """
         try:
-            latest = self.sys_docker.images.get(self.image)
+            origin = self.sys_docker.images.get(f"{self.image}:{self.version}")
         except docker.errors.DockerException:
             _LOGGER.warning("Can't find %s for cleanup", self.image)
             raise DockerAPIError() from None
 
         for image in self.sys_docker.images.list(name=self.image):
-            if latest.id == image.id:
+            if origin.id == image.id:
                 continue
 
             with suppress(docker.errors.DockerException):

hassio/docker/supervisor.py

@@ -25,7 +25,7 @@ class DockerSupervisor(DockerInterface, CoreSysAttributes):
         """Return IP address of this container."""
         return self.sys_docker.network.supervisor
 
-    def _attach(self) -> None:
+    def _attach(self, tag: str) -> None:
         """Attach to running docker container.
 
         Need run inside executor.

hassio/hassos.py

@@ -130,7 +130,7 @@ class HassOS(CoreSysAttributes):
 
         _LOGGER.info("Detect HassOS %s on host system", self.version)
         with suppress(DockerAPIError):
-            await self.instance.attach()
+            await self.instance.attach(tag="latest")
 
     def config_sync(self) -> Awaitable[None]:
         """Trigger a host config reload from usb.
@@ -187,7 +187,11 @@ class HassOS(CoreSysAttributes):
             return
 
         try:
-            await self.instance.update(version)
+            await self.instance.update(version, latest=True)
+
+            # Cleanup
+            with suppress(DockerAPIError):
+                await self.instance.cleanup()
         except DockerAPIError:
             _LOGGER.error("HassOS CLI update fails")
             raise HassOSUpdateError() from None

hassio/homeassistant.py

@@ -26,6 +26,7 @@ from .const import (
     ATTR_REFRESH_TOKEN,
     ATTR_SSL,
     ATTR_UUID,
+    ATTR_VERSION,
     ATTR_WAIT_BOOT,
     ATTR_WATCHDOG,
     FILE_HASSIO_HOMEASSISTANT,
@@ -41,7 +42,7 @@ from .exceptions import (
     HomeAssistantError,
     HomeAssistantUpdateError,
 )
-from .utils import convert_to_ascii, process_lock, check_port
+from .utils import check_port, convert_to_ascii, process_lock
 from .utils.json import JsonConfig
 from .validate import SCHEMA_HASS_CONFIG
 
@@ -76,7 +77,15 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
     async def load(self) -> None:
         """Prepare Home Assistant object."""
         with suppress(DockerAPIError):
-            await self.instance.attach()
+            # Evaluate Version if we lost this information
+            if not self.version:
+                if await self.instance.is_running():
+                    self.version = self.instance.version
+                else:
+                    self.version = await self.instance.get_latest_version()
+                self.save_data()
+
+            await self.instance.attach(tag=self.version)
             return
 
         _LOGGER.info("No Home Assistant Docker image %s found.", self.image)
@@ -159,11 +168,6 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
         """Set time to wait for Home Assistant startup."""
         self._data[ATTR_WAIT_BOOT] = value
 
-    @property
-    def version(self) -> str:
-        """Return version of running Home Assistant."""
-        return self.instance.version
-
     @property
     def latest_version(self) -> str:
         """Return last available version of Home Assistant."""
@@ -199,6 +203,16 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
         """Return True if a custom image is used."""
         return all(attr in self._data for attr in (ATTR_IMAGE, ATTR_LAST_VERSION))
 
+    @property
+    def version(self) -> Optional[str]:
+        """Return version of local version."""
+        return self._data.get(ATTR_VERSION)
+
+    @version.setter
+    def version(self, value: str) -> None:
+        """Set installed version."""
+        self._data[ATTR_VERSION] = value
+
     @property
     def boot(self) -> bool:
         """Return True if Home Assistant boot is enabled."""
@@ -234,11 +248,16 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
         """Install a landing page."""
         _LOGGER.info("Setup HomeAssistant landingpage")
         while True:
-            with suppress(DockerAPIError):
+            try:
                 await self.instance.install("landingpage")
-                return
+            except DockerAPIError:
-            _LOGGER.warning("Fails install landingpage, retry after 30sec")
+                _LOGGER.warning("Fails install landingpage, retry after 30sec")
-            await asyncio.sleep(30)
+                await asyncio.sleep(30)
+            else:
+                break
+
+        self.version = self.instance.version
+        self.save_data()
 
     @process_lock
     async def install(self) -> None:
@@ -257,21 +276,23 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
             _LOGGER.warning("Error on install Home Assistant. Retry in 30sec")
             await asyncio.sleep(30)
 
-        # finishing
         _LOGGER.info("Home Assistant docker now installed")
+        self.version = self.instance.version
+        self.save_data()
+
+        # finishing
         try:
-            if not self.boot:
-                return
             _LOGGER.info("Start Home Assistant")
             await self._start()
         except HomeAssistantError:
             _LOGGER.error("Can't start Home Assistant!")
-        finally:
+
-            with suppress(DockerAPIError):
+        # Cleanup
-                await self.instance.cleanup()
+        with suppress(DockerAPIError):
+            await self.instance.cleanup()
 
     @process_lock
-    async def update(self, version=None) -> None:
+    async def update(self, version: Optional[str] = None) -> None:
         """Update HomeAssistant version."""
         version = version or self.latest_version
         rollback = self.version if not self.error_state else None
@@ -283,7 +304,7 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
             return
 
         # process an update
-        async def _update(to_version):
+        async def _update(to_version: str) -> None:
             """Run Home Assistant update."""
             _LOGGER.info("Update Home Assistant to version %s", to_version)
             try:
@@ -291,10 +312,16 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
             except DockerAPIError:
                 _LOGGER.warning("Update Home Assistant image fails")
                 raise HomeAssistantUpdateError() from None
+            else:
+                self.version = self.instance.version
 
             if running:
                 await self._start()
+
             _LOGGER.info("Successful run Home Assistant %s", to_version)
+            self.save_data()
+            with suppress(DockerAPIError):
+                await self.instance.cleanup()
 
         # Update Home Assistant
         with suppress(HomeAssistantError):

hassio/misc/dns.py

@@ -24,7 +24,7 @@ class DNSForward:
                 *shlex.split(COMMAND),
                 stdin=asyncio.subprocess.DEVNULL,
                 stdout=asyncio.subprocess.DEVNULL,
-                stderr=asyncio.subprocess.DEVNULL
+                stderr=asyncio.subprocess.DEVNULL,
             )
         except OSError as err:
             _LOGGER.error("Can't start DNS forwarding: %s", err)

hassio/supervisor.py

@@ -34,7 +34,7 @@ class Supervisor(CoreSysAttributes):
     async def load(self) -> None:
         """Prepare Home Assistant object."""
         try:
-            await self.instance.attach()
+            await self.instance.attach(tag="latest")
         except DockerAPIError:
             _LOGGER.fatal("Can't setup Supervisor Docker container!")
 
@@ -109,7 +109,7 @@ class Supervisor(CoreSysAttributes):
 
         _LOGGER.info("Update Supervisor to version %s", version)
         try:
-            await self.instance.install(version)
+            await self.instance.install(version, latest=True)
         except DockerAPIError:
             _LOGGER.error("Update of Hass.io fails!")
             raise SupervisorUpdateError() from None

hassio/validate.py

@@ -27,6 +27,7 @@ from .const import (
     ATTR_SSL,
     ATTR_TIMEZONE,
     ATTR_UUID,
+    ATTR_VERSION,
     ATTR_WAIT_BOOT,
     ATTR_WATCHDOG,
     CHANNEL_BETA,
@@ -82,6 +83,7 @@ DOCKER_PORTS_DESCRIPTION = vol.Schema(
 SCHEMA_HASS_CONFIG = vol.Schema(
     {
         vol.Optional(ATTR_UUID, default=lambda: uuid.uuid4().hex): UUID_MATCH,
+        vol.Optional(ATTR_VERSION): vol.Maybe(vol.Coerce(str)),
         vol.Optional(ATTR_ACCESS_TOKEN): TOKEN,
         vol.Optional(ATTR_BOOT, default=True): vol.Boolean(),
         vol.Inclusive(ATTR_IMAGE, "custom_hass"): DOCKER_IMAGE,