From d142ea5d2396346cb5b89080b3ce4d0e035859b3 Mon Sep 17 00:00:00 2001 From: pvizeli Date: Thu, 27 Apr 2017 17:06:30 +0200 Subject: [PATCH 01/13] Allow custome repository / improve config validate --- API.md | 3 ++ hassio/addons/__init__.py | 54 +++++++++++++++++++++++++++++--- hassio/addons/data.py | 54 ++++++++++++++++++++++++-------- hassio/addons/git.py | 60 ++++++++++++++++++++++++++++++------ hassio/addons/validate.py | 13 ++++---- hassio/api/supervisor.py | 17 +++++++++- hassio/config.py | 65 ++++++++++++++++++++++++++++++++++----- hassio/const.py | 9 ++++-- hassio/dock/addon.py | 52 +++++++++++++++++++++---------- 9 files changed, 268 insertions(+), 59 deletions(-) diff --git a/API.md b/API.md index d5e6dcb2c..c2dba0c9c 100644 --- a/API.md +++ b/API.md @@ -40,6 +40,9 @@ On success "dedicated": "bool", "description": "description" } + ], + "addons_repositories": [ + "REPO_URL" ] } ``` diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index bf2431b03..cc68829ab 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -5,7 +5,7 @@ import os import shutil from .data import AddonsData -from .git import AddonsRepo +from .git import AddonsRepoHassIO, AddonsRepoCustom from ..const import STATE_STOPPED, STATE_STARTED from ..dock.addon import DockerAddon @@ -21,32 +21,76 @@ class AddonManager(AddonsData): self.loop = loop self.dock = dock - self.repo = AddonsRepo(config, loop) + self.repositories = [] self.dockers = {} async def prepare(self, arch): """Startup addon management.""" self.arch = arch + # init hassio repository + self.repository.append(AddonsRepoHassIO(self.config, self.loop)) + + # init custom repositories + for url, slug in self.config.addons_repositories.items(): + self.repositories.append( + AddonsRepoCustom(self.config, self.loop, url, slug)) + # load addon repository - if await self.repo.load(): + tasks = [addon.load() for addon in self.repositories] + if tasks: + await asyncio.wait(tasks, loop=self.loop) self.read_addons_repo() + # Update config + self.merge_local_config() + # load installed addons for addon in self.list_installed: self.dockers[addon] = DockerAddon( self.config, self.loop, self.dock, self, addon) await self.dockers[addon].attach() + async def add_custom_repository(self, url): + """Add a new custom repository.""" + if url in self.config.addons_repositories.keys(): + _LOGGER.warning("Repository already exists %s", url) + return False + + repo = AddonsRepoCustom(self.config, self.loop, url) + + if not await repo.load(): + _LOGGER.error("Can't load from repository %s", url) + return False + + self.repositories.append(repo) + return True + + def drop_custom_repository(self, url): + """Remove a custom repository.""" + for repo in self.repositories: + if repo.url == url: + repo = self.repositories.pop(repo, repo) + repo.remove() + return True + + return False + async def reload(self): """Update addons from repo and reload list.""" - if not await self.repo.pull(): + tasks = [addon.pull() for addon in self.repositories] + if not tasks: return + + await asyncio.wait(tasks, loop=self.loop) + + # Update config self.read_addons_repo() + self.merge_local_config() # remove stalled addons for addon in self.list_removed: - _LOGGER.warning("Dedicated addon '%s' found!", addon) + _LOGGER.warning("Dedicated addon '%s' found!", addon) async def auto_boot(self, start_type): """Boot addons with mode auto.""" diff --git a/hassio/addons/data.py b/hassio/addons/data.py index 93d3f303f..d43131f88 100644 --- a/hassio/addons/data.py +++ b/hassio/addons/data.py @@ -1,4 +1,5 @@ """Init file for HassIO addons.""" +import copy import logging import glob @@ -8,15 +9,15 @@ from voluptuous.humanize import humanize_error from .validate import validate_options, SCHEMA_ADDON_CONFIG from ..const import ( FILE_HASSIO_ADDONS, ATTR_NAME, ATTR_VERSION, ATTR_SLUG, ATTR_DESCRIPTON, - ATTR_STARTUP, ATTR_BOOT, ATTR_MAP_SSL, ATTR_MAP_CONFIG, ATTR_OPTIONS, - ATTR_PORTS, BOOT_AUTO, DOCKER_REPO, ATTR_INSTALLED, ATTR_SCHEMA, - ATTR_IMAGE, ATTR_DEDICATED) + ATTR_STARTUP, ATTR_BOOT, ATTR_MAP, ATTR_OPTIONS, ATTR_PORTS, BOOT_AUTO, + DOCKER_REPO, ATTR_INSTALLED, ATTR_SCHEMA, ATTR_IMAGE, ATTR_DEDICATED, + MAP_CONFIG, MAP_SSL, MAP_ADDONS, MAP_BACKUP) from ..config import Config from ..tools import read_json_file, write_json_file _LOGGER = logging.getLogger(__name__) -ADDONS_REPO_PATTERN = "{}/*/config.json" +ADDONS_REPO_PATTERN = "{}/**/config.json" SYSTEM = "system" USER = "user" @@ -52,7 +53,7 @@ class AddonsData(Config): """Read data from addons folder.""" pattern = ADDONS_REPO_PATTERN.format(folder) - for addon in glob.iglob(pattern): + for addon in glob.iglob(pattern, recursive=True): try: addon_config = read_json_file(addon) @@ -66,6 +67,27 @@ class AddonsData(Config): _LOGGER.warning("Can't read %s -> %s", addon, humanize_error(addon_config, ex)) + def merge_local_config(self): + """Update local config if they have update. + + It need to be the same version as the local version is. + """ + have_change = False + + for addon, data in self._system_data.items(): + # dedicated + if addon not in self._current_data: + continue + + current = self._current_data[addon] + if data[ATTR_VERSION] == current[ATTR_VERSION]: + if data != current[addon]: + self._system_data[addon] = copy.deepcopy(current[data]) + have_change = True + + if have_change: + self.save() + @property def list_installed(self): """Return a list of installed addons.""" @@ -132,7 +154,7 @@ class AddonsData(Config): def set_addon_install(self, addon, version): """Set addon as installed.""" - self._system_data[addon] = self._current_data[addon] + self._system_data[addon] = copy.deepcopy(self._current_data[addon]) self._user_data[addon] = { ATTR_OPTIONS: {}, ATTR_VERSION: version, @@ -147,13 +169,13 @@ class AddonsData(Config): def set_addon_update(self, addon, version): """Update version of addon.""" - self._system_data[addon] = self._current_data[addon] + self._system_data[addon] = copy.deepcopy(self._current_data[addon]) self._user_data[addon][ATTR_VERSION] = version self.save() def set_options(self, addon, options): """Store user addon options.""" - self._user_data[addon][ATTR_OPTIONS] = options + self._user_data[addon][ATTR_OPTIONS] = copy.deepcopy(options) self.save() def set_boot(self, addon, boot): @@ -202,13 +224,21 @@ class AddonsData(Config): return addon_data[ATTR_IMAGE] - def need_config(self, addon): + def map_config(self, addon): """Return True if config map is needed.""" - return self._system_data[addon][ATTR_MAP_CONFIG] + return MAP_CONFIG in self._system_data[addon][ATTR_MAP] - def need_ssl(self, addon): + def map_ssl(self, addon): """Return True if ssl map is needed.""" - return self._system_data[addon][ATTR_MAP_SSL] + return MAP_SSL in self._system_data[addon][ATTR_MAP] + + def map_addons(self, addon): + """Return True if addons map is needed.""" + return MAP_ADDONS in self._system_data[addon][ATTR_MAP] + + def map_backup(self, addon): + """Return True if backup map is needed.""" + return MAP_BACKUP in self._system_data[addon][ATTR_MAP] def path_data(self, addon): """Return addon data path inside supervisor.""" diff --git a/hassio/addons/git.py b/hassio/addons/git.py index 5b665e876..8195d42fa 100644 --- a/hassio/addons/git.py +++ b/hassio/addons/git.py @@ -2,6 +2,8 @@ import asyncio import logging import os +import shutil +import tempfile import git @@ -13,26 +15,28 @@ _LOGGER = logging.getLogger(__name__) class AddonsRepo(object): """Manage addons git repo.""" - def __init__(self, config, loop): - """Initialize docker base wrapper.""" + def __init__(self, config, loop, path, url): + """Initialize git base wrapper.""" self.config = config self.loop = loop self.repo = None + self.path = path + self.url = url self._lock = asyncio.Lock(loop=loop) async def load(self): """Init git addon repo.""" - if not os.path.isdir(self.config.path_addons_repo): + if not os.path.isdir(self.path): return await self.clone() async with self._lock: try: _LOGGER.info("Load addons repository") self.repo = await self.loop.run_in_executor( - None, git.Repo, self.config.path_addons_repo) + None, git.Repo, self.path) except (git.InvalidGitRepositoryError, git.NoSuchPathError) as err: - _LOGGER.error("Can't load addons repo: %s.", err) + _LOGGER.error("Can't load %s repo: %s.", self.path, err) return False return True @@ -43,11 +47,10 @@ class AddonsRepo(object): try: _LOGGER.info("Clone addons repository") self.repo = await self.loop.run_in_executor( - None, git.Repo.clone_from, URL_HASSIO_ADDONS, - self.config.path_addons_repo) + None, git.Repo.clone_from, self.url, self.path) except (git.InvalidGitRepositoryError, git.NoSuchPathError) as err: - _LOGGER.error("Can't clone addons repo: %s.", err) + _LOGGER.error("Can't clone %s repo: %s.", self.url, err) return False return True @@ -65,7 +68,46 @@ class AddonsRepo(object): None, self.repo.remotes.origin.pull) except (git.InvalidGitRepositoryError, git.NoSuchPathError) as err: - _LOGGER.error("Can't pull addons repo: %s.", err) + _LOGGER.error("Can't pull %s repo: %s.", self.url, err) return False return True + + +class AddonsRepoHassIO(AddonsRepo) + """HassIO addons repository.""" + + def __init__(self, config, loop): + """Initialize git hassio addon repository.""" + super()__init__( + config, loop, config.path_addons_repo, URL_HASSIO_ADDONS) + + +class AddonsRepoCustom(AddonsRepo) + """Custom addons repository.""" + + def __init__(self, config, loop, url, slug=None): + """Initialize git hassio addon repository.""" + if slug is None: + _LOGGER("Init new custom addon repository %s", url) + with tempfile.TemporaryDirectory(dir=config.path_addons_custom) \ + temp_dir: + slug = temp_dir.name + + config.add_addons_repository(url, slug) + + path = os.path.join(config.path_addons_custom, slug) + super()__init__(config, loop, path, url) + + def remove(self): + """Remove a custom addon.""" + if os.path.isdir(self.path): + _LOGGER.info("Remove custom addon repository %s", self.url) + + def log_err(funct, path, _): + """Log error.""" + _LOGGER.warning("Can't remove %s", path) + + shutil.rmtree(self.path, onerror=log_err) + + self.config.drop_addons_repository(self.url) diff --git a/hassio/addons/validate.py b/hassio/addons/validate.py index c6950e734..a8527c808 100644 --- a/hassio/addons/validate.py +++ b/hassio/addons/validate.py @@ -3,9 +3,9 @@ import voluptuous as vol from ..const import ( ATTR_NAME, ATTR_VERSION, ATTR_SLUG, ATTR_DESCRIPTON, ATTR_STARTUP, - ATTR_BOOT, ATTR_MAP_SSL, ATTR_MAP_CONFIG, ATTR_OPTIONS, - ATTR_PORTS, STARTUP_ONCE, STARTUP_AFTER, STARTUP_BEFORE, BOOT_AUTO, - BOOT_MANUAL, ATTR_SCHEMA, ATTR_IMAGE) + ATTR_BOOT, ATTR_MAP, ATTR_OPTIONS, ATTR_PORTS, STARTUP_ONCE, STARTUP_AFTER, + STARTUP_BEFORE, BOOT_AUTO, BOOT_MANUAL, ATTR_SCHEMA, ATTR_IMAGE, MAP_SSL, + MAP_CONFIG, MAP_ADDONS, MAP_BACKUP) V_STR = 'str' V_INT = 'int' @@ -27,8 +27,9 @@ SCHEMA_ADDON_CONFIG = vol.Schema({ vol.Required(ATTR_BOOT): vol.In([BOOT_AUTO, BOOT_MANUAL]), vol.Optional(ATTR_PORTS): dict, - vol.Optional(ATTR_MAP_CONFIG, default=False): vol.Boolean(), - vol.Optional(ATTR_MAP_SSL, default=False): vol.Boolean(), + vol.Optional(ATTR_MAP, default=[]): [ + vol.In([MAP_CONFIG, MAP_SSL, MAP_ADDONS, MAP_BACKUP]) + ], vol.Required(ATTR_OPTIONS): dict, vol.Required(ATTR_SCHEMA): { vol.Coerce(str): vol.Any(ADDON_ELEMENT, [ @@ -36,7 +37,7 @@ SCHEMA_ADDON_CONFIG = vol.Schema({ ]) }, vol.Optional(ATTR_IMAGE): vol.Match(r"\w*/\w*"), -}) +}, extra=vol.ALLOW_EXTRA) def validate_options(raw_schema): diff --git a/hassio/api/supervisor.py b/hassio/api/supervisor.py index feb83eb16..28e3e03d0 100644 --- a/hassio/api/supervisor.py +++ b/hassio/api/supervisor.py @@ -7,13 +7,14 @@ import voluptuous as vol from .util import api_process, api_process_raw, api_validate from ..const import ( ATTR_ADDONS, ATTR_VERSION, ATTR_LAST_VERSION, ATTR_BETA_CHANNEL, - HASSIO_VERSION) + HASSIO_VERSION, ATTR_ADDONS_REPOSITORIES) _LOGGER = logging.getLogger(__name__) SCHEMA_OPTIONS = vol.Schema({ # pylint: disable=no-value-for-parameter vol.Optional(ATTR_BETA_CHANNEL): vol.Boolean(), + vol.Optional(ATTR_ADDONS_REPOSITORIES): [vol.Url()], }) SCHEMA_VERSION = vol.Schema({ @@ -45,6 +46,8 @@ class APISupervisor(object): ATTR_LAST_VERSION: self.config.last_hassio, ATTR_BETA_CHANNEL: self.config.upstream_beta, ATTR_ADDONS: self.addons.list_api, + ATTR_ADDONS_REPOSITORIES: + list(self.config.addons_repositories.keys()), } @api_process @@ -55,6 +58,18 @@ class APISupervisor(object): if ATTR_BETA_CHANNEL in body: self.config.upstream_beta = body[ATTR_BETA_CHANNEL] + if ATTR_ADDONS_REPOSITORIES in body: + new = set(body[ATTR_ADDONS_REPOSITORIES]) + old = set(self.config.addons_repositories.keys()) + + # add new repositories + for url in set(new - old): + await self.addons.add_custom_repository(url) + + # remove old repositories + for url in set(old - new): + self.addons.drop_custom_repository(url) + return self.config.save() @api_process diff --git a/hassio/config.py b/hassio/config.py index e5f03bea3..4848fe5e0 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -2,6 +2,9 @@ import logging import os +import voluptuous as vol +from voluptuous.humanize import humanize_error + from .const import FILE_HASSIO_CONFIG, HASSIO_SHARE from .tools import ( fetch_current_versions, write_json_file, read_json_file) @@ -19,12 +22,30 @@ HASSIO_CLEANUP = 'hassio_cleanup' ADDONS_REPO = "{}/addons" ADDONS_DATA = "{}/addons_data" ADDONS_CUSTOM = "{}/addons_custom" +ADDONS_CUSTOM_LIST = 'addons_custom_list' + +BACKUP_DATA = "{}/backup" UPSTREAM_BETA = 'upstream_beta' API_ENDPOINT = 'api_endpoint' +SCHEMA_CONFIG = vol.Schema({ + vol.Optional( + HOMEASSISTANT_IMAGE default=os.environ['HOMEASSISTANT_REPOSITORY']): + vol.Coerce(str), + vol.Optional(UPSTREAM_BETA, default=False): vol.Boolean(), + vol.Optional(API_ENDPOINT): vol.Coerce(str), + vol.Optional(HOMEASSISTANT_LAST): vol.Coerce(str), + vol.Optional(HASSIO_LAST): vol.Coerce(str), + vol.Optional(HASSIO_CLEANUP): vol.Coerce(str), + vol.Optional(ADDONS_CUSTOM_LIST, default={}): { + vol.Url(): vol.Coerce(str), + } +}, extra=vol.REMOVE_EXTRA) + + class Config(object): """Hold all config data.""" @@ -57,13 +78,13 @@ class CoreConfig(Config): super().__init__(FILE_HASSIO_CONFIG) - # init data - if not self._data: - self._data.update({ - HOMEASSISTANT_IMAGE: os.environ['HOMEASSISTANT_REPOSITORY'], - UPSTREAM_BETA: False, - }) + # validate data + try: + self._data = SCHEMA_CONFIG(self._data) self.save() + except vol.Invalid as ex: + _LOGGER.warning( + "Invalid config %s", humanize_error(self._data, ex)) async def fetch_update_infos(self): """Read current versions from web.""" @@ -93,7 +114,7 @@ class CoreConfig(Config): @property def upstream_beta(self): """Return True if we run in beta upstream.""" - return self._data.get(UPSTREAM_BETA, False) + return self._data[UPSTREAM_BETA] @upstream_beta.setter def upstream_beta(self, value): @@ -164,6 +185,11 @@ class CoreConfig(Config): """Return path for customs addons.""" return ADDONS_CUSTOM.format(HASSIO_SHARE) + @property + def path_addons_custom_docker(self): + """Return path for customs addons.""" + return ADDONS_CUSTOM.format(self.path_hassio_docker) + @property def path_addons_data(self): """Return root addon data folder.""" @@ -173,3 +199,28 @@ class CoreConfig(Config): def path_addons_data_docker(self): """Return root addon data folder extern for docker.""" return ADDONS_DATA.format(self.path_hassio_docker) + + @property + def path_backup(self): + """Return root backup data folder.""" + return BACKUP_DATA.format(HASSIO_SHARE) + + @property + def path_backup_docker(self): + """Return root backup data folder extern for docker.""" + return BACKUP_DATA.format(self.path_hassio_docker) + + def add_addons_repository(self, repo, slug): + """Add a custom repository to list.""" + self._data[ADDONS_CUSTOM_LIST][repo][slug] + self.save() + + def drop_addons_repository(self, repo): + """Remove a custom repository from list.""" + if self._data[ADDONS_CUSTOM_LIST].pop(repo, None): + self.save() + + @property + def addons_repositories(self): + """Return list of addons custom repositories.""" + return self._data[ADDONS_CUSTOM_LIST] diff --git a/hassio/const.py b/hassio/const.py index a4b586626..cc509649d 100644 --- a/hassio/const.py +++ b/hassio/const.py @@ -45,14 +45,14 @@ ATTR_DESCRIPTON = 'description' ATTR_STARTUP = 'startup' ATTR_BOOT = 'boot' ATTR_PORTS = 'ports' -ATTR_MAP_CONFIG = 'map_config' -ATTR_MAP_SSL = 'map_ssl' +ATTR_MAP = 'map' ATTR_OPTIONS = 'options' ATTR_INSTALLED = 'installed' ATTR_DEDICATED = 'dedicated' ATTR_STATE = 'state' ATTR_SCHEMA = 'schema' ATTR_IMAGE = 'image' +ATTR_ADDONS_REPOSITORIES = 'addons_repositories' STARTUP_BEFORE = 'before' STARTUP_AFTER = 'after' @@ -63,3 +63,8 @@ BOOT_MANUAL = 'manual' STATE_STARTED = 'started' STATE_STOPPED = 'stopped' + +MAP_CONFIG = 'config' +MAP_SSL = 'ssl' +MAP_ADDONS = 'addons' +MAP_BACKUP = 'backup' diff --git a/hassio/dock/addon.py b/hassio/dock/addon.py index 0163faef4..afdb6092a 100644 --- a/hassio/dock/addon.py +++ b/hassio/dock/addon.py @@ -26,6 +26,40 @@ class DockerAddon(DockerBase): """Return name of docker container.""" return "addon_{}".format(self.addon) + @property + def volumes(self): + """Generate volumes for mappings.""" + volumes = { + self.addons_data.path_data_docker(self.addon): { + 'bind': '/data', 'mode': 'rw' + }} + + if self.addons_data.map_config(self.addon): + volumes.update({ + self.config.path_config_docker: { + 'bind': '/config', 'mode': 'rw' + }}) + + if self.addons_data.map_ssl(self.addon): + volumes.update({ + self.config.path_ssl_docker: { + 'bind': '/ssl', 'mode': 'rw' + }}) + + if self.addons_data.map_addons(self.addon): + volumes.update({ + self.config.path_addons_custom_docker: { + 'bind': '/addons', 'mode': 'rw' + }}) + + if self.addons_data.map_backup(self.addon): + volumes.update({ + self.config.path_backup_docker: { + 'bind': '/backup', 'mode': 'rw' + }}) + + return volumes + def _run(self): """Run docker image. @@ -37,22 +71,6 @@ class DockerAddon(DockerBase): # cleanup old container self._stop() - # volumes - volumes = { - self.addons_data.path_data_docker(self.addon): { - 'bind': '/data', 'mode': 'rw' - }} - if self.addons_data.need_config(self.addon): - volumes.update({ - self.config.path_config_docker: { - 'bind': '/config', 'mode': 'rw' - }}) - if self.addons_data.need_ssl(self.addon): - volumes.update({ - self.config.path_ssl_docker: { - 'bind': '/ssl', 'mode': 'rw' - }}) - try: self.container = self.dock.containers.run( self.image, @@ -60,7 +78,7 @@ class DockerAddon(DockerBase): detach=True, network_mode='bridge', ports=self.addons_data.get_ports(self.addon), - volumes=volumes, + volumes=self.volumes, ) self.version = get_version_from_env( From db20ea95d9620452c73e1ba942cb1207c78fb28d Mon Sep 17 00:00:00 2001 From: pvizeli Date: Thu, 27 Apr 2017 17:18:06 +0200 Subject: [PATCH 02/13] Fix lint --- hassio/addons/__init__.py | 2 +- hassio/addons/git.py | 2 +- hassio/config.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index cc68829ab..f3e674da2 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -90,7 +90,7 @@ class AddonManager(AddonsData): # remove stalled addons for addon in self.list_removed: - _LOGGER.warning("Dedicated addon '%s' found!", addon) + _LOGGER.warning("Dedicated addon '%s' found!", addon) async def auto_boot(self, start_type): """Boot addons with mode auto.""" diff --git a/hassio/addons/git.py b/hassio/addons/git.py index 8195d42fa..5088787a5 100644 --- a/hassio/addons/git.py +++ b/hassio/addons/git.py @@ -74,7 +74,7 @@ class AddonsRepo(object): return True -class AddonsRepoHassIO(AddonsRepo) +class AddonsRepoHassIO(AddonsRepo): """HassIO addons repository.""" def __init__(self, config, loop): diff --git a/hassio/config.py b/hassio/config.py index 4848fe5e0..5980530d2 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -33,7 +33,8 @@ API_ENDPOINT = 'api_endpoint' SCHEMA_CONFIG = vol.Schema({ vol.Optional( - HOMEASSISTANT_IMAGE default=os.environ['HOMEASSISTANT_REPOSITORY']): + HOMEASSISTANT_IMAGE, + default=os.environ.get('HOMEASSISTANT_REPOSITORY'): vol.Coerce(str), vol.Optional(UPSTREAM_BETA, default=False): vol.Boolean(), vol.Optional(API_ENDPOINT): vol.Coerce(str), From 906616e224a003644332c53abf90f79542287de4 Mon Sep 17 00:00:00 2001 From: pvizeli Date: Thu, 27 Apr 2017 17:19:11 +0200 Subject: [PATCH 03/13] Update description --- API.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/API.md b/API.md index c2dba0c9c..1077ee721 100644 --- a/API.md +++ b/API.md @@ -58,7 +58,10 @@ Optional: - POST `/supervisor/options` ```json { - "beta_channel": "true|false" + "beta_channel": "true|false", + "addons_repositories": [ + "REPO_URL" + ] } ``` From e0dcce58952c21dbc954cb5014aacbd29e855f16 Mon Sep 17 00:00:00 2001 From: pvizeli Date: Thu, 27 Apr 2017 17:22:48 +0200 Subject: [PATCH 04/13] Fix lint p2 --- hassio/addons/git.py | 4 ++-- hassio/config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hassio/addons/git.py b/hassio/addons/git.py index 5088787a5..e62817ab9 100644 --- a/hassio/addons/git.py +++ b/hassio/addons/git.py @@ -79,11 +79,11 @@ class AddonsRepoHassIO(AddonsRepo): def __init__(self, config, loop): """Initialize git hassio addon repository.""" - super()__init__( + super().__init__( config, loop, config.path_addons_repo, URL_HASSIO_ADDONS) -class AddonsRepoCustom(AddonsRepo) +class AddonsRepoCustom(AddonsRepo): """Custom addons repository.""" def __init__(self, config, loop, url, slug=None): diff --git a/hassio/config.py b/hassio/config.py index 5980530d2..eafc6a494 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -34,7 +34,7 @@ API_ENDPOINT = 'api_endpoint' SCHEMA_CONFIG = vol.Schema({ vol.Optional( HOMEASSISTANT_IMAGE, - default=os.environ.get('HOMEASSISTANT_REPOSITORY'): + default=os.environ.get('HOMEASSISTANT_REPOSITORY')): vol.Coerce(str), vol.Optional(UPSTREAM_BETA, default=False): vol.Boolean(), vol.Optional(API_ENDPOINT): vol.Coerce(str), From c916314704b9d64eb67f09334c447c57dc108f02 Mon Sep 17 00:00:00 2001 From: pvizeli Date: Thu, 27 Apr 2017 17:31:02 +0200 Subject: [PATCH 05/13] Fix lint p3 --- hassio/addons/__init__.py | 2 +- hassio/addons/git.py | 2 +- hassio/config.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index f3e674da2..f432ac96c 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -29,7 +29,7 @@ class AddonManager(AddonsData): self.arch = arch # init hassio repository - self.repository.append(AddonsRepoHassIO(self.config, self.loop)) + self.repositories.append(AddonsRepoHassIO(self.config, self.loop)) # init custom repositories for url, slug in self.config.addons_repositories.items(): diff --git a/hassio/addons/git.py b/hassio/addons/git.py index e62817ab9..130eba27b 100644 --- a/hassio/addons/git.py +++ b/hassio/addons/git.py @@ -91,7 +91,7 @@ class AddonsRepoCustom(AddonsRepo): if slug is None: _LOGGER("Init new custom addon repository %s", url) with tempfile.TemporaryDirectory(dir=config.path_addons_custom) \ - temp_dir: + as temp_dir: slug = temp_dir.name config.add_addons_repository(url, slug) diff --git a/hassio/config.py b/hassio/config.py index eafc6a494..217b4929a 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -31,11 +31,11 @@ UPSTREAM_BETA = 'upstream_beta' API_ENDPOINT = 'api_endpoint' +# pylint: disable=no-value-for-parameter SCHEMA_CONFIG = vol.Schema({ - vol.Optional( - HOMEASSISTANT_IMAGE, - default=os.environ.get('HOMEASSISTANT_REPOSITORY')): - vol.Coerce(str), + vol.Optional(HOMEASSISTANT_IMAGE, + default=os.environ.get('HOMEASSISTANT_REPOSITORY')): + vol.Coerce(str), vol.Optional(UPSTREAM_BETA, default=False): vol.Boolean(), vol.Optional(API_ENDPOINT): vol.Coerce(str), vol.Optional(HOMEASSISTANT_LAST): vol.Coerce(str), @@ -213,7 +213,7 @@ class CoreConfig(Config): def add_addons_repository(self, repo, slug): """Add a custom repository to list.""" - self._data[ADDONS_CUSTOM_LIST][repo][slug] + self._data[ADDONS_CUSTOM_LIST][repo] = slug self.save() def drop_addons_repository(self, repo): From dd38c73b859d7295ede95c3430747588737b3b01 Mon Sep 17 00:00:00 2001 From: pvizeli Date: Thu, 27 Apr 2017 17:48:25 +0200 Subject: [PATCH 06/13] Fix lint p4 --- hassio/addons/git.py | 2 +- hassio/config.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hassio/addons/git.py b/hassio/addons/git.py index 130eba27b..16ace9e08 100644 --- a/hassio/addons/git.py +++ b/hassio/addons/git.py @@ -97,7 +97,7 @@ class AddonsRepoCustom(AddonsRepo): config.add_addons_repository(url, slug) path = os.path.join(config.path_addons_custom, slug) - super()__init__(config, loop, path, url) + super().__init__(config, loop, path, url) def remove(self): """Remove a custom addon.""" diff --git a/hassio/config.py b/hassio/config.py index 217b4929a..86b8d61c9 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -31,11 +31,14 @@ UPSTREAM_BETA = 'upstream_beta' API_ENDPOINT = 'api_endpoint' +def hass_image(): + """Return HomeAssistant docker Image.""" + return os.environ.get('HOMEASSISTANT_REPOSITORY') + + # pylint: disable=no-value-for-parameter SCHEMA_CONFIG = vol.Schema({ - vol.Optional(HOMEASSISTANT_IMAGE, - default=os.environ.get('HOMEASSISTANT_REPOSITORY')): - vol.Coerce(str), + vol.Optional(HOMEASSISTANT_IMAGE, default=hass_image): vol.Coerce(str), vol.Optional(UPSTREAM_BETA, default=False): vol.Boolean(), vol.Optional(API_ENDPOINT): vol.Coerce(str), vol.Optional(HOMEASSISTANT_LAST): vol.Coerce(str), From c6cc8adbb71746f5836e326c5ca2cfc2108fe320 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 27 Apr 2017 21:58:21 +0200 Subject: [PATCH 07/13] Change handling with repo list --- hassio/addons/__init__.py | 16 +++++++++------- hassio/addons/data.py | 4 ++-- hassio/addons/git.py | 16 ++++------------ hassio/addons/util.py | 8 ++++++++ hassio/api/supervisor.py | 5 ++--- hassio/config.py | 22 ++++++++++++---------- 6 files changed, 37 insertions(+), 34 deletions(-) create mode 100644 hassio/addons/util.py diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index f432ac96c..a2891d21f 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -40,10 +40,10 @@ class AddonManager(AddonsData): tasks = [addon.load() for addon in self.repositories] if tasks: await asyncio.wait(tasks, loop=self.loop) - self.read_addons_repo() - # Update config - self.merge_local_config() + # read data from repositories + self.read_data_from_repositories() + self.merge_update_config() # load installed addons for addon in self.list_installed: @@ -53,7 +53,7 @@ class AddonManager(AddonsData): async def add_custom_repository(self, url): """Add a new custom repository.""" - if url in self.config.addons_repositories.keys(): + if url in self.config.addons_repositories: _LOGGER.warning("Repository already exists %s", url) return False @@ -63,6 +63,7 @@ class AddonManager(AddonsData): _LOGGER.error("Can't load from repository %s", url) return False + self.config.addons_repositories = repo self.repositories.append(repo) return True @@ -71,6 +72,7 @@ class AddonManager(AddonsData): for repo in self.repositories: if repo.url == url: repo = self.repositories.pop(repo, repo) + self.config.drop_addon_repository(url) repo.remove() return True @@ -84,9 +86,9 @@ class AddonManager(AddonsData): await asyncio.wait(tasks, loop=self.loop) - # Update config - self.read_addons_repo() - self.merge_local_config() + # read data from repositories + self.read_data_from_repositories() + self.merge_update_config() # remove stalled addons for addon in self.list_removed: diff --git a/hassio/addons/data.py b/hassio/addons/data.py index d43131f88..e42d6f630 100644 --- a/hassio/addons/data.py +++ b/hassio/addons/data.py @@ -42,7 +42,7 @@ class AddonsData(Config): } super().save() - def read_addons_repo(self): + def read_data_from_repositories(self): """Read data from addons repository.""" self._current_data = {} @@ -67,7 +67,7 @@ class AddonsData(Config): _LOGGER.warning("Can't read %s -> %s", addon, humanize_error(addon_config, ex)) - def merge_local_config(self): + def merge_update_config(self): """Update local config if they have update. It need to be the same version as the local version is. diff --git a/hassio/addons/git.py b/hassio/addons/git.py index 16ace9e08..02f015a97 100644 --- a/hassio/addons/git.py +++ b/hassio/addons/git.py @@ -3,10 +3,10 @@ import asyncio import logging import os import shutil -import tempfile import git +from .util import get_hash_from_repository from ..const import URL_HASSIO_ADDONS _LOGGER = logging.getLogger(__name__) @@ -86,17 +86,11 @@ class AddonsRepoHassIO(AddonsRepo): class AddonsRepoCustom(AddonsRepo): """Custom addons repository.""" - def __init__(self, config, loop, url, slug=None): + def __init__(self, config, loop, url): """Initialize git hassio addon repository.""" - if slug is None: - _LOGGER("Init new custom addon repository %s", url) - with tempfile.TemporaryDirectory(dir=config.path_addons_custom) \ - as temp_dir: - slug = temp_dir.name + path = os.path.join( + config.path_addons_custom, get_hash_from_repository(url)) - config.add_addons_repository(url, slug) - - path = os.path.join(config.path_addons_custom, slug) super().__init__(config, loop, path, url) def remove(self): @@ -109,5 +103,3 @@ class AddonsRepoCustom(AddonsRepo): _LOGGER.warning("Can't remove %s", path) shutil.rmtree(self.path, onerror=log_err) - - self.config.drop_addons_repository(self.url) diff --git a/hassio/addons/util.py b/hassio/addons/util.py new file mode 100644 index 000000000..688291246 --- /dev/null +++ b/hassio/addons/util.py @@ -0,0 +1,8 @@ +"""Util addons functions.""" +import hashlib + + +def get_hash_from_repository(repo): + """Generate a hash from repository.""" + key = repo.lower().encode() + return hashlib.sha1(key).hexdigest() diff --git a/hassio/api/supervisor.py b/hassio/api/supervisor.py index 28e3e03d0..938b7f685 100644 --- a/hassio/api/supervisor.py +++ b/hassio/api/supervisor.py @@ -46,8 +46,7 @@ class APISupervisor(object): ATTR_LAST_VERSION: self.config.last_hassio, ATTR_BETA_CHANNEL: self.config.upstream_beta, ATTR_ADDONS: self.addons.list_api, - ATTR_ADDONS_REPOSITORIES: - list(self.config.addons_repositories.keys()), + ATTR_ADDONS_REPOSITORIES: list(self.config.addons_repositories), } @api_process @@ -70,7 +69,7 @@ class APISupervisor(object): for url in set(old - new): self.addons.drop_custom_repository(url) - return self.config.save() + return True @api_process async def update(self, request): diff --git a/hassio/config.py b/hassio/config.py index 86b8d61c9..f8bba1749 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -44,9 +44,7 @@ SCHEMA_CONFIG = vol.Schema({ vol.Optional(HOMEASSISTANT_LAST): vol.Coerce(str), vol.Optional(HASSIO_LAST): vol.Coerce(str), vol.Optional(HASSIO_CLEANUP): vol.Coerce(str), - vol.Optional(ADDONS_CUSTOM_LIST, default={}): { - vol.Url(): vol.Coerce(str), - } + vol.Optional(ADDONS_CUSTOM_LIST, default=[]): [vol.Url()], }, extra=vol.REMOVE_EXTRA) @@ -214,17 +212,21 @@ class CoreConfig(Config): """Return root backup data folder extern for docker.""" return BACKUP_DATA.format(self.path_hassio_docker) - def add_addons_repository(self, repo, slug): + @addons_repositories.setter + def addons_repositories(self, repo): """Add a custom repository to list.""" - self._data[ADDONS_CUSTOM_LIST][repo] = slug - self.save() + if repo in self._data[ADDONS_CUSTOM_LIST]: + return - def drop_addons_repository(self, repo): - """Remove a custom repository from list.""" - if self._data[ADDONS_CUSTOM_LIST].pop(repo, None): - self.save() + self._data[ADDONS_CUSTOM_LIST].append(repo) + self.save() @property def addons_repositories(self): """Return list of addons custom repositories.""" return self._data[ADDONS_CUSTOM_LIST] + + def drop_addon_repository(self, repo): + """Remove a custom repository from list.""" + if self._data[ADDONS_CUSTOM_LIST].pop(repo, False): + self.save() From 645a8e2372660ff801db7f96dc2cb33a904df382 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 27 Apr 2017 23:09:52 +0200 Subject: [PATCH 08/13] Add id to addons slug --- hassio/addons/data.py | 19 ++++++++++++++----- hassio/addons/util.py | 26 ++++++++++++++++++++++++++ hassio/api/supervisor.py | 2 +- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/hassio/addons/data.py b/hassio/addons/data.py index e42d6f630..2a1017198 100644 --- a/hassio/addons/data.py +++ b/hassio/addons/data.py @@ -6,6 +6,7 @@ import glob import voluptuous as vol from voluptuous.humanize import humanize_error +from .util import extract_hash_from_path from .validate import validate_options, SCHEMA_ADDON_CONFIG from ..const import ( FILE_HASSIO_ADDONS, ATTR_NAME, ATTR_VERSION, ATTR_SLUG, ATTR_DESCRIPTON, @@ -47,9 +48,9 @@ class AddonsData(Config): self._current_data = {} self._read_addons_folder(self.config.path_addons_repo) - self._read_addons_folder(self.config.path_addons_custom) + self._read_addons_folder(self.config.path_addons_custom, custom=True) - def _read_addons_folder(self, folder): + def _read_addons_folder(self, folder, custom=False): """Read data from addons folder.""" pattern = ADDONS_REPO_PATTERN.format(folder) @@ -58,9 +59,17 @@ class AddonsData(Config): addon_config = read_json_file(addon) addon_config = SCHEMA_ADDON_CONFIG(addon_config) - self._current_data[addon_config[ATTR_SLUG]] = addon_config + if custom: + addon_slug = "{}_{}".format( + extract_hash_from_path(folder, addon), + addon_config[ATTR_SLUG], + ) + else: + addon_slug = addon_config[ATTR_SLUG] - except (OSError, KeyError): + self._current_data[addon_slug] = addon_config + + except OSError: _LOGGER.warning("Can't read %s", addon) except vol.Invalid as ex: @@ -105,7 +114,7 @@ class AddonsData(Config): data.append({ ATTR_NAME: values[ATTR_NAME], - ATTR_SLUG: values[ATTR_SLUG], + ATTR_SLUG: addon, ATTR_DESCRIPTON: values[ATTR_DESCRIPTON], ATTR_VERSION: values[ATTR_VERSION], ATTR_INSTALLED: i_version, diff --git a/hassio/addons/util.py b/hassio/addons/util.py index 688291246..bf9393f98 100644 --- a/hassio/addons/util.py +++ b/hassio/addons/util.py @@ -1,8 +1,34 @@ """Util addons functions.""" import hashlib +import pathlib +import re +import unicodedata + +RE_SLUGIFY = re.compile(r'[^a-z0-9_]+') + + +def slugify(text): + """Slugify a given text.""" + text = unicodedata.normalize('NFKD', text) + text = text.lower() + text = text.replace(" ", "_") + text = RE_SLUGIFY.sub("", text) + + return text def get_hash_from_repository(repo): """Generate a hash from repository.""" key = repo.lower().encode() return hashlib.sha1(key).hexdigest() + + +def extract_hash_from_path(base_path, options_path): + """Extract repo id from path.""" + base_dir = pathlib.PurePosixPath(base_path).parts[-1] + + dirlist = iter(pathlib.PurePosixPath(options_path).parts) + for obj in dirlist: + if obj != base_dir: + continue + return slugify(next(dirlist)) diff --git a/hassio/api/supervisor.py b/hassio/api/supervisor.py index 938b7f685..3c09cc99b 100644 --- a/hassio/api/supervisor.py +++ b/hassio/api/supervisor.py @@ -59,7 +59,7 @@ class APISupervisor(object): if ATTR_ADDONS_REPOSITORIES in body: new = set(body[ATTR_ADDONS_REPOSITORIES]) - old = set(self.config.addons_repositories.keys()) + old = set(self.config.addons_repositories) # add new repositories for url in set(new - old): From 0e258a4ae003e72e3d7b8b8deb548cedc3b8a5ce Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 27 Apr 2017 23:18:05 +0200 Subject: [PATCH 09/13] short hash --- hassio/addons/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hassio/addons/util.py b/hassio/addons/util.py index bf9393f98..d5223a214 100644 --- a/hassio/addons/util.py +++ b/hassio/addons/util.py @@ -20,7 +20,7 @@ def slugify(text): def get_hash_from_repository(repo): """Generate a hash from repository.""" key = repo.lower().encode() - return hashlib.sha1(key).hexdigest() + return hashlib.sha1(key).hexdigest()[:8] def extract_hash_from_path(base_path, options_path): From 3c0ebdf643240b3c0d8cc2461b3bbacc60cedf38 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 27 Apr 2017 23:28:58 +0200 Subject: [PATCH 10/13] fix lint --- hassio/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hassio/config.py b/hassio/config.py index f8bba1749..1595ee499 100644 --- a/hassio/config.py +++ b/hassio/config.py @@ -212,6 +212,11 @@ class CoreConfig(Config): """Return root backup data folder extern for docker.""" return BACKUP_DATA.format(self.path_hassio_docker) + @property + def addons_repositories(self): + """Return list of addons custom repositories.""" + return self._data[ADDONS_CUSTOM_LIST] + @addons_repositories.setter def addons_repositories(self, repo): """Add a custom repository to list.""" @@ -221,11 +226,6 @@ class CoreConfig(Config): self._data[ADDONS_CUSTOM_LIST].append(repo) self.save() - @property - def addons_repositories(self): - """Return list of addons custom repositories.""" - return self._data[ADDONS_CUSTOM_LIST] - def drop_addon_repository(self, repo): """Remove a custom repository from list.""" if self._data[ADDONS_CUSTOM_LIST].pop(repo, False): From 871721f04b90927cc12cc4ded589c548171d7eb6 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 27 Apr 2017 23:31:34 +0200 Subject: [PATCH 11/13] Fix lint p2 --- hassio/addons/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index a2891d21f..6b837e0d9 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -34,7 +34,7 @@ class AddonManager(AddonsData): # init custom repositories for url, slug in self.config.addons_repositories.items(): self.repositories.append( - AddonsRepoCustom(self.config, self.loop, url, slug)) + AddonsRepoCustom(self.config, self.loop, url)) # load addon repository tasks = [addon.load() for addon in self.repositories] From 76952db3eba75398d45587822fcd155f9aed97b4 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 27 Apr 2017 23:37:35 +0200 Subject: [PATCH 12/13] fix old code --- hassio/addons/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index 6b837e0d9..6444a7ae5 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -32,7 +32,7 @@ class AddonManager(AddonsData): self.repositories.append(AddonsRepoHassIO(self.config, self.loop)) # init custom repositories - for url, slug in self.config.addons_repositories.items(): + for url in self.config.addons_repositories: self.repositories.append( AddonsRepoCustom(self.config, self.loop, url)) @@ -97,10 +97,7 @@ class AddonManager(AddonsData): async def auto_boot(self, start_type): """Boot addons with mode auto.""" boot_list = self.list_startup(start_type) - tasks = [] - - for addon in boot_list: - tasks.append(self.loop.create_task(self.start(addon))) + tasks = [self.start(addon) from addon in boot_list] _LOGGER.info("Startup %s run %d addons", start_type, len(tasks)) if tasks: From a287f52e47d84358d5a6fcd6bbe8be06ab768775 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Thu, 27 Apr 2017 23:43:18 +0200 Subject: [PATCH 13/13] fix spell --- hassio/addons/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hassio/addons/__init__.py b/hassio/addons/__init__.py index 6444a7ae5..67a963ef4 100644 --- a/hassio/addons/__init__.py +++ b/hassio/addons/__init__.py @@ -97,7 +97,7 @@ class AddonManager(AddonsData): async def auto_boot(self, start_type): """Boot addons with mode auto.""" boot_list = self.list_startup(start_type) - tasks = [self.start(addon) from addon in boot_list] + tasks = [self.start(addon) for addon in boot_list] _LOGGER.info("Startup %s run %d addons", start_type, len(tasks)) if tasks: