From 5fe73cf33eb5b27a9317e33979639524fe8799ab Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Fri, 28 Nov 2014 15:34:42 -0800 Subject: [PATCH] Made dependency loading more robust --- homeassistant/bootstrap.py | 104 +++---------------------------------- homeassistant/loader.py | 98 ++++++++++++++++++++++++++++++++++ homeassistant/util.py | 71 +++++++++++++++++++++++++ test/helper.py | 10 ++++ test/test_loader.py | 54 ++++++++++++++++++- 5 files changed, 238 insertions(+), 99 deletions(-) diff --git a/homeassistant/bootstrap.py b/homeassistant/bootstrap.py index 81eaefdba24..3875c23b4ce 100644 --- a/homeassistant/bootstrap.py +++ b/homeassistant/bootstrap.py @@ -13,12 +13,10 @@ import os import configparser import logging from collections import defaultdict -from itertools import chain import homeassistant import homeassistant.loader as loader import homeassistant.components as core_components -import homeassistant.components.group as group # pylint: disable=too-many-branches, too-many-statements @@ -33,112 +31,22 @@ def from_config_dict(config, hass=None): logger = logging.getLogger(__name__) + loader.prepare(hass) + # Make a copy because we are mutating it. # Convert it to defaultdict so components can always have config dict config = defaultdict(dict, config) - # List of loaded components - components = {} - - # List of components to validate - to_validate = [] - - # List of validated components - validated = [] - - # List of components we are going to load - to_load = [key for key in config.keys() if key != homeassistant.DOMAIN] - - loader.prepare(hass) - - # Load required components - while to_load: - domain = to_load.pop() - - component = loader.get_component(domain) - - # if None it does not exist, error already thrown by get_component - if component is not None: - components[domain] = component - - # Special treatment for GROUP, we want to load it as late as - # possible. We do this by loading it if all other to be loaded - # modules depend on it. - if component.DOMAIN == group.DOMAIN: - pass - - # Components with no dependencies are valid - elif not component.DEPENDENCIES: - validated.append(domain) - - # If dependencies we'll validate it later - else: - to_validate.append(domain) - - # Make sure to load all dependencies that are not being loaded - for dependency in component.DEPENDENCIES: - if dependency not in chain(components.keys(), to_load): - to_load.append(dependency) - - # Validate dependencies - group_added = False - - while to_validate: - newly_validated = [] - - for domain in to_validate: - if all(domain in validated for domain - in components[domain].DEPENDENCIES): - - newly_validated.append(domain) - - # We validated new domains this iteration, add them to validated - if newly_validated: - - # Add newly validated domains to validated - validated.extend(newly_validated) - - # remove domains from to_validate - for domain in newly_validated: - to_validate.remove(domain) - - newly_validated.clear() - - # Nothing validated this iteration. Add group dependency and try again. - elif not group_added: - group_added = True - validated.append(group.DOMAIN) - - # Group has already been added and we still can't validate all. - # Report missing deps as error and skip loading of these domains - else: - for domain in to_validate: - missing_deps = [dep for dep in components[domain].DEPENDENCIES - if dep not in validated] - - logger.error( - "Could not validate all dependencies for %s: %s", - domain, ", ".join(missing_deps)) - - break - - # Make sure we load groups if not in list yet. - if not group_added: - validated.append(group.DOMAIN) - - if group.DOMAIN not in components: - components[group.DOMAIN] = \ - loader.get_component(group.DOMAIN) + # Filter out the common config section [homeassistant] + components = (key for key in config.keys() if key != homeassistant.DOMAIN) # Setup the components if core_components.setup(hass, config): logger.info("Home Assistant core initialized") - for domain in validated: - component = components[domain] - + for domain in loader.load_order_components(components): try: - if component.setup(hass, config): + if loader.get_component(domain).setup(hass, config): logger.info("component %s initialized", domain) else: logger.error("component %s failed to initialize", domain) diff --git a/homeassistant/loader.py b/homeassistant/loader.py index 87a76d9eb7d..e537eb96450 100644 --- a/homeassistant/loader.py +++ b/homeassistant/loader.py @@ -19,6 +19,10 @@ import pkgutil import importlib import logging +from homeassistant.util import OrderedSet + +PREPARED = False + # List of available components AVAILABLE_COMPONENTS = [] @@ -30,6 +34,8 @@ _LOGGER = logging.getLogger(__name__) def prepare(hass): """ Prepares the loading of components. """ + global PREPARED # pylint: disable=global-statement + # Load the built-in components import homeassistant.components as components @@ -62,9 +68,13 @@ def prepare(hass): AVAILABLE_COMPONENTS.append( 'custom_components.{}'.format(fil[0:-3])) + PREPARED = True + def set_component(comp_name, component): """ Sets a component in the cache. """ + _check_prepared() + _COMPONENT_CACHE[comp_name] = component @@ -76,6 +86,8 @@ def get_component(comp_name): if comp_name in _COMPONENT_CACHE: return _COMPONENT_CACHE[comp_name] + _check_prepared() + # If we ie. try to load custom_components.switch.wemo but the parent # custom_components.switch does not exist, importing it will trigger # an exception because it will try to import the parent. @@ -125,3 +137,89 @@ def get_component(comp_name): _LOGGER.error("Unable to find component %s", comp_name) return None + + +def load_order_components(components): + """ + Takes in a list of components we want to load: + - filters out components we cannot load + - filters out components that have invalid/circular dependencies + - Will ensure that all components that do not directly depend on + the group component will be loaded before the group component. + - returns an OrderedSet load order. + """ + _check_prepared() + + group = get_component('group') + + load_order = OrderedSet() + + # Sort the list of modules on if they depend on group component or not. + # We do this because the components that do not depend on the group + # component usually set up states that the group component requires to be + # created before it can group them. + # This does not matter in the future if we can setup groups without the + # states existing yet. + for comp_load_order in sorted((load_order_component(component) + for component in components), + # Test if group component exists in case + # above get_component call had an error. + key=lambda order: + group and group.DOMAIN in order): + load_order.update(comp_load_order) + + return load_order + + +def load_order_component(comp_name): + """ + Returns an OrderedSet of components in the correct order of loading. + Raises HomeAssistantError if a circular dependency is detected. + Returns an empty list if component could not be loaded. + """ + return _load_order_component(comp_name, OrderedSet(), set()) + + +def _load_order_component(comp_name, load_order, loading): + """ Recursive function to get load order of components. """ + component = get_component(comp_name) + + # if None it does not exist, error already thrown by get_component + if component is None: + return OrderedSet() + + loading.add(comp_name) + + for dependency in component.DEPENDENCIES: + # Check not already loaded + if dependency not in load_order: + # If we are already loading it, we have a circular dependency + if dependency in loading: + _LOGGER.error('Circular dependency detected: %s -> %s', + comp_name, dependency) + + return OrderedSet() + + dep_load_order = _load_order_component( + dependency, load_order, loading) + + # length == 0 means error loading dependency or children + if len(dep_load_order) == 0: + _LOGGER.error('Error loading %s dependency: %s', + comp_name, dependency) + return OrderedSet() + + load_order.update(dep_load_order) + + load_order.add(comp_name) + loading.remove(comp_name) + + return load_order + + +def _check_prepared(): + """ Issues a warning if loader.prepare() has never been called. """ + if not PREPARED: + _LOGGER.warning(( + "You did not call loader.prepare() yet. " + "Certain functionality might not be working.")) diff --git a/homeassistant/util.py b/homeassistant/util.py index 67188926546..42b6e668096 100644 --- a/homeassistant/util.py +++ b/homeassistant/util.py @@ -4,6 +4,8 @@ homeassistant.util Helper methods for various modules. """ +import collections +from itertools import chain import threading import queue import datetime @@ -176,6 +178,75 @@ class OrderedEnum(enum.Enum): return NotImplemented +class OrderedSet(collections.MutableSet): + """ Ordered set taken from http://code.activestate.com/recipes/576694/ """ + + def __init__(self, iterable=None): + self.end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.map = {} # key --> [key, prev, next] + if iterable is not None: + self |= iterable + + def __len__(self): + return len(self.map) + + def __contains__(self, key): + return key in self.map + + def add(self, key): + """ Add an element to the set. """ + if key not in self.map: + end = self.end + curr = end[1] + curr[2] = end[1] = self.map[key] = [key, curr, end] + + def discard(self, key): + """ Discard an element from the set. """ + if key in self.map: + key, prev_item, next_item = self.map.pop(key) + prev_item[2] = next_item + next_item[1] = prev_item + + def __iter__(self): + end = self.end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def pop(self, last=True): # pylint: disable=arguments-differ + """ Pops element of the beginning of the set. + Set last=True to pop from the back. """ + if not self: + raise KeyError('set is empty') + key = self.end[1][0] if last else self.end[2][0] + self.discard(key) + return key + + def update(self, *args): + """ Add elements from args to the set. """ + for item in chain(*args): + self.add(item) + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, list(self)) + + def __eq__(self, other): + if isinstance(other, OrderedSet): + return len(self) == len(other) and list(self) == list(other) + return set(self) == set(other) + + def validate_config(config, items, logger): """ Validates if all items are available in the configuration. diff --git a/test/helper.py b/test/helper.py index 3e5c9caa761..ed0c2c74e82 100644 --- a/test/helper.py +++ b/test/helper.py @@ -28,3 +28,13 @@ def mock_service(hass, domain, service): domain, service, lambda call: calls.append(call)) return calls + + +class MockModule(object): + """ Provides a fake module. """ + + def __init__(self, domain, dependencies=[], setup=None): + self.DOMAIN = domain + self.DEPENDENCIES = dependencies + # Setup a mock setup if none given. + self.setup = lambda hass, config: False if setup is None else setup diff --git a/test/test_loader.py b/test/test_loader.py index 8d9cad71303..ceb398815b4 100644 --- a/test/test_loader.py +++ b/test/test_loader.py @@ -6,13 +6,14 @@ Provides tests to verify that we can load components. """ # pylint: disable=too-many-public-methods,protected-access import unittest +from collections import namedtuple import homeassistant as ha import homeassistant.loader as loader import homeassistant.components.http as http import mock_toggledevice_platform -from helper import get_test_home_assistant +from helper import get_test_home_assistant, MockModule class TestLoader(unittest.TestCase): @@ -37,3 +38,54 @@ class TestLoader(unittest.TestCase): self.assertEqual(http, loader.get_component('http')) self.assertIsNotNone(loader.get_component('custom_one')) + + def test_load_order_component(self): + """ Test if we can get the proper load order of components. """ + loader.set_component('mod1', MockModule('mod1')) + loader.set_component('mod2', MockModule('mod2', ['mod1'])) + loader.set_component('mod3', MockModule('mod3', ['mod2'])) + + self.assertEqual( + ['mod1', 'mod2', 'mod3'], loader.load_order_component('mod3')) + + # Create circular dependency + loader.set_component('mod1', MockModule('mod1', ['mod3'])) + + self.assertEqual([], loader.load_order_component('mod3')) + + # Depend on non-existing component + loader.set_component('mod1', MockModule('mod1', ['nonexisting'])) + + self.assertEqual([], loader.load_order_component('mod1')) + + # Try to get load order for non-existing component + self.assertEqual([], loader.load_order_component('mod1')) + + def test_load_order_components(self): + loader.set_component('mod1', MockModule('mod1', ['group'])) + loader.set_component('mod2', MockModule('mod2', ['mod1', 'sun'])) + loader.set_component('mod3', MockModule('mod3', ['mod2'])) + loader.set_component('mod4', MockModule('mod4', ['group'])) + + self.assertEqual( + ['group', 'mod4', 'mod1', 'sun', 'mod2', 'mod3'], + loader.load_order_components(['mod4', 'mod3', 'mod2'])) + + loader.set_component('mod1', MockModule('mod1')) + loader.set_component('mod2', MockModule('mod2', ['group'])) + + self.assertEqual( + ['mod1', 'group', 'mod2'], + loader.load_order_components(['mod2', 'mod1'])) + + # Add a non existing one + self.assertEqual( + ['mod1', 'group', 'mod2'], + loader.load_order_components(['mod2', 'nonexisting', 'mod1'])) + + # Depend on a non existing one + loader.set_component('mod1', MockModule('mod1', ['nonexisting'])) + + self.assertEqual( + ['group', 'mod2'], + loader.load_order_components(['mod2', 'mod1']))