Made dependency loading more robust

homeassistant/bootstrap.py

@@ -13,12 +13,10 @@ import os
 import configparser
 import logging
 from collections import defaultdict
-from itertools import chain
 
 import homeassistant
 import homeassistant.loader as loader
 import homeassistant.components as core_components
-import homeassistant.components.group as group
 
 
 # pylint: disable=too-many-branches, too-many-statements
@@ -33,112 +31,22 @@ def from_config_dict(config, hass=None):
 
     logger = logging.getLogger(__name__)
 
+    loader.prepare(hass)
+
     # Make a copy because we are mutating it.
     # Convert it to defaultdict so components can always have config dict
     config = defaultdict(dict, config)
 
-    # List of loaded components
+    # Filter out the common config section [homeassistant]
-    components = {}
+    components = (key for key in config.keys() if key != homeassistant.DOMAIN)
-
-    # List of components to validate
-    to_validate = []
-
-    # List of validated components
-    validated = []
-
-    # List of components we are going to load
-    to_load = [key for key in config.keys() if key != homeassistant.DOMAIN]
-
-    loader.prepare(hass)
-
-    # Load required components
-    while to_load:
-        domain = to_load.pop()
-
-        component = loader.get_component(domain)
-
-        # if None it does not exist, error already thrown by get_component
-        if component is not None:
-            components[domain] = component
-
-            # Special treatment for GROUP, we want to load it as late as
-            # possible. We do this by loading it if all other to be loaded
-            # modules depend on it.
-            if component.DOMAIN == group.DOMAIN:
-                pass
-
-            # Components with no dependencies are valid
-            elif not component.DEPENDENCIES:
-                validated.append(domain)
-
-            # If dependencies we'll validate it later
-            else:
-                to_validate.append(domain)
-
-                # Make sure to load all dependencies that are not being loaded
-                for dependency in component.DEPENDENCIES:
-                    if dependency not in chain(components.keys(), to_load):
-                        to_load.append(dependency)
-
-    # Validate dependencies
-    group_added = False
-
-    while to_validate:
-        newly_validated = []
-
-        for domain in to_validate:
-            if all(domain in validated for domain
-                   in components[domain].DEPENDENCIES):
-
-                newly_validated.append(domain)
-
-        # We validated new domains this iteration, add them to validated
-        if newly_validated:
-
-            # Add newly validated domains to validated
-            validated.extend(newly_validated)
-
-            # remove domains from to_validate
-            for domain in newly_validated:
-                to_validate.remove(domain)
-
-            newly_validated.clear()
-
-        # Nothing validated this iteration. Add group dependency and try again.
-        elif not group_added:
-            group_added = True
-            validated.append(group.DOMAIN)
-
-        # Group has already been added and we still can't validate all.
-        # Report missing deps as error and skip loading of these domains
-        else:
-            for domain in to_validate:
-                missing_deps = [dep for dep in components[domain].DEPENDENCIES
-                                if dep not in validated]
-
-                logger.error(
-                    "Could not validate all dependencies for %s: %s",
-                    domain, ", ".join(missing_deps))
-
-            break
-
-    # Make sure we load groups if not in list yet.
-    if not group_added:
-        validated.append(group.DOMAIN)
-
-        if group.DOMAIN not in components:
-            components[group.DOMAIN] = \
-                loader.get_component(group.DOMAIN)
 
     # Setup the components
     if core_components.setup(hass, config):
         logger.info("Home Assistant core initialized")
 
-        for domain in validated:
+        for domain in loader.load_order_components(components):
-            component = components[domain]
-
             try:
-                if component.setup(hass, config):
+                if loader.get_component(domain).setup(hass, config):
                     logger.info("component %s initialized", domain)
                 else:
                     logger.error("component %s failed to initialize", domain)

homeassistant/loader.py

@@ -19,6 +19,10 @@ import pkgutil
 import importlib
 import logging
 
+from homeassistant.util import OrderedSet
+
+PREPARED = False
+
 # List of available components
 AVAILABLE_COMPONENTS = []
 
@@ -30,6 +34,8 @@ _LOGGER = logging.getLogger(__name__)
 
 def prepare(hass):
     """ Prepares the loading of components. """
+    global PREPARED  # pylint: disable=global-statement
+
     # Load the built-in components
     import homeassistant.components as components
 
@@ -62,9 +68,13 @@ def prepare(hass):
                 AVAILABLE_COMPONENTS.append(
                     'custom_components.{}'.format(fil[0:-3]))
 
+    PREPARED = True
+
 
 def set_component(comp_name, component):
     """ Sets a component in the cache. """
+    _check_prepared()
+
     _COMPONENT_CACHE[comp_name] = component
 
 
@@ -76,6 +86,8 @@ def get_component(comp_name):
     if comp_name in _COMPONENT_CACHE:
         return _COMPONENT_CACHE[comp_name]
 
+    _check_prepared()
+
     # If we ie. try to load custom_components.switch.wemo but the parent
     # custom_components.switch does not exist, importing it will trigger
     # an exception because it will try to import the parent.
@@ -125,3 +137,89 @@ def get_component(comp_name):
     _LOGGER.error("Unable to find component %s", comp_name)
 
     return None
+
+
+def load_order_components(components):
+    """
+    Takes in a list of components we want to load:
+     - filters out components we cannot load
+     - filters out components that have invalid/circular dependencies
+     - Will ensure that all components that do not directly depend on
+       the group component will be loaded before the group component.
+     - returns an OrderedSet load order.
+    """
+    _check_prepared()
+
+    group = get_component('group')
+
+    load_order = OrderedSet()
+
+    # Sort the list of modules on if they depend on group component or not.
+    # We do this because the components that do not depend on the group
+    # component usually set up states that the group component requires to be
+    # created before it can group them.
+    # This does not matter in the future if we can setup groups without the
+    # states existing yet.
+    for comp_load_order in sorted((load_order_component(component)
+                                   for component in components),
+                                  # Test if group component exists in case
+                                  # above get_component call had an error.
+                                  key=lambda order:
+                                  group and group.DOMAIN in order):
+        load_order.update(comp_load_order)
+
+    return load_order
+
+
+def load_order_component(comp_name):
+    """
+    Returns an OrderedSet of components in the correct order of loading.
+    Raises HomeAssistantError if a circular dependency is detected.
+    Returns an empty list if component could not be loaded.
+    """
+    return _load_order_component(comp_name, OrderedSet(), set())
+
+
+def _load_order_component(comp_name, load_order, loading):
+    """ Recursive function to get load order of components. """
+    component = get_component(comp_name)
+
+    # if None it does not exist, error already thrown by get_component
+    if component is None:
+        return OrderedSet()
+
+    loading.add(comp_name)
+
+    for dependency in component.DEPENDENCIES:
+        # Check not already loaded
+        if dependency not in load_order:
+            # If we are already loading it, we have a circular dependency
+            if dependency in loading:
+                _LOGGER.error('Circular dependency detected: %s -> %s',
+                              comp_name, dependency)
+
+                return OrderedSet()
+
+            dep_load_order = _load_order_component(
+                dependency, load_order, loading)
+
+            # length == 0 means error loading dependency or children
+            if len(dep_load_order) == 0:
+                _LOGGER.error('Error loading %s dependency: %s',
+                              comp_name, dependency)
+                return OrderedSet()
+
+            load_order.update(dep_load_order)
+
+    load_order.add(comp_name)
+    loading.remove(comp_name)
+
+    return load_order
+
+
+def _check_prepared():
+    """ Issues a warning if loader.prepare() has never been called. """
+    if not PREPARED:
+        _LOGGER.warning((
+            "You did not call loader.prepare() yet. "
+            "Certain functionality might not be working."))

homeassistant/util.py

@@ -4,6 +4,8 @@ homeassistant.util
 
 Helper methods for various modules.
 """
+import collections
+from itertools import chain
 import threading
 import queue
 import datetime
@@ -176,6 +178,75 @@ class OrderedEnum(enum.Enum):
         return NotImplemented
 
 
+class OrderedSet(collections.MutableSet):
+    """ Ordered set taken from http://code.activestate.com/recipes/576694/ """
+
+    def __init__(self, iterable=None):
+        self.end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.map = {}                   # key --> [key, prev, next]
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        return len(self.map)
+
+    def __contains__(self, key):
+        return key in self.map
+
+    def add(self, key):
+        """ Add an element to the set. """
+        if key not in self.map:
+            end = self.end
+            curr = end[1]
+            curr[2] = end[1] = self.map[key] = [key, curr, end]
+
+    def discard(self, key):
+        """ Discard an element from the set. """
+        if key in self.map:
+            key, prev_item, next_item = self.map.pop(key)
+            prev_item[2] = next_item
+            next_item[1] = prev_item
+
+    def __iter__(self):
+        end = self.end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def pop(self, last=True):  # pylint: disable=arguments-differ
+        """ Pops element of the beginning of the set.
+            Set last=True to pop from the back. """
+        if not self:
+            raise KeyError('set is empty')
+        key = self.end[1][0] if last else self.end[2][0]
+        self.discard(key)
+        return key
+
+    def update(self, *args):
+        """ Add elements from args to the set. """
+        for item in chain(*args):
+            self.add(item)
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedSet):
+            return len(self) == len(other) and list(self) == list(other)
+        return set(self) == set(other)
+
+
 def validate_config(config, items, logger):
     """
     Validates if all items are available in the configuration.

test/helper.py

@@ -28,3 +28,13 @@ def mock_service(hass, domain, service):
         domain, service, lambda call: calls.append(call))
 
     return calls
+
+
+class MockModule(object):
+    """ Provides a fake module. """
+
+    def __init__(self, domain, dependencies=[], setup=None):
+        self.DOMAIN = domain
+        self.DEPENDENCIES = dependencies
+        # Setup a mock setup if none given.
+        self.setup = lambda hass, config: False if setup is None else setup

test/test_loader.py

@@ -6,13 +6,14 @@ Provides tests to verify that we can load components.
 """
 # pylint: disable=too-many-public-methods,protected-access
 import unittest
+from collections import namedtuple
 
 import homeassistant as ha
 import homeassistant.loader as loader
 import homeassistant.components.http as http
 
 import mock_toggledevice_platform
-from helper import get_test_home_assistant
+from helper import get_test_home_assistant, MockModule
 
 
 class TestLoader(unittest.TestCase):
@@ -37,3 +38,54 @@ class TestLoader(unittest.TestCase):
         self.assertEqual(http, loader.get_component('http'))
 
         self.assertIsNotNone(loader.get_component('custom_one'))
+
+    def test_load_order_component(self):
+        """ Test if we can get the proper load order of components. """
+        loader.set_component('mod1', MockModule('mod1'))
+        loader.set_component('mod2', MockModule('mod2', ['mod1']))
+        loader.set_component('mod3', MockModule('mod3', ['mod2']))
+
+        self.assertEqual(
+            ['mod1', 'mod2', 'mod3'], loader.load_order_component('mod3'))
+
+        # Create circular dependency
+        loader.set_component('mod1', MockModule('mod1', ['mod3']))
+
+        self.assertEqual([], loader.load_order_component('mod3'))
+
+        # Depend on non-existing component
+        loader.set_component('mod1', MockModule('mod1', ['nonexisting']))
+
+        self.assertEqual([], loader.load_order_component('mod1'))
+
+        # Try to get load order for non-existing component
+        self.assertEqual([], loader.load_order_component('mod1'))
+
+    def test_load_order_components(self):
+        loader.set_component('mod1', MockModule('mod1', ['group']))
+        loader.set_component('mod2', MockModule('mod2', ['mod1', 'sun']))
+        loader.set_component('mod3', MockModule('mod3', ['mod2']))
+        loader.set_component('mod4', MockModule('mod4', ['group']))
+
+        self.assertEqual(
+            ['group', 'mod4', 'mod1', 'sun', 'mod2', 'mod3'],
+            loader.load_order_components(['mod4', 'mod3', 'mod2']))
+
+        loader.set_component('mod1', MockModule('mod1'))
+        loader.set_component('mod2', MockModule('mod2', ['group']))
+
+        self.assertEqual(
+            ['mod1', 'group', 'mod2'],
+            loader.load_order_components(['mod2', 'mod1']))
+
+        # Add a non existing one
+        self.assertEqual(
+            ['mod1', 'group', 'mod2'],
+            loader.load_order_components(['mod2', 'nonexisting', 'mod1']))
+
+        # Depend on a non existing one
+        loader.set_component('mod1', MockModule('mod1', ['nonexisting']))
+
+        self.assertEqual(
+            ['group', 'mod2'],
+            loader.load_order_components(['mod2', 'mod1']))