Load as many components in parallel as possible (#20806)

* Load as many components in parallel as possible

* Lint
This commit is contained in:
Paulus Schoutsen 2019-02-07 13:56:40 -08:00 committed by Pascal Vizeli
parent f3b20d138e
commit a9672b0d52
6 changed files with 92 additions and 227 deletions

View File

@ -10,7 +10,8 @@ from typing import Any, Optional, Dict
import voluptuous as vol
from homeassistant import (
core, config as conf_util, config_entries, components as core_components)
core, config as conf_util, config_entries, components as core_components,
loader)
from homeassistant.components import persistent_notification
from homeassistant.const import EVENT_HOMEASSISTANT_CLOSE
from homeassistant.setup import async_setup_component
@ -124,6 +125,15 @@ async def async_from_config_dict(config: Dict[str, Any],
if key != core.DOMAIN)
components.update(hass.config_entries.async_domains())
# Resolve all dependencies of all components.
for component in list(components):
try:
components.update(loader.component_dependencies(hass, component))
except loader.LoaderError:
# Ignore it, or we'll break startup
# It will be properly handled during setup.
pass
# setup components
res = await core_components.async_setup(hass, config)
if not res:

View File

@ -18,7 +18,6 @@ from types import ModuleType
from typing import Optional, Set, TYPE_CHECKING, Callable, Any, TypeVar # noqa pylint: disable=unused-import
from homeassistant.const import PLATFORM_FORMAT
from homeassistant.util import OrderedSet
# Typing imports that create a circular dependency
# pylint: disable=using-constant-test,unused-import
@ -39,6 +38,30 @@ PATH_CUSTOM_COMPONENTS = 'custom_components'
PACKAGE_COMPONENTS = 'homeassistant.components'
class LoaderError(Exception):
"""Loader base error."""
class ComponentNotFound(LoaderError):
"""Raised when a component is not found."""
def __init__(self, domain: str) -> None:
"""Initialize a component not found error."""
super().__init__("Component {} not found.".format(domain))
self.domain = domain
class CircularDependency(LoaderError):
"""Raised when a circular dependency is found when resolving components."""
def __init__(self, from_domain: str, to_domain: str) -> None:
"""Initialize circular dependency error."""
super().__init__("Circular dependency detected: {} -> {}.".format(
from_domain, to_domain))
self.from_domain = from_domain
self.to_domain = to_domain
def set_component(hass, # type: HomeAssistant
comp_name: str, component: Optional[ModuleType]) -> None:
"""Set a component in the cache.
@ -235,57 +258,46 @@ def bind_hass(func: CALLABLE_T) -> CALLABLE_T:
return func
def load_order_component(hass, # type: HomeAssistant
comp_name: str) -> OrderedSet:
"""Return an OrderedSet of components in the correct order of loading.
def component_dependencies(hass, # type: HomeAssistant
comp_name: str) -> Set[str]:
"""Return all dependencies and subdependencies of components.
Returns an empty list if a circular dependency is detected
or the component could not be loaded. In both cases, the error is
logged.
Raises CircularDependency if a circular dependency is found.
Async friendly.
"""
return _load_order_component(hass, comp_name, OrderedSet(), set())
return _component_dependencies(hass, comp_name, set(), set())
def _load_order_component(hass, # type: HomeAssistant
comp_name: str, load_order: OrderedSet,
loading: Set) -> OrderedSet:
"""Recursive function to get load order of components.
def _component_dependencies(hass, # type: HomeAssistant
comp_name: str, loaded: Set[str],
loading: Set) -> Set[str]:
"""Recursive function to get component dependencies.
Async friendly.
"""
component = get_component(hass, comp_name)
# If None it does not exist, error already thrown by get_component.
if component is None:
return OrderedSet()
raise ComponentNotFound(comp_name)
loading.add(comp_name)
for dependency in getattr(component, 'DEPENDENCIES', []):
# Check not already loaded
if dependency in load_order:
if dependency in loaded:
continue
# If we are already loading it, we have a circular dependency.
if dependency in loading:
_LOGGER.error("Circular dependency detected: %s -> %s",
comp_name, dependency)
return OrderedSet()
raise CircularDependency(comp_name, dependency)
dep_load_order = _load_order_component(
hass, dependency, load_order, loading)
dep_loaded = _component_dependencies(
hass, dependency, loaded, loading)
# length == 0 means error loading dependency or children
if not dep_load_order:
_LOGGER.error("Error loading %s dependency: %s",
comp_name, dependency)
return OrderedSet()
loaded.update(dep_loaded)
load_order.update(dep_load_order)
load_order.add(comp_name)
loaded.add(comp_name)
loading.remove(comp_name)
return load_order
return loaded

View File

@ -106,12 +106,18 @@ async def _async_setup_component(hass: core.HomeAssistant,
log_error("Component not found.", False)
return False
# Validate no circular dependencies
components = loader.load_order_component(hass, domain)
# OrderedSet is empty if component or dependencies could not be resolved
if not components:
log_error("Unable to resolve component or dependencies.")
# Validate all dependencies exist and there are no circular dependencies
try:
loader.component_dependencies(hass, domain)
except loader.ComponentNotFound as err:
_LOGGER.error(
"Not setting up %s because we are unable to resolve "
"(sub)dependency %s", domain, err.domain)
return False
except loader.CircularDependency as err:
_LOGGER.error(
"Not setting up %s because it contains a circular dependency: "
"%s -> %s", domain, err.from_domain, err.to_domain)
return False
processed_config = \

View File

@ -1,7 +1,6 @@
"""Helper methods for various modules."""
import asyncio
from datetime import datetime, timedelta
from itertools import chain
import threading
import re
import enum
@ -141,96 +140,6 @@ class OrderedEnum(enum.Enum):
return NotImplemented
class OrderedSet(MutableSet[T]):
"""Ordered set taken from http://code.activestate.com/recipes/576694/."""
def __init__(self, iterable: Optional[Iterable[T]] = None) -> None:
"""Initialize the set."""
self.end = end = [] # type: List[Any]
end += [None, end, end] # sentinel node for doubly linked list
self.map = {} # type: Dict[T, List] # key --> [key, prev, next]
if iterable is not None:
self |= iterable # type: ignore
def __len__(self) -> int:
"""Return the length of the set."""
return len(self.map)
def __contains__(self, key: T) -> bool: # type: ignore
"""Check if key is in set."""
return key in self.map
# pylint: disable=arguments-differ
def add(self, key: T) -> None:
"""Add an element to the end of the set."""
if key not in self.map:
end = self.end
curr = end[1]
curr[2] = end[1] = self.map[key] = [key, curr, end]
def promote(self, key: T) -> None:
"""Promote element to beginning of the set, add if not there."""
if key in self.map:
self.discard(key)
begin = self.end[2]
curr = begin[1]
curr[2] = begin[1] = self.map[key] = [key, curr, begin]
# pylint: disable=arguments-differ
def discard(self, key: T) -> None:
"""Discard an element from the set."""
if key in self.map:
key, prev_item, next_item = self.map.pop(key)
prev_item[2] = next_item
next_item[1] = prev_item
def __iter__(self) -> Iterator[T]:
"""Iterate of the set."""
end = self.end
curr = end[2]
while curr is not end:
yield curr[0]
curr = curr[2]
def __reversed__(self) -> Iterator[T]:
"""Reverse the ordering."""
end = self.end
curr = end[1]
while curr is not end:
yield curr[0]
curr = curr[1]
# pylint: disable=arguments-differ
def pop(self, last: bool = True) -> T:
"""Pop element of the end of the set.
Set last=False to pop from the beginning.
"""
if not self:
raise KeyError('set is empty')
key = self.end[1][0] if last else self.end[2][0]
self.discard(key)
return key # type: ignore
def update(self, *args: Any) -> None:
"""Add elements from args to the set."""
for item in chain(*args):
self.add(item)
def __repr__(self) -> str:
"""Return the representation."""
if not self:
return '%s()' % (self.__class__.__name__,)
return '%s(%r)' % (self.__class__.__name__, list(self))
def __eq__(self, other: Any) -> bool:
"""Return the comparison."""
if isinstance(other, OrderedSet):
return len(self) == len(other) and list(self) == list(other)
return set(self) == set(other)
class Throttle:
"""A class for throttling the execution of tasks.

View File

@ -1,63 +1,52 @@
"""Test to verify that we can load components."""
# pylint: disable=protected-access
import asyncio
import unittest
import pytest
import homeassistant.loader as loader
import homeassistant.components.http as http
from tests.common import (
get_test_home_assistant, MockModule, async_mock_service)
from tests.common import MockModule, async_mock_service
class TestLoader(unittest.TestCase):
"""Test the loader module."""
def test_set_component(hass):
"""Test if set_component works."""
comp = object()
loader.set_component(hass, 'switch.test_set', comp)
# pylint: disable=invalid-name
def setUp(self):
"""Set up tests."""
self.hass = get_test_home_assistant()
assert loader.get_component(hass, 'switch.test_set') is comp
# pylint: disable=invalid-name
def tearDown(self):
"""Stop everything that was started."""
self.hass.stop()
def test_set_component(self):
"""Test if set_component works."""
comp = object()
loader.set_component(self.hass, 'switch.test_set', comp)
def test_get_component(hass):
"""Test if get_component works."""
assert http == loader.get_component(hass, 'http')
assert loader.get_component(self.hass, 'switch.test_set') is comp
def test_get_component(self):
"""Test if get_component works."""
assert http == loader.get_component(self.hass, 'http')
def test_component_dependencies(hass):
"""Test if we can get the proper load order of components."""
loader.set_component(hass, 'mod1', MockModule('mod1'))
loader.set_component(hass, 'mod2', MockModule('mod2', ['mod1']))
loader.set_component(hass, 'mod3', MockModule('mod3', ['mod2']))
def test_load_order_component(self):
"""Test if we can get the proper load order of components."""
loader.set_component(self.hass, 'mod1', MockModule('mod1'))
loader.set_component(self.hass, 'mod2', MockModule('mod2', ['mod1']))
loader.set_component(self.hass, 'mod3', MockModule('mod3', ['mod2']))
assert {'mod1', 'mod2', 'mod3'} == \
loader.component_dependencies(hass, 'mod3')
assert ['mod1', 'mod2', 'mod3'] == \
loader.load_order_component(self.hass, 'mod3')
# Create circular dependency
loader.set_component(hass, 'mod1', MockModule('mod1', ['mod3']))
# Create circular dependency
loader.set_component(self.hass, 'mod1', MockModule('mod1', ['mod3']))
with pytest.raises(loader.CircularDependency):
print(loader.component_dependencies(hass, 'mod3'))
assert [] == loader.load_order_component(self.hass, 'mod3')
# Depend on non-existing component
loader.set_component(hass, 'mod1',
MockModule('mod1', ['nonexisting']))
# Depend on non-existing component
loader.set_component(self.hass, 'mod1',
MockModule('mod1', ['nonexisting']))
with pytest.raises(loader.ComponentNotFound):
print(loader.component_dependencies(hass, 'mod1'))
assert [] == loader.load_order_component(self.hass, 'mod1')
# Try to get load order for non-existing component
assert [] == loader.load_order_component(self.hass, 'mod1')
# Try to get dependencies for non-existing component
with pytest.raises(loader.ComponentNotFound):
print(loader.component_dependencies(hass, 'nonexisting'))
def test_component_loader(hass):

View File

@ -105,67 +105,6 @@ class TestUtil(unittest.TestCase):
with pytest.raises(TypeError):
TestEnum.FIRST >= 1
def test_ordered_set(self):
"""Test ordering of set."""
set1 = util.OrderedSet([1, 2, 3, 4])
set2 = util.OrderedSet([3, 4, 5])
assert 4 == len(set1)
assert 3 == len(set2)
assert 1 in set1
assert 2 in set1
assert 3 in set1
assert 4 in set1
assert 5 not in set1
assert 1 not in set2
assert 2 not in set2
assert 3 in set2
assert 4 in set2
assert 5 in set2
set1.add(5)
assert 5 in set1
set1.discard(5)
assert 5 not in set1
# Try again while key is not in
set1.discard(5)
assert 5 not in set1
assert [1, 2, 3, 4] == list(set1)
assert [4, 3, 2, 1] == list(reversed(set1))
assert 1 == set1.pop(False)
assert [2, 3, 4] == list(set1)
assert 4 == set1.pop()
assert [2, 3] == list(set1)
assert 'OrderedSet()' == str(util.OrderedSet())
assert 'OrderedSet([2, 3])' == str(set1)
assert set1 == util.OrderedSet([2, 3])
assert set1 != util.OrderedSet([3, 2])
assert set1 == set([2, 3])
assert set1 == {3, 2}
assert set1 == [2, 3]
assert set1 == [3, 2]
assert set1 != {2}
set3 = util.OrderedSet(set1)
set3.update(set2)
assert [3, 4, 5, 2] == set3
assert [3, 4, 5, 2] == set1 | set2
assert [3] == set1 & set2
assert [2] == set1 - set2
set1.update([1, 2], [5, 6])
assert [2, 3, 1, 5, 6] == set1
def test_throttle(self):
"""Test the add cooldown decorator."""
calls1 = []