mirror of
https://github.com/home-assistant/core.git
synced 2025-04-25 17:57:55 +00:00
Load as many components in parallel as possible (#20806)
* Load as many components in parallel as possible * Lint
This commit is contained in:
Paulus Schoutsen
Pascal Vizeli
parent
f3b20d138e
commit
a9672b0d52
@ -10,7 +10,8 @@ from typing import Any, Optional, Dict
|
||||
import voluptuous as vol
|
||||
|
||||
from homeassistant import (
|
||||
core, config as conf_util, config_entries, components as core_components)
|
||||
core, config as conf_util, config_entries, components as core_components,
|
||||
loader)
|
||||
from homeassistant.components import persistent_notification
|
||||
from homeassistant.const import EVENT_HOMEASSISTANT_CLOSE
|
||||
from homeassistant.setup import async_setup_component
|
||||
@ -124,6 +125,15 @@ async def async_from_config_dict(config: Dict[str, Any],
|
||||
if key != core.DOMAIN)
|
||||
components.update(hass.config_entries.async_domains())
|
||||
|
||||
# Resolve all dependencies of all components.
|
||||
for component in list(components):
|
||||
try:
|
||||
components.update(loader.component_dependencies(hass, component))
|
||||
except loader.LoaderError:
|
||||
# Ignore it, or we'll break startup
|
||||
# It will be properly handled during setup.
|
||||
pass
|
||||
|
||||
# setup components
|
||||
res = await core_components.async_setup(hass, config)
|
||||
if not res:
|
||||
|
||||
@ -18,7 +18,6 @@ from types import ModuleType
|
||||
from typing import Optional, Set, TYPE_CHECKING, Callable, Any, TypeVar # noqa pylint: disable=unused-import
|
||||
|
||||
from homeassistant.const import PLATFORM_FORMAT
|
||||
from homeassistant.util import OrderedSet
|
||||
|
||||
# Typing imports that create a circular dependency
|
||||
# pylint: disable=using-constant-test,unused-import
|
||||
@ -39,6 +38,30 @@ PATH_CUSTOM_COMPONENTS = 'custom_components'
|
||||
PACKAGE_COMPONENTS = 'homeassistant.components'
|
||||
|
||||
|
||||
class LoaderError(Exception):
|
||||
"""Loader base error."""
|
||||
|
||||
|
||||
class ComponentNotFound(LoaderError):
|
||||
"""Raised when a component is not found."""
|
||||
|
||||
def __init__(self, domain: str) -> None:
|
||||
"""Initialize a component not found error."""
|
||||
super().__init__("Component {} not found.".format(domain))
|
||||
self.domain = domain
|
||||
|
||||
|
||||
class CircularDependency(LoaderError):
|
||||
"""Raised when a circular dependency is found when resolving components."""
|
||||
|
||||
def __init__(self, from_domain: str, to_domain: str) -> None:
|
||||
"""Initialize circular dependency error."""
|
||||
super().__init__("Circular dependency detected: {} -> {}.".format(
|
||||
from_domain, to_domain))
|
||||
self.from_domain = from_domain
|
||||
self.to_domain = to_domain
|
||||
|
||||
|
||||
def set_component(hass, # type: HomeAssistant
|
||||
comp_name: str, component: Optional[ModuleType]) -> None:
|
||||
"""Set a component in the cache.
|
||||
@ -235,57 +258,46 @@ def bind_hass(func: CALLABLE_T) -> CALLABLE_T:
|
||||
return func
|
||||
|
||||
|
||||
def load_order_component(hass, # type: HomeAssistant
|
||||
comp_name: str) -> OrderedSet:
|
||||
"""Return an OrderedSet of components in the correct order of loading.
|
||||
def component_dependencies(hass, # type: HomeAssistant
|
||||
comp_name: str) -> Set[str]:
|
||||
"""Return all dependencies and subdependencies of components.
|
||||
|
||||
Returns an empty list if a circular dependency is detected
|
||||
or the component could not be loaded. In both cases, the error is
|
||||
logged.
|
||||
Raises CircularDependency if a circular dependency is found.
|
||||
|
||||
Async friendly.
|
||||
"""
|
||||
return _load_order_component(hass, comp_name, OrderedSet(), set())
|
||||
return _component_dependencies(hass, comp_name, set(), set())
|
||||
|
||||
|
||||
def _load_order_component(hass, # type: HomeAssistant
|
||||
comp_name: str, load_order: OrderedSet,
|
||||
loading: Set) -> OrderedSet:
|
||||
"""Recursive function to get load order of components.
|
||||
def _component_dependencies(hass, # type: HomeAssistant
|
||||
comp_name: str, loaded: Set[str],
|
||||
loading: Set) -> Set[str]:
|
||||
"""Recursive function to get component dependencies.
|
||||
|
||||
Async friendly.
|
||||
"""
|
||||
component = get_component(hass, comp_name)
|
||||
|
||||
# If None it does not exist, error already thrown by get_component.
|
||||
if component is None:
|
||||
return OrderedSet()
|
||||
raise ComponentNotFound(comp_name)
|
||||
|
||||
loading.add(comp_name)
|
||||
|
||||
for dependency in getattr(component, 'DEPENDENCIES', []):
|
||||
# Check not already loaded
|
||||
if dependency in load_order:
|
||||
if dependency in loaded:
|
||||
continue
|
||||
|
||||
# If we are already loading it, we have a circular dependency.
|
||||
if dependency in loading:
|
||||
_LOGGER.error("Circular dependency detected: %s -> %s",
|
||||
comp_name, dependency)
|
||||
return OrderedSet()
|
||||
raise CircularDependency(comp_name, dependency)
|
||||
|
||||
dep_load_order = _load_order_component(
|
||||
hass, dependency, load_order, loading)
|
||||
dep_loaded = _component_dependencies(
|
||||
hass, dependency, loaded, loading)
|
||||
|
||||
# length == 0 means error loading dependency or children
|
||||
if not dep_load_order:
|
||||
_LOGGER.error("Error loading %s dependency: %s",
|
||||
comp_name, dependency)
|
||||
return OrderedSet()
|
||||
loaded.update(dep_loaded)
|
||||
|
||||
load_order.update(dep_load_order)
|
||||
|
||||
load_order.add(comp_name)
|
||||
loaded.add(comp_name)
|
||||
loading.remove(comp_name)
|
||||
|
||||
return load_order
|
||||
return loaded
|
||||
|
||||
@ -106,12 +106,18 @@ async def _async_setup_component(hass: core.HomeAssistant,
|
||||
log_error("Component not found.", False)
|
||||
return False
|
||||
|
||||
# Validate no circular dependencies
|
||||
components = loader.load_order_component(hass, domain)
|
||||
|
||||
# OrderedSet is empty if component or dependencies could not be resolved
|
||||
if not components:
|
||||
log_error("Unable to resolve component or dependencies.")
|
||||
# Validate all dependencies exist and there are no circular dependencies
|
||||
try:
|
||||
loader.component_dependencies(hass, domain)
|
||||
except loader.ComponentNotFound as err:
|
||||
_LOGGER.error(
|
||||
"Not setting up %s because we are unable to resolve "
|
||||
"(sub)dependency %s", domain, err.domain)
|
||||
return False
|
||||
except loader.CircularDependency as err:
|
||||
_LOGGER.error(
|
||||
"Not setting up %s because it contains a circular dependency: "
|
||||
"%s -> %s", domain, err.from_domain, err.to_domain)
|
||||
return False
|
||||
|
||||
processed_config = \
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
"""Helper methods for various modules."""
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from itertools import chain
|
||||
import threading
|
||||
import re
|
||||
import enum
|
||||
@ -141,96 +140,6 @@ class OrderedEnum(enum.Enum):
|
||||
return NotImplemented
|
||||
|
||||
|
||||
class OrderedSet(MutableSet[T]):
|
||||
"""Ordered set taken from http://code.activestate.com/recipes/576694/."""
|
||||
|
||||
def __init__(self, iterable: Optional[Iterable[T]] = None) -> None:
|
||||
"""Initialize the set."""
|
||||
self.end = end = [] # type: List[Any]
|
||||
end += [None, end, end] # sentinel node for doubly linked list
|
||||
self.map = {} # type: Dict[T, List] # key --> [key, prev, next]
|
||||
if iterable is not None:
|
||||
self |= iterable # type: ignore
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Return the length of the set."""
|
||||
return len(self.map)
|
||||
|
||||
def __contains__(self, key: T) -> bool: # type: ignore
|
||||
"""Check if key is in set."""
|
||||
return key in self.map
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
def add(self, key: T) -> None:
|
||||
"""Add an element to the end of the set."""
|
||||
if key not in self.map:
|
||||
end = self.end
|
||||
curr = end[1]
|
||||
curr[2] = end[1] = self.map[key] = [key, curr, end]
|
||||
|
||||
def promote(self, key: T) -> None:
|
||||
"""Promote element to beginning of the set, add if not there."""
|
||||
if key in self.map:
|
||||
self.discard(key)
|
||||
|
||||
begin = self.end[2]
|
||||
curr = begin[1]
|
||||
curr[2] = begin[1] = self.map[key] = [key, curr, begin]
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
def discard(self, key: T) -> None:
|
||||
"""Discard an element from the set."""
|
||||
if key in self.map:
|
||||
key, prev_item, next_item = self.map.pop(key)
|
||||
prev_item[2] = next_item
|
||||
next_item[1] = prev_item
|
||||
|
||||
def __iter__(self) -> Iterator[T]:
|
||||
"""Iterate of the set."""
|
||||
end = self.end
|
||||
curr = end[2]
|
||||
while curr is not end:
|
||||
yield curr[0]
|
||||
curr = curr[2]
|
||||
|
||||
def __reversed__(self) -> Iterator[T]:
|
||||
"""Reverse the ordering."""
|
||||
end = self.end
|
||||
curr = end[1]
|
||||
while curr is not end:
|
||||
yield curr[0]
|
||||
curr = curr[1]
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
def pop(self, last: bool = True) -> T:
|
||||
"""Pop element of the end of the set.
|
||||
|
||||
Set last=False to pop from the beginning.
|
||||
"""
|
||||
if not self:
|
||||
raise KeyError('set is empty')
|
||||
key = self.end[1][0] if last else self.end[2][0]
|
||||
self.discard(key)
|
||||
return key # type: ignore
|
||||
|
||||
def update(self, *args: Any) -> None:
|
||||
"""Add elements from args to the set."""
|
||||
for item in chain(*args):
|
||||
self.add(item)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Return the representation."""
|
||||
if not self:
|
||||
return '%s()' % (self.__class__.__name__,)
|
||||
return '%s(%r)' % (self.__class__.__name__, list(self))
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
"""Return the comparison."""
|
||||
if isinstance(other, OrderedSet):
|
||||
return len(self) == len(other) and list(self) == list(other)
|
||||
return set(self) == set(other)
|
||||
|
||||
|
||||
class Throttle:
|
||||
"""A class for throttling the execution of tasks.
|
||||
|
||||
|
||||
@ -1,63 +1,52 @@
|
||||
"""Test to verify that we can load components."""
|
||||
# pylint: disable=protected-access
|
||||
import asyncio
|
||||
import unittest
|
||||
|
||||
import pytest
|
||||
|
||||
import homeassistant.loader as loader
|
||||
import homeassistant.components.http as http
|
||||
|
||||
from tests.common import (
|
||||
get_test_home_assistant, MockModule, async_mock_service)
|
||||
from tests.common import MockModule, async_mock_service
|
||||
|
||||
|
||||
class TestLoader(unittest.TestCase):
|
||||
"""Test the loader module."""
|
||||
def test_set_component(hass):
|
||||
"""Test if set_component works."""
|
||||
comp = object()
|
||||
loader.set_component(hass, 'switch.test_set', comp)
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
def setUp(self):
|
||||
"""Set up tests."""
|
||||
self.hass = get_test_home_assistant()
|
||||
assert loader.get_component(hass, 'switch.test_set') is comp
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
def tearDown(self):
|
||||
"""Stop everything that was started."""
|
||||
self.hass.stop()
|
||||
|
||||
def test_set_component(self):
|
||||
"""Test if set_component works."""
|
||||
comp = object()
|
||||
loader.set_component(self.hass, 'switch.test_set', comp)
|
||||
def test_get_component(hass):
|
||||
"""Test if get_component works."""
|
||||
assert http == loader.get_component(hass, 'http')
|
||||
|
||||
assert loader.get_component(self.hass, 'switch.test_set') is comp
|
||||
|
||||
def test_get_component(self):
|
||||
"""Test if get_component works."""
|
||||
assert http == loader.get_component(self.hass, 'http')
|
||||
def test_component_dependencies(hass):
|
||||
"""Test if we can get the proper load order of components."""
|
||||
loader.set_component(hass, 'mod1', MockModule('mod1'))
|
||||
loader.set_component(hass, 'mod2', MockModule('mod2', ['mod1']))
|
||||
loader.set_component(hass, 'mod3', MockModule('mod3', ['mod2']))
|
||||
|
||||
def test_load_order_component(self):
|
||||
"""Test if we can get the proper load order of components."""
|
||||
loader.set_component(self.hass, 'mod1', MockModule('mod1'))
|
||||
loader.set_component(self.hass, 'mod2', MockModule('mod2', ['mod1']))
|
||||
loader.set_component(self.hass, 'mod3', MockModule('mod3', ['mod2']))
|
||||
assert {'mod1', 'mod2', 'mod3'} == \
|
||||
loader.component_dependencies(hass, 'mod3')
|
||||
|
||||
assert ['mod1', 'mod2', 'mod3'] == \
|
||||
loader.load_order_component(self.hass, 'mod3')
|
||||
# Create circular dependency
|
||||
loader.set_component(hass, 'mod1', MockModule('mod1', ['mod3']))
|
||||
|
||||
# Create circular dependency
|
||||
loader.set_component(self.hass, 'mod1', MockModule('mod1', ['mod3']))
|
||||
with pytest.raises(loader.CircularDependency):
|
||||
print(loader.component_dependencies(hass, 'mod3'))
|
||||
|
||||
assert [] == loader.load_order_component(self.hass, 'mod3')
|
||||
# Depend on non-existing component
|
||||
loader.set_component(hass, 'mod1',
|
||||
MockModule('mod1', ['nonexisting']))
|
||||
|
||||
# Depend on non-existing component
|
||||
loader.set_component(self.hass, 'mod1',
|
||||
MockModule('mod1', ['nonexisting']))
|
||||
with pytest.raises(loader.ComponentNotFound):
|
||||
print(loader.component_dependencies(hass, 'mod1'))
|
||||
|
||||
assert [] == loader.load_order_component(self.hass, 'mod1')
|
||||
|
||||
# Try to get load order for non-existing component
|
||||
assert [] == loader.load_order_component(self.hass, 'mod1')
|
||||
# Try to get dependencies for non-existing component
|
||||
with pytest.raises(loader.ComponentNotFound):
|
||||
print(loader.component_dependencies(hass, 'nonexisting'))
|
||||
|
||||
|
||||
def test_component_loader(hass):
|
||||
|
||||
@ -105,67 +105,6 @@ class TestUtil(unittest.TestCase):
|
||||
with pytest.raises(TypeError):
|
||||
TestEnum.FIRST >= 1
|
||||
|
||||
def test_ordered_set(self):
|
||||
"""Test ordering of set."""
|
||||
set1 = util.OrderedSet([1, 2, 3, 4])
|
||||
set2 = util.OrderedSet([3, 4, 5])
|
||||
|
||||
assert 4 == len(set1)
|
||||
assert 3 == len(set2)
|
||||
|
||||
assert 1 in set1
|
||||
assert 2 in set1
|
||||
assert 3 in set1
|
||||
assert 4 in set1
|
||||
assert 5 not in set1
|
||||
|
||||
assert 1 not in set2
|
||||
assert 2 not in set2
|
||||
assert 3 in set2
|
||||
assert 4 in set2
|
||||
assert 5 in set2
|
||||
|
||||
set1.add(5)
|
||||
assert 5 in set1
|
||||
|
||||
set1.discard(5)
|
||||
assert 5 not in set1
|
||||
|
||||
# Try again while key is not in
|
||||
set1.discard(5)
|
||||
assert 5 not in set1
|
||||
|
||||
assert [1, 2, 3, 4] == list(set1)
|
||||
assert [4, 3, 2, 1] == list(reversed(set1))
|
||||
|
||||
assert 1 == set1.pop(False)
|
||||
assert [2, 3, 4] == list(set1)
|
||||
|
||||
assert 4 == set1.pop()
|
||||
assert [2, 3] == list(set1)
|
||||
|
||||
assert 'OrderedSet()' == str(util.OrderedSet())
|
||||
assert 'OrderedSet([2, 3])' == str(set1)
|
||||
|
||||
assert set1 == util.OrderedSet([2, 3])
|
||||
assert set1 != util.OrderedSet([3, 2])
|
||||
assert set1 == set([2, 3])
|
||||
assert set1 == {3, 2}
|
||||
assert set1 == [2, 3]
|
||||
assert set1 == [3, 2]
|
||||
assert set1 != {2}
|
||||
|
||||
set3 = util.OrderedSet(set1)
|
||||
set3.update(set2)
|
||||
|
||||
assert [3, 4, 5, 2] == set3
|
||||
assert [3, 4, 5, 2] == set1 | set2
|
||||
assert [3] == set1 & set2
|
||||
assert [2] == set1 - set2
|
||||
|
||||
set1.update([1, 2], [5, 6])
|
||||
assert [2, 3, 1, 5, 6] == set1
|
||||
|
||||
def test_throttle(self):
|
||||
"""Test the add cooldown decorator."""
|
||||
calls1 = []
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user