diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index b81f7423f30..8ae1023e1a9 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -4,6 +4,7 @@ async_timeout==3.0.1 attrs==19.1.0 bcrypt==3.1.6 certifi>=2018.04.16 +importlib-metadata==0.15 jinja2>=2.10 PyJWT==1.7.1 cryptography==2.6.1 diff --git a/homeassistant/requirements.py b/homeassistant/requirements.py index 5039fbbd41e..3a1081e4b87 100644 --- a/homeassistant/requirements.py +++ b/homeassistant/requirements.py @@ -3,11 +3,7 @@ import asyncio from functools import partial import logging import os -import sys from typing import Any, Dict, List, Optional -from urllib.parse import urlparse - -import pkg_resources import homeassistant.util.package as pkg_util from homeassistant.core import HomeAssistant @@ -28,16 +24,12 @@ async def async_process_requirements(hass: HomeAssistant, name: str, if pip_lock is None: pip_lock = hass.data[DATA_PIP_LOCK] = asyncio.Lock() - pkg_cache = hass.data.get(DATA_PKG_CACHE) - if pkg_cache is None: - pkg_cache = hass.data[DATA_PKG_CACHE] = PackageLoadable(hass) - pip_install = partial(pkg_util.install_package, **pip_kwargs(hass.config.config_dir)) async with pip_lock: for req in requirements: - if await pkg_cache.loadable(req): + if pkg_util.is_installed(req): continue ret = await hass.async_add_executor_job(pip_install, req) @@ -58,50 +50,3 @@ def pip_kwargs(config_dir: Optional[str]) -> Dict[str, Any]: if not (config_dir is None or pkg_util.is_virtual_env()): kwargs['target'] = os.path.join(config_dir, 'deps') return kwargs - - -class PackageLoadable: - """Class to check if a package is loadable, with built-in cache.""" - - def __init__(self, hass: HomeAssistant) -> None: - """Initialize the PackageLoadable class.""" - self.dist_cache = {} # type: Dict[str, pkg_resources.Distribution] - self.hass = hass - - async def loadable(self, package: str) -> bool: - """Check if a package is what will be loaded when we import it. - - Returns True when the requirement is met. - Returns False when the package is not installed or doesn't meet req. - """ - dist_cache = self.dist_cache - - try: - req = pkg_resources.Requirement.parse(package) - except ValueError: - # This is a zip file. We no longer use this in Home Assistant, - # leaving it in for custom components. - req = pkg_resources.Requirement.parse(urlparse(package).fragment) - - req_proj_name = req.project_name.lower() - dist = dist_cache.get(req_proj_name) - - if dist is not None: - return dist in req - - for path in sys.path: - # We read the whole mount point as we're already here - # Caching it on first call makes subsequent calls a lot faster. - await self.hass.async_add_executor_job(self._fill_cache, path) - - dist = dist_cache.get(req_proj_name) - if dist is not None: - return dist in req - - return False - - def _fill_cache(self, path: str) -> None: - """Add packages from a path to the cache.""" - dist_cache = self.dist_cache - for dist in pkg_resources.find_distributions(path): - dist_cache.setdefault(dist.project_name.lower(), dist) diff --git a/homeassistant/scripts/__init__.py b/homeassistant/scripts/__init__.py index 070d907a7d9..961ce5a9d13 100644 --- a/homeassistant/scripts/__init__.py +++ b/homeassistant/scripts/__init__.py @@ -9,9 +9,9 @@ from typing import List from homeassistant.bootstrap import async_mount_local_lib_path from homeassistant.config import get_default_config_dir -from homeassistant.core import HomeAssistant -from homeassistant.requirements import pip_kwargs, PackageLoadable -from homeassistant.util.package import install_package, is_virtual_env +from homeassistant.requirements import pip_kwargs +from homeassistant.util.package import ( + install_package, is_virtual_env, is_installed) def run(args: List) -> int: @@ -49,10 +49,8 @@ def run(args: List) -> int: logging.basicConfig(stream=sys.stdout, level=logging.INFO) - hass = HomeAssistant(loop) - pkgload = PackageLoadable(hass) for req in getattr(script, 'REQUIREMENTS', []): - if loop.run_until_complete(pkgload.loadable(req)): + if is_installed(req): continue if not install_package(req, **_pip_kwargs): diff --git a/homeassistant/util/package.py b/homeassistant/util/package.py index 925755eb741..3a34ab0a365 100644 --- a/homeassistant/util/package.py +++ b/homeassistant/util/package.py @@ -5,6 +5,11 @@ import os from subprocess import PIPE, Popen import sys from typing import Optional +from urllib.parse import urlparse + +import pkg_resources +from importlib_metadata import version, PackageNotFoundError + _LOGGER = logging.getLogger(__name__) @@ -16,6 +21,25 @@ def is_virtual_env() -> bool: hasattr(sys, 'real_prefix')) +def is_installed(package: str) -> bool: + """Check if a package is installed and will be loaded when we import it. + + Returns True when the requirement is met. + Returns False when the package is not installed or doesn't meet req. + """ + try: + req = pkg_resources.Requirement.parse(package) + except ValueError: + # This is a zip file. We no longer use this in Home Assistant, + # leaving it in for custom components. + req = pkg_resources.Requirement.parse(urlparse(package).fragment) + + try: + return version(req.project_name) in req + except PackageNotFoundError: + return False + + def install_package(package: str, upgrade: bool = True, target: Optional[str] = None, constraints: Optional[str] = None) -> bool: diff --git a/requirements_all.txt b/requirements_all.txt index 81582926d95..31569b62a47 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -5,6 +5,7 @@ async_timeout==3.0.1 attrs==19.1.0 bcrypt==3.1.6 certifi>=2018.04.16 +importlib-metadata==0.15 jinja2>=2.10 PyJWT==1.7.1 cryptography==2.6.1 diff --git a/setup.py b/setup.py index b1b66e0ca01..2ae5d8e8c3b 100755 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ REQUIRES = [ 'attrs==19.1.0', 'bcrypt==3.1.6', 'certifi>=2018.04.16', + 'importlib-metadata==0.15', 'jinja2>=2.10', 'PyJWT==1.7.1', # PyJWT has loose dependency. We want the latest one. diff --git a/tests/test_requirements.py b/tests/test_requirements.py index dcc107ea07e..c061e37ca0a 100644 --- a/tests/test_requirements.py +++ b/tests/test_requirements.py @@ -4,21 +4,11 @@ from unittest.mock import patch, call from homeassistant import setup from homeassistant.requirements import ( - CONSTRAINT_FILE, PackageLoadable, async_process_requirements) - -import pkg_resources + CONSTRAINT_FILE, async_process_requirements) from tests.common import ( get_test_home_assistant, MockModule, mock_coro, mock_integration) -RESOURCE_DIR = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', 'resources')) - -TEST_NEW_REQ = 'pyhelloworld3==1.0.0' - -TEST_ZIP_REQ = 'file://{}#{}' \ - .format(os.path.join(RESOURCE_DIR, 'pyhelloworld3.zip'), TEST_NEW_REQ) - class TestRequirements: """Test the requirements module.""" @@ -80,47 +70,10 @@ async def test_install_existing_package(hass): assert len(mock_inst.mock_calls) == 1 - with patch('homeassistant.requirements.PackageLoadable.loadable', - return_value=mock_coro(True)), \ + with patch('homeassistant.util.package.is_installed', return_value=True), \ patch( 'homeassistant.util.package.install_package') as mock_inst: assert await async_process_requirements( hass, 'test_component', ['hello==1.0.0']) assert len(mock_inst.mock_calls) == 0 - - -async def test_check_package_global(hass): - """Test for an installed package.""" - installed_package = list(pkg_resources.working_set)[0].project_name - assert await PackageLoadable(hass).loadable(installed_package) - - -async def test_check_package_zip(hass): - """Test for an installed zip package.""" - assert not await PackageLoadable(hass).loadable(TEST_ZIP_REQ) - - -async def test_package_loadable_installed_twice(hass): - """Test that a package is loadable when installed twice. - - If a package is installed twice, only the first version will be imported. - Test that package_loadable will only compare with the first package. - """ - v1 = pkg_resources.Distribution(project_name='hello', version='1.0.0') - v2 = pkg_resources.Distribution(project_name='hello', version='2.0.0') - - with patch('pkg_resources.find_distributions', side_effect=[[v1]]): - assert not await PackageLoadable(hass).loadable('hello==2.0.0') - - with patch('pkg_resources.find_distributions', side_effect=[[v1], [v2]]): - assert not await PackageLoadable(hass).loadable('hello==2.0.0') - - with patch('pkg_resources.find_distributions', side_effect=[[v2], [v1]]): - assert await PackageLoadable(hass).loadable('hello==2.0.0') - - with patch('pkg_resources.find_distributions', side_effect=[[v2]]): - assert await PackageLoadable(hass).loadable('hello==2.0.0') - - with patch('pkg_resources.find_distributions', side_effect=[[v2]]): - assert await PackageLoadable(hass).loadable('Hello==2.0.0') diff --git a/tests/util/test_package.py b/tests/util/test_package.py index 5422140c232..41af56265d2 100644 --- a/tests/util/test_package.py +++ b/tests/util/test_package.py @@ -6,13 +6,20 @@ import sys from subprocess import PIPE from unittest.mock import MagicMock, call, patch +import pkg_resources import pytest import homeassistant.util.package as package +RESOURCE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', 'resources')) + TEST_NEW_REQ = 'pyhelloworld3==1.0.0' +TEST_ZIP_REQ = 'file://{}#{}' \ + .format(os.path.join(RESOURCE_DIR, 'pyhelloworld3.zip'), TEST_NEW_REQ) + @pytest.fixture def mock_sys(): @@ -176,3 +183,14 @@ def test_async_get_user_site(mock_env_copy): stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env) assert ret == os.path.join(deps_dir, 'lib_dir') + + +def test_check_package_global(): + """Test for an installed package.""" + installed_package = list(pkg_resources.working_set)[0].project_name + assert package.is_installed(installed_package) + + +def test_check_package_zip(): + """Test for an installed zip package.""" + assert not package.is_installed(TEST_ZIP_REQ)