Install requirements for all deps with tests (#27362)

* Install requirements for all deps with tests

* Remove unused REQUIREMENTS var

* Print diff if not the same

* Simplify

* Update command line

* Fix detecting empty dirs

* Install non-integration

* Fix upnp tests

* Lint

* Fix ZHA test
This commit is contained in:
Paulus Schoutsen 2019-10-09 16:16:29 -07:00 committed by GitHub
parent 74ef1358da
commit 54c24de158
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 284 additions and 252 deletions

View File

@ -10,8 +10,6 @@ from homeassistant.exceptions import PlatformNotReady
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity import Entity
REQUIREMENTS = ["epsonprinter==0.0.9"]
_LOGGER = logging.getLogger(__name__)
MONITORED_CONDITIONS = {
"black": ["Ink level Black", "%", "mdi:water"],

View File

@ -19,8 +19,6 @@ import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.dispatcher import async_dispatcher_connect, dispatcher_send
from homeassistant.helpers.event import track_time_interval
REQUIREMENTS = ["georss_ign_sismologia_client==0.2"]
_LOGGER = logging.getLogger(__name__)
ATTR_EXTERNAL_ID = "external_id"

View File

@ -9,8 +9,6 @@ import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.discovery import load_platform
from homeassistant.helpers.entity import Entity
REQUIREMENTS = ["pysupla==0.0.3"]
_LOGGER = logging.getLogger(__name__)
DOMAIN = "supla"

View File

@ -3,6 +3,7 @@ import asyncio
from ipaddress import IPv4Address
import aiohttp
from async_upnp_client.profiles.igd import IgdDevice
from homeassistant.helpers.aiohttp_client import async_get_clientsession
from homeassistant.helpers.typing import HomeAssistantType
@ -29,9 +30,6 @@ class Device:
if local_ip:
local_ip = IPv4Address(local_ip)
# discover devices
from async_upnp_client.profiles.igd import IgdDevice
discovery_infos = await IgdDevice.async_search(source_ip=local_ip, timeout=10)
# add extra info and store devices
@ -61,9 +59,6 @@ class Device:
factory = UpnpFactory(requester, disable_state_variable_validation=True)
upnp_device = await factory.async_create_device(ssdp_description)
# wrap with async_upnp_client.IgdDevice
from async_upnp_client.profiles.igd import IgdDevice
igd_device = IgdDevice(upnp_device, None)
return cls(igd_device)

View File

@ -33,6 +33,3 @@ enum34==1000000000.0.0
# This is a old unmaintained library and is replaced with pycryptodome
pycrypto==1000000000.0.0
# Contains code to modify Home Assistant to work around our rules
python-systemair-savecair==1000000000.0.0

View File

@ -30,15 +30,24 @@ HAP-python==2.6.0
# homeassistant.components.owntracks
PyNaCl==1.3.0
# homeassistant.auth.mfa_modules.totp
PyQRCode==1.2.1
# homeassistant.components.rmvtransport
PyRMVtransport==0.1.3
# homeassistant.components.transport_nsw
PyTransportNSW==0.1.1
# homeassistant.components.remember_the_milk
RtmAPI==0.7.0
# homeassistant.components.yessssms
YesssSMS==0.4.1
# homeassistant.components.androidtv
adb-shell==0.0.4
# homeassistant.components.adguard
adguardhome==0.2.1
@ -48,6 +57,9 @@ aio_geojson_geonetnz_quakes==0.10
# homeassistant.components.ambient_station
aioambient==0.3.2
# homeassistant.components.asuswrt
aioasuswrt==1.1.21
# homeassistant.components.automatic
aioautomatic==0.6.5
@ -91,6 +103,13 @@ apns2==0.3.0
# homeassistant.components.aprs
aprslib==0.6.46
# homeassistant.components.arcam_fmj
arcam-fmj==0.4.3
# homeassistant.components.dlna_dmr
# homeassistant.components.upnp
async-upnp-client==0.14.11
# homeassistant.components.stream
av==6.1.2
@ -100,17 +119,46 @@ axis==25
# homeassistant.components.zha
bellows-homeassistant==0.10.0
# homeassistant.components.bom
bomradarloop==0.1.3
# homeassistant.components.broadlink
broadlink==0.12.0
# homeassistant.components.buienradar
buienradar==1.0.1
# homeassistant.components.caldav
caldav==0.6.1
# homeassistant.components.coinmarketcap
coinmarketcap==5.0.3
# homeassistant.scripts.check_config
colorlog==4.0.2
# homeassistant.components.eddystone_temperature
# homeassistant.components.eq3btsmart
# homeassistant.components.xiaomi_miio
construct==2.9.45
# homeassistant.scripts.credstash
# credstash==1.15.0
# homeassistant.components.datadog
datadog==0.15.0
# homeassistant.components.ihc
# homeassistant.components.namecheapdns
# homeassistant.components.ohmconnect
defusedxml==0.6.0
# homeassistant.components.directv
directpy==0.5
# homeassistant.components.updater
distro==1.4.0
# homeassistant.components.dsmr
dsmr_parser==0.12
@ -120,9 +168,6 @@ eebrightbox==0.0.4
# homeassistant.components.emulated_roku
emulated_roku==0.1.8
# homeassistant.components.enocean
enocean==0.50
# homeassistant.components.season
ephem==3.7.6.0
@ -160,6 +205,9 @@ getmac==0.8.1
# homeassistant.components.google
google-api-python-client==1.6.4
# homeassistant.components.google_pubsub
google-cloud-pubsub==0.39.1
# homeassistant.components.ffmpeg
ha-ffmpeg==2.0
@ -187,6 +235,9 @@ holidays==0.9.11
# homeassistant.components.frontend
home-assistant-frontend==20191002.2
# homeassistant.components.zwave
homeassistant-pyozw==0.1.4
# homeassistant.components.homekit_controller
homekit[IP]==0.15.0
@ -209,12 +260,21 @@ influxdb==5.2.3
# homeassistant.components.verisure
jsonpath==0.75
# homeassistant.scripts.keyring
keyring==17.1.1
# homeassistant.scripts.keyring
keyrings.alt==3.1.1
# homeassistant.components.dyson
libpurecool==0.5.0
# homeassistant.components.soundtouch
libsoundtouch==0.7.2
# homeassistant.components.logi_circle
logi_circle==0.2.2
# homeassistant.components.luftdaten
luftdaten==0.6.3
@ -227,10 +287,22 @@ mficlient==0.3.0
# homeassistant.components.minio
minio==4.0.9
# homeassistant.components.tts
mutagen==1.42.0
# homeassistant.components.ness_alarm
nessclient==0.9.15
# homeassistant.components.discovery
# homeassistant.components.ssdp
netdisco==2.6.0
# homeassistant.components.nsw_fuel_station
nsw-fuel-api-client==1.0.10
# homeassistant.components.nuheat
nuheat==0.3.0
# homeassistant.components.iqvia
# homeassistant.components.opencv
# homeassistant.components.tensorflow
@ -268,6 +340,12 @@ plexauth==0.0.4
# homeassistant.components.serial_pm
pmsensor==0.4
# homeassistant.components.reddit
praw==6.3.1
# homeassistant.components.islamic_prayer_times
prayer_times_calculator==0.0.3
# homeassistant.components.prometheus
prometheus_client==0.7.1
@ -280,6 +358,9 @@ pushbullet.py==0.11.0
# homeassistant.components.canary
py-canary==0.5.0
# homeassistant.components.melissa
py-melissa-climate==2.0.0
# homeassistant.components.seventeentrack
py17track==2.2.2
@ -290,6 +371,15 @@ pyHS100==0.3.5
# homeassistant.components.norway_air
pyMetno==0.4.6
# homeassistant.components.rfxtrx
pyRFXtrx==0.23
# homeassistant.components.nextbus
py_nextbusnext==0.1.4
# homeassistant.components.arlo
pyarlo==0.2.3
# homeassistant.components.blackbird
pyblackbird==0.5
@ -299,30 +389,69 @@ pybotvac==0.0.16
# homeassistant.components.cast
pychromecast==4.0.1
# homeassistant.components.daikin
pydaikin==1.6.1
# homeassistant.components.deconz
pydeconz==64
# homeassistant.components.zwave
pydispatcher==2.0.5
# homeassistant.components.everlights
pyeverlights==0.1.0
# homeassistant.components.fido
pyfido==2.1.1
# homeassistant.components.fritzbox
pyfritzhome==0.4.0
# homeassistant.components.ifttt
pyfttt==0.3
# homeassistant.components.version
pyhaversion==3.1.0
# homeassistant.components.heos
pyheos==0.6.0
# homeassistant.components.homematic
pyhomematic==0.1.60
# homeassistant.components.hydroquebec
pyhydroquebec==2.2.2
# homeassistant.components.ipma
pyipma==1.2.1
# homeassistant.components.iqvia
pyiqvia==0.2.1
# homeassistant.components.kira
pykira==0.1.1
# homeassistant.components.webostv
pylgtv==0.1.9
# homeassistant.components.linky
pylinky==0.4.0
# homeassistant.components.litejet
pylitejet==0.1
# homeassistant.components.mailgun
pymailgunner==1.4
# homeassistant.components.somfy
pymfy==0.5.2
# homeassistant.components.mochad
pymochad==0.2.0
# homeassistant.components.modbus
pymodbus==1.5.2
# homeassistant.components.monoprice
pymonoprice==0.3
@ -343,6 +472,9 @@ pyotgw==0.5b0
# homeassistant.components.otp
pyotp==2.3.0
# homeassistant.components.point
pypoint==1.1.1
# homeassistant.components.ps4
pyps4-2ndscreen==1.0.1
@ -376,6 +508,9 @@ python-forecastio==1.4.0
# homeassistant.components.izone
python-izone==1.1.1
# homeassistant.components.xiaomi_miio
python-miio==0.4.6
# homeassistant.components.nest
python-nest==4.1.0
@ -385,6 +520,9 @@ python-velbus==2.0.27
# homeassistant.components.awair
python_awair==0.0.4
# homeassistant.components.traccar
pytraccar==0.9.0
# homeassistant.components.tradfri
pytradfri[async]==6.3.1
@ -409,6 +547,9 @@ ring_doorbell==0.2.3
# homeassistant.components.yamaha
rxv==0.6.0
# homeassistant.components.samsungtv
samsungctl[websocket]==0.7.1
# homeassistant.components.simplisafe
simplisafe-python==5.0.1
@ -431,15 +572,29 @@ sqlalchemy==1.3.9
# homeassistant.components.statsd
statsd==3.2.1
# homeassistant.components.solaredge
# homeassistant.components.thermoworks_smoke
# homeassistant.components.traccar
stringcase==1.2.0
# homeassistant.components.tellduslive
tellduslive==0.10.10
# homeassistant.components.toon
toonapilib==3.2.4
# homeassistant.components.tplink
tplink==0.2.1
# homeassistant.components.transmission
transmissionrpc==0.11
# homeassistant.components.twentemilieu
twentemilieu==0.1.0
# homeassistant.components.twilio
twilio==6.19.1
# homeassistant.components.uvc
uvcclient==0.11.0
@ -454,11 +609,42 @@ vultr==0.1.2
# homeassistant.components.wake_on_lan
wakeonlan==1.1.6
# homeassistant.components.folder_watcher
watchdog==0.8.3
# homeassistant.components.webostv
websockets==6.0
# homeassistant.components.withings
withings-api==2.0.0b7
# homeassistant.components.bluesound
# homeassistant.components.startca
# homeassistant.components.ted5000
# homeassistant.components.yr
# homeassistant.components.zestimate
xmltodict==0.12.0
# homeassistant.components.yandex_transport
ya_ma==0.3.7
# homeassistant.components.yweather
yahooweather==0.10
# homeassistant.components.zeroconf
zeroconf==0.23.0
# homeassistant.components.zha
zha-quirks==0.0.26
# homeassistant.components.zha
zigpy-deconz==0.5.0
# homeassistant.components.zha
zigpy-homeassistant==0.9.0
# homeassistant.components.zha
zigpy-xbee-homeassistant==0.5.0
# homeassistant.components.zha
zigpy-zigate==0.4.1

View File

@ -1,8 +1,9 @@
#!/usr/bin/env python3
"""Generate an updated requirements_all.txt."""
import difflib
import importlib
import os
import pathlib
from pathlib import Path
import pkgutil
import re
import sys
@ -41,159 +42,8 @@ COMMENT_REQUIREMENTS = (
"VL53L1X2",
)
TEST_REQUIREMENTS = (
"adguardhome",
"aio_geojson_geonetnz_quakes",
"aioambient",
"aioautomatic",
"aiobotocore",
"aioesphomeapi",
"aiohttp_cors",
"aiohue",
"aionotion",
"aioswitcher",
"aiounifi",
"aiowwlln",
"airly",
"ambiclimate",
"androidtv",
"apns2",
"aprslib",
"av",
"axis",
"bellows-homeassistant",
"caldav",
"coinmarketcap",
"defusedxml",
"dsmr_parser",
"eebrightbox",
"emulated_roku",
"enocean",
"ephem",
"evohomeclient",
"feedparser-homeassistant",
"foobot_async",
"geojson_client",
"geopy",
"georss_generic_client",
"georss_ign_sismologia_client",
"georss_qld_bushfire_alert_client",
"getmac",
"google-api-python-client",
"gTTS-token",
"ha-ffmpeg",
"hangups",
"HAP-python",
"hass-nabucasa",
"haversine",
"hbmqtt",
"hdate",
"herepy",
"hole",
"holidays",
"home-assistant-frontend",
"homekit[IP]",
"homematicip",
"httplib2",
"huawei-lte-api",
"iaqualink",
"influxdb",
"jsonpath",
"libpurecool",
"libsoundtouch",
"luftdaten",
"mbddns",
"mficlient",
"minio",
"netdisco",
"numpy",
"oauth2client",
"paho-mqtt",
"pexpect",
"pilight",
"pillow",
"plexapi",
"plexauth",
"pmsensor",
"prometheus_client",
"ptvsd",
"pushbullet.py",
"py-canary",
"py17track",
"pyblackbird",
"pybotvac",
"pychromecast",
"pydeconz",
"pydispatcher",
"pyheos",
"pyhomematic",
"pyHS100",
"pyiqvia",
"pylinky",
"pylitejet",
"pyMetno",
"pymfy",
"pymonoprice",
"PyNaCl",
"pynws",
"pynx584",
"pyopenuv",
"pyotgw",
"pyotp",
"pyps4-2ndscreen",
"pyqwikswitch",
"PyRMVtransport",
"pysma",
"pysmartapp",
"pysmartthings",
"pysoma",
"pysonos",
"pyspcwebgw",
"python_awair",
"python-ecobee-api",
"python-forecastio",
"python-izone",
"python-nest",
"python-velbus",
"pythonwhois",
"pytradfri[async]",
"PyTransportNSW",
"pyunifi",
"pyupnp-async",
"pyvesync",
"pywebpush",
"regenmaschine",
"restrictedpython",
"rflink",
"ring_doorbell",
"ruamel.yaml",
"rxv",
"simplisafe-python",
"sleepyq",
"smhi-pkg",
"solaredge",
"somecomfort",
"sqlalchemy",
"srpenergy",
"statsd",
"toonapilib",
"transmissionrpc",
"twentemilieu",
"uvcclient",
"vsure",
"vultr",
"wakeonlan",
"warrant",
"withings-api",
"YesssSMS",
"zeroconf",
"zigpy-homeassistant",
)
IGNORE_PIN = ("colorlog>2.1,<3", "keyring>=9.3,<10.0", "urllib3")
IGNORE_REQ = ("colorama<=1",) # Windows only requirement in check_config
URL_PIN = (
"https://developers.home-assistant.io/docs/"
"creating_platform_code_review.html#1-requirements"
@ -211,12 +61,31 @@ enum34==1000000000.0.0
# This is a old unmaintained library and is replaced with pycryptodome
pycrypto==1000000000.0.0
# Contains code to modify Home Assistant to work around our rules
python-systemair-savecair==1000000000.0.0
"""
def has_tests(module: str):
"""Test if a module has tests.
Module format: homeassistant.components.hue
Test if exists: tests/components/hue
"""
path = Path(module.replace(".", "/").replace("homeassistant", "tests"))
if not path.exists():
return False
if not path.is_dir():
return True
# Dev environments might have stale directories around
# from removed tests. Check for that.
content = [f.name for f in path.glob("*")]
# Directories need to contain more than `__pycache__`
# to exist in Git and so be seen by CI.
return content != ["__pycache__"]
def explore_module(package, explore_children):
"""Explore the modules."""
module = importlib.import_module(package)
@ -237,8 +106,9 @@ def explore_module(package, explore_children):
def core_requirements():
"""Gather core requirements out of setup.py."""
with open("setup.py") as inp:
reqs_raw = re.search(r"REQUIRES = \[(.*?)\]", inp.read(), re.S).group(1)
reqs_raw = re.search(
r"REQUIRES = \[(.*?)\]", Path("setup.py").read_text(), re.S
).group(1)
return [x[1] for x in re.findall(r"(['\"])(.*?)\1", reqs_raw)]
@ -248,7 +118,7 @@ def gather_recursive_requirements(domain, seen=None):
seen = set()
seen.add(domain)
integration = Integration(pathlib.Path(f"homeassistant/components/{domain}"))
integration = Integration(Path(f"homeassistant/components/{domain}"))
integration.load_manifest()
reqs = set(integration.manifest["requirements"])
for dep_domain in integration.manifest["dependencies"]:
@ -283,7 +153,7 @@ def gather_modules():
def gather_requirements_from_manifests(errors, reqs):
"""Gather all of the requirements from manifests."""
integrations = Integration.load_dir(pathlib.Path("homeassistant/components"))
integrations = Integration.load_dir(Path("homeassistant/components"))
for domain in sorted(integrations):
integration = integrations[domain]
@ -319,8 +189,6 @@ def gather_requirements_from_modules(errors, reqs):
def process_requirements(errors, module_requirements, package, reqs):
"""Process all of the requirements."""
for req in module_requirements:
if req in IGNORE_REQ:
continue
if "://" in req:
errors.append(f"{package}[Only pypi dependencies are allowed: {req}]")
if req.partition("==")[1] == "" and req not in IGNORE_PIN:
@ -359,15 +227,18 @@ def requirements_test_output(reqs):
output = []
output.append("# Home Assistant test")
output.append("\n")
with open("requirements_test.txt") as test_file:
output.append(test_file.read())
output.append(Path("requirements_test.txt").read_text())
output.append("\n")
filtered = {
key: value
for key, value in reqs.items()
requirement: modules
for requirement, modules in reqs.items()
if any(
re.search(r"(^|#){}($|[=><])".format(re.escape(ign)), key) is not None
for ign in TEST_REQUIREMENTS
# Always install requirements that are not part of integrations
not mdl.startswith("homeassistant.components.") or
# Install tests for integrations that have tests
has_tests(mdl)
for mdl in modules
)
}
output.append(generate_requirements_list(filtered))
@ -377,48 +248,28 @@ def requirements_test_output(reqs):
def gather_constraints():
"""Construct output for constraint file."""
return "\n".join(
sorted(
core_requirements() + list(gather_recursive_requirements("default_config"))
return (
"\n".join(
sorted(
core_requirements()
+ list(gather_recursive_requirements("default_config"))
)
+ [""]
)
+ [""]
+ CONSTRAINT_BASE
)
def write_requirements_file(data):
"""Write the modules to the requirements_all.txt."""
with open("requirements_all.txt", "w+", newline="\n") as req_file:
req_file.write(data)
def write_test_requirements_file(data):
"""Write the modules to the requirements_test_all.txt."""
with open("requirements_test_all.txt", "w+", newline="\n") as req_file:
req_file.write(data)
def write_constraints_file(data):
"""Write constraints to a file."""
with open(CONSTRAINT_PATH, "w+", newline="\n") as req_file:
req_file.write(data + CONSTRAINT_BASE)
def validate_requirements_file(data):
"""Validate if requirements_all.txt is up to date."""
with open("requirements_all.txt", "r") as req_file:
return data == req_file.read()
def validate_requirements_test_file(data):
"""Validate if requirements_test_all.txt is up to date."""
with open("requirements_test_all.txt", "r") as req_file:
return data == req_file.read()
def validate_constraints_file(data):
"""Validate if constraints is up to date."""
with open(CONSTRAINT_PATH, "r") as req_file:
return data + CONSTRAINT_BASE == req_file.read()
def diff_file(filename, content):
"""Diff a file."""
return list(
difflib.context_diff(
[line + "\n" for line in Path(filename).read_text().split("\n")],
[line + "\n" for line in content.split("\n")],
filename,
"generated",
)
)
def main(validate):
@ -432,33 +283,38 @@ def main(validate):
if data is None:
return 1
constraints = gather_constraints()
reqs_file = requirements_all_output(data)
reqs_test_file = requirements_test_output(data)
constraints = gather_constraints()
files = (
("requirements_all.txt", reqs_file),
("requirements_test_all.txt", reqs_test_file),
("homeassistant/package_constraints.txt", constraints),
)
if validate:
errors = []
if not validate_requirements_file(reqs_file):
errors.append("requirements_all.txt is not up to date")
if not validate_requirements_test_file(reqs_test_file):
errors.append("requirements_test_all.txt is not up to date")
if not validate_constraints_file(constraints):
errors.append("home-assistant/package_constraints.txt is not up to date")
for filename, content in files:
diff = diff_file(filename, content)
if diff:
errors.append("".join(diff))
if errors:
print("******* ERROR")
print("\n".join(errors))
print("Please run script/gen_requirements_all.py")
print("ERROR - FOUND THE FOLLOWING DIFFERENCES")
print()
print()
print("\n\n".join(errors))
print()
print("Please run python3 -m script.gen_requirements_all")
return 1
return 0
write_requirements_file(reqs_file)
write_test_requirements_file(reqs_test_file)
write_constraints_file(constraints)
for filename, content in files:
Path(filename).write_text(content)
return 0

View File

@ -59,17 +59,20 @@ async def test_async_setup_entry_default(hass):
}
with MockDependency("netdisco.discovery"), patch(
"homeassistant.components.upnp.get_local_ip", return_value="192.168.1.10"
):
), patch.object(Device, "async_create_device") as create_device, patch.object(
Device, "async_create_device"
) as create_device, patch.object(
Device, "async_discover", return_value=mock_coro([])
) as async_discover:
await async_setup_component(hass, "http", config)
await async_setup_component(hass, "upnp", config)
await hass.async_block_till_done()
# mock homeassistant.components.upnp.device.Device
mock_device = MockDevice(udn)
discovery_infos = [{"udn": udn, "ssdp_description": "http://192.168.1.1/desc.xml"}]
with patch.object(Device, "async_create_device") as create_device, patch.object(
Device, "async_discover"
) as async_discover: # noqa:E125
# mock homeassistant.components.upnp.device.Device
mock_device = MockDevice(udn)
discovery_infos = [
{"udn": udn, "ssdp_description": "http://192.168.1.1/desc.xml"}
]
create_device.return_value = mock_coro(return_value=mock_device)
async_discover.return_value = mock_coro(return_value=discovery_infos)
@ -100,16 +103,17 @@ async def test_async_setup_entry_port_mapping(hass):
}
with MockDependency("netdisco.discovery"), patch(
"homeassistant.components.upnp.get_local_ip", return_value="192.168.1.10"
):
), patch.object(Device, "async_create_device") as create_device, patch.object(
Device, "async_discover", return_value=mock_coro([])
) as async_discover:
await async_setup_component(hass, "http", config)
await async_setup_component(hass, "upnp", config)
await hass.async_block_till_done()
mock_device = MockDevice(udn)
discovery_infos = [{"udn": udn, "ssdp_description": "http://192.168.1.1/desc.xml"}]
with patch.object(Device, "async_create_device") as create_device, patch.object(
Device, "async_discover"
) as async_discover: # noqa:E125
mock_device = MockDevice(udn)
discovery_infos = [
{"udn": udn, "ssdp_description": "http://192.168.1.1/desc.xml"}
]
create_device.return_value = mock_coro(return_value=mock_device)
async_discover.return_value = mock_coro(return_value=discovery_infos)

View File

@ -41,7 +41,7 @@ async def test_get_actions(hass, config_entry, zha_gateway):
zha_gateway,
)
await hass.config_entries.async_forward_entry_setup(config_entry, DOMAIN)
await hass.config_entries.async_forward_entry_setup(config_entry, "binary_sensor")
await hass.async_block_till_done()
hass.config_entries._entries.append(config_entry)