From 6e92931087b27b04854d4a806136abc52a15c9d3 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Thu, 18 Aug 2022 12:02:12 -0400 Subject: [PATCH] Add file selector and file upload integration (#76672) --- .strict-typing | 1 + CODEOWNERS | 2 + .../components/file_upload/__init__.py | 182 ++++++++++++++++++ .../components/file_upload/manifest.json | 8 + .../components/frontend/manifest.json | 1 + homeassistant/helpers/selector.py | 40 +++- mypy.ini | 10 + script/hassfest/manifest.py | 1 + tests/components/file_upload/__init__.py | 1 + tests/components/file_upload/test_init.py | 66 +++++++ tests/components/image/__init__.py | 3 + tests/components/image/test_init.py | 7 +- tests/helpers/test_selector.py | 16 ++ 13 files changed, 332 insertions(+), 6 deletions(-) create mode 100644 homeassistant/components/file_upload/__init__.py create mode 100644 homeassistant/components/file_upload/manifest.json create mode 100644 tests/components/file_upload/__init__.py create mode 100644 tests/components/file_upload/test_init.py diff --git a/.strict-typing b/.strict-typing index d9cc4ffb55a..f8a0579433b 100644 --- a/.strict-typing +++ b/.strict-typing @@ -98,6 +98,7 @@ homeassistant.components.energy.* homeassistant.components.evil_genius_labs.* homeassistant.components.fan.* homeassistant.components.fastdotcom.* +homeassistant.components.file_upload.* homeassistant.components.filesize.* homeassistant.components.fitbit.* homeassistant.components.flunearyou.* diff --git a/CODEOWNERS b/CODEOWNERS index 26d1a8da2f8..f952ae22d9b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -329,6 +329,8 @@ build.json @home-assistant/supervisor /tests/components/fibaro/ @rappenze /homeassistant/components/file/ @fabaff /tests/components/file/ @fabaff +/homeassistant/components/file_upload/ @home-assistant/core +/tests/components/file_upload/ @home-assistant/core /homeassistant/components/filesize/ @gjohansson-ST /tests/components/filesize/ @gjohansson-ST /homeassistant/components/filter/ @dgomes diff --git a/homeassistant/components/file_upload/__init__.py b/homeassistant/components/file_upload/__init__.py new file mode 100644 index 00000000000..9f548e14459 --- /dev/null +++ b/homeassistant/components/file_upload/__init__.py @@ -0,0 +1,182 @@ +"""The File Upload integration.""" +from __future__ import annotations + +import asyncio +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +import shutil +import tempfile + +from aiohttp import web +import voluptuous as vol + +from homeassistant.components.http import HomeAssistantView +from homeassistant.components.http.data_validator import RequestDataValidator +from homeassistant.const import EVENT_HOMEASSISTANT_STOP +from homeassistant.core import Event, HomeAssistant, callback +from homeassistant.helpers.typing import ConfigType +from homeassistant.util import raise_if_invalid_filename +from homeassistant.util.ulid import ulid_hex + +DOMAIN = "file_upload" + +# If increased, change upload view to streaming +# https://docs.aiohttp.org/en/stable/web_quickstart.html#file-uploads +MAX_SIZE = 1024 * 1024 * 10 +TEMP_DIR_NAME = f"home-assistant-{DOMAIN}" + + +@contextmanager +def process_uploaded_file(hass: HomeAssistant, file_id: str) -> Iterator[Path]: + """Get an uploaded file. + + File is removed at the end of the context. + """ + if DOMAIN not in hass.data: + raise ValueError("File does not exist") + + file_upload_data: FileUploadData = hass.data[DOMAIN] + + if not file_upload_data.has_file(file_id): + raise ValueError("File does not exist") + + try: + yield file_upload_data.file_path(file_id) + finally: + file_upload_data.files.pop(file_id) + shutil.rmtree(file_upload_data.file_dir(file_id)) + + +async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool: + """Set up File Upload.""" + hass.http.register_view(FileUploadView) + return True + + +@dataclass(frozen=True) +class FileUploadData: + """File upload data.""" + + temp_dir: Path + files: dict[str, str] + + @classmethod + async def create(cls, hass: HomeAssistant) -> FileUploadData: + """Initialize the file upload data.""" + + def _create_temp_dir() -> Path: + """Create temporary directory.""" + temp_dir = Path(tempfile.gettempdir()) / TEMP_DIR_NAME + + # If it exists, it's an old one and Home Assistant didn't shut down correctly. + if temp_dir.exists(): + shutil.rmtree(temp_dir) + + temp_dir.mkdir(0o700) + return temp_dir + + temp_dir = await hass.async_add_executor_job(_create_temp_dir) + + def cleanup_unused_files(ev: Event) -> None: + """Clean up unused files.""" + shutil.rmtree(temp_dir) + + hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, cleanup_unused_files) + + return cls(temp_dir, {}) + + def has_file(self, file_id: str) -> bool: + """Return if file exists.""" + return file_id in self.files + + def file_dir(self, file_id: str) -> Path: + """Return the file directory.""" + return self.temp_dir / file_id + + def file_path(self, file_id: str) -> Path: + """Return the file path.""" + return self.file_dir(file_id) / self.files[file_id] + + +class FileUploadView(HomeAssistantView): + """HTTP View to upload files.""" + + url = "/api/file_upload" + name = "api:file_upload" + + _upload_lock: asyncio.Lock | None = None + + @callback + def _get_upload_lock(self) -> asyncio.Lock: + """Get upload lock.""" + if self._upload_lock is None: + self._upload_lock = asyncio.Lock() + + return self._upload_lock + + async def post(self, request: web.Request) -> web.Response: + """Upload a file.""" + async with self._get_upload_lock(): + return await self._upload_file(request) + + async def _upload_file(self, request: web.Request) -> web.Response: + """Handle uploaded file.""" + # Increase max payload + request._client_max_size = MAX_SIZE # pylint: disable=protected-access + + data = await request.post() + file_field = data.get("file") + + if not isinstance(file_field, web.FileField): + raise vol.Invalid("Expected a file") + + try: + raise_if_invalid_filename(file_field.filename) + except ValueError as err: + raise web.HTTPBadRequest from err + + hass: HomeAssistant = request.app["hass"] + file_id = ulid_hex() + + if DOMAIN not in hass.data: + hass.data[DOMAIN] = await FileUploadData.create(hass) + + file_upload_data: FileUploadData = hass.data[DOMAIN] + file_dir = file_upload_data.file_dir(file_id) + + def _sync_work() -> None: + file_dir.mkdir() + + # MyPy forgets about the isinstance check because we're in a function scope + assert isinstance(file_field, web.FileField) + + with (file_dir / file_field.filename).open("wb") as target_fileobj: + shutil.copyfileobj(file_field.file, target_fileobj) + + await hass.async_add_executor_job(_sync_work) + + file_upload_data.files[file_id] = file_field.filename + + return self.json({"file_id": file_id}) + + @RequestDataValidator({vol.Required("file_id"): str}) + async def delete(self, request: web.Request, data: dict[str, str]) -> web.Response: + """Delete a file.""" + hass: HomeAssistant = request.app["hass"] + + if DOMAIN not in hass.data: + raise web.HTTPNotFound() + + file_id = data["file_id"] + file_upload_data: FileUploadData = hass.data[DOMAIN] + + if file_upload_data.files.pop(file_id, None) is None: + raise web.HTTPNotFound() + + await hass.async_add_executor_job( + lambda: shutil.rmtree(file_upload_data.file_dir(file_id)) + ) + + return self.json_message("File deleted") diff --git a/homeassistant/components/file_upload/manifest.json b/homeassistant/components/file_upload/manifest.json new file mode 100644 index 00000000000..6e190ba3712 --- /dev/null +++ b/homeassistant/components/file_upload/manifest.json @@ -0,0 +1,8 @@ +{ + "domain": "file_upload", + "name": "File Upload", + "documentation": "https://www.home-assistant.io/integrations/file_upload", + "dependencies": ["http"], + "codeowners": ["@home-assistant/core"], + "quality_scale": "internal" +} diff --git a/homeassistant/components/frontend/manifest.json b/homeassistant/components/frontend/manifest.json index 207b57babb2..f1e6f31fd4b 100644 --- a/homeassistant/components/frontend/manifest.json +++ b/homeassistant/components/frontend/manifest.json @@ -9,6 +9,7 @@ "config", "device_automation", "diagnostics", + "file_upload", "http", "lovelace", "onboarding", diff --git a/homeassistant/helpers/selector.py b/homeassistant/helpers/selector.py index ccb7ac67dfb..deacc821672 100644 --- a/homeassistant/helpers/selector.py +++ b/homeassistant/helpers/selector.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Callable, Sequence from typing import Any, TypedDict, cast +from uuid import UUID import voluptuous as vol @@ -10,7 +11,7 @@ from homeassistant.backports.enum import StrEnum from homeassistant.const import CONF_MODE, CONF_UNIT_OF_MEASUREMENT from homeassistant.core import split_entity_id, valid_entity_id from homeassistant.util import decorator -from homeassistant.util.yaml.dumper import add_representer, represent_odict +from homeassistant.util.yaml import dumper from . import config_validation as cv @@ -888,9 +889,42 @@ class TimeSelector(Selector): return cast(str, data) -add_representer( +class FileSelectorConfig(TypedDict): + """Class to represent a file selector config.""" + + accept: str # required + + +@SELECTORS.register("file") +class FileSelector(Selector): + """Selector of a file.""" + + selector_type = "file" + + CONFIG_SCHEMA = vol.Schema( + { + # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file#accept + vol.Required("accept"): str, + } + ) + + def __init__(self, config: FileSelectorConfig | None = None) -> None: + """Instantiate a selector.""" + super().__init__(config) + + def __call__(self, data: Any) -> str: + """Validate the passed selection.""" + if not isinstance(data, str): + raise vol.Invalid("Value should be a string") + + UUID(data) + + return data + + +dumper.add_representer( Selector, - lambda dumper, value: represent_odict( + lambda dumper, value: dumper.represent_odict( dumper, "tag:yaml.org,2002:map", value.serialize() ), ) diff --git a/mypy.ini b/mypy.ini index 570004a14dd..051c1065423 100644 --- a/mypy.ini +++ b/mypy.ini @@ -739,6 +739,16 @@ disallow_untyped_defs = true warn_return_any = true warn_unreachable = true +[mypy-homeassistant.components.file_upload.*] +check_untyped_defs = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +disallow_untyped_defs = true +warn_return_any = true +warn_unreachable = true + [mypy-homeassistant.components.filesize.*] check_untyped_defs = true disallow_incomplete_defs = true diff --git a/script/hassfest/manifest.py b/script/hassfest/manifest.py index 1e6bd03f457..338682bbe76 100644 --- a/script/hassfest/manifest.py +++ b/script/hassfest/manifest.py @@ -51,6 +51,7 @@ NO_IOT_CLASS = [ "discovery", "downloader", "ffmpeg", + "file_upload", "frontend", "hardkernel", "hardware", diff --git a/tests/components/file_upload/__init__.py b/tests/components/file_upload/__init__.py new file mode 100644 index 00000000000..2630811ffc5 --- /dev/null +++ b/tests/components/file_upload/__init__.py @@ -0,0 +1 @@ +"""Tests for the File Upload integration.""" diff --git a/tests/components/file_upload/test_init.py b/tests/components/file_upload/test_init.py new file mode 100644 index 00000000000..ba3485c96e1 --- /dev/null +++ b/tests/components/file_upload/test_init.py @@ -0,0 +1,66 @@ +"""Test the File Upload integration.""" +from pathlib import Path +from random import getrandbits +from unittest.mock import patch + +import pytest + +from homeassistant.components import file_upload +from homeassistant.core import HomeAssistant +from homeassistant.setup import async_setup_component + +from tests.components.image import TEST_IMAGE + + +@pytest.fixture +async def uploaded_file_dir(hass: HomeAssistant, hass_client) -> Path: + """Test uploading and using a file.""" + assert await async_setup_component(hass, "file_upload", {}) + client = await hass_client() + + with patch( + # Patch temp dir name to avoid tests fail running in parallel + "homeassistant.components.file_upload.TEMP_DIR_NAME", + file_upload.TEMP_DIR_NAME + f"-{getrandbits(10):03x}", + ), TEST_IMAGE.open("rb") as fp: + res = await client.post("/api/file_upload", data={"file": fp}) + + assert res.status == 200 + response = await res.json() + + file_dir = hass.data[file_upload.DOMAIN].file_dir(response["file_id"]) + assert file_dir.is_dir() + return file_dir + + +async def test_using_file(hass: HomeAssistant, uploaded_file_dir): + """Test uploading and using a file.""" + # Test we can use it + with file_upload.process_uploaded_file(hass, uploaded_file_dir.name) as file_path: + assert file_path.is_file() + assert file_path.parent == uploaded_file_dir + assert file_path.read_bytes() == TEST_IMAGE.read_bytes() + + # Test it's removed + assert not uploaded_file_dir.exists() + + +async def test_removing_file(hass: HomeAssistant, hass_client, uploaded_file_dir): + """Test uploading and using a file.""" + client = await hass_client() + + response = await client.delete( + "/api/file_upload", json={"file_id": uploaded_file_dir.name} + ) + assert response.status == 200 + + # Test it's removed + assert not uploaded_file_dir.exists() + + +async def test_removed_on_stop(hass: HomeAssistant, hass_client, uploaded_file_dir): + """Test uploading and using a file.""" + await hass.async_stop() + + # Test it's removed + assert not uploaded_file_dir.exists() diff --git a/tests/components/image/__init__.py b/tests/components/image/__init__.py index 8bf90c4f516..b04214669aa 100644 --- a/tests/components/image/__init__.py +++ b/tests/components/image/__init__.py @@ -1 +1,4 @@ """Tests for the Image integration.""" +import pathlib + +TEST_IMAGE = pathlib.Path(__file__).parent / "logo.png" diff --git a/tests/components/image/test_init.py b/tests/components/image/test_init.py index ab73bb71286..d62717cb894 100644 --- a/tests/components/image/test_init.py +++ b/tests/components/image/test_init.py @@ -9,11 +9,12 @@ from homeassistant.components.websocket_api import const as ws_const from homeassistant.setup import async_setup_component from homeassistant.util import dt as util_dt +from . import TEST_IMAGE + async def test_upload_image(hass, hass_client, hass_ws_client): """Test we can upload an image.""" now = util_dt.utcnow() - test_image = pathlib.Path(__file__).parent / "logo.png" with tempfile.TemporaryDirectory() as tempdir, patch.object( hass.config, "path", return_value=tempdir @@ -22,7 +23,7 @@ async def test_upload_image(hass, hass_client, hass_ws_client): ws_client: ClientWebSocketResponse = await hass_ws_client() client: ClientSession = await hass_client() - with test_image.open("rb") as fp: + with TEST_IMAGE.open("rb") as fp: res = await client.post("/api/image/upload", data={"file": fp}) assert res.status == 200 @@ -36,7 +37,7 @@ async def test_upload_image(hass, hass_client, hass_ws_client): tempdir = pathlib.Path(tempdir) item_folder: pathlib.Path = tempdir / item["id"] - assert (item_folder / "original").read_bytes() == test_image.read_bytes() + assert (item_folder / "original").read_bytes() == TEST_IMAGE.read_bytes() # fetch non-existing image res = await client.get("/api/image/serve/non-existing/256x256") diff --git a/tests/helpers/test_selector.py b/tests/helpers/test_selector.py index 4cb924a520e..d1018299d96 100644 --- a/tests/helpers/test_selector.py +++ b/tests/helpers/test_selector.py @@ -646,3 +646,19 @@ def test_datetime_selector_schema(schema, valid_selections, invalid_selections): def test_template_selector_schema(schema, valid_selections, invalid_selections): """Test template selector.""" _test_selector("template", schema, valid_selections, invalid_selections) + + +@pytest.mark.parametrize( + "schema,valid_selections,invalid_selections", + ( + ( + {"accept": "image/*"}, + ("0182a1b99dbc5ae24aecd90c346605fa",), + (None, "not-a-uuid", "abcd", 1), + ), + ), +) +def test_file_selector_schema(schema, valid_selections, invalid_selections): + """Test file selector.""" + + _test_selector("file", schema, valid_selections, invalid_selections)