From 994c9812285f563ee07916564d9ddec59d6b9127 Mon Sep 17 00:00:00 2001 From: Mike Degatano Date: Wed, 11 Oct 2023 02:52:19 -0400 Subject: [PATCH] Allow home assistant backups to exclude database (#4591) * Allow home assistant backups to exclude database * Tweak Co-authored-by: Pascal Vizeli --------- Co-authored-by: Franck Nijhof Co-authored-by: Pascal Vizeli --- supervisor/api/backups.py | 3 ++ supervisor/api/homeassistant.py | 8 ++++ supervisor/backups/backup.py | 23 ++++++++--- supervisor/backups/manager.py | 19 +++++++-- supervisor/backups/validate.py | 4 ++ supervisor/const.py | 3 ++ supervisor/homeassistant/module.py | 46 +++++++++++++++++++--- supervisor/homeassistant/validate.py | 2 + supervisor/utils/__init__.py | 26 ++++++++++++- tests/api/test_backups.py | 34 +++++++++++++++- tests/api/test_homeassistant.py | 26 ++++++++++++- tests/backups/test_manager.py | 58 ++++++++++++++++++++++++++++ 12 files changed, 236 insertions(+), 16 deletions(-) diff --git a/supervisor/api/backups.py b/supervisor/api/backups.py index 93f266875..bbc116501 100644 --- a/supervisor/api/backups.py +++ b/supervisor/api/backups.py @@ -20,6 +20,7 @@ from ..const import ( ATTR_DAYS_UNTIL_STALE, ATTR_FOLDERS, ATTR_HOMEASSISTANT, + ATTR_HOMEASSISTANT_EXCLUDE_DATABASE, ATTR_LOCATON, ATTR_NAME, ATTR_PASSWORD, @@ -64,6 +65,7 @@ SCHEMA_BACKUP_FULL = vol.Schema( vol.Optional(ATTR_PASSWORD): vol.Maybe(str), vol.Optional(ATTR_COMPRESSED): vol.Maybe(vol.Boolean()), vol.Optional(ATTR_LOCATON): vol.Maybe(str), + vol.Optional(ATTR_HOMEASSISTANT_EXCLUDE_DATABASE): vol.Boolean(), } ) @@ -184,6 +186,7 @@ class APIBackups(CoreSysAttributes): ATTR_ADDONS: data_addons, ATTR_REPOSITORIES: backup.repositories, ATTR_FOLDERS: backup.folders, + ATTR_HOMEASSISTANT_EXCLUDE_DATABASE: backup.homeassistant_exclude_database, } def _location_to_mount(self, body: dict[str, Any]) -> dict[str, Any]: diff --git a/supervisor/api/homeassistant.py b/supervisor/api/homeassistant.py index a9ea2b96e..9ed870a0d 100644 --- a/supervisor/api/homeassistant.py +++ b/supervisor/api/homeassistant.py @@ -12,6 +12,7 @@ from ..const import ( ATTR_AUDIO_INPUT, ATTR_AUDIO_OUTPUT, ATTR_BACKUP, + ATTR_BACKUPS_EXCLUDE_DATABASE, ATTR_BLK_READ, ATTR_BLK_WRITE, ATTR_BOOT, @@ -51,6 +52,7 @@ SCHEMA_OPTIONS = vol.Schema( vol.Optional(ATTR_REFRESH_TOKEN): vol.Maybe(str), vol.Optional(ATTR_AUDIO_OUTPUT): vol.Maybe(str), vol.Optional(ATTR_AUDIO_INPUT): vol.Maybe(str), + vol.Optional(ATTR_BACKUPS_EXCLUDE_DATABASE): vol.Boolean(), } ) @@ -82,6 +84,7 @@ class APIHomeAssistant(CoreSysAttributes): ATTR_WATCHDOG: self.sys_homeassistant.watchdog, ATTR_AUDIO_INPUT: self.sys_homeassistant.audio_input, ATTR_AUDIO_OUTPUT: self.sys_homeassistant.audio_output, + ATTR_BACKUPS_EXCLUDE_DATABASE: self.sys_homeassistant.backups_exclude_database, } @api_process @@ -113,6 +116,11 @@ class APIHomeAssistant(CoreSysAttributes): if ATTR_AUDIO_OUTPUT in body: self.sys_homeassistant.audio_output = body[ATTR_AUDIO_OUTPUT] + if ATTR_BACKUPS_EXCLUDE_DATABASE in body: + self.sys_homeassistant.backups_exclude_database = body[ + ATTR_BACKUPS_EXCLUDE_DATABASE + ] + self.sys_homeassistant.save_data() @api_process diff --git a/supervisor/backups/backup.py b/supervisor/backups/backup.py index 250636787..3c2e2ce7f 100644 --- a/supervisor/backups/backup.py +++ b/supervisor/backups/backup.py @@ -26,6 +26,7 @@ from ..const import ( ATTR_CRYPTO, ATTR_DATE, ATTR_DOCKER, + ATTR_EXCLUDE_DATABASE, ATTR_FOLDERS, ATTR_HOMEASSISTANT, ATTR_NAME, @@ -130,7 +131,14 @@ class Backup(CoreSysAttributes): """Return backup Home Assistant version.""" if self.homeassistant is None: return None - return self._data[ATTR_HOMEASSISTANT][ATTR_VERSION] + return self.homeassistant[ATTR_VERSION] + + @property + def homeassistant_exclude_database(self) -> bool: + """Return whether database was excluded from Home Assistant backup.""" + if self.homeassistant is None: + return None + return self.homeassistant[ATTR_EXCLUDE_DATABASE] @property def homeassistant(self): @@ -539,9 +547,12 @@ class Backup(CoreSysAttributes): except Exception as err: # pylint: disable=broad-except _LOGGER.warning("Can't restore folder %s: %s", folder, err) - async def store_homeassistant(self): + async def store_homeassistant(self, exclude_database: bool = False): """Backup Home Assistant Core configuration folder.""" - self._data[ATTR_HOMEASSISTANT] = {ATTR_VERSION: self.sys_homeassistant.version} + self._data[ATTR_HOMEASSISTANT] = { + ATTR_VERSION: self.sys_homeassistant.version, + ATTR_EXCLUDE_DATABASE: exclude_database, + } # Backup Home Assistant Core config directory tar_name = Path( @@ -551,7 +562,7 @@ class Backup(CoreSysAttributes): tar_name, "w", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE ) - await self.sys_homeassistant.backup(homeassistant_file) + await self.sys_homeassistant.backup(homeassistant_file, exclude_database) # Store size self.homeassistant[ATTR_SIZE] = homeassistant_file.size @@ -568,7 +579,9 @@ class Backup(CoreSysAttributes): tar_name, "r", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE ) - await self.sys_homeassistant.restore(homeassistant_file) + await self.sys_homeassistant.restore( + homeassistant_file, self.homeassistant_exclude_database + ) # Generate restore task async def _core_update(): diff --git a/supervisor/backups/manager.py b/supervisor/backups/manager.py index 2017cfacb..ba1cf583d 100644 --- a/supervisor/backups/manager.py +++ b/supervisor/backups/manager.py @@ -226,6 +226,7 @@ class BackupManager(FileConfiguration, JobGroup): addon_list: list[Addon], folder_list: list[str], homeassistant: bool, + homeassistant_exclude_database: bool | None, ) -> Backup | None: """Create a backup. @@ -245,7 +246,11 @@ class BackupManager(FileConfiguration, JobGroup): # HomeAssistant Folder is for v1 if homeassistant: self._change_stage(BackupJobStage.HOME_ASSISTANT, backup) - await backup.store_homeassistant() + await backup.store_homeassistant( + self.sys_homeassistant.backups_exclude_database + if homeassistant_exclude_database is None + else homeassistant_exclude_database + ) # Backup folders if folder_list: @@ -282,6 +287,7 @@ class BackupManager(FileConfiguration, JobGroup): password: str | None = None, compressed: bool = True, location: Mount | type[DEFAULT] | None = DEFAULT, + homeassistant_exclude_database: bool | None = None, ) -> Backup | None: """Create a full backup.""" if self._get_base_path(location) == self.sys_config.path_backup: @@ -295,7 +301,11 @@ class BackupManager(FileConfiguration, JobGroup): _LOGGER.info("Creating new full backup with slug %s", backup.slug) backup = await self._do_backup( - backup, self.sys_addons.installed, ALL_FOLDERS, True + backup, + self.sys_addons.installed, + ALL_FOLDERS, + True, + homeassistant_exclude_database, ) if backup: _LOGGER.info("Creating full backup with slug %s completed", backup.slug) @@ -316,6 +326,7 @@ class BackupManager(FileConfiguration, JobGroup): homeassistant: bool = False, compressed: bool = True, location: Mount | type[DEFAULT] | None = DEFAULT, + homeassistant_exclude_database: bool | None = None, ) -> Backup | None: """Create a partial backup.""" if self._get_base_path(location) == self.sys_config.path_backup: @@ -347,7 +358,9 @@ class BackupManager(FileConfiguration, JobGroup): continue _LOGGER.warning("Add-on %s not found/installed", addon_slug) - backup = await self._do_backup(backup, addon_list, folders, homeassistant) + backup = await self._do_backup( + backup, addon_list, folders, homeassistant, homeassistant_exclude_database + ) if backup: _LOGGER.info("Creating partial backup with slug %s completed", backup.slug) return backup diff --git a/supervisor/backups/validate.py b/supervisor/backups/validate.py index dbc0b4656..c7c98c303 100644 --- a/supervisor/backups/validate.py +++ b/supervisor/backups/validate.py @@ -14,6 +14,7 @@ from ..const import ( ATTR_DATE, ATTR_DAYS_UNTIL_STALE, ATTR_DOCKER, + ATTR_EXCLUDE_DATABASE, ATTR_FOLDERS, ATTR_HOMEASSISTANT, ATTR_NAME, @@ -103,6 +104,9 @@ SCHEMA_BACKUP = vol.Schema( { vol.Required(ATTR_VERSION): version_tag, vol.Optional(ATTR_SIZE, default=0): vol.Coerce(float), + vol.Optional( + ATTR_EXCLUDE_DATABASE, default=False + ): vol.Boolean(), }, extra=vol.REMOVE_EXTRA, ) diff --git a/supervisor/const.py b/supervisor/const.py index 95628e0ca..5898acf21 100644 --- a/supervisor/const.py +++ b/supervisor/const.py @@ -115,6 +115,7 @@ ATTR_BACKUP_EXCLUDE = "backup_exclude" ATTR_BACKUP_POST = "backup_post" ATTR_BACKUP_PRE = "backup_pre" ATTR_BACKUPS = "backups" +ATTR_BACKUPS_EXCLUDE_DATABASE = "backups_exclude_database" ATTR_BLK_READ = "blk_read" ATTR_BLK_WRITE = "blk_write" ATTR_BOARD = "board" @@ -167,6 +168,7 @@ ATTR_ENABLE = "enable" ATTR_ENABLED = "enabled" ATTR_ENVIRONMENT = "environment" ATTR_EVENT = "event" +ATTR_EXCLUDE_DATABASE = "exclude_database" ATTR_FEATURES = "features" ATTR_FILENAME = "filename" ATTR_FLAGS = "flags" @@ -182,6 +184,7 @@ ATTR_HASSOS = "hassos" ATTR_HEALTHY = "healthy" ATTR_HEARTBEAT_LED = "heartbeat_led" ATTR_HOMEASSISTANT = "homeassistant" +ATTR_HOMEASSISTANT_EXCLUDE_DATABASE = "homeassistant_exclude_database" ATTR_HOMEASSISTANT_API = "homeassistant_api" ATTR_HOST = "host" ATTR_HOST_DBUS = "host_dbus" diff --git a/supervisor/homeassistant/module.py b/supervisor/homeassistant/module.py index 280f00c31..9b715cc48 100644 --- a/supervisor/homeassistant/module.py +++ b/supervisor/homeassistant/module.py @@ -18,6 +18,7 @@ from ..const import ( ATTR_ACCESS_TOKEN, ATTR_AUDIO_INPUT, ATTR_AUDIO_OUTPUT, + ATTR_BACKUPS_EXCLUDE_DATABASE, ATTR_BOOT, ATTR_IMAGE, ATTR_PORT, @@ -62,6 +63,10 @@ HOMEASSISTANT_BACKUP_EXCLUDE = [ "*.log.*", "OZW_Log.txt", ] +HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE = [ + "home-assistant_v?.db", + "home-assistant_v?.db-wal", +] class HomeAssistant(FileConfiguration, CoreSysAttributes): @@ -258,6 +263,16 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes): except (AwesomeVersionException, TypeError): return False + @property + def backups_exclude_database(self) -> bool: + """Exclude database from core backups by default.""" + return self._data[ATTR_BACKUPS_EXCLUDE_DATABASE] + + @backups_exclude_database.setter + def backups_exclude_database(self, value: bool) -> None: + """Set whether backups should exclude database by default.""" + self._data[ATTR_BACKUPS_EXCLUDE_DATABASE] = value + async def load(self) -> None: """Prepare Home Assistant object.""" await asyncio.wait( @@ -327,7 +342,9 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes): ) @Job(name="home_assistant_module_backup") - async def backup(self, tar_file: tarfile.TarFile) -> None: + async def backup( + self, tar_file: tarfile.TarFile, exclude_database: bool = False + ) -> None: """Backup Home Assistant Core config/ directory.""" await self.begin_backup() try: @@ -351,11 +368,16 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes): # Backup metadata backup.add(temp, arcname=".") + # Set excludes + excludes = HOMEASSISTANT_BACKUP_EXCLUDE.copy() + if exclude_database: + excludes += HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE + # Backup data atomic_contents_add( backup, self.sys_config.path_homeassistant, - excludes=HOMEASSISTANT_BACKUP_EXCLUDE, + excludes=excludes, arcname="data", ) @@ -371,7 +393,10 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes): finally: await self.end_backup() - async def restore(self, tar_file: tarfile.TarFile) -> None: + @Job(name="home_assistant_module_restore") + async def restore( + self, tar_file: tarfile.TarFile, exclude_database: bool = False + ) -> None: """Restore Home Assistant Core config/ directory.""" with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp: temp_path = Path(temp) @@ -399,11 +424,22 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes): def _restore_data(): """Restore data.""" shutil.copytree( - temp_data, self.sys_config.path_homeassistant, symlinks=True + temp_data, + self.sys_config.path_homeassistant, + symlinks=True, + dirs_exist_ok=bool(excludes), ) _LOGGER.info("Restore Home Assistant Core config folder") - await remove_folder(self.sys_config.path_homeassistant) + excludes = ( + HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE if exclude_database else None + ) + await remove_folder( + self.sys_config.path_homeassistant, + content_only=bool(excludes), + excludes=excludes, + tmp_dir=self.sys_config.path_tmp, + ) try: await self.sys_run_in_executor(_restore_data) except shutil.Error as err: diff --git a/supervisor/homeassistant/validate.py b/supervisor/homeassistant/validate.py index 85be5d283..90669eef8 100644 --- a/supervisor/homeassistant/validate.py +++ b/supervisor/homeassistant/validate.py @@ -7,6 +7,7 @@ from ..const import ( ATTR_ACCESS_TOKEN, ATTR_AUDIO_INPUT, ATTR_AUDIO_OUTPUT, + ATTR_BACKUPS_EXCLUDE_DATABASE, ATTR_BOOT, ATTR_IMAGE, ATTR_PORT, @@ -32,6 +33,7 @@ SCHEMA_HASS_CONFIG = vol.Schema( vol.Optional(ATTR_WATCHDOG, default=True): vol.Boolean(), vol.Optional(ATTR_AUDIO_OUTPUT, default=None): vol.Maybe(str), vol.Optional(ATTR_AUDIO_INPUT, default=None): vol.Maybe(str), + vol.Optional(ATTR_BACKUPS_EXCLUDE_DATABASE, default=False): vol.Boolean(), }, extra=vol.REMOVE_EXTRA, ) diff --git a/supervisor/utils/__init__.py b/supervisor/utils/__init__.py index e351c9a18..1f893742c 100644 --- a/supervisor/utils/__init__.py +++ b/supervisor/utils/__init__.py @@ -6,6 +6,7 @@ import os from pathlib import Path import re import socket +from tempfile import TemporaryDirectory from typing import Any _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -76,13 +77,31 @@ def get_message_from_exception_chain(err: Exception) -> str: return get_message_from_exception_chain(err.__context__) -async def remove_folder(folder: Path, content_only: bool = False) -> None: +async def remove_folder( + folder: Path, + content_only: bool = False, + excludes: list[str] | None = None, + tmp_dir: Path | None = None, +) -> None: """Remove folder and reset privileged. Is needed to avoid issue with: - CAP_DAC_OVERRIDE - CAP_DAC_READ_SEARCH """ + if excludes: + if not tmp_dir: + raise ValueError("tmp_dir is required if excludes are provided") + if not content_only: + raise ValueError("Cannot delete the folder if excludes are provided") + + temp = TemporaryDirectory(dir=tmp_dir) + temp_path = Path(temp.name) + moved_files: list[Path] = [] + for item in folder.iterdir(): + if any(item.match(exclude) for exclude in excludes): + moved_files.append(item.rename(temp_path / item.name)) + del_folder = f"{folder}" + "/{,.[!.],..?}*" if content_only else f"{folder}" try: proc = await asyncio.create_subprocess_exec( @@ -99,6 +118,11 @@ async def remove_folder(folder: Path, content_only: bool = False) -> None: else: if proc.returncode == 0: return + finally: + if excludes: + for item in moved_files: + item.rename(folder / item.name) + temp.cleanup() _LOGGER.error("Can't remove folder %s: %s", folder, error_msg) diff --git a/tests/api/test_backups.py b/tests/api/test_backups.py index d6003ba6c..63fc9621f 100644 --- a/tests/api/test_backups.py +++ b/tests/api/test_backups.py @@ -2,7 +2,7 @@ import asyncio from pathlib import Path, PurePath -from unittest.mock import AsyncMock, patch +from unittest.mock import ANY, AsyncMock, patch from aiohttp.test_utils import TestClient from awesomeversion import AwesomeVersion @@ -11,6 +11,7 @@ import pytest from supervisor.backups.backup import Backup from supervisor.const import CoreState from supervisor.coresys import CoreSys +from supervisor.homeassistant.module import HomeAssistant from supervisor.mounts.mount import Mount @@ -167,3 +168,34 @@ async def test_api_freeze_thaw( call.args[0] == {"type": "backup/end"} for call in ha_ws_client.async_send_command.call_args_list ) + + +@pytest.mark.parametrize( + "partial_backup,exclude_db_setting", + [(False, True), (True, True), (False, False), (True, False)], +) +async def test_api_backup_exclude_database( + api_client: TestClient, + coresys: CoreSys, + partial_backup: bool, + exclude_db_setting: bool, + tmp_supervisor_data, + path_extern, +): + """Test backups exclude the database when specified.""" + coresys.core.state = CoreState.RUNNING + coresys.hardware.disk.get_disk_free_space = lambda x: 5000 + coresys.homeassistant.version = AwesomeVersion("2023.09.0") + coresys.homeassistant.backups_exclude_database = exclude_db_setting + + json = {} if exclude_db_setting else {"homeassistant_exclude_database": True} + with patch.object(HomeAssistant, "backup") as backup: + if partial_backup: + resp = await api_client.post( + "/backups/new/partial", json={"homeassistant": True} | json + ) + else: + resp = await api_client.post("/backups/new/full", json=json) + + backup.assert_awaited_once_with(ANY, True) + assert resp.status == 200 diff --git a/tests/api/test_homeassistant.py b/tests/api/test_homeassistant.py index 409cd4d84..12e835a6d 100644 --- a/tests/api/test_homeassistant.py +++ b/tests/api/test_homeassistant.py @@ -1,11 +1,12 @@ """Test homeassistant api.""" -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from aiohttp.test_utils import TestClient import pytest from supervisor.coresys import CoreSys +from supervisor.homeassistant.module import HomeAssistant from tests.common import load_json_fixture @@ -39,3 +40,26 @@ async def test_api_stats(api_client: TestClient, coresys: CoreSys): assert result["data"]["memory_usage"] == 59700000 assert result["data"]["memory_limit"] == 4000000000 assert result["data"]["memory_percent"] == 1.49 + + +async def test_api_set_options(api_client: TestClient, coresys: CoreSys): + """Test setting options for homeassistant.""" + resp = await api_client.get("/homeassistant/info") + assert resp.status == 200 + result = await resp.json() + assert result["data"]["watchdog"] is True + assert result["data"]["backups_exclude_database"] is False + + with patch.object(HomeAssistant, "save_data") as save_data: + resp = await api_client.post( + "/homeassistant/options", + json={"backups_exclude_database": True, "watchdog": False}, + ) + assert resp.status == 200 + save_data.assert_called_once() + + resp = await api_client.get("/homeassistant/info") + assert resp.status == 200 + result = await resp.json() + assert result["data"]["watchdog"] is False + assert result["data"]["backups_exclude_database"] is True diff --git a/tests/backups/test_manager.py b/tests/backups/test_manager.py index bbc323543..816480e85 100644 --- a/tests/backups/test_manager.py +++ b/tests/backups/test_manager.py @@ -24,7 +24,9 @@ from supervisor.exceptions import AddonsError, BackupError, BackupJobError, Dock from supervisor.homeassistant.api import HomeAssistantAPI from supervisor.homeassistant.core import HomeAssistantCore from supervisor.homeassistant.module import HomeAssistant +from supervisor.jobs.const import JobCondition from supervisor.mounts.mount import Mount +from supervisor.utils.json import read_json_file, write_json_file from tests.const import TEST_ADDON_SLUG from tests.dbus_service_mocks.base import DBusServiceMock @@ -1441,3 +1443,59 @@ async def test_backup_to_mount_bypasses_free_space_condition( # These succeed because local free space does not matter when using a mount await coresys.backups.do_backup_full(location=mount) await coresys.backups.do_backup_partial(folders=["media"], location=mount) + + +@pytest.mark.parametrize( + "partial_backup,exclude_db_setting", + [(False, True), (True, True), (False, False), (True, False)], +) +async def test_skip_homeassistant_database( + coresys: CoreSys, + container: MagicMock, + partial_backup: bool, + exclude_db_setting: bool | None, + tmp_supervisor_data, + path_extern, +): + """Test exclude database option skips database in backup.""" + coresys.core.state = CoreState.RUNNING + coresys.hardware.disk.get_disk_free_space = lambda x: 5000 + coresys.jobs.ignore_conditions = [ + JobCondition.INTERNET_HOST, + JobCondition.INTERNET_SYSTEM, + ] + coresys.homeassistant.version = AwesomeVersion("2023.09.0") + coresys.homeassistant.backups_exclude_database = exclude_db_setting + + test_file = coresys.config.path_homeassistant / "configuration.yaml" + (test_db := coresys.config.path_homeassistant / "home-assistant_v2.db").touch() + ( + test_db_wal := coresys.config.path_homeassistant / "home-assistant_v2.db-wal" + ).touch() + ( + test_db_shm := coresys.config.path_homeassistant / "home-assistant_v2.db-shm" + ).touch() + + write_json_file(test_file, {"default_config": {}}) + + kwargs = {} if exclude_db_setting else {"homeassistant_exclude_database": True} + if partial_backup: + backup: Backup = await coresys.backups.do_backup_partial( + homeassistant=True, **kwargs + ) + else: + backup: Backup = await coresys.backups.do_backup_full(**kwargs) + + test_file.unlink() + write_json_file(test_db, {"hello": "world"}) + write_json_file(test_db_wal, {"hello": "world"}) + + with patch.object(HomeAssistantCore, "update"), patch.object( + HomeAssistantCore, "start" + ): + await coresys.backups.do_restore_partial(backup, homeassistant=True) + + assert read_json_file(test_file) == {"default_config": {}} + assert read_json_file(test_db) == {"hello": "world"} + assert read_json_file(test_db_wal) == {"hello": "world"} + assert not test_db_shm.exists()