From 1799c765b45ce5a011bdd6455984075f31b4c36a Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Wed, 5 Jan 2022 11:57:41 +0100 Subject: [PATCH] Notify HA Core when backup is being executed (#3305) * Notify HA Core when backup is being executed Notify Home Assistant Core using the Websocket API when the Home Assistant config directory is about to get backed up. This makes sure the Core can prepare the SQLite database (if active) so it safe to make a backup. * Only close WebSocket connection on connection errors Let regular WebSocket errors bubble to the caller so they can be handled explicitly. * Add version restriction to backup/start/end WS commands * Restore Home Assistant metadata when Home Assistant is selected --- supervisor/backups/backup.py | 34 ++++++++----- supervisor/backups/manager.py | 12 +++-- supervisor/homeassistant/const.py | 4 +- supervisor/homeassistant/module.py | 71 +++++++++++++++++++++++++++ supervisor/homeassistant/websocket.py | 12 +++-- tests/backups/conftest.py | 5 +- tests/backups/test_manager.py | 13 +++-- 7 files changed, 126 insertions(+), 25 deletions(-) diff --git a/supervisor/backups/backup.py b/supervisor/backups/backup.py index 601fc0f47..8f59eddb0 100644 --- a/supervisor/backups/backup.py +++ b/supervisor/backups/backup.py @@ -52,17 +52,6 @@ from .validate import SCHEMA_BACKUP _LOGGER: logging.Logger = logging.getLogger(__name__) -MAP_FOLDER_EXCLUDE = { - FOLDER_HOMEASSISTANT: [ - "*.db-wal", - "*.db-shm", - "__pycache__/*", - "*.log", - "*.log.*", - "OZW_Log.txt", - ] -} - class Backup(CoreSysAttributes): """A single Supervisor backup.""" @@ -395,7 +384,7 @@ class Backup(CoreSysAttributes): atomic_contents_add( tar_file, origin_dir, - excludes=MAP_FOLDER_EXCLUDE.get(name, []), + excludes=[], arcname=".", ) @@ -486,6 +475,27 @@ class Backup(CoreSysAttributes): # save self.sys_homeassistant.save_data() + async def store_homeassistant_config_dir(self): + """Backup Home Assitant Core configuration folder.""" + + # Backup Home Assistant Core config directory + homeassistant_file = SecureTarFile( + Path(self._tmp.name, "homeassistant.tar.gz"), "w", key=self._key + ) + + await self.sys_homeassistant.backup(homeassistant_file) + self._data[ATTR_FOLDERS].append(FOLDER_HOMEASSISTANT) + + async def restore_homeassistant_config_dir(self): + """Restore Home Assitant Core configuration folder.""" + + # Restore Home Assistant Core config directory + homeassistant_file = SecureTarFile( + Path(self._tmp.name, "homeassistant.tar.gz"), "r", key=self._key + ) + + await self.sys_homeassistant.restore(homeassistant_file) + def store_repositories(self): """Store repository list into backup.""" self.repositories = self.sys_config.addons_repositories diff --git a/supervisor/backups/manager.py b/supervisor/backups/manager.py index a7e10f5fb..c3b0ccfc1 100644 --- a/supervisor/backups/manager.py +++ b/supervisor/backups/manager.py @@ -145,6 +145,11 @@ class BackupManager(CoreSysAttributes): await backup.store_addons(addon_list) # Backup folders + if FOLDER_HOMEASSISTANT in folder_list: + await backup.store_homeassistant_config_dir() + folder_list = list(folder_list) + folder_list.remove(FOLDER_HOMEASSISTANT) + if folder_list: _LOGGER.info("Backing up %s store folders", backup.slug) await backup.store_folders(folder_list) @@ -153,11 +158,9 @@ class BackupManager(CoreSysAttributes): _LOGGER.exception("Backup %s error", backup.slug) self.sys_capture_exception(err) return None - else: self._backups[backup.slug] = backup return backup - finally: self.sys_core.state = CoreState.RUNNING @@ -232,7 +235,9 @@ class BackupManager(CoreSysAttributes): backup.restore_dockerconfig() if FOLDER_HOMEASSISTANT in folder_list: - backup.restore_homeassistant() + await backup.restore_homeassistant_config_dir() + folder_list = list(folder_list) + folder_list.remove(FOLDER_HOMEASSISTANT) # Process folders if folder_list: @@ -243,6 +248,7 @@ class BackupManager(CoreSysAttributes): task_hass = None if homeassistant: _LOGGER.info("Restoring %s Home Assistant Core", backup.slug) + backup.restore_homeassistant() task_hass = self._update_core_task(backup.homeassistant_version) if addon_list: diff --git a/supervisor/homeassistant/const.py b/supervisor/homeassistant/const.py index 09318e768..bdf3fb703 100644 --- a/supervisor/homeassistant/const.py +++ b/supervisor/homeassistant/const.py @@ -7,8 +7,6 @@ from ..const import CoreState LANDINGPAGE: AwesomeVersion = AwesomeVersion("landingpage") -MIN_VERSION = {"supervisor/event": "2021.2.4"} - CLOSING_STATES = [ CoreState.SHUTDOWN, CoreState.STOPPING, @@ -21,6 +19,8 @@ class WSType(str, Enum): AUTH = "auth" SUPERVISOR_EVENT = "supervisor/event" + BACKUP_START = "backup/start" + BACKUP_END = "backup/end" class WSEvent(str, Enum): diff --git a/supervisor/homeassistant/module.py b/supervisor/homeassistant/module.py index 3ae85156a..174c35a46 100644 --- a/supervisor/homeassistant/module.py +++ b/supervisor/homeassistant/module.py @@ -4,11 +4,16 @@ from ipaddress import IPv4Address import logging from pathlib import Path import shutil +import tarfile from typing import Optional from uuid import UUID from awesomeversion import AwesomeVersion, AwesomeVersionException +from supervisor.exceptions import HomeAssistantWSError +from supervisor.homeassistant.const import WSType +from supervisor.utils.tar import SecureTarFile, atomic_contents_add + from ..const import ( ATTR_ACCESS_TOKEN, ATTR_AUDIO_INPUT, @@ -24,6 +29,7 @@ from ..const import ( ATTR_WAIT_BOOT, ATTR_WATCHDOG, FILE_HASSIO_HOMEASSISTANT, + FOLDER_HOMEASSISTANT, BusEvent, ) from ..coresys import CoreSys, CoreSysAttributes @@ -39,6 +45,15 @@ from .websocket import HomeAssistantWebSocket _LOGGER: logging.Logger = logging.getLogger(__name__) +HOMEASSISTANT_BACKUP_EXCLUDE = [ + "*.db-shm", + "__pycache__/*", + "*.log", + "*.log.*", + "OZW_Log.txt", +] + + class HomeAssistant(FileConfiguration, CoreSysAttributes): """Home Assistant core object for handle it.""" @@ -284,3 +299,59 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes): return self.sys_homeassistant.websocket.send_message({ATTR_TYPE: "usb/scan"}) + + async def backup(self, secure_tar_file: SecureTarFile) -> None: + """Backup Home Assistant Core config/ directory.""" + + # Let Home Assistant Core know we are about to backup + try: + await self.websocket.async_send_command({ATTR_TYPE: WSType.BACKUP_START}) + + except HomeAssistantWSError: + _LOGGER.warning( + "Preparing backup of Home Assistant Core failed. Check HA Core logs." + ) + + def _write_tarfile(): + with secure_tar_file as tar_file: + # Backup system + origin_dir = Path(self.sys_config.path_supervisor, FOLDER_HOMEASSISTANT) + + # Backup data + atomic_contents_add( + tar_file, + origin_dir, + excludes=HOMEASSISTANT_BACKUP_EXCLUDE, + arcname=".", + ) + + try: + _LOGGER.info("Backing up Home Assistant Core config folder") + await self.sys_run_in_executor(_write_tarfile) + _LOGGER.info("Backup Home Assistant Core config folder done") + finally: + try: + await self.sys_homeassistant.websocket.async_send_command( + {ATTR_TYPE: WSType.BACKUP_END} + ) + except HomeAssistantWSError: + _LOGGER.warning( + "Error during Home Assistant Core backup. Check HA Core logs." + ) + + async def restore(self, secure_tar_file: SecureTarFile) -> None: + """Restore Home Assistant Core config/ directory.""" + + # Perform a restore + def _restore_tarfile(): + origin_dir = Path(self.sys_config.path_supervisor, FOLDER_HOMEASSISTANT) + + with secure_tar_file as tar_file: + tar_file.extractall(path=origin_dir, members=tar_file) + + try: + _LOGGER.info("Restore Home Assistant Core config folder") + await self.sys_run_in_executor(_restore_tarfile) + _LOGGER.info("Restore Home Assistant Core config folder done") + except (tarfile.TarError, OSError) as err: + _LOGGER.warning("Can't restore Home Assistant Core config folder: %s", err) diff --git a/supervisor/homeassistant/websocket.py b/supervisor/homeassistant/websocket.py index 78c475dc8..a2106e476 100644 --- a/supervisor/homeassistant/websocket.py +++ b/supervisor/homeassistant/websocket.py @@ -17,7 +17,13 @@ from ..exceptions import ( HomeAssistantWSError, HomeAssistantWSNotSupported, ) -from .const import CLOSING_STATES, MIN_VERSION, WSEvent, WSType +from .const import CLOSING_STATES, WSEvent, WSType + +MIN_VERSION = { + WSType.SUPERVISOR_EVENT: "2021.2.4", + WSType.BACKUP_START: "2021.12.0", + WSType.BACKUP_END: "2021.12.0", +} _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -214,7 +220,7 @@ class HomeAssistantWebSocket(CoreSysAttributes): try: await self._client.async_send_command(message) - except HomeAssistantWSError: + except HomeAssistantWSConnectionError: await self._client.close() self._client = None @@ -225,7 +231,7 @@ class HomeAssistantWebSocket(CoreSysAttributes): try: return await self._client.async_send_command(message) - except HomeAssistantWSError: + except HomeAssistantWSConnectionError: await self._client.close() self._client = None diff --git a/tests/backups/conftest.py b/tests/backups/conftest.py index 0c1fce259..c16f2c367 100644 --- a/tests/backups/conftest.py +++ b/tests/backups/conftest.py @@ -19,8 +19,11 @@ def fixture_backup_mock(): backup_instance.store_addons = AsyncMock(return_value=None) backup_instance.store_folders = AsyncMock(return_value=None) - backup_instance.restore_addons = AsyncMock(return_value=None) + backup_instance.store_homeassistant_config_dir = AsyncMock(return_value=None) + backup_instance.store_addons = AsyncMock(return_value=None) backup_instance.restore_folders = AsyncMock(return_value=None) + backup_instance.restore_homeassistant_config_dir = AsyncMock(return_value=None) + backup_instance.restore_addons = AsyncMock(return_value=None) backup_instance.restore_repositories = AsyncMock(return_value=None) yield backup_mock diff --git a/tests/backups/test_manager.py b/tests/backups/test_manager.py index d32d4d635..6c80e7be6 100644 --- a/tests/backups/test_manager.py +++ b/tests/backups/test_manager.py @@ -4,7 +4,7 @@ from unittest.mock import AsyncMock, MagicMock from supervisor.backups.const import BackupType from supervisor.backups.manager import BackupManager -from supervisor.const import FOLDER_HOMEASSISTANT, CoreState +from supervisor.const import FOLDER_HOMEASSISTANT, FOLDER_SHARE, CoreState from supervisor.coresys import CoreSys from tests.const import TEST_ADDON_SLUG @@ -32,7 +32,8 @@ async def test_do_backup_full(coresys: CoreSys, backup_mock, install_addon_ssh): assert install_addon_ssh in backup_instance.store_addons.call_args[0][0] backup_instance.store_folders.assert_called_once() - assert len(backup_instance.store_folders.call_args[0][0]) == 5 + assert len(backup_instance.store_folders.call_args[0][0]) == 4 + backup_instance.store_homeassistant_config_dir.assert_called_once() assert coresys.core.state == CoreState.RUNNING @@ -75,7 +76,9 @@ async def test_do_backup_partial_maximal( # backup_mock fixture causes Backup() to be a MagicMock backup_instance: MagicMock = await manager.do_backup_partial( - addons=[TEST_ADDON_SLUG], folders=["/test"], homeassistant=True + addons=[TEST_ADDON_SLUG], + folders=[FOLDER_SHARE, FOLDER_HOMEASSISTANT], + homeassistant=True, ) # Check Backup has been created without password @@ -91,6 +94,7 @@ async def test_do_backup_partial_maximal( backup_instance.store_folders.assert_called_once() assert len(backup_instance.store_folders.call_args[0][0]) == 1 + backup_instance.store_homeassistant_config_dir.assert_called_once() assert coresys.core.state == CoreState.RUNNING @@ -190,7 +194,7 @@ async def test_do_restore_partial_maximal(coresys: CoreSys, partial_backup_mock) await manager.do_restore_partial( backup_instance, addons=[TEST_ADDON_SLUG], - folders=[FOLDER_HOMEASSISTANT], + folders=[FOLDER_SHARE, FOLDER_HOMEASSISTANT], homeassistant=True, ) @@ -201,5 +205,6 @@ async def test_do_restore_partial_maximal(coresys: CoreSys, partial_backup_mock) backup_instance.restore_addons.assert_called_once() backup_instance.restore_folders.assert_called_once() + backup_instance.restore_homeassistant_config_dir.assert_called_once() assert coresys.core.state == CoreState.RUNNING