Allow home assistant backups to exclude database (#4591)

* Allow home assistant backups to exclude database

* Tweak

Co-authored-by: Pascal Vizeli <pvizeli@syshack.ch>

---------

Co-authored-by: Franck Nijhof <git@frenck.dev>
Co-authored-by: Pascal Vizeli <pvizeli@syshack.ch>

supervisor/api/backups.py

@@ -20,6 +20,7 @@ from ..const import (
     ATTR_DAYS_UNTIL_STALE,
     ATTR_FOLDERS,
     ATTR_HOMEASSISTANT,
+    ATTR_HOMEASSISTANT_EXCLUDE_DATABASE,
     ATTR_LOCATON,
     ATTR_NAME,
     ATTR_PASSWORD,
@@ -64,6 +65,7 @@ SCHEMA_BACKUP_FULL = vol.Schema(
         vol.Optional(ATTR_PASSWORD): vol.Maybe(str),
         vol.Optional(ATTR_COMPRESSED): vol.Maybe(vol.Boolean()),
         vol.Optional(ATTR_LOCATON): vol.Maybe(str),
+        vol.Optional(ATTR_HOMEASSISTANT_EXCLUDE_DATABASE): vol.Boolean(),
     }
 )
 
@@ -184,6 +186,7 @@ class APIBackups(CoreSysAttributes):
             ATTR_ADDONS: data_addons,
             ATTR_REPOSITORIES: backup.repositories,
             ATTR_FOLDERS: backup.folders,
+            ATTR_HOMEASSISTANT_EXCLUDE_DATABASE: backup.homeassistant_exclude_database,
         }
 
     def _location_to_mount(self, body: dict[str, Any]) -> dict[str, Any]:

supervisor/api/homeassistant.py

@@ -12,6 +12,7 @@ from ..const import (
     ATTR_AUDIO_INPUT,
     ATTR_AUDIO_OUTPUT,
     ATTR_BACKUP,
+    ATTR_BACKUPS_EXCLUDE_DATABASE,
     ATTR_BLK_READ,
     ATTR_BLK_WRITE,
     ATTR_BOOT,
@@ -51,6 +52,7 @@ SCHEMA_OPTIONS = vol.Schema(
         vol.Optional(ATTR_REFRESH_TOKEN): vol.Maybe(str),
         vol.Optional(ATTR_AUDIO_OUTPUT): vol.Maybe(str),
         vol.Optional(ATTR_AUDIO_INPUT): vol.Maybe(str),
+        vol.Optional(ATTR_BACKUPS_EXCLUDE_DATABASE): vol.Boolean(),
     }
 )
 
@@ -82,6 +84,7 @@ class APIHomeAssistant(CoreSysAttributes):
             ATTR_WATCHDOG: self.sys_homeassistant.watchdog,
             ATTR_AUDIO_INPUT: self.sys_homeassistant.audio_input,
             ATTR_AUDIO_OUTPUT: self.sys_homeassistant.audio_output,
+            ATTR_BACKUPS_EXCLUDE_DATABASE: self.sys_homeassistant.backups_exclude_database,
         }
 
     @api_process
@@ -113,6 +116,11 @@ class APIHomeAssistant(CoreSysAttributes):
         if ATTR_AUDIO_OUTPUT in body:
             self.sys_homeassistant.audio_output = body[ATTR_AUDIO_OUTPUT]
 
+        if ATTR_BACKUPS_EXCLUDE_DATABASE in body:
+            self.sys_homeassistant.backups_exclude_database = body[
+                ATTR_BACKUPS_EXCLUDE_DATABASE
+            ]
+
         self.sys_homeassistant.save_data()
 
     @api_process

supervisor/backups/backup.py

@@ -26,6 +26,7 @@ from ..const import (
     ATTR_CRYPTO,
     ATTR_DATE,
     ATTR_DOCKER,
+    ATTR_EXCLUDE_DATABASE,
     ATTR_FOLDERS,
     ATTR_HOMEASSISTANT,
     ATTR_NAME,
@@ -130,7 +131,14 @@ class Backup(CoreSysAttributes):
         """Return backup Home Assistant version."""
         if self.homeassistant is None:
             return None
-        return self._data[ATTR_HOMEASSISTANT][ATTR_VERSION]
+        return self.homeassistant[ATTR_VERSION]
+
+    @property
+    def homeassistant_exclude_database(self) -> bool:
+        """Return whether database was excluded from Home Assistant backup."""
+        if self.homeassistant is None:
+            return None
+        return self.homeassistant[ATTR_EXCLUDE_DATABASE]
 
     @property
     def homeassistant(self):
@@ -539,9 +547,12 @@ class Backup(CoreSysAttributes):
             except Exception as err:  # pylint: disable=broad-except
                 _LOGGER.warning("Can't restore folder %s: %s", folder, err)
 
-    async def store_homeassistant(self):
+    async def store_homeassistant(self, exclude_database: bool = False):
         """Backup Home Assistant Core configuration folder."""
-        self._data[ATTR_HOMEASSISTANT] = {ATTR_VERSION: self.sys_homeassistant.version}
+        self._data[ATTR_HOMEASSISTANT] = {
+            ATTR_VERSION: self.sys_homeassistant.version,
+            ATTR_EXCLUDE_DATABASE: exclude_database,
+        }
 
         # Backup Home Assistant Core config directory
         tar_name = Path(
@@ -551,7 +562,7 @@ class Backup(CoreSysAttributes):
             tar_name, "w", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
         )
 
-        await self.sys_homeassistant.backup(homeassistant_file)
+        await self.sys_homeassistant.backup(homeassistant_file, exclude_database)
 
         # Store size
         self.homeassistant[ATTR_SIZE] = homeassistant_file.size
@@ -568,7 +579,9 @@ class Backup(CoreSysAttributes):
             tar_name, "r", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
         )
 
-        await self.sys_homeassistant.restore(homeassistant_file)
+        await self.sys_homeassistant.restore(
+            homeassistant_file, self.homeassistant_exclude_database
+        )
 
         # Generate restore task
         async def _core_update():

supervisor/backups/manager.py

@@ -226,6 +226,7 @@ class BackupManager(FileConfiguration, JobGroup):
         addon_list: list[Addon],
         folder_list: list[str],
         homeassistant: bool,
+        homeassistant_exclude_database: bool | None,
     ) -> Backup | None:
         """Create a backup.
 
@@ -245,7 +246,11 @@ class BackupManager(FileConfiguration, JobGroup):
                 # HomeAssistant Folder is for v1
                 if homeassistant:
                     self._change_stage(BackupJobStage.HOME_ASSISTANT, backup)
-                    await backup.store_homeassistant()
+                    await backup.store_homeassistant(
+                        self.sys_homeassistant.backups_exclude_database
+                        if homeassistant_exclude_database is None
+                        else homeassistant_exclude_database
+                    )
 
                 # Backup folders
                 if folder_list:
@@ -282,6 +287,7 @@ class BackupManager(FileConfiguration, JobGroup):
         password: str | None = None,
         compressed: bool = True,
         location: Mount | type[DEFAULT] | None = DEFAULT,
+        homeassistant_exclude_database: bool | None = None,
     ) -> Backup | None:
         """Create a full backup."""
         if self._get_base_path(location) == self.sys_config.path_backup:
@@ -295,7 +301,11 @@ class BackupManager(FileConfiguration, JobGroup):
 
         _LOGGER.info("Creating new full backup with slug %s", backup.slug)
         backup = await self._do_backup(
-            backup, self.sys_addons.installed, ALL_FOLDERS, True
+            backup,
+            self.sys_addons.installed,
+            ALL_FOLDERS,
+            True,
+            homeassistant_exclude_database,
         )
         if backup:
             _LOGGER.info("Creating full backup with slug %s completed", backup.slug)
@@ -316,6 +326,7 @@ class BackupManager(FileConfiguration, JobGroup):
         homeassistant: bool = False,
         compressed: bool = True,
         location: Mount | type[DEFAULT] | None = DEFAULT,
+        homeassistant_exclude_database: bool | None = None,
     ) -> Backup | None:
         """Create a partial backup."""
         if self._get_base_path(location) == self.sys_config.path_backup:
@@ -347,7 +358,9 @@ class BackupManager(FileConfiguration, JobGroup):
                 continue
             _LOGGER.warning("Add-on %s not found/installed", addon_slug)
 
-        backup = await self._do_backup(backup, addon_list, folders, homeassistant)
+        backup = await self._do_backup(
+            backup, addon_list, folders, homeassistant, homeassistant_exclude_database
+        )
         if backup:
             _LOGGER.info("Creating partial backup with slug %s completed", backup.slug)
         return backup

supervisor/backups/validate.py

@@ -14,6 +14,7 @@ from ..const import (
     ATTR_DATE,
     ATTR_DAYS_UNTIL_STALE,
     ATTR_DOCKER,
+    ATTR_EXCLUDE_DATABASE,
     ATTR_FOLDERS,
     ATTR_HOMEASSISTANT,
     ATTR_NAME,
@@ -103,6 +104,9 @@ SCHEMA_BACKUP = vol.Schema(
                     {
                         vol.Required(ATTR_VERSION): version_tag,
                         vol.Optional(ATTR_SIZE, default=0): vol.Coerce(float),
+                        vol.Optional(
+                            ATTR_EXCLUDE_DATABASE, default=False
+                        ): vol.Boolean(),
                     },
                     extra=vol.REMOVE_EXTRA,
                 )

supervisor/const.py

@@ -115,6 +115,7 @@ ATTR_BACKUP_EXCLUDE = "backup_exclude"
 ATTR_BACKUP_POST = "backup_post"
 ATTR_BACKUP_PRE = "backup_pre"
 ATTR_BACKUPS = "backups"
+ATTR_BACKUPS_EXCLUDE_DATABASE = "backups_exclude_database"
 ATTR_BLK_READ = "blk_read"
 ATTR_BLK_WRITE = "blk_write"
 ATTR_BOARD = "board"
@@ -167,6 +168,7 @@ ATTR_ENABLE = "enable"
 ATTR_ENABLED = "enabled"
 ATTR_ENVIRONMENT = "environment"
 ATTR_EVENT = "event"
+ATTR_EXCLUDE_DATABASE = "exclude_database"
 ATTR_FEATURES = "features"
 ATTR_FILENAME = "filename"
 ATTR_FLAGS = "flags"
@@ -182,6 +184,7 @@ ATTR_HASSOS = "hassos"
 ATTR_HEALTHY = "healthy"
 ATTR_HEARTBEAT_LED = "heartbeat_led"
 ATTR_HOMEASSISTANT = "homeassistant"
+ATTR_HOMEASSISTANT_EXCLUDE_DATABASE = "homeassistant_exclude_database"
 ATTR_HOMEASSISTANT_API = "homeassistant_api"
 ATTR_HOST = "host"
 ATTR_HOST_DBUS = "host_dbus"

supervisor/homeassistant/module.py

@@ -18,6 +18,7 @@ from ..const import (
     ATTR_ACCESS_TOKEN,
     ATTR_AUDIO_INPUT,
     ATTR_AUDIO_OUTPUT,
+    ATTR_BACKUPS_EXCLUDE_DATABASE,
     ATTR_BOOT,
     ATTR_IMAGE,
     ATTR_PORT,
@@ -62,6 +63,10 @@ HOMEASSISTANT_BACKUP_EXCLUDE = [
     "*.log.*",
     "OZW_Log.txt",
 ]
+HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE = [
+    "home-assistant_v?.db",
+    "home-assistant_v?.db-wal",
+]
 
 
 class HomeAssistant(FileConfiguration, CoreSysAttributes):
@@ -258,6 +263,16 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
         except (AwesomeVersionException, TypeError):
             return False
 
+    @property
+    def backups_exclude_database(self) -> bool:
+        """Exclude database from core backups by default."""
+        return self._data[ATTR_BACKUPS_EXCLUDE_DATABASE]
+
+    @backups_exclude_database.setter
+    def backups_exclude_database(self, value: bool) -> None:
+        """Set whether backups should exclude database by default."""
+        self._data[ATTR_BACKUPS_EXCLUDE_DATABASE] = value
+
     async def load(self) -> None:
         """Prepare Home Assistant object."""
         await asyncio.wait(
@@ -327,7 +342,9 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
             )
 
     @Job(name="home_assistant_module_backup")
-    async def backup(self, tar_file: tarfile.TarFile) -> None:
+    async def backup(
+        self, tar_file: tarfile.TarFile, exclude_database: bool = False
+    ) -> None:
         """Backup Home Assistant Core config/ directory."""
         await self.begin_backup()
         try:
@@ -351,11 +368,16 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
                         # Backup metadata
                         backup.add(temp, arcname=".")
 
+                        # Set excludes
+                        excludes = HOMEASSISTANT_BACKUP_EXCLUDE.copy()
+                        if exclude_database:
+                            excludes += HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE
+
                         # Backup data
                         atomic_contents_add(
                             backup,
                             self.sys_config.path_homeassistant,
-                            excludes=HOMEASSISTANT_BACKUP_EXCLUDE,
+                            excludes=excludes,
                             arcname="data",
                         )
 
@@ -371,7 +393,10 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
         finally:
             await self.end_backup()
 
-    async def restore(self, tar_file: tarfile.TarFile) -> None:
+    @Job(name="home_assistant_module_restore")
+    async def restore(
+        self, tar_file: tarfile.TarFile, exclude_database: bool = False
+    ) -> None:
         """Restore Home Assistant Core config/ directory."""
         with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
             temp_path = Path(temp)
@@ -399,11 +424,22 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
             def _restore_data():
                 """Restore data."""
                 shutil.copytree(
-                    temp_data, self.sys_config.path_homeassistant, symlinks=True
+                    temp_data,
+                    self.sys_config.path_homeassistant,
+                    symlinks=True,
+                    dirs_exist_ok=bool(excludes),
                 )
 
             _LOGGER.info("Restore Home Assistant Core config folder")
-            await remove_folder(self.sys_config.path_homeassistant)
+            excludes = (
+                HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE if exclude_database else None
+            )
+            await remove_folder(
+                self.sys_config.path_homeassistant,
+                content_only=bool(excludes),
+                excludes=excludes,
+                tmp_dir=self.sys_config.path_tmp,
+            )
             try:
                 await self.sys_run_in_executor(_restore_data)
             except shutil.Error as err:

supervisor/homeassistant/validate.py

@@ -7,6 +7,7 @@ from ..const import (
     ATTR_ACCESS_TOKEN,
     ATTR_AUDIO_INPUT,
     ATTR_AUDIO_OUTPUT,
+    ATTR_BACKUPS_EXCLUDE_DATABASE,
     ATTR_BOOT,
     ATTR_IMAGE,
     ATTR_PORT,
@@ -32,6 +33,7 @@ SCHEMA_HASS_CONFIG = vol.Schema(
         vol.Optional(ATTR_WATCHDOG, default=True): vol.Boolean(),
         vol.Optional(ATTR_AUDIO_OUTPUT, default=None): vol.Maybe(str),
         vol.Optional(ATTR_AUDIO_INPUT, default=None): vol.Maybe(str),
+        vol.Optional(ATTR_BACKUPS_EXCLUDE_DATABASE, default=False): vol.Boolean(),
     },
     extra=vol.REMOVE_EXTRA,
 )

supervisor/utils/__init__.py

@@ -6,6 +6,7 @@ import os
 from pathlib import Path
 import re
 import socket
+from tempfile import TemporaryDirectory
 from typing import Any
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
@@ -76,13 +77,31 @@ def get_message_from_exception_chain(err: Exception) -> str:
     return get_message_from_exception_chain(err.__context__)
 
 
-async def remove_folder(folder: Path, content_only: bool = False) -> None:
+async def remove_folder(
+    folder: Path,
+    content_only: bool = False,
+    excludes: list[str] | None = None,
+    tmp_dir: Path | None = None,
+) -> None:
     """Remove folder and reset privileged.
 
     Is needed to avoid issue with:
         - CAP_DAC_OVERRIDE
         - CAP_DAC_READ_SEARCH
     """
+    if excludes:
+        if not tmp_dir:
+            raise ValueError("tmp_dir is required if excludes are provided")
+        if not content_only:
+            raise ValueError("Cannot delete the folder if excludes are provided")
+
+        temp = TemporaryDirectory(dir=tmp_dir)
+        temp_path = Path(temp.name)
+        moved_files: list[Path] = []
+        for item in folder.iterdir():
+            if any(item.match(exclude) for exclude in excludes):
+                moved_files.append(item.rename(temp_path / item.name))
+
     del_folder = f"{folder}" + "/{,.[!.],..?}*" if content_only else f"{folder}"
     try:
         proc = await asyncio.create_subprocess_exec(
@@ -99,6 +118,11 @@ async def remove_folder(folder: Path, content_only: bool = False) -> None:
     else:
         if proc.returncode == 0:
             return
+    finally:
+        if excludes:
+            for item in moved_files:
+                item.rename(folder / item.name)
+            temp.cleanup()
 
     _LOGGER.error("Can't remove folder %s: %s", folder, error_msg)
 

tests/api/test_backups.py

@@ -2,7 +2,7 @@
 
 import asyncio
 from pathlib import Path, PurePath
-from unittest.mock import AsyncMock, patch
+from unittest.mock import ANY, AsyncMock, patch
 
 from aiohttp.test_utils import TestClient
 from awesomeversion import AwesomeVersion
@@ -11,6 +11,7 @@ import pytest
 from supervisor.backups.backup import Backup
 from supervisor.const import CoreState
 from supervisor.coresys import CoreSys
+from supervisor.homeassistant.module import HomeAssistant
 from supervisor.mounts.mount import Mount
 
 
@@ -167,3 +168,34 @@ async def test_api_freeze_thaw(
         call.args[0] == {"type": "backup/end"}
         for call in ha_ws_client.async_send_command.call_args_list
     )
+
+
+@pytest.mark.parametrize(
+    "partial_backup,exclude_db_setting",
+    [(False, True), (True, True), (False, False), (True, False)],
+)
+async def test_api_backup_exclude_database(
+    api_client: TestClient,
+    coresys: CoreSys,
+    partial_backup: bool,
+    exclude_db_setting: bool,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test backups exclude the database when specified."""
+    coresys.core.state = CoreState.RUNNING
+    coresys.hardware.disk.get_disk_free_space = lambda x: 5000
+    coresys.homeassistant.version = AwesomeVersion("2023.09.0")
+    coresys.homeassistant.backups_exclude_database = exclude_db_setting
+
+    json = {} if exclude_db_setting else {"homeassistant_exclude_database": True}
+    with patch.object(HomeAssistant, "backup") as backup:
+        if partial_backup:
+            resp = await api_client.post(
+                "/backups/new/partial", json={"homeassistant": True} | json
+            )
+        else:
+            resp = await api_client.post("/backups/new/full", json=json)
+
+        backup.assert_awaited_once_with(ANY, True)
+        assert resp.status == 200

tests/api/test_homeassistant.py

@@ -1,11 +1,12 @@
 """Test homeassistant api."""
 
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 from aiohttp.test_utils import TestClient
 import pytest
 
 from supervisor.coresys import CoreSys
+from supervisor.homeassistant.module import HomeAssistant
 
 from tests.common import load_json_fixture
 
@@ -39,3 +40,26 @@ async def test_api_stats(api_client: TestClient, coresys: CoreSys):
     assert result["data"]["memory_usage"] == 59700000
     assert result["data"]["memory_limit"] == 4000000000
     assert result["data"]["memory_percent"] == 1.49
+
+
+async def test_api_set_options(api_client: TestClient, coresys: CoreSys):
+    """Test setting options for homeassistant."""
+    resp = await api_client.get("/homeassistant/info")
+    assert resp.status == 200
+    result = await resp.json()
+    assert result["data"]["watchdog"] is True
+    assert result["data"]["backups_exclude_database"] is False
+
+    with patch.object(HomeAssistant, "save_data") as save_data:
+        resp = await api_client.post(
+            "/homeassistant/options",
+            json={"backups_exclude_database": True, "watchdog": False},
+        )
+        assert resp.status == 200
+        save_data.assert_called_once()
+
+    resp = await api_client.get("/homeassistant/info")
+    assert resp.status == 200
+    result = await resp.json()
+    assert result["data"]["watchdog"] is False
+    assert result["data"]["backups_exclude_database"] is True

tests/backups/test_manager.py

@@ -24,7 +24,9 @@ from supervisor.exceptions import AddonsError, BackupError, BackupJobError, Dock
 from supervisor.homeassistant.api import HomeAssistantAPI
 from supervisor.homeassistant.core import HomeAssistantCore
 from supervisor.homeassistant.module import HomeAssistant
+from supervisor.jobs.const import JobCondition
 from supervisor.mounts.mount import Mount
+from supervisor.utils.json import read_json_file, write_json_file
 
 from tests.const import TEST_ADDON_SLUG
 from tests.dbus_service_mocks.base import DBusServiceMock
@@ -1441,3 +1443,59 @@ async def test_backup_to_mount_bypasses_free_space_condition(
     # These succeed because local free space does not matter when using a mount
     await coresys.backups.do_backup_full(location=mount)
     await coresys.backups.do_backup_partial(folders=["media"], location=mount)
+
+
+@pytest.mark.parametrize(
+    "partial_backup,exclude_db_setting",
+    [(False, True), (True, True), (False, False), (True, False)],
+)
+async def test_skip_homeassistant_database(
+    coresys: CoreSys,
+    container: MagicMock,
+    partial_backup: bool,
+    exclude_db_setting: bool | None,
+    tmp_supervisor_data,
+    path_extern,
+):
+    """Test exclude database option skips database in backup."""
+    coresys.core.state = CoreState.RUNNING
+    coresys.hardware.disk.get_disk_free_space = lambda x: 5000
+    coresys.jobs.ignore_conditions = [
+        JobCondition.INTERNET_HOST,
+        JobCondition.INTERNET_SYSTEM,
+    ]
+    coresys.homeassistant.version = AwesomeVersion("2023.09.0")
+    coresys.homeassistant.backups_exclude_database = exclude_db_setting
+
+    test_file = coresys.config.path_homeassistant / "configuration.yaml"
+    (test_db := coresys.config.path_homeassistant / "home-assistant_v2.db").touch()
+    (
+        test_db_wal := coresys.config.path_homeassistant / "home-assistant_v2.db-wal"
+    ).touch()
+    (
+        test_db_shm := coresys.config.path_homeassistant / "home-assistant_v2.db-shm"
+    ).touch()
+
+    write_json_file(test_file, {"default_config": {}})
+
+    kwargs = {} if exclude_db_setting else {"homeassistant_exclude_database": True}
+    if partial_backup:
+        backup: Backup = await coresys.backups.do_backup_partial(
+            homeassistant=True, **kwargs
+        )
+    else:
+        backup: Backup = await coresys.backups.do_backup_full(**kwargs)
+
+    test_file.unlink()
+    write_json_file(test_db, {"hello": "world"})
+    write_json_file(test_db_wal, {"hello": "world"})
+
+    with patch.object(HomeAssistantCore, "update"), patch.object(
+        HomeAssistantCore, "start"
+    ):
+        await coresys.backups.do_restore_partial(backup, homeassistant=True)
+
+    assert read_json_file(test_file) == {"default_config": {}}
+    assert read_json_file(test_db) == {"hello": "world"}
+    assert read_json_file(test_db_wal) == {"hello": "world"}
+    assert not test_db_shm.exists()