jeans/supervisor
jeans/supervisor
1
0
Fork 0
mirror of https://github.com/home-assistant/supervisor.git synced 2025-07-24 09:36:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow home assistant backups to exclude database (#4591)

Browse Source 
* Allow home assistant backups to exclude database

* Tweak

Co-authored-by: Pascal Vizeli <pvizeli@syshack.ch>

---------

Co-authored-by: Franck Nijhof <git@frenck.dev>
Co-authored-by: Pascal Vizeli <pvizeli@syshack.ch>
This commit is contained in:
Mike Degatano 2023-10-11 02:52:19 -04:00 committed by GitHub
parent 5bbfbf44ae
commit 994c981228
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 236 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
supervisor/api/backups.py
View File

@ -20,6 +20,7 @@ from ..const import (
ATTR_DAYS_UNTIL_STALE,
ATTR_FOLDERS,
ATTR_HOMEASSISTANT,
ATTR_HOMEASSISTANT_EXCLUDE_DATABASE,
ATTR_LOCATON,
ATTR_NAME,
ATTR_PASSWORD,
@ -64,6 +65,7 @@ SCHEMA_BACKUP_FULL = vol.Schema(
vol.Optional(ATTR_PASSWORD): vol.Maybe(str),
vol.Optional(ATTR_COMPRESSED): vol.Maybe(vol.Boolean()),
vol.Optional(ATTR_LOCATON): vol.Maybe(str),
vol.Optional(ATTR_HOMEASSISTANT_EXCLUDE_DATABASE): vol.Boolean(),
}
)
@ -184,6 +186,7 @@ class APIBackups(CoreSysAttributes):
ATTR_ADDONS: data_addons,
ATTR_REPOSITORIES: backup.repositories,
ATTR_FOLDERS: backup.folders,
ATTR_HOMEASSISTANT_EXCLUDE_DATABASE: backup.homeassistant_exclude_database,
}
def _location_to_mount(self, body: dict[str, Any]) -> dict[str, Any]:

8
supervisor/api/homeassistant.py
View File

@ -12,6 +12,7 @@ from ..const import (
ATTR_AUDIO_INPUT,
ATTR_AUDIO_OUTPUT,
ATTR_BACKUP,
ATTR_BACKUPS_EXCLUDE_DATABASE,
ATTR_BLK_READ,
ATTR_BLK_WRITE,
ATTR_BOOT,
@ -51,6 +52,7 @@ SCHEMA_OPTIONS = vol.Schema(
vol.Optional(ATTR_REFRESH_TOKEN): vol.Maybe(str),
vol.Optional(ATTR_AUDIO_OUTPUT): vol.Maybe(str),
vol.Optional(ATTR_AUDIO_INPUT): vol.Maybe(str),
vol.Optional(ATTR_BACKUPS_EXCLUDE_DATABASE): vol.Boolean(),
}
)
@ -82,6 +84,7 @@ class APIHomeAssistant(CoreSysAttributes):
ATTR_WATCHDOG: self.sys_homeassistant.watchdog,
ATTR_AUDIO_INPUT: self.sys_homeassistant.audio_input,
ATTR_AUDIO_OUTPUT: self.sys_homeassistant.audio_output,
ATTR_BACKUPS_EXCLUDE_DATABASE: self.sys_homeassistant.backups_exclude_database,
}
@api_process
@ -113,6 +116,11 @@ class APIHomeAssistant(CoreSysAttributes):
if ATTR_AUDIO_OUTPUT in body:
self.sys_homeassistant.audio_output = body[ATTR_AUDIO_OUTPUT]
if ATTR_BACKUPS_EXCLUDE_DATABASE in body:
self.sys_homeassistant.backups_exclude_database = body[
ATTR_BACKUPS_EXCLUDE_DATABASE
]
self.sys_homeassistant.save_data()
@api_process

23
supervisor/backups/backup.py
View File

@ -26,6 +26,7 @@ from ..const import (
ATTR_CRYPTO,
ATTR_DATE,
ATTR_DOCKER,
ATTR_EXCLUDE_DATABASE,
ATTR_FOLDERS,
ATTR_HOMEASSISTANT,
ATTR_NAME,
@ -130,7 +131,14 @@ class Backup(CoreSysAttributes):
"""Return backup Home Assistant version."""
if self.homeassistant is None:
return None
return self._data[ATTR_HOMEASSISTANT][ATTR_VERSION]
return self.homeassistant[ATTR_VERSION]
@property
def homeassistant_exclude_database(self) -> bool:
"""Return whether database was excluded from Home Assistant backup."""
if self.homeassistant is None:
return None
return self.homeassistant[ATTR_EXCLUDE_DATABASE]
@property
def homeassistant(self):
@ -539,9 +547,12 @@ class Backup(CoreSysAttributes):
except Exception as err: # pylint: disable=broad-except
_LOGGER.warning("Can't restore folder %s: %s", folder, err)
async def store_homeassistant(self):
async def store_homeassistant(self, exclude_database: bool = False):
"""Backup Home Assistant Core configuration folder."""
self._data[ATTR_HOMEASSISTANT] = {ATTR_VERSION: self.sys_homeassistant.version}
self._data[ATTR_HOMEASSISTANT] = {
ATTR_VERSION: self.sys_homeassistant.version,
ATTR_EXCLUDE_DATABASE: exclude_database,
}
# Backup Home Assistant Core config directory
tar_name = Path(
@ -551,7 +562,7 @@ class Backup(CoreSysAttributes):
tar_name, "w", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
)
await self.sys_homeassistant.backup(homeassistant_file)
await self.sys_homeassistant.backup(homeassistant_file, exclude_database)
# Store size
self.homeassistant[ATTR_SIZE] = homeassistant_file.size
@ -568,7 +579,9 @@ class Backup(CoreSysAttributes):
tar_name, "r", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
)
await self.sys_homeassistant.restore(homeassistant_file)
await self.sys_homeassistant.restore(
homeassistant_file, self.homeassistant_exclude_database
)
# Generate restore task
async def _core_update():

19
supervisor/backups/manager.py
View File

@ -226,6 +226,7 @@ class BackupManager(FileConfiguration, JobGroup):
addon_list: list[Addon],
folder_list: list[str],
homeassistant: bool,
homeassistant_exclude_database: bool | None,
) -> Backup | None:
"""Create a backup.
@ -245,7 +246,11 @@ class BackupManager(FileConfiguration, JobGroup):
# HomeAssistant Folder is for v1
if homeassistant:
self._change_stage(BackupJobStage.HOME_ASSISTANT, backup)
await backup.store_homeassistant()
await backup.store_homeassistant(
self.sys_homeassistant.backups_exclude_database
if homeassistant_exclude_database is None
else homeassistant_exclude_database
)
# Backup folders
if folder_list:
@ -282,6 +287,7 @@ class BackupManager(FileConfiguration, JobGroup):
password: str | None = None,
compressed: bool = True,
location: Mount | type[DEFAULT] | None = DEFAULT,
homeassistant_exclude_database: bool | None = None,
) -> Backup | None:
"""Create a full backup."""
if self._get_base_path(location) == self.sys_config.path_backup:
@ -295,7 +301,11 @@ class BackupManager(FileConfiguration, JobGroup):
_LOGGER.info("Creating new full backup with slug %s", backup.slug)
backup = await self._do_backup(
backup, self.sys_addons.installed, ALL_FOLDERS, True
backup,
self.sys_addons.installed,
ALL_FOLDERS,
True,
homeassistant_exclude_database,
)
if backup:
_LOGGER.info("Creating full backup with slug %s completed", backup.slug)
@ -316,6 +326,7 @@ class BackupManager(FileConfiguration, JobGroup):
homeassistant: bool = False,
compressed: bool = True,
location: Mount | type[DEFAULT] | None = DEFAULT,
homeassistant_exclude_database: bool | None = None,
) -> Backup | None:
"""Create a partial backup."""
if self._get_base_path(location) == self.sys_config.path_backup:
@ -347,7 +358,9 @@ class BackupManager(FileConfiguration, JobGroup):
continue
_LOGGER.warning("Add-on %s not found/installed", addon_slug)
backup = await self._do_backup(backup, addon_list, folders, homeassistant)
backup = await self._do_backup(
backup, addon_list, folders, homeassistant, homeassistant_exclude_database
)
if backup:
_LOGGER.info("Creating partial backup with slug %s completed", backup.slug)
return backup

4
supervisor/backups/validate.py
View File

@ -14,6 +14,7 @@ from ..const import (
ATTR_DATE,
ATTR_DAYS_UNTIL_STALE,
ATTR_DOCKER,
ATTR_EXCLUDE_DATABASE,
ATTR_FOLDERS,
ATTR_HOMEASSISTANT,
ATTR_NAME,
@ -103,6 +104,9 @@ SCHEMA_BACKUP = vol.Schema(
{
vol.Required(ATTR_VERSION): version_tag,
vol.Optional(ATTR_SIZE, default=0): vol.Coerce(float),
vol.Optional(
ATTR_EXCLUDE_DATABASE, default=False
): vol.Boolean(),
},
extra=vol.REMOVE_EXTRA,
)

3
supervisor/const.py
View File

@ -115,6 +115,7 @@ ATTR_BACKUP_EXCLUDE = "backup_exclude"
ATTR_BACKUP_POST = "backup_post"
ATTR_BACKUP_PRE = "backup_pre"
ATTR_BACKUPS = "backups"
ATTR_BACKUPS_EXCLUDE_DATABASE = "backups_exclude_database"
ATTR_BLK_READ = "blk_read"
ATTR_BLK_WRITE = "blk_write"
ATTR_BOARD = "board"
@ -167,6 +168,7 @@ ATTR_ENABLE = "enable"
ATTR_ENABLED = "enabled"
ATTR_ENVIRONMENT = "environment"
ATTR_EVENT = "event"
ATTR_EXCLUDE_DATABASE = "exclude_database"
ATTR_FEATURES = "features"
ATTR_FILENAME = "filename"
ATTR_FLAGS = "flags"
@ -182,6 +184,7 @@ ATTR_HASSOS = "hassos"
ATTR_HEALTHY = "healthy"
ATTR_HEARTBEAT_LED = "heartbeat_led"
ATTR_HOMEASSISTANT = "homeassistant"
ATTR_HOMEASSISTANT_EXCLUDE_DATABASE = "homeassistant_exclude_database"
ATTR_HOMEASSISTANT_API = "homeassistant_api"
ATTR_HOST = "host"
ATTR_HOST_DBUS = "host_dbus"

46
supervisor/homeassistant/module.py
View File

@ -18,6 +18,7 @@ from ..const import (
ATTR_ACCESS_TOKEN,
ATTR_AUDIO_INPUT,
ATTR_AUDIO_OUTPUT,
ATTR_BACKUPS_EXCLUDE_DATABASE,
ATTR_BOOT,
ATTR_IMAGE,
ATTR_PORT,
@ -62,6 +63,10 @@ HOMEASSISTANT_BACKUP_EXCLUDE = [
"*.log.*",
"OZW_Log.txt",
]
HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE = [
"home-assistant_v?.db",
"home-assistant_v?.db-wal",
]
class HomeAssistant(FileConfiguration, CoreSysAttributes):
@ -258,6 +263,16 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
except (AwesomeVersionException, TypeError):
return False
@property
def backups_exclude_database(self) -> bool:
"""Exclude database from core backups by default."""
return self._data[ATTR_BACKUPS_EXCLUDE_DATABASE]
@backups_exclude_database.setter
def backups_exclude_database(self, value: bool) -> None:
"""Set whether backups should exclude database by default."""
self._data[ATTR_BACKUPS_EXCLUDE_DATABASE] = value
async def load(self) -> None:
"""Prepare Home Assistant object."""
await asyncio.wait(
@ -327,7 +342,9 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
)
@Job(name="home_assistant_module_backup")
async def backup(self, tar_file: tarfile.TarFile) -> None:
async def backup(
self, tar_file: tarfile.TarFile, exclude_database: bool = False
) -> None:
"""Backup Home Assistant Core config/ directory."""
await self.begin_backup()
try:
@ -351,11 +368,16 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
# Backup metadata
backup.add(temp, arcname=".")
# Set excludes
excludes = HOMEASSISTANT_BACKUP_EXCLUDE.copy()
if exclude_database:
excludes += HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE
# Backup data
atomic_contents_add(
backup,
self.sys_config.path_homeassistant,
excludes=HOMEASSISTANT_BACKUP_EXCLUDE,
excludes=excludes,
arcname="data",
)
@ -371,7 +393,10 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
finally:
await self.end_backup()
async def restore(self, tar_file: tarfile.TarFile) -> None:
@Job(name="home_assistant_module_restore")
async def restore(
self, tar_file: tarfile.TarFile, exclude_database: bool = False
) -> None:
"""Restore Home Assistant Core config/ directory."""
with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
temp_path = Path(temp)
@ -399,11 +424,22 @@ class HomeAssistant(FileConfiguration, CoreSysAttributes):
def _restore_data():
"""Restore data."""
shutil.copytree(
temp_data, self.sys_config.path_homeassistant, symlinks=True
temp_data,
self.sys_config.path_homeassistant,
symlinks=True,
dirs_exist_ok=bool(excludes),
)
_LOGGER.info("Restore Home Assistant Core config folder")
await remove_folder(self.sys_config.path_homeassistant)
excludes = (
HOMEASSISTANT_BACKUP_EXCLUDE_DATABASE if exclude_database else None
)
await remove_folder(
self.sys_config.path_homeassistant,
content_only=bool(excludes),
excludes=excludes,
tmp_dir=self.sys_config.path_tmp,
)
try:
await self.sys_run_in_executor(_restore_data)
except shutil.Error as err:

2
supervisor/homeassistant/validate.py
View File

@ -7,6 +7,7 @@ from ..const import (
ATTR_ACCESS_TOKEN,
ATTR_AUDIO_INPUT,
ATTR_AUDIO_OUTPUT,
ATTR_BACKUPS_EXCLUDE_DATABASE,
ATTR_BOOT,
ATTR_IMAGE,
ATTR_PORT,
@ -32,6 +33,7 @@ SCHEMA_HASS_CONFIG = vol.Schema(
vol.Optional(ATTR_WATCHDOG, default=True): vol.Boolean(),
vol.Optional(ATTR_AUDIO_OUTPUT, default=None): vol.Maybe(str),
vol.Optional(ATTR_AUDIO_INPUT, default=None): vol.Maybe(str),
vol.Optional(ATTR_BACKUPS_EXCLUDE_DATABASE, default=False): vol.Boolean(),
},
extra=vol.REMOVE_EXTRA,
)

26
supervisor/utils/__init__.py
View File

@ -6,6 +6,7 @@ import os
from pathlib import Path
import re
import socket
from tempfile import TemporaryDirectory
from typing import Any
_LOGGER: logging.Logger = logging.getLogger(__name__)
@ -76,13 +77,31 @@ def get_message_from_exception_chain(err: Exception) -> str:
return get_message_from_exception_chain(err.__context__)
async def remove_folder(folder: Path, content_only: bool = False) -> None:
async def remove_folder(
folder: Path,
content_only: bool = False,
excludes: list[str] | None = None,
tmp_dir: Path | None = None,
) -> None:
"""Remove folder and reset privileged.
Is needed to avoid issue with:
- CAP_DAC_OVERRIDE
- CAP_DAC_READ_SEARCH
"""
if excludes:
if not tmp_dir:
raise ValueError("tmp_dir is required if excludes are provided")
if not content_only:
raise ValueError("Cannot delete the folder if excludes are provided")
temp = TemporaryDirectory(dir=tmp_dir)
temp_path = Path(temp.name)
moved_files: list[Path] = []
for item in folder.iterdir():
if any(item.match(exclude) for exclude in excludes):
moved_files.append(item.rename(temp_path / item.name))
del_folder = f"{folder}" + "/{,.[!.],..?}*" if content_only else f"{folder}"
try:
proc = await asyncio.create_subprocess_exec(
@ -99,6 +118,11 @@ async def remove_folder(folder: Path, content_only: bool = False) -> None:
else:
if proc.returncode == 0:
return
finally:
if excludes:
for item in moved_files:
item.rename(folder / item.name)
temp.cleanup()
_LOGGER.error("Can't remove folder %s: %s", folder, error_msg)

34
tests/api/test_backups.py
View File

@ -2,7 +2,7 @@
import asyncio
from pathlib import Path, PurePath
from unittest.mock import AsyncMock, patch
from unittest.mock import ANY, AsyncMock, patch
from aiohttp.test_utils import TestClient
from awesomeversion import AwesomeVersion
@ -11,6 +11,7 @@ import pytest
from supervisor.backups.backup import Backup
from supervisor.const import CoreState
from supervisor.coresys import CoreSys
from supervisor.homeassistant.module import HomeAssistant
from supervisor.mounts.mount import Mount
@ -167,3 +168,34 @@ async def test_api_freeze_thaw(
call.args[0] == {"type": "backup/end"}
for call in ha_ws_client.async_send_command.call_args_list
)
@pytest.mark.parametrize(
"partial_backup,exclude_db_setting",
[(False, True), (True, True), (False, False), (True, False)],
)
async def test_api_backup_exclude_database(
api_client: TestClient,
coresys: CoreSys,
partial_backup: bool,
exclude_db_setting: bool,
tmp_supervisor_data,
path_extern,
):
"""Test backups exclude the database when specified."""
coresys.core.state = CoreState.RUNNING
coresys.hardware.disk.get_disk_free_space = lambda x: 5000
coresys.homeassistant.version = AwesomeVersion("2023.09.0")
coresys.homeassistant.backups_exclude_database = exclude_db_setting
json = {} if exclude_db_setting else {"homeassistant_exclude_database": True}
with patch.object(HomeAssistant, "backup") as backup:
if partial_backup:
resp = await api_client.post(
"/backups/new/partial", json={"homeassistant": True} | json
)
else:
resp = await api_client.post("/backups/new/full", json=json)
backup.assert_awaited_once_with(ANY, True)
assert resp.status == 200

26
tests/api/test_homeassistant.py
View File

@ -1,11 +1,12 @@
"""Test homeassistant api."""
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from aiohttp.test_utils import TestClient
import pytest
from supervisor.coresys import CoreSys
from supervisor.homeassistant.module import HomeAssistant
from tests.common import load_json_fixture
@ -39,3 +40,26 @@ async def test_api_stats(api_client: TestClient, coresys: CoreSys):
assert result["data"]["memory_usage"] == 59700000
assert result["data"]["memory_limit"] == 4000000000
assert result["data"]["memory_percent"] == 1.49
async def test_api_set_options(api_client: TestClient, coresys: CoreSys):
"""Test setting options for homeassistant."""
resp = await api_client.get("/homeassistant/info")
assert resp.status == 200
result = await resp.json()
assert result["data"]["watchdog"] is True
assert result["data"]["backups_exclude_database"] is False
with patch.object(HomeAssistant, "save_data") as save_data:
resp = await api_client.post(
"/homeassistant/options",
json={"backups_exclude_database": True, "watchdog": False},
)
assert resp.status == 200
save_data.assert_called_once()
resp = await api_client.get("/homeassistant/info")
assert resp.status == 200
result = await resp.json()
assert result["data"]["watchdog"] is False
assert result["data"]["backups_exclude_database"] is True

58
tests/backups/test_manager.py
View File

@ -24,7 +24,9 @@ from supervisor.exceptions import AddonsError, BackupError, BackupJobError, Dock
from supervisor.homeassistant.api import HomeAssistantAPI
from supervisor.homeassistant.core import HomeAssistantCore
from supervisor.homeassistant.module import HomeAssistant
from supervisor.jobs.const import JobCondition
from supervisor.mounts.mount import Mount
from supervisor.utils.json import read_json_file, write_json_file
from tests.const import TEST_ADDON_SLUG
from tests.dbus_service_mocks.base import DBusServiceMock
@ -1441,3 +1443,59 @@ async def test_backup_to_mount_bypasses_free_space_condition(
# These succeed because local free space does not matter when using a mount
await coresys.backups.do_backup_full(location=mount)
await coresys.backups.do_backup_partial(folders=["media"], location=mount)
@pytest.mark.parametrize(
"partial_backup,exclude_db_setting",
[(False, True), (True, True), (False, False), (True, False)],
)
async def test_skip_homeassistant_database(
coresys: CoreSys,
container: MagicMock,
partial_backup: bool,
exclude_db_setting: bool | None,
tmp_supervisor_data,
path_extern,
):
"""Test exclude database option skips database in backup."""
coresys.core.state = CoreState.RUNNING
coresys.hardware.disk.get_disk_free_space = lambda x: 5000
coresys.jobs.ignore_conditions = [
JobCondition.INTERNET_HOST,
JobCondition.INTERNET_SYSTEM,
]
coresys.homeassistant.version = AwesomeVersion("2023.09.0")
coresys.homeassistant.backups_exclude_database = exclude_db_setting
test_file = coresys.config.path_homeassistant / "configuration.yaml"
(test_db := coresys.config.path_homeassistant / "home-assistant_v2.db").touch()
(
test_db_wal := coresys.config.path_homeassistant / "home-assistant_v2.db-wal"
).touch()
(
test_db_shm := coresys.config.path_homeassistant / "home-assistant_v2.db-shm"
).touch()
write_json_file(test_file, {"default_config": {}})
kwargs = {} if exclude_db_setting else {"homeassistant_exclude_database": True}
if partial_backup:
backup: Backup = await coresys.backups.do_backup_partial(
homeassistant=True, **kwargs
)
else:
backup: Backup = await coresys.backups.do_backup_full(**kwargs)
test_file.unlink()
write_json_file(test_db, {"hello": "world"})
write_json_file(test_db_wal, {"hello": "world"})
with patch.object(HomeAssistantCore, "update"), patch.object(
HomeAssistantCore, "start"
):
await coresys.backups.do_restore_partial(backup, homeassistant=True)
assert read_json_file(test_file) == {"default_config": {}}
assert read_json_file(test_db) == {"hello": "world"}
assert read_json_file(test_db_wal) == {"hello": "world"}
assert not test_db_shm.exists()