jeans/supervisor
jeans/supervisor
1
0
Fork 0
mirror of https://github.com/home-assistant/supervisor.git synced 2025-07-13 20:26:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

backups: Add support for creating fully uncompressed backups (#3378)

Browse Source 
Hassio supervisor saves backups in tar files that contains compressed
tar archives, this is convenient when such backups are kept in the same
environment or need to be transferred remotely, but it's not convenient
when they will be processed using other backup tools such as borg or
restic that can handle compression, encryption and data deduplication
themselves.

In fact deduplication won't actually work at all with hassio compressed
backups as there's no way to find common streams for such tools (unless
we make them to export the archives during importing as borg's
import-tar can do), but this would lead to archives that are not easily
recoverable by the supervisor.

So, make possible to pass a "compressed" boolean parameter when creating
backups that will just archive all the data uncompressed.

It will be then up to other tools to manage the archives compression.
This commit is contained in:
Marco Trevisan 2022-02-03 10:24:44 +01:00 committed by GitHub
parent a52272a7fe
commit 80e67b3c57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 112 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
supervisor/api/backups.py
View File

@ -13,6 +13,7 @@ from ..backups.validate import ALL_FOLDERS
from ..const import (
ATTR_ADDONS,
ATTR_BACKUPS,
ATTR_COMPRESSED,
ATTR_CONTENT,
ATTR_DATE,
ATTR_FOLDERS,
@ -51,6 +52,7 @@ SCHEMA_BACKUP_FULL = vol.Schema(
{
vol.Optional(ATTR_NAME): str,
vol.Optional(ATTR_PASSWORD): vol.Maybe(str),
vol.Optional(ATTR_COMPRESSED): vol.Maybe(vol.Boolean()),
}
)
@ -86,6 +88,7 @@ class APIBackups(CoreSysAttributes):
ATTR_TYPE: backup.sys_type,
ATTR_SIZE: backup.size,
ATTR_PROTECTED: backup.protected,
ATTR_COMPRESSED: backup.compressed,
ATTR_CONTENT: {
ATTR_HOMEASSISTANT: backup.homeassistant_version is not None,
ATTR_ADDONS: backup.addon_list,
@ -128,6 +131,7 @@ class APIBackups(CoreSysAttributes):
ATTR_NAME: backup.name,
ATTR_DATE: backup.date,
ATTR_SIZE: backup.size,
ATTR_COMPRESSED: backup.compressed,
ATTR_PROTECTED: backup.protected,
ATTR_HOMEASSISTANT: backup.homeassistant_version,
ATTR_ADDONS: data_addons,

33
supervisor/backups/backup.py
View File

@ -19,6 +19,7 @@ from ..const import (
ATTR_AUDIO_INPUT,
ATTR_AUDIO_OUTPUT,
ATTR_BOOT,
ATTR_COMPRESSED,
ATTR_CRYPTO,
ATTR_DATE,
ATTR_DOCKER,
@ -90,6 +91,11 @@ class Backup(CoreSysAttributes):
"""Return backup date."""
return self._data.get(ATTR_PROTECTED) is not None
@property
def compressed(self):
"""Return whether backup is compressed."""
return self._data.get(ATTR_COMPRESSED)
@property
def addons(self):
"""Return backup date."""
@ -152,7 +158,7 @@ class Backup(CoreSysAttributes):
"""Return path to backup tarfile."""
return self._tarfile
def new(self, slug, name, date, sys_type, password=None):
def new(self, slug, name, date, sys_type, password=None, compressed=True):
"""Initialize a new backup."""
# Init metadata
self._data[ATTR_SLUG] = slug
@ -169,6 +175,9 @@ class Backup(CoreSysAttributes):
self._data[ATTR_PROTECTED] = password_for_validating(password)
self._data[ATTR_CRYPTO] = CRYPTO_AES128
if not compressed:
self._data[ATTR_COMPRESSED] = False
def set_password(self, password: str) -> bool:
"""Set the password for an existing backup."""
if not password:
@ -306,8 +315,9 @@ class Backup(CoreSysAttributes):
async def _addon_save(addon: Addon):
"""Task to store an add-on into backup."""
tar_name = f"{addon.slug}.tar{'.gz' if self.compressed else ''}"
addon_file = SecureTarFile(
Path(self._tmp.name, f"{addon.slug}.tar.gz"), "w", key=self._key
Path(self._tmp.name, tar_name), "w", key=self._key, gzip=self.compressed
)
# Take backup
@ -340,8 +350,9 @@ class Backup(CoreSysAttributes):
async def _addon_restore(addon_slug: str):
"""Task to restore an add-on into backup."""
tar_name = f"{addon_slug}.tar{'.gz' if self.compressed else ''}"
addon_file = SecureTarFile(
Path(self._tmp.name, f"{addon_slug}.tar.gz"), "r", key=self._key
Path(self._tmp.name, tar_name), "r", key=self._key, gzip=self.compressed
)
# If exists inside backup
@ -369,7 +380,9 @@ class Backup(CoreSysAttributes):
def _folder_save(name: str):
"""Take backup of a folder."""
slug_name = name.replace("/", "_")
tar_name = Path(self._tmp.name, f"{slug_name}.tar.gz")
tar_name = Path(
self._tmp.name, f"{slug_name}.tar{'.gz' if self.compressed else ''}"
)
origin_dir = Path(self.sys_config.path_supervisor, name)
# Check if exists
@ -380,7 +393,9 @@ class Backup(CoreSysAttributes):
# Take backup
try:
_LOGGER.info("Backing up folder %s", name)
with SecureTarFile(tar_name, "w", key=self._key) as tar_file:
with SecureTarFile(
tar_name, "w", key=self._key, gzip=self.compressed
) as tar_file:
atomic_contents_add(
tar_file,
origin_dir,
@ -407,7 +422,9 @@ class Backup(CoreSysAttributes):
def _folder_restore(name: str):
"""Intenal function to restore a folder."""
slug_name = name.replace("/", "_")
tar_name = Path(self._tmp.name, f"{slug_name}.tar.gz")
tar_name = Path(
self._tmp.name, f"{slug_name}.tar{'.gz' if self.compressed else ''}"
)
origin_dir = Path(self.sys_config.path_supervisor, name)
# Check if exists inside backup
@ -422,7 +439,9 @@ class Backup(CoreSysAttributes):
# Perform a restore
try:
_LOGGER.info("Restore folder %s", name)
with SecureTarFile(tar_name, "r", key=self._key) as tar_file:
with SecureTarFile(
tar_name, "r", key=self._key, gzip=self.compressed
) as tar_file:
tar_file.extractall(path=origin_dir, members=tar_file)
_LOGGER.info("Restore folder %s done", name)
except (tarfile.TarError, OSError) as err:

22
supervisor/backups/manager.py
View File

@ -40,7 +40,9 @@ class BackupManager(CoreSysAttributes):
"""Return backup object."""
return self._backups.get(slug)
def _create_backup(self, name, sys_type, password, homeassistant=True):
def _create_backup(
self, name, sys_type, password, compressed=True, homeassistant=True
):
"""Initialize a new backup object from name."""
date_str = utcnow().isoformat()
slug = create_slug(name, date_str)
@ -48,7 +50,7 @@ class BackupManager(CoreSysAttributes):
# init object
backup = Backup(self.coresys, tar_file)
backup.new(slug, name, date_str, sys_type, password)
backup.new(slug, name, date_str, sys_type, password, compressed)
# set general data
if homeassistant:
@ -165,13 +167,13 @@ class BackupManager(CoreSysAttributes):
self.sys_core.state = CoreState.RUNNING
@Job(conditions=[JobCondition.FREE_SPACE, JobCondition.RUNNING])
async def do_backup_full(self, name="", password=None):
async def do_backup_full(self, name="", password=None, compressed=True):
"""Create a full backup."""
if self.lock.locked():
_LOGGER.error("A backup/restore process is already running")
return None
backup = self._create_backup(name, BackupType.FULL, password)
backup = self._create_backup(name, BackupType.FULL, password, compressed)
_LOGGER.info("Creating new full backup with slug %s", backup.slug)
async with self.lock:
@ -184,7 +186,13 @@ class BackupManager(CoreSysAttributes):
@Job(conditions=[JobCondition.FREE_SPACE, JobCondition.RUNNING])
async def do_backup_partial(
self, name="", addons=None, folders=None, password=None, homeassistant=True
self,
name="",
addons=None,
folders=None,
password=None,
homeassistant=True,
compressed=True,
):
"""Create a partial backup."""
if self.lock.locked():
@ -197,7 +205,9 @@ class BackupManager(CoreSysAttributes):
if len(addons) == 0 and len(folders) == 0 and not homeassistant:
_LOGGER.error("Nothing to create backup for")
backup = self._create_backup(name, BackupType.PARTIAL, password, homeassistant)
backup = self._create_backup(
name, BackupType.PARTIAL, password, compressed, homeassistant
)
_LOGGER.info("Creating new partial backup with slug %s", backup.slug)
async with self.lock:

2
supervisor/backups/validate.py
View File

@ -7,6 +7,7 @@ from ..const import (
ATTR_AUDIO_INPUT,
ATTR_AUDIO_OUTPUT,
ATTR_BOOT,
ATTR_COMPRESSED,
ATTR_CRYPTO,
ATTR_DATE,
ATTR_DOCKER,
@ -65,6 +66,7 @@ SCHEMA_BACKUP = vol.Schema(
vol.Required(ATTR_TYPE): vol.Coerce(BackupType),
vol.Required(ATTR_NAME): str,
vol.Required(ATTR_DATE): str,
vol.Optional(ATTR_COMPRESSED, default=True): vol.Boolean(),
vol.Inclusive(ATTR_PROTECTED, "encrypted"): vol.All(
str, vol.Length(min=1, max=1)
),

1
supervisor/const.py
View File

@ -132,6 +132,7 @@ ATTR_CHANNEL = "channel"
ATTR_CHASSIS = "chassis"
ATTR_CHECKS = "checks"
ATTR_CLI = "cli"
ATTR_COMPRESSED = "compressed"
ATTR_CONFIG = "config"
ATTR_CONFIGURATION = "configuration"
ATTR_CONNECTED = "connected"

63
tests/backups/test_manager.py
View File

@ -23,6 +23,37 @@ async def test_do_backup_full(coresys: CoreSys, backup_mock, install_addon_ssh):
# Check Backup has been created without password
assert backup_instance.new.call_args[0][3] == BackupType.FULL
assert backup_instance.new.call_args[0][4] is None
assert backup_instance.new.call_args[0][5] is True
backup_instance.store_homeassistant.assert_called_once()
backup_instance.store_repositories.assert_called_once()
backup_instance.store_dockerconfig.assert_called_once()
backup_instance.store_addons.assert_called_once()
assert install_addon_ssh in backup_instance.store_addons.call_args[0][0]
backup_instance.store_folders.assert_called_once()
assert len(backup_instance.store_folders.call_args[0][0]) == 4
assert coresys.core.state == CoreState.RUNNING
async def test_do_backup_full_uncompressed(
coresys: CoreSys, backup_mock, install_addon_ssh
):
"""Test creating Backup."""
coresys.core.state = CoreState.RUNNING
coresys.hardware.disk.get_disk_free_space = lambda x: 5000
manager = BackupManager(coresys)
# backup_mock fixture causes Backup() to be a MagicMock
backup_instance: MagicMock = await manager.do_backup_full(compressed=False)
# Check Backup has been created without password
assert backup_instance.new.call_args[0][3] == BackupType.FULL
assert backup_instance.new.call_args[0][4] is None
assert backup_instance.new.call_args[0][5] is False
backup_instance.store_homeassistant.assert_called_once()
backup_instance.store_repositories.assert_called_once()
@ -53,6 +84,37 @@ async def test_do_backup_partial_minimal(
# Check Backup has been created without password
assert backup_instance.new.call_args[0][3] == BackupType.PARTIAL
assert backup_instance.new.call_args[0][4] is None
assert backup_instance.new.call_args[0][5] is True
backup_instance.store_homeassistant.assert_not_called()
backup_instance.store_repositories.assert_called_once()
backup_instance.store_dockerconfig.assert_called_once()
backup_instance.store_addons.assert_not_called()
backup_instance.store_folders.assert_not_called()
assert coresys.core.state == CoreState.RUNNING
async def test_do_backup_partial_minimal_uncompressed(
coresys: CoreSys, backup_mock, install_addon_ssh
):
"""Test creating minimal partial Backup."""
coresys.core.state = CoreState.RUNNING
coresys.hardware.disk.get_disk_free_space = lambda x: 5000
manager = BackupManager(coresys)
# backup_mock fixture causes Backup() to be a MagicMock
backup_instance: MagicMock = await manager.do_backup_partial(
homeassistant=False, compressed=False
)
# Check Backup has been created without password
assert backup_instance.new.call_args[0][3] == BackupType.PARTIAL
assert backup_instance.new.call_args[0][4] is None
assert backup_instance.new.call_args[0][5] is False
backup_instance.store_homeassistant.assert_not_called()
backup_instance.store_repositories.assert_called_once()
@ -84,6 +146,7 @@ async def test_do_backup_partial_maximal(
# Check Backup has been created without password
assert backup_instance.new.call_args[0][3] == BackupType.PARTIAL
assert backup_instance.new.call_args[0][4] is None
assert backup_instance.new.call_args[0][5] is True
backup_instance.store_homeassistant.assert_called_once()
backup_instance.store_repositories.assert_called_once()