Bugfix: No such file or directory: '/data/homeassistant/home-assistant_v2.db-shm' (#1795)

* Do not use `tar_file.add` to recursively add backup folder

As the folders might contain files which are being removed temporarily (e.g. shared memory file of sqlite database), relying on `tar_file.add` becomes problematic as it crashes the whole backup process if a file does not exist anymore.
This becomes annoying, if the file which causes the error should be excluded by the filter.

To workaround this issue, we manually iterating over the files/directories and apply filters before passing the file or directory to the `tar_file.add` method. As per [documentation](https://docs.python.org/3/library/pathlib.html#pure-paths), pure path does not access the file system.

Fixes #779

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Remove unused import

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Applied code review suggestions

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Applied codestyle

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Remove `pathlib` util and move `is_excluded_by_filter` into `tar` utils

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Rename method

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Rename `origin_dir` to `origin_path` and apply `Path` typehint

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Codestyle

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Add comment why we add the directory even if we are iterating over all its items

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Use `atomic_contents_add` from tar utils to archive addon data

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Remove unused function `exclude_filter`

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Remove unsecure default list value

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Some more codestyle

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Lowercase method name `Path.joinpath`

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Fix codestyle and use proper variable

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Add test for `_is_excluded_by_filter`

Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com>

* Update addon.py

Co-authored-by: Pascal Vizeli <pascal.vizeli@syshack.ch>
This commit is contained in:
Maximilian Bösing 2020-06-26 11:36:49 +02:00 committed by GitHub
parent 3f31979f66
commit a3cf445c93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 73 additions and 41 deletions

View File

@ -51,7 +51,7 @@ from ..exceptions import (
) )
from ..utils.apparmor import adjust_profile from ..utils.apparmor import adjust_profile
from ..utils.json import read_json_file, write_json_file from ..utils.json import read_json_file, write_json_file
from ..utils.tar import exclude_filter, secure_path from ..utils.tar import secure_path, atomic_contents_add
from .model import AddonModel, Data from .model import AddonModel, Data
from .utils import remove_data from .utils import remove_data
from .validate import SCHEMA_ADDON_SNAPSHOT, validate_options from .validate import SCHEMA_ADDON_SNAPSHOT, validate_options
@ -534,10 +534,12 @@ class Addon(AddonModel):
async def snapshot(self, tar_file: tarfile.TarFile) -> None: async def snapshot(self, tar_file: tarfile.TarFile) -> None:
"""Snapshot state of an add-on.""" """Snapshot state of an add-on."""
with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp: with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
temp_path = Path(temp)
# store local image # store local image
if self.need_build: if self.need_build:
try: try:
await self.instance.export_image(Path(temp, "image.tar")) await self.instance.export_image(temp_path.joinpath("image.tar"))
except DockerAPIError: except DockerAPIError:
raise AddonsError() from None raise AddonsError() from None
@ -550,14 +552,14 @@ class Addon(AddonModel):
# Store local configs/state # Store local configs/state
try: try:
write_json_file(Path(temp, "addon.json"), data) write_json_file(temp_path.joinpath("addon.json"), data)
except JsonFileError: except JsonFileError:
_LOGGER.error("Can't save meta for %s", self.slug) _LOGGER.error("Can't save meta for %s", self.slug)
raise AddonsError() from None raise AddonsError() from None
# Store AppArmor Profile # Store AppArmor Profile
if self.sys_host.apparmor.exists(self.slug): if self.sys_host.apparmor.exists(self.slug):
profile = Path(temp, "apparmor.txt") profile = temp_path.joinpath("apparmor.txt")
try: try:
self.sys_host.apparmor.backup_profile(self.slug, profile) self.sys_host.apparmor.backup_profile(self.slug, profile)
except HostAppArmorError: except HostAppArmorError:
@ -569,13 +571,15 @@ class Addon(AddonModel):
"""Write tar inside loop.""" """Write tar inside loop."""
with tar_file as snapshot: with tar_file as snapshot:
# Snapshot system # Snapshot system
snapshot.add(temp, arcname=".") snapshot.add(temp, arcname=".")
# Snapshot data # Snapshot data
snapshot.add( atomic_contents_add(
snapshot,
self.path_data, self.path_data,
excludes=self.snapshot_exclude,
arcname="data", arcname="data",
filter=exclude_filter(self.snapshot_exclude),
) )
try: try:

View File

@ -42,7 +42,11 @@ from ..const import (
from ..coresys import CoreSys, CoreSysAttributes from ..coresys import CoreSys, CoreSysAttributes
from ..exceptions import AddonsError from ..exceptions import AddonsError
from ..utils.json import write_json_file from ..utils.json import write_json_file
from ..utils.tar import SecureTarFile, exclude_filter, secure_path from ..utils.tar import (
SecureTarFile,
secure_path,
atomic_contents_add,
)
from .utils import key_to_iv, password_for_validating, password_to_key, remove_folder from .utils import key_to_iv, password_for_validating, password_to_key, remove_folder
from .validate import ALL_FOLDERS, SCHEMA_SNAPSHOT from .validate import ALL_FOLDERS, SCHEMA_SNAPSHOT
@ -370,10 +374,11 @@ class Snapshot(CoreSysAttributes):
try: try:
_LOGGER.info("Snapshot folder %s", name) _LOGGER.info("Snapshot folder %s", name)
with SecureTarFile(tar_name, "w", key=self._key) as tar_file: with SecureTarFile(tar_name, "w", key=self._key) as tar_file:
tar_file.add( atomic_contents_add(
tar_file,
origin_dir, origin_dir,
excludes=MAP_FOLDER_EXCLUDE.get(name, []),
arcname=".", arcname=".",
filter=exclude_filter(MAP_FOLDER_EXCLUDE.get(name, [])),
) )
_LOGGER.info("Snapshot folder %s done", name) _LOGGER.info("Snapshot folder %s done", name)

View File

@ -2,9 +2,9 @@
import hashlib import hashlib
import logging import logging
import os import os
from pathlib import Path from pathlib import Path, PurePath
import tarfile import tarfile
from typing import IO, Callable, Generator, List, Optional from typing import IO, Generator, List, Optional
from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives import padding
@ -134,20 +134,41 @@ def secure_path(tar: tarfile.TarFile) -> Generator[tarfile.TarInfo, None, None]:
yield member yield member
def exclude_filter( def _is_excluded_by_filter(path: PurePath, exclude_list: List[str]) -> bool:
exclude_list: List[str],
) -> Callable[[tarfile.TarInfo], Optional[tarfile.TarInfo]]:
"""Create callable filter function to check TarInfo for add."""
def my_filter(tar: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
"""Filter to filter excludes.""" """Filter to filter excludes."""
file_path = Path(tar.name)
for exclude in exclude_list: for exclude in exclude_list:
if not file_path.match(exclude): if not path.match(exclude):
continue continue
_LOGGER.debug("Ignore %s because of %s", file_path, exclude) _LOGGER.debug("Ignore %s because of %s", path, exclude)
return True
return False
def atomic_contents_add(
tar_file: tarfile.TarFile,
origin_path: Path,
excludes: List[str],
arcname: str = ".",
) -> None:
"""Append directories and/or files to the TarFile if excludes wont filter."""
if _is_excluded_by_filter(origin_path, excludes):
return None return None
return tar # Add directory only (recursive=False) to ensure we also archive empty directories
tar_file.add(origin_path.as_posix(), arcname, recursive=False)
return my_filter for directory_item in origin_path.iterdir():
if _is_excluded_by_filter(directory_item, excludes):
continue
arcpath = PurePath(arcname, directory_item.name).as_posix()
if directory_item.is_dir():
atomic_contents_add(tar_file, directory_item.as_posix(), excludes, arcpath)
continue
tar_file.add(directory_item.as_posix(), arcname=arcpath)
return None

View File

@ -2,7 +2,8 @@
import attr import attr
from supervisor.utils.tar import exclude_filter, secure_path from pathlib import PurePath
from supervisor.utils.tar import secure_path, _is_excluded_by_filter
@attr.s @attr.s
@ -33,28 +34,29 @@ def test_not_secure_path():
assert [] == list(secure_path(test_list)) assert [] == list(secure_path(test_list))
def test_exclude_filter_good(): def test_is_excluded_by_filter_good():
"""Test exclude filter.""" """Test exclude filter."""
filter_funct = exclude_filter(["not/match", "/dev/xy"]) filter_list = ["not/match", "/dev/xy"]
test_list = [ test_list = [
TarInfo("test.txt"), PurePath("test.txt"),
TarInfo("data/xy.blob"), PurePath("data/xy.blob"),
TarInfo("bla/blu/ble"), PurePath("bla/blu/ble"),
TarInfo("data/../xy.blob"), PurePath("data/../xy.blob"),
] ]
assert test_list == [filter_funct(result) for result in test_list] for path_object in test_list:
assert _is_excluded_by_filter(path_object, filter_list) is False
def test_exclude_filter_bad(): def test_is_exclude_by_filter_bad():
"""Test exclude filter.""" """Test exclude filter."""
filter_funct = exclude_filter(["*.txt", "data/*", "bla/blu/ble"]) filter_list = ["*.txt", "data/*", "bla/blu/ble"]
test_list = [ test_list = [
TarInfo("test.txt"), PurePath("test.txt"),
TarInfo("data/xy.blob"), PurePath("data/xy.blob"),
TarInfo("bla/blu/ble"), PurePath("bla/blu/ble"),
TarInfo("data/test_files/kk.txt"), PurePath("data/test_files/kk.txt"),
] ]
for info in [filter_funct(result) for result in test_list]: for path_object in test_list:
assert info is None assert _is_excluded_by_filter(path_object, filter_list) is True