mirror of
https://github.com/home-assistant/supervisor.git
synced 2025-07-10 18:56:30 +00:00
Bugfix: No such file or directory: '/data/homeassistant/home-assistant_v2.db-shm' (#1795)
* Do not use `tar_file.add` to recursively add backup folder As the folders might contain files which are being removed temporarily (e.g. shared memory file of sqlite database), relying on `tar_file.add` becomes problematic as it crashes the whole backup process if a file does not exist anymore. This becomes annoying, if the file which causes the error should be excluded by the filter. To workaround this issue, we manually iterating over the files/directories and apply filters before passing the file or directory to the `tar_file.add` method. As per [documentation](https://docs.python.org/3/library/pathlib.html#pure-paths), pure path does not access the file system. Fixes #779 Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Remove unused import Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Applied code review suggestions Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Applied codestyle Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Remove `pathlib` util and move `is_excluded_by_filter` into `tar` utils Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Rename method Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Rename `origin_dir` to `origin_path` and apply `Path` typehint Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Codestyle Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Add comment why we add the directory even if we are iterating over all its items Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Use `atomic_contents_add` from tar utils to archive addon data Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Remove unused function `exclude_filter` Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Remove unsecure default list value Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Some more codestyle Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Lowercase method name `Path.joinpath` Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Fix codestyle and use proper variable Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Add test for `_is_excluded_by_filter` Signed-off-by: Maximilian Bösing <2189546+boesing@users.noreply.github.com> * Update addon.py Co-authored-by: Pascal Vizeli <pascal.vizeli@syshack.ch>
This commit is contained in:
parent
3f31979f66
commit
a3cf445c93
@ -51,7 +51,7 @@ from ..exceptions import (
|
|||||||
)
|
)
|
||||||
from ..utils.apparmor import adjust_profile
|
from ..utils.apparmor import adjust_profile
|
||||||
from ..utils.json import read_json_file, write_json_file
|
from ..utils.json import read_json_file, write_json_file
|
||||||
from ..utils.tar import exclude_filter, secure_path
|
from ..utils.tar import secure_path, atomic_contents_add
|
||||||
from .model import AddonModel, Data
|
from .model import AddonModel, Data
|
||||||
from .utils import remove_data
|
from .utils import remove_data
|
||||||
from .validate import SCHEMA_ADDON_SNAPSHOT, validate_options
|
from .validate import SCHEMA_ADDON_SNAPSHOT, validate_options
|
||||||
@ -534,10 +534,12 @@ class Addon(AddonModel):
|
|||||||
async def snapshot(self, tar_file: tarfile.TarFile) -> None:
|
async def snapshot(self, tar_file: tarfile.TarFile) -> None:
|
||||||
"""Snapshot state of an add-on."""
|
"""Snapshot state of an add-on."""
|
||||||
with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
|
with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
|
||||||
|
temp_path = Path(temp)
|
||||||
|
|
||||||
# store local image
|
# store local image
|
||||||
if self.need_build:
|
if self.need_build:
|
||||||
try:
|
try:
|
||||||
await self.instance.export_image(Path(temp, "image.tar"))
|
await self.instance.export_image(temp_path.joinpath("image.tar"))
|
||||||
except DockerAPIError:
|
except DockerAPIError:
|
||||||
raise AddonsError() from None
|
raise AddonsError() from None
|
||||||
|
|
||||||
@ -550,14 +552,14 @@ class Addon(AddonModel):
|
|||||||
|
|
||||||
# Store local configs/state
|
# Store local configs/state
|
||||||
try:
|
try:
|
||||||
write_json_file(Path(temp, "addon.json"), data)
|
write_json_file(temp_path.joinpath("addon.json"), data)
|
||||||
except JsonFileError:
|
except JsonFileError:
|
||||||
_LOGGER.error("Can't save meta for %s", self.slug)
|
_LOGGER.error("Can't save meta for %s", self.slug)
|
||||||
raise AddonsError() from None
|
raise AddonsError() from None
|
||||||
|
|
||||||
# Store AppArmor Profile
|
# Store AppArmor Profile
|
||||||
if self.sys_host.apparmor.exists(self.slug):
|
if self.sys_host.apparmor.exists(self.slug):
|
||||||
profile = Path(temp, "apparmor.txt")
|
profile = temp_path.joinpath("apparmor.txt")
|
||||||
try:
|
try:
|
||||||
self.sys_host.apparmor.backup_profile(self.slug, profile)
|
self.sys_host.apparmor.backup_profile(self.slug, profile)
|
||||||
except HostAppArmorError:
|
except HostAppArmorError:
|
||||||
@ -569,13 +571,15 @@ class Addon(AddonModel):
|
|||||||
"""Write tar inside loop."""
|
"""Write tar inside loop."""
|
||||||
with tar_file as snapshot:
|
with tar_file as snapshot:
|
||||||
# Snapshot system
|
# Snapshot system
|
||||||
|
|
||||||
snapshot.add(temp, arcname=".")
|
snapshot.add(temp, arcname=".")
|
||||||
|
|
||||||
# Snapshot data
|
# Snapshot data
|
||||||
snapshot.add(
|
atomic_contents_add(
|
||||||
|
snapshot,
|
||||||
self.path_data,
|
self.path_data,
|
||||||
|
excludes=self.snapshot_exclude,
|
||||||
arcname="data",
|
arcname="data",
|
||||||
filter=exclude_filter(self.snapshot_exclude),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -42,7 +42,11 @@ from ..const import (
|
|||||||
from ..coresys import CoreSys, CoreSysAttributes
|
from ..coresys import CoreSys, CoreSysAttributes
|
||||||
from ..exceptions import AddonsError
|
from ..exceptions import AddonsError
|
||||||
from ..utils.json import write_json_file
|
from ..utils.json import write_json_file
|
||||||
from ..utils.tar import SecureTarFile, exclude_filter, secure_path
|
from ..utils.tar import (
|
||||||
|
SecureTarFile,
|
||||||
|
secure_path,
|
||||||
|
atomic_contents_add,
|
||||||
|
)
|
||||||
from .utils import key_to_iv, password_for_validating, password_to_key, remove_folder
|
from .utils import key_to_iv, password_for_validating, password_to_key, remove_folder
|
||||||
from .validate import ALL_FOLDERS, SCHEMA_SNAPSHOT
|
from .validate import ALL_FOLDERS, SCHEMA_SNAPSHOT
|
||||||
|
|
||||||
@ -370,10 +374,11 @@ class Snapshot(CoreSysAttributes):
|
|||||||
try:
|
try:
|
||||||
_LOGGER.info("Snapshot folder %s", name)
|
_LOGGER.info("Snapshot folder %s", name)
|
||||||
with SecureTarFile(tar_name, "w", key=self._key) as tar_file:
|
with SecureTarFile(tar_name, "w", key=self._key) as tar_file:
|
||||||
tar_file.add(
|
atomic_contents_add(
|
||||||
|
tar_file,
|
||||||
origin_dir,
|
origin_dir,
|
||||||
|
excludes=MAP_FOLDER_EXCLUDE.get(name, []),
|
||||||
arcname=".",
|
arcname=".",
|
||||||
filter=exclude_filter(MAP_FOLDER_EXCLUDE.get(name, [])),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
_LOGGER.info("Snapshot folder %s done", name)
|
_LOGGER.info("Snapshot folder %s done", name)
|
||||||
|
@ -2,9 +2,9 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path, PurePath
|
||||||
import tarfile
|
import tarfile
|
||||||
from typing import IO, Callable, Generator, List, Optional
|
from typing import IO, Generator, List, Optional
|
||||||
|
|
||||||
from cryptography.hazmat.backends import default_backend
|
from cryptography.hazmat.backends import default_backend
|
||||||
from cryptography.hazmat.primitives import padding
|
from cryptography.hazmat.primitives import padding
|
||||||
@ -134,20 +134,41 @@ def secure_path(tar: tarfile.TarFile) -> Generator[tarfile.TarInfo, None, None]:
|
|||||||
yield member
|
yield member
|
||||||
|
|
||||||
|
|
||||||
def exclude_filter(
|
def _is_excluded_by_filter(path: PurePath, exclude_list: List[str]) -> bool:
|
||||||
exclude_list: List[str],
|
"""Filter to filter excludes."""
|
||||||
) -> Callable[[tarfile.TarInfo], Optional[tarfile.TarInfo]]:
|
|
||||||
"""Create callable filter function to check TarInfo for add."""
|
|
||||||
|
|
||||||
def my_filter(tar: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
|
for exclude in exclude_list:
|
||||||
"""Filter to filter excludes."""
|
if not path.match(exclude):
|
||||||
file_path = Path(tar.name)
|
continue
|
||||||
for exclude in exclude_list:
|
_LOGGER.debug("Ignore %s because of %s", path, exclude)
|
||||||
if not file_path.match(exclude):
|
return True
|
||||||
continue
|
|
||||||
_LOGGER.debug("Ignore %s because of %s", file_path, exclude)
|
|
||||||
return None
|
|
||||||
|
|
||||||
return tar
|
return False
|
||||||
|
|
||||||
return my_filter
|
|
||||||
|
def atomic_contents_add(
|
||||||
|
tar_file: tarfile.TarFile,
|
||||||
|
origin_path: Path,
|
||||||
|
excludes: List[str],
|
||||||
|
arcname: str = ".",
|
||||||
|
) -> None:
|
||||||
|
"""Append directories and/or files to the TarFile if excludes wont filter."""
|
||||||
|
|
||||||
|
if _is_excluded_by_filter(origin_path, excludes):
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Add directory only (recursive=False) to ensure we also archive empty directories
|
||||||
|
tar_file.add(origin_path.as_posix(), arcname, recursive=False)
|
||||||
|
|
||||||
|
for directory_item in origin_path.iterdir():
|
||||||
|
if _is_excluded_by_filter(directory_item, excludes):
|
||||||
|
continue
|
||||||
|
|
||||||
|
arcpath = PurePath(arcname, directory_item.name).as_posix()
|
||||||
|
if directory_item.is_dir():
|
||||||
|
atomic_contents_add(tar_file, directory_item.as_posix(), excludes, arcpath)
|
||||||
|
continue
|
||||||
|
|
||||||
|
tar_file.add(directory_item.as_posix(), arcname=arcpath)
|
||||||
|
|
||||||
|
return None
|
||||||
|
@ -2,7 +2,8 @@
|
|||||||
|
|
||||||
import attr
|
import attr
|
||||||
|
|
||||||
from supervisor.utils.tar import exclude_filter, secure_path
|
from pathlib import PurePath
|
||||||
|
from supervisor.utils.tar import secure_path, _is_excluded_by_filter
|
||||||
|
|
||||||
|
|
||||||
@attr.s
|
@attr.s
|
||||||
@ -33,28 +34,29 @@ def test_not_secure_path():
|
|||||||
assert [] == list(secure_path(test_list))
|
assert [] == list(secure_path(test_list))
|
||||||
|
|
||||||
|
|
||||||
def test_exclude_filter_good():
|
def test_is_excluded_by_filter_good():
|
||||||
"""Test exclude filter."""
|
"""Test exclude filter."""
|
||||||
filter_funct = exclude_filter(["not/match", "/dev/xy"])
|
filter_list = ["not/match", "/dev/xy"]
|
||||||
test_list = [
|
test_list = [
|
||||||
TarInfo("test.txt"),
|
PurePath("test.txt"),
|
||||||
TarInfo("data/xy.blob"),
|
PurePath("data/xy.blob"),
|
||||||
TarInfo("bla/blu/ble"),
|
PurePath("bla/blu/ble"),
|
||||||
TarInfo("data/../xy.blob"),
|
PurePath("data/../xy.blob"),
|
||||||
]
|
]
|
||||||
|
|
||||||
assert test_list == [filter_funct(result) for result in test_list]
|
for path_object in test_list:
|
||||||
|
assert _is_excluded_by_filter(path_object, filter_list) is False
|
||||||
|
|
||||||
|
|
||||||
def test_exclude_filter_bad():
|
def test_is_exclude_by_filter_bad():
|
||||||
"""Test exclude filter."""
|
"""Test exclude filter."""
|
||||||
filter_funct = exclude_filter(["*.txt", "data/*", "bla/blu/ble"])
|
filter_list = ["*.txt", "data/*", "bla/blu/ble"]
|
||||||
test_list = [
|
test_list = [
|
||||||
TarInfo("test.txt"),
|
PurePath("test.txt"),
|
||||||
TarInfo("data/xy.blob"),
|
PurePath("data/xy.blob"),
|
||||||
TarInfo("bla/blu/ble"),
|
PurePath("bla/blu/ble"),
|
||||||
TarInfo("data/test_files/kk.txt"),
|
PurePath("data/test_files/kk.txt"),
|
||||||
]
|
]
|
||||||
|
|
||||||
for info in [filter_funct(result) for result in test_list]:
|
for path_object in test_list:
|
||||||
assert info is None
|
assert _is_excluded_by_filter(path_object, filter_list) is True
|
||||||
|
Loading…
x
Reference in New Issue
Block a user