Speed up backups by increasing buffer size (#4229)

* Speed up backups by increasing buffer size

This is the same change as https://github.com/home-assistant/core/pull/90613
but for supervisor

If the backup takes too long, core will release the lock on the database
and the backup will be no good

2fc34e7cce/homeassistant/components/recorder/core.py (L926)

cpython uses copyfileobj under the hood for fast copies but the default buffer size is quite low which increases the amount of time in python code when copying the sqlite database. As this is the usually the bulk of the backup, increasing the buffer can help reduce the backup time quite a bit.

Ideally this would all use sendfile under the hood as it would shift nearly all the burden out of userspace but tarfile doesn't currently try that 4664a7cf68/Lib/shutil.py (L106)

related:
In testing (non encrypted) improvement was at least as good as https://github.com/python/cpython/issues/71386

* add the const
This commit is contained in:
J. Nick Koston 2023-04-19 02:01:44 -10:00 committed by GitHub
parent b3ca08f2c2
commit 744cd4ea39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 7 deletions

View File

@ -44,7 +44,7 @@ from ..exceptions import AddonsError, BackupError
from ..utils import remove_folder
from ..utils.dt import parse_datetime, utcnow
from ..utils.json import write_json_file
from .const import BackupType
from .const import BUF_SIZE, BackupType
from .utils import key_to_iv, password_to_key
from .validate import SCHEMA_BACKUP
@ -330,7 +330,11 @@ class Backup(CoreSysAttributes):
"""Task to store an add-on into backup."""
tar_name = f"{addon.slug}.tar{'.gz' if self.compressed else ''}"
addon_file = SecureTarFile(
Path(self._tmp.name, tar_name), "w", key=self._key, gzip=self.compressed
Path(self._tmp.name, tar_name),
"w",
key=self._key,
gzip=self.compressed,
bufsize=BUF_SIZE,
)
# Take backup
@ -365,7 +369,11 @@ class Backup(CoreSysAttributes):
"""Task to restore an add-on into backup."""
tar_name = f"{addon_slug}.tar{'.gz' if self.compressed else ''}"
addon_file = SecureTarFile(
Path(self._tmp.name, tar_name), "r", key=self._key, gzip=self.compressed
Path(self._tmp.name, tar_name),
"r",
key=self._key,
gzip=self.compressed,
bufsize=BUF_SIZE,
)
# If exists inside backup
@ -406,7 +414,7 @@ class Backup(CoreSysAttributes):
# Take backup
_LOGGER.info("Backing up folder %s", name)
with SecureTarFile(
tar_name, "w", key=self._key, gzip=self.compressed
tar_name, "w", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
) as tar_file:
atomic_contents_add(
tar_file,
@ -453,7 +461,11 @@ class Backup(CoreSysAttributes):
try:
_LOGGER.info("Restore folder %s", name)
with SecureTarFile(
tar_name, "r", key=self._key, gzip=self.compressed
tar_name,
"r",
key=self._key,
gzip=self.compressed,
bufsize=BUF_SIZE,
) as tar_file:
tar_file.extractall(path=origin_dir, members=tar_file)
_LOGGER.info("Restore folder %s done", name)
@ -479,7 +491,7 @@ class Backup(CoreSysAttributes):
self._tmp.name, f"homeassistant.tar{'.gz' if self.compressed else ''}"
)
homeassistant_file = SecureTarFile(
tar_name, "w", key=self._key, gzip=self.compressed
tar_name, "w", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
)
await self.sys_homeassistant.backup(homeassistant_file)
@ -496,7 +508,7 @@ class Backup(CoreSysAttributes):
self._tmp.name, f"homeassistant.tar{'.gz' if self.compressed else ''}"
)
homeassistant_file = SecureTarFile(
tar_name, "r", key=self._key, gzip=self.compressed
tar_name, "r", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
)
await self.sys_homeassistant.restore(homeassistant_file)

View File

@ -1,6 +1,8 @@
"""Backup consts."""
from enum import Enum
BUF_SIZE = 2**20 * 4 # 4MB
class BackupType(str, Enum):
"""Backup type enum."""