From 744cd4ea39339c7d4015e55cc1d3850e69b9cfa0 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 19 Apr 2023 02:01:44 -1000
Subject: [PATCH] Speed up backups by increasing buffer size (#4229)

* Speed up backups by increasing buffer size

This is the same change as https://github.com/home-assistant/core/pull/90613
but for supervisor

If the backup takes too long, core will release the lock on the database
and the backup will be no good

https://github.com/home-assistant/core/blob/2fc34e7cced87a8e042919e059d3a07bb760c77f/homeassistant/components/recorder/core.py#L926

cpython uses copyfileobj under the hood for fast copies but the default buffer size is quite low which increases the amount of time in python code when copying the sqlite database. As this is the usually the bulk of the backup, increasing the buffer can help reduce the backup time quite a bit.

Ideally this would all use sendfile under the hood as it would shift nearly all the burden out of userspace but tarfile doesn't currently try that https://github.com/python/cpython/blob/4664a7cf689946f0c9854cadee7c6aa9c276a8cf/Lib/shutil.py#L106

related:
In testing (non encrypted) improvement was at least as good as https://github.com/python/cpython/issues/71386

* add the const
---
 supervisor/backups/backup.py | 26 +++++++++++++++++++-------
 supervisor/backups/const.py  |  2 ++
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/supervisor/backups/backup.py b/supervisor/backups/backup.py
index 2d8560792..67cb14352 100644
--- a/supervisor/backups/backup.py
+++ b/supervisor/backups/backup.py
@@ -44,7 +44,7 @@ from ..exceptions import AddonsError, BackupError
 from ..utils import remove_folder
 from ..utils.dt import parse_datetime, utcnow
 from ..utils.json import write_json_file
-from .const import BackupType
+from .const import BUF_SIZE, BackupType
 from .utils import key_to_iv, password_to_key
 from .validate import SCHEMA_BACKUP
 
@@ -330,7 +330,11 @@ class Backup(CoreSysAttributes):
             """Task to store an add-on into backup."""
             tar_name = f"{addon.slug}.tar{'.gz' if self.compressed else ''}"
             addon_file = SecureTarFile(
-                Path(self._tmp.name, tar_name), "w", key=self._key, gzip=self.compressed
+                Path(self._tmp.name, tar_name),
+                "w",
+                key=self._key,
+                gzip=self.compressed,
+                bufsize=BUF_SIZE,
             )
 
             # Take backup
@@ -365,7 +369,11 @@ class Backup(CoreSysAttributes):
             """Task to restore an add-on into backup."""
             tar_name = f"{addon_slug}.tar{'.gz' if self.compressed else ''}"
             addon_file = SecureTarFile(
-                Path(self._tmp.name, tar_name), "r", key=self._key, gzip=self.compressed
+                Path(self._tmp.name, tar_name),
+                "r",
+                key=self._key,
+                gzip=self.compressed,
+                bufsize=BUF_SIZE,
             )
 
             # If exists inside backup
@@ -406,7 +414,7 @@ class Backup(CoreSysAttributes):
             # Take backup
             _LOGGER.info("Backing up folder %s", name)
             with SecureTarFile(
-                tar_name, "w", key=self._key, gzip=self.compressed
+                tar_name, "w", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
             ) as tar_file:
                 atomic_contents_add(
                     tar_file,
@@ -453,7 +461,11 @@ class Backup(CoreSysAttributes):
                 try:
                     _LOGGER.info("Restore folder %s", name)
                     with SecureTarFile(
-                        tar_name, "r", key=self._key, gzip=self.compressed
+                        tar_name,
+                        "r",
+                        key=self._key,
+                        gzip=self.compressed,
+                        bufsize=BUF_SIZE,
                     ) as tar_file:
                         tar_file.extractall(path=origin_dir, members=tar_file)
                     _LOGGER.info("Restore folder %s done", name)
@@ -479,7 +491,7 @@ class Backup(CoreSysAttributes):
             self._tmp.name, f"homeassistant.tar{'.gz' if self.compressed else ''}"
         )
         homeassistant_file = SecureTarFile(
-            tar_name, "w", key=self._key, gzip=self.compressed
+            tar_name, "w", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
         )
 
         await self.sys_homeassistant.backup(homeassistant_file)
@@ -496,7 +508,7 @@ class Backup(CoreSysAttributes):
             self._tmp.name, f"homeassistant.tar{'.gz' if self.compressed else ''}"
         )
         homeassistant_file = SecureTarFile(
-            tar_name, "r", key=self._key, gzip=self.compressed
+            tar_name, "r", key=self._key, gzip=self.compressed, bufsize=BUF_SIZE
         )
 
         await self.sys_homeassistant.restore(homeassistant_file)
diff --git a/supervisor/backups/const.py b/supervisor/backups/const.py
index ff6e37347..e3762163c 100644
--- a/supervisor/backups/const.py
+++ b/supervisor/backups/const.py
@@ -1,6 +1,8 @@
 """Backup consts."""
 from enum import Enum
 
+BUF_SIZE = 2**20 * 4  # 4MB
+
 
 class BackupType(str, Enum):
     """Backup type enum."""