Configuration file save and restore .xdrvsetXXX

Configuration file save and restore now backup and restore ``.xdrvsetXXX`` files too (#18295)

CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 - ESP32 shutter driver support up to 16 shutters (#18295)
+- Configuration file save and restore now backup and restore ``.xdrvsetXXX`` files too (#18295)
 
 ### Fixed
 

RELEASENOTES.md

@@ -127,6 +127,7 @@ The latter links can be used for OTA upgrades too like ``OtaUrl https://ota.tasm
 
 ### Changed
 - ESP32 LVGL library from v8.3.7 to v8.3.8 (no functional change)
+- Configuration file save and restore now backup and restore ``.xdrvsetXXX`` files too [#18295](https://github.com/arendst/Tasmota/issues/18295)
 - ESP32 shutter driver support up to 16 shutters [#18295](https://github.com/arendst/Tasmota/issues/18295)
 - Matter support for temperature in Fahrenheit (`SetOption8 1`) [#18987](https://github.com/arendst/Tasmota/issues/18987)
 - Matter improve responsiveness [#19002](https://github.com/arendst/Tasmota/issues/19002)

tasmota/tasmota_support/settings.ino

@@ -217,6 +217,7 @@ const uint8_t CFG_ROTATES = 7;      // Number of flash sectors used (handles upl
 
 uint32_t settings_location = EEPROM_LOCATION;
 uint32_t settings_crc32 = 0;
+uint32_t settings_size = 0;
 uint8_t *settings_buffer = nullptr;
 uint8_t config_xor_on_set = CONFIG_FILE_XOR;
 
@@ -375,22 +376,6 @@ void SettingsSaveAll(void) {
  * Settings backup and restore
 \*********************************************************************************************/
 
-void SettingsBufferFree(void) {
-  if (settings_buffer != nullptr) {
-    free(settings_buffer);
-    settings_buffer = nullptr;
-  }
-}
-
-bool SettingsBufferAlloc(void) {
-  SettingsBufferFree();
-  if (!(settings_buffer = (uint8_t *)malloc(sizeof(TSettings)))) {
-    AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_APPLICATION D_UPLOAD_ERR_2));  // Not enough (memory) space
-    return false;
-  }
-  return true;
-}
-
 String SettingsConfigFilename(void) {
   char filename[TOPSZ];
   char hostname[sizeof(TasmotaGlobal.hostname)];
@@ -398,33 +383,117 @@ String SettingsConfigFilename(void) {
   return String(filename);
 }
 
+void SettingsBufferXor(void) {
+  if (config_xor_on_set) {
+    uint32_t xor_index = (settings_size > sizeof(TSettings)) ? 18 : 2;
+    for (uint32_t i = xor_index; i < settings_size; i++) {
+      settings_buffer[i] ^= (config_xor_on_set +i);
+    }
+  }
+}
+
+void SettingsBufferFree(void) {
+  if (settings_buffer != nullptr) {
+    free(settings_buffer);
+    settings_buffer = nullptr;
+  }
+  settings_size = 0;
+}
+
+bool SettingsBufferAlloc(uint32_t upload_size = 0);
+bool SettingsBufferAlloc(uint32_t upload_size) {
+  SettingsBufferFree();
+
+  settings_size = sizeof(TSettings);
+  if (upload_size >= sizeof(TSettings)) {
+    uint32_t mem = ESP_getFreeHeap();
+    if ((mem - upload_size) < (8 * 1024)) {
+      AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_APPLICATION D_UPLOAD_ERR_2));  // Not enough (memory) space
+      return false;
+    }
+    settings_size = upload_size;
+  } else {  
+
+#ifdef USE_UFILESYS
+  char filename[14];
+  for (uint32_t i = 0; i < 129; i++) {
+    snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), i);
+    uint32_t fsize = TfsFileSize(filename);
+    if (fsize) {
+      if (settings_size == sizeof(TSettings)) {
+        settings_size += 16;                     // Add tar header for total file size
+      }
+      fsize = ((fsize / 16) * 16) + 16;          // Use 16-byte boundary
+      settings_size += (16 + fsize);             // Tar header size is 16 bytes
+    }
+  }
+#endif  // USE_UFILESYS
+
+  }
+
+  if (!(settings_buffer = (uint8_t *)calloc(settings_size, 1))) {
+    AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_APPLICATION D_UPLOAD_ERR_2));  // Not enough (memory) space
+    return false;
+  }
+  return true;
+}
+
 uint32_t SettingsConfigBackup(void) {
   if (!SettingsBufferAlloc()) { return 0; }
 
+  uint8_t *filebuf_ptr = settings_buffer;
+
+#ifdef USE_UFILESYS
+  if (settings_size > sizeof(TSettings)) {
+    snprintf_P((char*)filebuf_ptr, 14, PSTR(TASM_FILE_SETTINGS));  // /.settings
+    filebuf_ptr += 14;
+    *filebuf_ptr = settings_size;
+    filebuf_ptr++;
+    *filebuf_ptr = (settings_size >> 8);
+    filebuf_ptr++;
+  }
+#endif  // USE_UFILESYS
+
   uint32_t cfg_crc32 = Settings->cfg_crc32;
   Settings->cfg_crc32 = GetSettingsCrc32();  // Calculate crc (again) as it might be wrong when savedata = 0 (#3918)
-
+  memcpy(filebuf_ptr, Settings, sizeof(TSettings));
-  uint32_t config_len = sizeof(TSettings);
-  memcpy(settings_buffer, Settings, config_len);
-
   Settings->cfg_crc32 = cfg_crc32;  // Restore crc in case savedata = 0 to make sure settings will be noted as changed
 
-  if (config_xor_on_set) {
+#ifdef USE_UFILESYS
-    for (uint32_t i = 2; i < config_len; i++) {
+  if (settings_size > sizeof(TSettings)) {
-      settings_buffer[i] ^= (config_xor_on_set +i);
+    filebuf_ptr += sizeof(TSettings);
+    char filename[14];
+    for (uint32_t i = 0; i < 129; i++) {
+      snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), i);  // /.drvset012
+      uint32_t fsize = TfsFileSize(filename);
+      if (fsize) {
+        memcpy(filebuf_ptr, filename, 14);
+        filebuf_ptr += 14;
+        *filebuf_ptr = fsize;
+        filebuf_ptr++;
+        *filebuf_ptr = (fsize >> 8);
+        filebuf_ptr++;
+        AddLog(LOG_LEVEL_DEBUG, PSTR("CFG: Backup file %s (%d)"), (char*)filebuf_ptr -16, fsize);
+        TfsLoadFile((const char*)filebuf_ptr -16, (uint8_t*)filebuf_ptr, fsize);
+        filebuf_ptr += ((fsize / 16) * 16) + 16;
+      }
     }
   }
-  return config_len;
+#endif  // USE_UFILESYS
+
+  SettingsBufferXor();
+  return settings_size;
 }
 
 bool SettingsConfigRestore(void) {
-  uint32_t config_len = sizeof(TSettings);
+  SettingsBufferXor();
 
-  if (config_xor_on_set) {
+#ifdef USE_UFILESYS
-    for (uint32_t i = 2; i < config_len; i++) {
+  if (settings_size > sizeof(TSettings)) {
-      settings_buffer[i] ^= (config_xor_on_set +i);
+    settings_size -= 16;
-    }
+    memmove(settings_buffer, settings_buffer +16, settings_size);  // Skip tar header
   }
+#endif  // USE_UFILESYS
 
   bool valid_settings = false;
 
@@ -434,11 +503,11 @@ bool SettingsConfigRestore(void) {
     // uint16_t      cfg_size;                  // 002
     uint32_t buffer_size = settings_buffer[3] << 8 | settings_buffer[2];
     if (buffer_version > 0x0606000A) {
-      // uint32_t      cfg_crc32;                 // FFC
+      // uint32_t      cfg_crc32;               // FFC
       uint32_t buffer_crc32 = settings_buffer[4095] << 24 | settings_buffer[4094] << 16 | settings_buffer[4093] << 8 | settings_buffer[4092];
       valid_settings = (GetCfgCrc32(settings_buffer, buffer_size -4) == buffer_crc32);
     } else {
-      // uint16_t      cfg_crc;                   // 00E
+      // uint16_t      cfg_crc;                 // 00E
       uint16_t buffer_crc16 = settings_buffer[15] << 8 | settings_buffer[14];
       valid_settings = (GetCfgCrc16(settings_buffer, buffer_size) == buffer_crc16);
     }
@@ -467,12 +536,27 @@ bool SettingsConfigRestore(void) {
 
   if (valid_settings) {
     SettingsDefaultSet2();
-    memcpy((char*)Settings +16, settings_buffer +16, config_len -16);
+    memcpy((char*)Settings +16, settings_buffer +16, sizeof(TSettings) -16);
     Settings->version = buffer_version;  // Restore version and auto upgrade after restart
   }
 
-  SettingsBufferFree();
+#ifdef USE_UFILESYS
+  if (settings_size > sizeof(TSettings)) {
+    uint8_t *filebuf_ptr = settings_buffer + sizeof(TSettings);
+    while ((filebuf_ptr - settings_buffer) < settings_size) {
+      filebuf_ptr += 14;
+      uint32_t fsize = *filebuf_ptr;
+      filebuf_ptr++;
+      fsize += *filebuf_ptr << 8;
+      filebuf_ptr++;
+      AddLog(LOG_LEVEL_DEBUG, PSTR("CFG: Restore file %s (%d)"), (char*)filebuf_ptr -16, fsize);
+      TfsSaveFile((const char*)filebuf_ptr -16, (uint8_t*)filebuf_ptr, fsize);
+      filebuf_ptr += ((fsize / 16) * 16) + 16;
+    }
+  }
+#endif  // USE_UFILESYS
 
+  SettingsBufferFree();
   return valid_settings;
 }
 

tasmota/tasmota_xdrv_driver/xdrv_01_9_webserver.ino

@@ -2236,7 +2236,7 @@ void HandleBackupConfiguration(void)
   AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_HTTP D_BACKUP_CONFIGURATION));
 
   uint32_t config_len = SettingsConfigBackup();
-  if (!config_len) { return; }
+  if (!config_len) { return; }    // Unable to allocate 4k buffer
 
   WiFiClient myClient = Webserver->client();
   Webserver->setContentLength(config_len);
@@ -2731,14 +2731,8 @@ void HandleUploadLoop(void) {
 
     AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_UPLOAD D_FILE " %s"), upload.filename.c_str());
 
-    if (UPL_SETTINGS == Web.upload_file_type) {
-      if (!SettingsBufferAlloc()) {
-        Web.upload_error = 2;  // Not enough space
-        return;
-      }
-    }
 #ifdef USE_UFILESYS
-    else if (UPL_UFSFILE == Web.upload_file_type) {
+    if (UPL_UFSFILE == Web.upload_file_type) {
       if (!UfsUploadFileOpen(upload.filename.c_str())) {
         Web.upload_error = 2;
         return;
@@ -2751,6 +2745,16 @@ void HandleUploadLoop(void) {
   else if (UPLOAD_FILE_WRITE == upload.status) {
     if (0 == upload.totalSize) {  // First block received
       if (UPL_SETTINGS == Web.upload_file_type) {
+        uint32_t set_size = sizeof(TSettings);
+#ifdef USE_UFILESYS
+        if (('s' == upload.buf[2]) && ('e' == upload.buf[3])) {  // /.settings
+          set_size = upload.buf[14] + (upload.buf[15] << 8);
+        }
+#endif  // USE_UFILESYS
+        if (!SettingsBufferAlloc(set_size)) {
+          Web.upload_error = 2;  // Not enough space
+          return;
+        }
         Web.config_block_count = 0;
       }
 #ifdef USE_WEB_FW_UPGRADE
@@ -2820,7 +2824,7 @@ void HandleUploadLoop(void) {
     }  // First block received
 
     if (UPL_SETTINGS == Web.upload_file_type) {
-      if (upload.currentSize > (sizeof(TSettings) - (Web.config_block_count * HTTP_UPLOAD_BUFLEN))) {
+      if (upload.currentSize > (settings_size - (Web.config_block_count * HTTP_UPLOAD_BUFLEN))) {
         Web.upload_error = 9;  // File too large
         return;
       }
@@ -3368,9 +3372,10 @@ int WebGetConfig(char *buffer) {
       if (http_code == HTTP_CODE_OK || http_code == HTTP_CODE_MOVED_PERMANENTLY) {
         WiFiClient *stream = http.getStreamPtr();
         int len = http.getSize();
-        if ((len <= sizeof(TSettings)) && SettingsBufferAlloc()) {
+//        if ((len <= sizeof(TSettings)) && SettingsBufferAlloc()) {
+        if (SettingsBufferAlloc(len)) {
           uint8_t *buff = settings_buffer;
-          if (len == -1) { len = sizeof(TSettings); }
+          if (len == -1) { len = settings_size; }
           while (http.connected() && (len > 0)) {
             size_t size = stream->available();
             if (size) {

tasmota/tasmota_xdrv_driver/xdrv_02_1_mqtt_file.ino

@@ -96,9 +96,9 @@ uint32_t MqttFileUploadValidate(uint32_t rcv_id) {
 
     // Check buffer size
     if (UPL_SETTINGS == FMqtt.file_type) {
-      if (FMqtt.file_size > sizeof(TSettings)) {
+//      if (FMqtt.file_size > sizeof(TSettings)) {
-        return 2;                                            // Settings supports max 4k size
+//        return 2;                                            // Settings supports max 4k size
-      }
+//      }
     } else {                                                 // Check enough flash space for intermediate upload
       uint32_t head_room = (FlashWriteMaxSector() - FlashWriteStartSector()) * SPI_FLASH_SEC_SIZE;
       uint32_t rounded_size = (FMqtt.file_size + SPI_FLASH_SEC_SIZE -1) & (~(SPI_FLASH_SEC_SIZE - 1));
@@ -109,8 +109,11 @@ uint32_t MqttFileUploadValidate(uint32_t rcv_id) {
 
     // Init file_buffer
     if (UPL_SETTINGS == FMqtt.file_type) {
-      if (SettingsConfigBackup()) {
+//      if (SettingsConfigBackup()) {
+      if (SettingsBufferAlloc(FMqtt.file_size)) {
         FMqtt.file_buffer = settings_buffer;
+      } else {
+        return 2;                                            // Settings supports max size
       }
     }
     else {