Add internal support for persistent JSON settings using single file

CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
 
 ## [13.3.0.5]
 ### Added
+- Internal support for persistent JSON settings using single file
 
 ### Breaking Changed
 - ESP32 LVGL library from v8.3.11 to v9.0.0, some small breaking changes in C, none in HASPmota (#20659)

tasmota/tasmota_support/settings.ino

@@ -420,7 +420,7 @@ bool SettingsBufferAlloc(uint32_t upload_size) {
   } else {  
     char filename[14];
     for (uint32_t i = 0; i < 129; i++) {
-      snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), i);
+      snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), i);      // /.drvset012
       uint32_t fsize = TfsFileSize(filename);
       if (fsize) {
         if (settings_size == sizeof(TSettings)) {
@@ -466,7 +466,7 @@ uint32_t SettingsConfigBackup(void) {
     filebuf_ptr += sizeof(TSettings);
     char filename[14];
     for (uint32_t i = 0; i < 129; i++) {
-      snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), i);  // /.drvset012
+      snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), i);      // /.drvset012
       uint32_t fsize = TfsFileSize(filename);
       if (fsize) {
         // Add tar header with file size
@@ -474,7 +474,7 @@ uint32_t SettingsConfigBackup(void) {
         filebuf_ptr[14] = fsize;
         filebuf_ptr[15] = fsize >> 8;
         filebuf_ptr += 16;
-        if (XdrvCallDriver(i, FUNC_RESTORE_SETTINGS)) {  // Enabled driver
+        if (i && (XdrvCallDriver(i, FUNC_RESTORE_SETTINGS))) {  // Enabled driver
           // Use most relevant config data which might not have been saved to file
 //          AddLog(LOG_LEVEL_DEBUG, PSTR("CFG: Backup driver %d"), i);
           uint32_t data_size = fsize;              // Fix possible buffer overflow
@@ -565,10 +565,13 @@ bool SettingsConfigRestore(void) {
       uint32_t driver = atoi((const char*)filebuf_ptr +8);      // /.drvset012 = 12
       uint32_t fsize = filebuf_ptr[15] << 8 | filebuf_ptr[14];  // Tar header settings size
       filebuf_ptr += 16;                           // Start of file settings
-      uint32_t buffer_crc32 = filebuf_ptr[3] << 24 | filebuf_ptr[2] << 16 | filebuf_ptr[1] << 8 | filebuf_ptr[0];
+      bool valid_buffer = true;
-      bool valid_buffer = (GetCfgCrc32(filebuf_ptr +4, fsize -4) == buffer_crc32);
+      if (driver) {
+        uint32_t buffer_crc32 = filebuf_ptr[3] << 24 | filebuf_ptr[2] << 16 | filebuf_ptr[1] << 8 | filebuf_ptr[0];
+        valid_buffer = (GetCfgCrc32(filebuf_ptr +4, fsize -4) == buffer_crc32);
+      }
       if (valid_buffer) {
-        if (XdrvCallDriver(driver, FUNC_RESTORE_SETTINGS)) {
+        if (driver && (XdrvCallDriver(driver, FUNC_RESTORE_SETTINGS))) {
           // Restore live config data which will be saved to file before restart
 //          AddLog(LOG_LEVEL_DEBUG, PSTR("CFG: Restore driver %d"), driver);
           filebuf_ptr[1]++;                        // Force invalid crc32 to enable auto upgrade after restart
@@ -1120,8 +1123,8 @@ void SettingsDefaultSet2(void) {
   flag5.mqtt_switches |= MQTT_SWITCHES;
   flag5.mqtt_persistent |= ~MQTT_CLEAN_SESSION;
   flag6.mqtt_disable_sserialrec |= MQTT_DISABLE_SSERIALRECEIVED;
-//  flag.mqtt_serial |= 0;
   flag6.mqtt_disable_modbus |= MQTT_DISABLE_MODBUSRECEIVED;
+//  flag.mqtt_serial |= 0;
   flag.device_index_enable |= MQTT_POWER_FORMAT;
   flag3.time_append_timezone |= MQTT_APPEND_TIMEZONE;
   flag3.button_switch_force_local |= MQTT_BUTTON_SWITCH_FORCE_LOCAL;

tasmota/tasmota_xdrv_driver/xdrv_122_file_json_settings_demo.ino

@@ -0,0 +1,230 @@
+/*
+  xdrv_122_file_json_settings_demo.ino - Demo for Tasmota
+
+  Copyright (C) 2024  Theo Arends
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// Enable this define to use this demo
+//#define USE_DRV_FILE_JSON_DEMO
+
+#ifdef USE_DRV_FILE_JSON_DEMO
+/*********************************************************************************************\
+ * JSON settings load and save demo using Tasmota file system
+ *
+ * To test this file:
+ * - Have hardware with at least 2M flash
+ * - Enable a board with at least 256k filesystem in platform_override.ini
+\*********************************************************************************************/
+#warning **** USE_DRV_FILE_JSON_DEMO is enabled ****
+
+#define XDRV_122               122
+#define XDRV_KEY               "drvset122"
+
+#define DRV_DEMO_MAX_DRV_TEXT  16
+
+const uint16_t DRV_DEMO_VERSION = 0x0105;       // Latest driver version (See settings deltas below)
+
+// Global structure containing driver saved variables
+struct {
+  uint32_t  crc32;    // To detect file changes
+  uint16_t  version;  // To detect driver function changes
+  char      drv_text[DRV_DEMO_MAX_DRV_TEXT][10];
+} DrvDemoSettings;
+
+// Global structure containing driver non-saved variables
+struct {
+  uint32_t any_value;
+} DrvDemoGlobal;
+
+/*********************************************************************************************\
+ * Driver Settings load and save
+\*********************************************************************************************/
+
+bool DrvDemoLoadData(void) {
+  char key[20];
+  snprintf_P(key, sizeof(key), PSTR(XDRV_KEY));
+  String json = UfsJsonSettingsRead(key);
+  if (json.length() == 0) { return false; }
+
+  JsonParser parser((char*)json.c_str());
+  JsonParserObject root = parser.getRootObject();
+  if (!root) { return false; }
+
+  JsonParserToken val = root[PSTR("Crc")];
+  if (val) {
+    DrvDemoSettings.crc32 = val.getUInt();
+  }
+  val = root[PSTR("Version")];
+  if (val) {
+    DrvDemoSettings.version = val.getInt();
+  }
+  JsonParserArray arr = root[PSTR("Text")];
+  if (arr) {
+    for (uint32_t i = 0; i < DRV_DEMO_MAX_DRV_TEXT; i++) {
+      if (arr[i]) {
+        snprintf(DrvDemoSettings.drv_text[i], 10, arr[i].getStr());
+      }
+    }
+  }
+  return true;
+}
+
+bool DrvDemoSaveData(void) {
+  Response_P(PSTR("{\"" XDRV_KEY "\":{\"Crc\":%u,\"Version\":%u,\"Text\":["), DrvDemoSettings.crc32, DrvDemoSettings.version);
+  for (uint32_t i = 0; i < DRV_DEMO_MAX_DRV_TEXT; i++) {
+    ResponseAppend_P(PSTR("%s\"%s\""), (i)?",":"", DrvDemoSettings.drv_text[i]);
+  }
+  ResponseAppend_P(PSTR("]}}"));
+  return UfsJsonSettingsWrite(ResponseData());
+}
+
+void DrvDemoDeleteData(void) {
+  char key[20];
+  snprintf_P(key, sizeof(key), PSTR(XDRV_KEY));
+  UfsJsonSettingsDelete(key);  // Use defaults
+}
+
+/*********************************************************************************************/
+
+void DrvDemoSettingsLoad(bool erase) {
+  // Called from FUNC_PRE_INIT (erase = 0) once at restart
+  // Called from FUNC_RESET_SETTINGS (erase = 1) after command reset 4, 5, or 6
+
+  // *** Start init default values in case key is not found ***
+  AddLog(LOG_LEVEL_INFO, PSTR("DRV: " D_USE_DEFAULTS));
+
+  memset(&DrvDemoSettings, 0x00, sizeof(DrvDemoSettings));
+  DrvDemoSettings.version = DRV_DEMO_VERSION;
+  // Init any other parameter in struct DrvDemoSettings
+  snprintf_P(DrvDemoSettings.drv_text[0], sizeof(DrvDemoSettings.drv_text[0]), PSTR("Azalea"));
+
+  // *** End Init default values ***
+
+#ifndef USE_UFILESYS
+  AddLog(LOG_LEVEL_INFO, PSTR("CFG: Demo use defaults as file system not enabled"));
+#else
+  // Try to load key
+  if (erase) {
+    DrvDemoDeleteData();
+  }
+  else if (DrvDemoLoadData()) {
+    if (DrvDemoSettings.version != DRV_DEMO_VERSION) {      // Fix version dependent changes
+
+      // *** Start fix possible setting deltas ***
+      if (DrvDemoSettings.version < 0x0103) {
+        AddLog(LOG_LEVEL_INFO, PSTR("CFG: Update oldest version restore"));
+
+        snprintf_P(DrvDemoSettings.drv_text[1], sizeof(DrvDemoSettings.drv_text[1]), PSTR("Begonia"));
+      }
+      if (DrvDemoSettings.version < 0x0104) {
+        AddLog(LOG_LEVEL_INFO, PSTR("CFG: Update old version restore"));
+
+      }
+
+      // *** End setting deltas ***
+
+      // Set current version and save settings
+      DrvDemoSettings.version = DRV_DEMO_VERSION;
+      DrvDemoSettingsSave();
+    }
+    AddLog(LOG_LEVEL_INFO, PSTR("CFG: Demo loaded from file"));
+  }
+  else {
+    // File system not ready: No flash space reserved for file system
+    AddLog(LOG_LEVEL_DEBUG, PSTR("CFG: Demo use defaults as file system not ready or key not found"));
+  }
+#endif  // USE_UFILESYS
+}
+
+void DrvDemoSettingsSave(void) {
+  // Called from FUNC_SAVE_SETTINGS every SaveData second and at restart
+#ifdef USE_UFILESYS
+  uint32_t crc32 = GetCfgCrc32((uint8_t*)&DrvDemoSettings +4, sizeof(DrvDemoSettings) -4);  // Skip crc32
+  if (crc32 != DrvDemoSettings.crc32) {
+    // Try to save file /.drvset122
+    DrvDemoSettings.crc32 = crc32;
+
+    if (DrvDemoSaveData()) {
+      AddLog(LOG_LEVEL_DEBUG, PSTR("CFG: Demo saved to file"));
+    } else {
+      // File system not ready: No flash space reserved for file system
+      AddLog(LOG_LEVEL_DEBUG, PSTR("CFG: ERROR Demo file system not ready or unable to save file"));
+    }
+  }
+#endif  // USE_UFILESYS
+}
+
+/*********************************************************************************************\
+ * Commands
+\*********************************************************************************************/
+
+// Demo command line commands
+const char kDrvDemoCommands[] PROGMEM = "Drv|"  // Prefix
+  "Text";
+
+void (* const DrvDemoCommand[])(void) PROGMEM = {
+  &CmndDrvText };
+
+void CmndDrvText(void) {
+  if ((XdrvMailbox.index > 0) && (XdrvMailbox.index <= DRV_DEMO_MAX_DRV_TEXT)) {
+    if (!XdrvMailbox.usridx) {
+      // Command DrvText
+      for (uint32_t i = 0; i < DRV_DEMO_MAX_DRV_TEXT; i++) {
+        AddLog(LOG_LEVEL_DEBUG, PSTR("DRV: DrvText%02d %s"), i, DrvDemoSettings.drv_text[i]);
+      }
+      ResponseCmndDone();
+    } else {
+      // Command DrvText<index> <text>
+      uint32_t index = XdrvMailbox.index -1;
+      if (XdrvMailbox.data_len > 0) {
+        snprintf_P(DrvDemoSettings.drv_text[index], sizeof(DrvDemoSettings.drv_text[index]), XdrvMailbox.data);
+      }
+      ResponseCmndIdxChar(DrvDemoSettings.drv_text[index]);
+    }
+  }
+}
+
+/*********************************************************************************************\
+ * Interface
+\*********************************************************************************************/
+
+bool Xdrv122(uint32_t function) {
+  bool result = false;
+
+  switch (function) {
+    case FUNC_RESET_SETTINGS:
+      DrvDemoSettingsLoad(1);
+      break;
+    case FUNC_SAVE_SETTINGS:
+      DrvDemoSettingsSave();
+      break;
+    case FUNC_COMMAND:
+      result = DecodeCommand(kDrvDemoCommands, DrvDemoCommand);
+      break;
+    case FUNC_PRE_INIT:
+      DrvDemoSettingsLoad(0);
+      break;
+    case FUNC_SAVE_BEFORE_RESTART:
+      // !!! DO NOT USE AS IT'S FUNCTION IS BETTER HANDLED BY FUNC_SAVE_SETTINGS !!!
+      break;
+    case FUNC_ACTIVE:
+      result = true;
+      break;
+  }
+  return result;
+}
+
+#endif  // USE_DRV_FILE_DEMO

tasmota/tasmota_xdrv_driver/xdrv_122_file_settings_demo.ino

@@ -109,11 +109,11 @@ void DrvDemoSettingsLoad(bool erase) {
     if (DrvDemoSettings.version != DRV_DEMO_VERSION) {      // Fix version dependent changes
 
       // *** Start fix possible setting deltas ***
-      if (Settings->version < 0x01010100) {
+      if (DrvDemoSettings.version < 0x01010100) {
         AddLog(LOG_LEVEL_INFO, PSTR("CFG: Update oldest version restore"));
 
       }
-      if (Settings->version < 0x01010101) {
+      if (DrvDemoSettings.version < 0x01010101) {
         AddLog(LOG_LEVEL_INFO, PSTR("CFG: Update old version restore"));
 
       }

tasmota/tasmota_xdrv_driver/xdrv_50_filesystem.ino

@@ -538,6 +538,225 @@ bool UfsExecuteCommandFile(const char *fname) {
   return false;
 }
 
+/*********************************************************************************************\
+ * File JSON settings support using file /.drvset000
+ * 
+ * {"UserSet1":{"Param1":123,"Param2":"Text1"},"UserSet2":{"Param1":123,"Param2":"Text2"},"UserSet3":{"Param1":123,"Param2":"Text3"}}
+\*********************************************************************************************/
+
+bool _UfsJsonSettingsUpdate(const char* data) {
+  // Delete: Input UserSet2
+  // Append: Input {"UserSet2":{"Param1":123,"Param2":"Text2"}}
+
+  char filename[14];
+  snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), 0);  // /.drvset000
+  if (!TfsFileExists(filename)) { return false; }  // Error - File not found
+
+  char bfname[14];
+  strcpy_P(bfname, PSTR("/settmp"));
+  File ofile = ffsp->open(bfname, "w");
+  if (!ofile) { return false; }        // Error - unable to open temporary file
+  File ifile = ffsp->open(filename, "r");
+  if (!ifile) { 
+    ofile.close();
+    ffsp->remove(bfname);
+    return false;                      // Error - unable to open settings file
+  }
+
+  bool append = false;
+  char* key = (char*)data;
+  char key_pos[32];                    // Max key length
+  char *p = strchr(data, '"');
+  if (p) {
+    append = true;
+    char *q = strchr(++p, '"');
+    if (!q) { return false; }          // Error - No valid key provided in data to append
+    uint32_t len = (uint32_t)q - (uint32_t)p;
+    memcpy(key_pos, p, len);
+    key_pos[len] = '\0';               // key = UserSet2
+    key = key_pos;
+  }
+
+  char buffer[32];                     // Max key length
+  uint8_t buf[1];
+  uint32_t index = 0;
+  uint32_t bracket_count = 0;
+  int entries = 0;
+  bool quote = false;
+  bool mine = false;
+  bool deleted = false;
+  while (ifile.available()) {          // Process file
+    ifile.read(buf, 1);
+    if (bracket_count > 1) {           // Copy or skip old data
+      if (!mine) {
+        ofile.write(buf, 1);           // Copy data
+      }
+      if (buf[0] == '}') {
+        bracket_count--;
+      }
+    } else {
+      if (buf[0] == '}') {             // Last bracket
+        break;                         // End of file
+      }
+      else if (buf[0] == '{') {
+        bracket_count++;
+        if (bracket_count > 1) {       // Skip first bracket
+          entries++;
+        }
+      }
+      else if (buf[0] == '"') {
+        quote ^= 1;
+        if (quote) {
+          index = 0;
+        } else {
+          buffer[index] = '\0';        // End of key name
+          mine = (!strcasecmp(buffer, key));
+          if (mine) {
+            entries--;                 // Skip old data
+            deleted = true;
+          } else {
+            ofile.write((entries) ? (uint8_t*)",\"" : (uint8_t*)"{\"", 2);
+            ofile.write((uint8_t*)buffer, strlen(buffer));
+            ofile.write((uint8_t*)"\":{", 3);
+          }
+        }
+      }
+      else {
+        buffer[index++] = buf[0];      // Add key name
+        if (index > sizeof(buffer) -2) {
+          break;                       // Key name too long
+        }
+      }
+    }
+  }
+  ifile.close();
+  if (append) {
+    // Append new data
+    ofile.write((entries) ? (uint8_t*)"," : (uint8_t*)"{", 1);
+    ofile.write((uint8_t*)data +1, strlen(data) -1);
+  } else {
+    // Delete data
+    if (entries) {
+      ofile.write((uint8_t*)"}", 1);
+    }
+  }
+  ofile.close();
+
+  if (index > sizeof(buffer) -2) { 
+    // No changes
+    ffsp->remove(bfname);
+    return false;                      // Error - Key name too long
+  }
+  ffsp->remove(filename);
+  ffsp->rename(bfname, filename);
+  if (!append) {
+    // Delete data
+    if (!entries) {
+      ffsp->remove(filename);
+    }
+    return deleted;                    // State - 0 = Not found, 1 = Deleted
+  }
+  return true;                         // State - Append success
+}
+
+bool UfsJsonSettingsDelete(const char* key) {
+  // Delete: Input UserSet2
+  //         Output 0 = Not found, 1 = Deleted
+  return _UfsJsonSettingsUpdate(key);  // State - 0 = Not found, 1 = Deleted
+}
+
+bool UfsJsonSettingsWrite(const char* data) {
+  // Add new UserSet replacing present UserSet
+  // Input {"UserSet2":{"Param1":123,"Param2":"Text2"}}
+  // Output 0 = Error, 1 = Append success
+
+  char filename[14];
+  snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), 0);  // /.drvset000
+  if (!TfsFileExists(filename)) {
+    File ofile = ffsp->open(filename, "w");
+    if (!ofile) { return false; }      // Error - unable to open settings file
+    ofile.write((uint8_t*)data, strlen(data));
+    ofile.close();
+    return true;                       // State - Append success
+  }
+  return _UfsJsonSettingsUpdate(data); // State - 0 = Error, 1 = Append success
+}
+
+String UfsJsonSettingsRead(const char* key) {
+  // Read: Input UserSet2
+  //       Output "" = Error, {"Param1":123,"Param2":"Text2"} = Data
+
+  String data = "";
+  char filename[14];
+  snprintf_P(filename, sizeof(filename), PSTR(TASM_FILE_DRIVER), 0);      // /.drvset000
+  if (!TfsFileExists(filename)) { return data; }  // Error - File not found
+  File file = ffsp->open(filename, "r");
+  if (!file) { return data; }          // Error - unable to open settings file
+
+  Trim((char*)key);
+  char buffer[128];
+  uint8_t buf[1] = { 0 };
+  uint32_t index = 0;
+  uint32_t bracket_count = 0;
+  bool quote = false;
+  bool mine = false;
+  while (file.available()) {           // Process file
+    file.read(buf, 1);
+    if (bracket_count > 1) {           // Build JSON
+      if (mine) {
+        buffer[index++] = buf[0];      // Add key data
+        if (index > sizeof(buffer) -2) {
+          buffer[index] = '\0'; 
+          data += buffer;              // Add buffer to result
+          index = 0;
+        }
+      }
+      if (buf[0] == '}') {
+        bracket_count--;
+        if (1 == bracket_count) {
+          if (mine) {
+            break;                     // End of key data
+          } else {
+            index = 0;                 // End of data which is not mine
+          }
+        }
+      }
+    } else {
+      if (buf[0] == '}') {             // Last bracket
+        index = 0;
+        break;                         // End of file - key not found
+      }
+      else if (buf[0] == '{') {
+        bracket_count++;
+        if (bracket_count > 1) {       // Skip first bracket
+          index = 0;
+          buffer[index++] = buf[0];    // Start of key data
+        }
+      }
+      else if (buf[0] == '"') {
+        quote ^= 1;
+        if (quote) {
+          index = 0;
+        } else {
+          buffer[index] = '\0';        // End of key name
+          mine = (!strcasecmp(buffer, key));
+        }
+      }
+      else {
+        buffer[index++] = buf[0];      // Add key name
+        if (index > sizeof(buffer) -2) {
+          index = 0;
+          break;                       // Key name too long
+        }
+      }
+    }
+  }
+  file.close();
+  buffer[index] = '\0';
+  data += buffer;
+  return data;                         // State - "" = Error, {"Param1":123,"Param2":"Text2"} = Data
+}
+
 /*********************************************************************************************\
  * Commands
 \*********************************************************************************************/