Merge pull request #9933 from s-hadinger/zigbee_data_persistence

Zigbee persistence of device/sensir data in EEPROM (only ZBBridge)

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ## [9.1.0.2]
 ### Added
 - KNX read reply for Power (#9236, #9891)
+- Zigbee persistence of device/sensir data in EEPROM (only ZBBridge)
 
 ### Breaking Changed
 - KNX DPT9 (16-bit float) to DPT14 (32-bit float) by Adrian Scillato (#9811, #9888)

tasmota/xdrv_23_zigbee_1_headers.ino

@@ -89,6 +89,7 @@ public:
   bool init_phase = true;             // initialization phase, before accepting zigbee traffic
   bool recv_until = false;            // ignore all messages until the received frame fully matches
   bool eeprom_present = false;        // is the ZBBridge EEPROM present?
+  bool eeprom_ready = false;          // is the ZBBridge EEPROM formatted and ready?
 
   uint8_t on_error_goto = ZIGBEE_LABEL_ABORT;         // on error goto label, 99 default to abort
   uint8_t on_timeout_goto = ZIGBEE_LABEL_ABORT;       // on timeout goto label, 99 default to abort

tasmota/xdrv_23_zigbee_4_persistence.ino

@@ -121,6 +121,7 @@ public:
 
 const static uint32_t ZIGB_NAME1 = 0x3167697A; // 'zig1' little endian
 const static uint32_t ZIGB_NAME2 = 0x3267697A; // 'zig2' little endian, v2
+const static uint32_t ZIGB_DATA2 = 0x32746164; // 'dat2' little endian, v2
 const static size_t   Z_MAX_FLASH = z_block_len - sizeof(Z_Flashentry);  // 2040
 
 bool hibernateDeviceConfiguration(SBuffer & buf, const class Z_Data_Set & data, uint8_t endpoint) {
@@ -318,7 +319,7 @@ void loadZigbeeDevices(bool dump_only = false) {
     // parse what seems to be a valid entry
     SBuffer buf(buf_len);
     buf.addBuffer(z_dev_start + sizeof(Z_Flashentry), buf_len);
-    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee devices data in Flash v%d (%d bytes)"), version, buf_len);
+    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee device information in Flash (%d bytes)"), buf_len);
 
     if (dump_only) {
       size_t buf_len = buf.len();
@@ -332,7 +333,7 @@ void loadZigbeeDevices(bool dump_only = false) {
       zigbee_devices.clean();   // don't write back to Flash what we just loaded
     }
   } else {
-    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No zigbee devices data in Flash"));
+    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No Zigbee device information in Flash"));
   }
 #ifdef ESP32
   free(spi_buffer);

tasmota/xdrv_23_zigbee_4a_nano_fs.ino

@@ -0,0 +1,456 @@
+/*
+  xdrv_23_zigbee_4a_eeprom.ino - zigbee support for Tasmota - nano filesystem for EEPROM, with anti-weavering
+
+  Copyright (C) 2020  Theo Arends and Stephan Hadinger
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef USE_ZIGBEE
+
+#include <memory>
+#define Z_EEPROM_DEBUG
+
+// The EEPROM is 64KB in size with individually writable bytes.
+// They are conveniently organized in pages of 128 bytes to accelerate
+// data transfer, but unlike flash memory, you don't need to erase an entire page.
+// The chip spec says it supports over 2 million writes per byte.
+
+// EEPROM LAYOUT:
+// ==============
+// 64KB EEPROM is divided in 256 block of 256 bytes.
+// The internal page size is 128 bytes, so we're grouping 2 pages in one block
+// The advantage is that any pointer to a block is a single byte
+//
+// Block 0, 1 and 255 are reserved.
+//
+// BLock 0 contains the directory of files
+// Block 1 contains the linked list of blocks for each file
+// Block 255 contains the bitmap of block and ageing information
+
+// File structure
+// Each file has :
+//  - a name of 4 chars (no extension) that conveniently fit in uint32_t.
+//  - a length in bytes, encoded with 16 bits (uint16_t)
+//  - 1 byte indicating the first block of the file
+//  - 1 byte reserved
+//
+// Then blocks are a linked-list of content. The next block is indicated in Block 1
+//
+// Note: the linked list could cause a circular reference loop and potentially an infinite loop.
+// This is why the content lenght is used to check that the block count does not exceed
+// the content length hence cannot cause an infinite loop.
+// Any pointer to blocks 1 or 255 is considered invalid and means a corruption of the file system.
+
+// Signature entry:
+// - 4 bytes of signature, currently 'Tasm'. Any other entry indicates that the EEPROM was not formatted
+// - 1 byte version number, currently 0x00
+// - all other bytes (5..7) are reserved and filled with 0s
+
+// DIRECTORY
+// =========
+// Block 0 is the directory. There is no support for folders.
+// Each file entry is 8 bytes.
+// First entry is a signature marker and version
+// Entries 1..30 are for files
+// Entry 31 (last entry) is reserved and filled with 0s
+
+// BITMAP
+// ==========
+// Block 255:
+// Each byte represents a block, remember there are 256 blocks in total
+// Each byte is set as follows:
+// bit 7 - block is used (1) or free (0) - note that blocks 0, 1 and 255 are always used
+// bit 6 - block is damaged - not implemented yet but may be useful
+// bit 0..5 - generation number for anti-weavering
+//
+// Caveat: this bitmap system may lead to wasted blocked marked as used but actually unused
+// Periodical garbage collection and sanity checks can occur, for ex at boot.
+//
+// If the generation number overflows, all blocks start at generation `0`
+// meaning that the entire bitmap block is overwritten.
+
+// Version 0:
+// Many features are not yet implemented.
+// We start with hardcoded values:
+//  - the two entries for files 'zig2' and 'dat2' are predefined
+//  - the starting block for each file is fixed.
+//    'Zig2' uses 32 blocks (8kb max) - starting at block 32
+//    'Dat2' uses 32 blocks (8kb max) - starting at block 64
+//  - the bitmap marks those blocks as used
+//  - version number only uses first entry that doesn't get re-written
+//  - only file size actually changes
+
+/*********************************************************************************************\
+ * 
+ * Constants
+ * 
+\*********************************************************************************************/
+const size_t ZFS_BLOCK_SIZE      = 256;
+const size_t ZFS_ENTRY_SIZE      = 8;    // each entry is 32 bytes
+const size_t ZFS_ENTRIES         = 30;
+const uint32_t ZFS_SIGNATURE     = 0x6D736154;    // 'Tasm'
+
+
+/*********************************************************************************************\
+ * Specific to v2 (limited support)
+\*********************************************************************************************/
+const size_t ZFS_FILE_BLOCKS     = 31;    // 31 blocks
+
+/*********************************************************************************************\
+ * ZFS_File_Entry
+\*********************************************************************************************/
+class ZFS_File_Entry {
+public:
+  uint32_t        name;       // file name representing 4 chars, 0x00000000 means empty entry
+  uint16_t        length;     // length of file in bytes
+  uint8_t         blk_start;
+  uint8_t         reserved;  // reserved for future use
+
+  ZFS_File_Entry() :
+    name(0),
+    length(0),
+    blk_start(0),
+    reserved(0)
+    {}
+  
+  inline static bool validIdx(uint8_t blk_start) { return ((blk_start != 0x00) && (blk_start != 0x01) && (blk_start != 0xFF)); };
+  static uint16_t getAddress(uint8_t entry_idx);
+  void read(uint8_t entry_idx);
+  void write(uint8_t entry_idx) const ;
+};
+
+/*********************************************************************************************\
+ * ZFS_File_Entry
+\*********************************************************************************************/
+class ZFS_Root_Entry {
+public:
+  uint32_t      signature;    // Signature that the block is correctly formatted
+  uint8_t       version;      // version of file system structure
+  uint8_t       reserved[3];
+
+  ZFS_Root_Entry() :
+    signature(ZFS_SIGNATURE),    // 'Tasm'
+    version(0),
+    reserved{}
+    {};
+};
+
+/*********************************************************************************************\
+ * ZFS_File_Entry
+\*********************************************************************************************/
+class ZFS_Dir_Block {
+public:
+  ZFS_Root_Entry    b0;               // signature entry
+  ZFS_File_Entry    e[ZFS_ENTRIES];   // 7 entries for files
+  ZFS_File_Entry    reserved;         // reserved for future use
+
+  void format(void);                  // prepare default values for formatting
+};
+
+/*********************************************************************************************\
+ * ZFS_Bitmap at block 0xFF
+\*********************************************************************************************/
+// Individual block
+union ZFS_Bitmap_Entry {
+  uint8_t   raw;
+  struct {
+    uint8_t   gen : 6;
+    bool      damaged : 1;
+    bool      used : 1;
+  };
+};
+
+class ZFS_Bitmap {
+public:
+  ZFS_Bitmap_Entry         block[ZFS_BLOCK_SIZE];
+
+  void format(void);
+};
+
+/*********************************************************************************************\
+ * ZFS_Map, linked list of blocks, at block 1
+\*********************************************************************************************/
+class ZFS_Map {
+public:
+  uint8_t                  next_blk[ZFS_BLOCK_SIZE];
+
+  void format(void);
+};
+
+/*********************************************************************************************\
+ * 
+ * Formatting implementations
+ * 
+\*********************************************************************************************/
+
+void ZFS_Dir_Block::format(void) {
+  // entry 0 - 'zig2'
+  e[0].name = ZIGB_NAME2;
+  e[0].length = 0;
+  e[0].blk_start = 2;             // start at block 2 to 32
+  // entry 1 - 'dat2'
+  e[1].name = ZIGB_DATA2;
+  e[1].length = 0;
+  e[1].blk_start = 2 + 31;        // start at block 33 to 63
+}
+
+void ZFS_Bitmap::format(void) {
+  ZFS_Bitmap_Entry val_used;
+  val_used.gen = 0;
+  val_used.damaged = false;
+  val_used.used = true;
+  // block 0, 1, 255
+  // block[0x00] = val_used;    // already in loop
+  // block[0x01] = val_used;
+  block[0xFF] = val_used;
+  // reserve block 32->63 for file 0 and 64->95 for file 1
+  for (uint32_t i = 0; i < 64; i++) {
+    block[i] = val_used;
+  }
+}
+
+void ZFS_Map::format(void) {
+  // map a linear linked list for v1
+  for (uint32_t i = 2; i < ZFS_BLOCK_SIZE - 2; i++) {
+    next_blk[i] = i+1;
+  }
+}
+
+/*********************************************************************************************\
+ * 
+ * Writing a file
+ * 
+\*********************************************************************************************/
+
+class ZFS_Write_File {
+public:
+  // file info
+  uint32_t  name;
+  uint16_t  cursor;
+  uint16_t  length;
+  uint8_t   blk_start;      // if 0x00 then file does not exist
+  uint8_t   entry_idx;      // entry number in the directory
+
+  ZFS_Write_File(uint32_t _name) : name(_name), cursor(0), length(0), blk_start(0) { findOrCreate(); }
+
+  inline bool valid(void) const { return blk_start != 0; }       // does the file exist?
+
+  int32_t addBytes(void* buffer, size_t buffer_len);
+  int32_t close(void);
+
+protected:
+  void findOrCreate(void);
+};
+
+
+/*********************************************************************************************\
+ * 
+ * Check that the EEPROM is formatted
+ * 
+\*********************************************************************************************/
+
+// Main class for the Zigbee filesystem
+class ZFS {
+public:
+
+  static void initOrFormat(void);     // <0 means error
+  static void format(void);           // format EEPROM
+
+  static int32_t getLength(uint32_t name);
+  static bool findFileEntry(uint32_t name, ZFS_File_Entry & entry, uint8_t * entry_idx);
+  static void erase(void);              // erase EEPROM
+
+  // read file
+  static int32_t readBytes(uint32_t name, uint8_t* buffer, size_t buffer_len, uint16_t start, uint16_t len);
+};
+
+/*********************************************************************************************\
+ * 
+ * Check that the EEPROM is formatted
+ * 
+\*********************************************************************************************/
+
+bool ZFS::findFileEntry(uint32_t name, ZFS_File_Entry & entry, uint8_t * _entry_idx) {
+  if (!zigbee.eeprom_ready) { return false; }
+  for (uint32_t entry_idx = 0; entry_idx < ZFS_ENTRIES; entry_idx++) {
+    // read entry from EEPROM
+    uint16_t  entry_addr = 0x0000 + sizeof(ZFS_Root_Entry) + sizeof(ZFS_File_Entry) * entry_idx;
+    zigbee.eeprom.readBytes(entry_addr, sizeof(ZFS_File_Entry), (byte*)&entry);
+#ifdef Z_EEPROM_DEBUG
+    // {
+    //   char hex_char[(sizeof(ZFS_File_Entry) * 2) + 2];
+    //   AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Read entry %d at address 0x%04X contains %s"), entry_idx, entry_addr, ToHex_P((uint8_t*)&entry, sizeof(entry), hex_char, sizeof(hex_char)));
+    // }
+#endif
+    if (entry.name == name) {
+      if (_entry_idx) { *_entry_idx = entry_idx; }
+      return true;
+    }
+  }
+  return false;
+}
+
+int32_t ZFS::getLength(uint32_t name) {
+  ZFS_File_Entry entry;
+  if (ZFS::findFileEntry(name, entry, nullptr)) {
+    return entry.length;
+  }
+  return -1;
+}
+
+void ZFS::erase(void) {
+  if (!zigbee.eeprom_present) { return; }
+  uint32_t zero = 0;
+  zigbee.eeprom.writeBytes(0x0000, sizeof(zero), (byte*)&zero);
+}
+
+/*********************************************************************************************\
+ * 
+ * Reading a file
+ * 
+\*********************************************************************************************/
+int32_t ZFS::readBytes(uint32_t name, uint8_t* buffer, size_t buffer_len, uint16_t read_start, uint16_t read_len) {
+  if (!zigbee.eeprom_ready) { return -1; }
+#ifdef Z_EEPROM_DEBUG
+    // AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "readBytes name=%08X, buffer_len=%d, read_start=0x%04X, read_len=%d"), name, buffer_len, read_start, read_len);
+#endif
+  if (name == 0x00000000) { return -1; }
+  if (buffer_len == 0) { return 0; }
+
+  // look for file
+  ZFS_File_Entry entry;
+  uint8_t entry_idx;
+  if (!findFileEntry(name, entry, &entry_idx)) { return -1; }   // file not found
+
+  if (read_start >= entry.length) { return 0; }    // start of read is beyond end of file, return nothing
+  uint16_t max_read_len = entry.length - read_start;   // we know it's > 0
+  if (read_len > max_read_len) { read_len = max_read_len; }
+  if (read_len > buffer_len) { read_len = buffer_len; }
+  // we know read_len is the correct max value now
+
+  // compute the start block for the file
+  // V1 it's the first one
+  uint8_t  blk = entry.blk_start;
+
+  zigbee.eeprom.readBytes((blk << 8) + read_start, read_len, (byte*) buffer);
+  return read_len;
+}
+
+/*********************************************************************************************\
+ * 
+ * Check that the EEPROM is formatted
+ * 
+\*********************************************************************************************/
+
+void ZFS::initOrFormat(void) {
+  if (!zigbee.eeprom_present) { return; }
+
+#ifdef Z_EEPROM_DEBUG
+  // AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "sizeof(ZFS_Bitmap)=%d sizeof(ZFS_File_Entry)=%d sizeof(ZFS_Root_Entry)=%d sizeof(ZFS_Dir_Block)=%d"), sizeof(ZFS_Bitmap), sizeof(ZFS_File_Entry), sizeof(ZFS_Root_Entry), sizeof(ZFS_Dir_Block));
+  {
+    byte map[256];
+    char hex_char[(256 * 2) + 2];
+    zigbee.eeprom.readBytes(0x0000, 256, map);
+    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "BLK 00 %s"), ToHex_P(map, sizeof(map), hex_char, sizeof(hex_char)));
+    zigbee.eeprom.readBytes(0x0100, 256, map);
+    // AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "BLK 01 %s"), ToHex_P(map, sizeof(map), hex_char, sizeof(hex_char)));
+    // zigbee.eeprom.readBytes(0x0200, 256, map);
+    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "BLK 02 %s"), ToHex_P(map, sizeof(map), hex_char, sizeof(hex_char)));
+    zigbee.eeprom.readBytes(0x2100, 256, map);
+    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "BLK 21 %s"), ToHex_P(map, sizeof(map), hex_char, sizeof(hex_char)));
+    // zigbee.eeprom.readBytes(0xFF00, 256, map);
+    // AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "BLK FF %s"), ToHex_P(map, sizeof(map), hex_char, sizeof(hex_char)));
+  }
+#endif
+
+  ZFS_Dir_Block * dir = new ZFS_Dir_Block();
+  zigbee.eeprom.readBytes(0, sizeof(ZFS_Dir_Block), (byte*) dir);
+
+  if (dir->b0.signature == ZFS_SIGNATURE) {
+    // Good
+    AddLog_P(LOG_LEVEL_DEBUG, PSTR(D_LOG_ZIGBEE "EEPROM signature 0x%08X is correct"), dir->b0.signature);
+  } else {
+    AddLog_P(LOG_LEVEL_DEBUG, PSTR(D_LOG_ZIGBEE "EEPROM signature 0x%08X is incorrect, formatting"), dir->b0.signature);
+    format();
+  }
+  delete dir;
+  
+  zigbee.eeprom_ready = true;
+}
+
+//
+// Format EEPROM
+//
+void ZFS::format(void) {
+  AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Formatting EEPROM"));
+
+  // First write the bitmap
+  ZFS_Bitmap * bitmap = new ZFS_Bitmap();
+  bitmap->format();
+  zigbee.eeprom.writeBytes(0xFF00, 256, (byte*) bitmap);
+  delete bitmap;
+
+  // Map
+  ZFS_Map * map = new ZFS_Map();
+  map->format();
+  zigbee.eeprom.writeBytes(0x0100, 256, (byte*) map);
+  delete map;
+
+  // Dir 
+  ZFS_Dir_Block * dir = new ZFS_Dir_Block();
+  dir->format();
+  zigbee.eeprom.writeBytes(0x0000, 256, (byte*) dir);
+  delete dir;
+}
+
+uint16_t ZFS_File_Entry::getAddress(uint8_t entry_idx) {
+  return sizeof(ZFS_Root_Entry) + sizeof(ZFS_File_Entry) * entry_idx;
+}
+
+void ZFS_File_Entry::read(uint8_t entry_idx) {
+  if (!zigbee.eeprom_ready) { return; }
+  zigbee.eeprom.readBytes(getAddress(entry_idx), sizeof(ZFS_File_Entry), (byte*)this);
+}
+
+void ZFS_Write_File::findOrCreate(void) {
+  ZFS_File_Entry entry;
+
+  if (ZFS::findFileEntry(name, entry, &entry_idx)) {
+    blk_start = entry.blk_start;
+  }
+};
+
+int32_t ZFS_Write_File::addBytes(void* buffer, size_t buffer_len) {
+  if (!zigbee.eeprom_ready) { return -1; }
+  if ((buffer == nullptr) || (buffer_len == 0)) { return 0; }
+  if (length + buffer_len > ZFS_FILE_BLOCKS * 256) { return -1; }   // exceeded max size
+
+// #ifdef Z_EEPROM_DEBUG
+//   AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "eeprom.writeBytes address=0x%04X, len=%d"), (blk_start << 8) + length, buffer_len);
+// #endif
+  zigbee.eeprom.writeBytes((blk_start << 8) + length, buffer_len, (byte*)buffer);
+  length += buffer_len;
+  return length;
+}
+
+int32_t ZFS_Write_File::close(void) {
+  if (!zigbee.eeprom_ready) { return -1; }
+  // write the final length
+  uint16_t address = ZFS_File_Entry::getAddress(entry_idx);
+  zigbee.eeprom.writeBytes(address + sizeof(name), 2, (byte*)&length);
+  return length;
+}
+
+#endif // USE_ZIGBEE

tasmota/xdrv_23_zigbee_4b_eeprom.ino

@@ -24,10 +24,7 @@
 // ZbData v1
 // File structure:
 //
-// uint8 - number of devices, 0=none, 0xFF=invalid entry (probably Flash was erased)
-//
 // [Array of devices]
-// [Offset = 2]
 // uint8  - length of device record (excluding the length byte)
 // uint16 - short address
 //
@@ -40,17 +37,6 @@
 // uint8[] - list of data
 //
 
-void dumpZigbeeDevicesData(void) {
-#ifdef USE_ZIGBEE_EZSP
-  if (zigbee.eeprom_present) {
-    SBuffer buf(192);
-
-    zigbee.eeprom.readBytes(64, 192, buf.getBuffer());
-    AddLogBuffer(LOG_LEVEL_INFO, buf.getBuffer(), 192);
-  }
-#endif // USE_ZIGBEE_EZSP
-}
-
 // returns the lenght of consumed buffer, or -1 if error
 int32_t hydrateDeviceWideData(class Z_Device & device, const SBuffer & buf, size_t start, size_t len) {
   size_t segment_len = buf.get8(start);
@@ -70,10 +56,16 @@ bool hydrateDeviceData(class Z_Device & device, const SBuffer & buf, size_t star
   int32_t ret = hydrateDeviceWideData(device, buf, start, len);
   if (ret < 0) { return false; }
 
-  size_t offset = 0 + ret;
+  size_t offset = ret;
   while (offset + 5 <= len) {    // each entry is at least 5 bytes
     uint8_t data_len = buf.get8(start + offset);
-    Z_Data & data_elt = device.data.createFromBuffer(buf, offset + 1, data_len);
+// #ifdef Z_EEPROM_DEBUG
+//   {
+//     char hex_char[((data_len+1) * 2) + 2];
+//     AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "hydrateDeviceData data_len=%d contains %s"), data_len, ToHex_P(buf.buf(start+offset+1), data_len, hex_char, sizeof(hex_char)));
+//   }
+// #endif
+    Z_Data & data_elt = device.data.createFromBuffer(buf, start + offset + 1, data_len);
     (void)data_elt;   // avoid compiler warning
     offset += data_len + 1;
   }
@@ -94,7 +86,14 @@ int32_t hydrateSingleDevice(const class SBuffer & buf, size_t start, size_t len)
     AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "invalid shortaddr=0x%04X"), shortaddr);
     return -1;
   }
-
+#ifdef Z_EEPROM_DEBUG
+  {
+    if (segment_len > 3) {
+      char hex_char[((segment_len+1) * 2) + 2];
+      AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData 0x%04X,%s"), shortaddr, ToHex_P(buf.buf(start+3), segment_len+1-3, hex_char, sizeof(hex_char)));
+    }
+  }
+#endif
   // check if the device exists, if not skip the record
   Z_Device & device = zigbee_devices.findShortAddr(shortaddr);
   if (&device != nullptr) {
@@ -107,28 +106,53 @@ int32_t hydrateSingleDevice(const class SBuffer & buf, size_t start, size_t len)
   return segment_len + 1;
 }
 
+/*********************************************************************************************\
+ * 
+ * Hydrate data from the EEPROM
+ * 
+\*********************************************************************************************/
 // Parse the entire blob
 // return true if ok
-bool hydrateDevicesDataBlob(const class SBuffer & buf, size_t start, size_t len) {
+bool hydrateDevicesDataFromEEPROM(void) {
-  // read number of devices
+  if (!zigbee.eeprom_ready) { return false; }
-  uint8_t num_devices = buf.get8(start);
+
-  if (num_devices > 0x80) {
+  int32_t file_length = ZFS::getLength(ZIGB_DATA2);
-    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "wrong number of devices=%d"), num_devices);
+  if (file_length > 0) {
+    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee device data in EEPROM (%d bytes)"), file_length);
+  } else {
+    AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No Zigbee device data in EEPROM"));
     return false;
   }
 
-  size_t offset = 0;
+  const uint16_t READ_BUFFER = 192;
-  for (uint32_t cur_dev_num = 0; (cur_dev_num < num_devices) && (offset + 4 <= len); cur_dev_num++) {
+  uint16_t cursor = 0x0000;         // cursor in the file
-    int32_t segment_len = hydrateSingleDevice(buf, offset, len);
+  bool read_more = true;
 
-    // advance buffer
+  SBuffer buf(READ_BUFFER);
+  while (read_more) {
+    buf.setLen(buf.size());         // set to max size and fill with zeros
+    int32_t bytes_read = ZFS::readBytes(ZIGB_DATA2, buf.getBuffer(), buf.size(), cursor, READ_BUFFER);
+// #ifdef Z_EEPROM_DEBUG
+//     AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "readBytes buffer_len=%d, read_start=%d, read_len=%d, actual_read=%d"), buf.size(), cursor, length, bytes_read);
+// #endif
+    if (bytes_read > 0) {
+      buf.setLen(bytes_read);       // adjust to actual size
+      int32_t segment_len = hydrateSingleDevice(buf, 0, buf.len());
+// #ifdef Z_EEPROM_DEBUG
+//       AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "hydrateSingleDevice segment_len=%d"), segment_len);
+// #endif
       if (segment_len <= 0) { return false; }
-    offset += segment_len;
+
+      cursor += segment_len;
+    } else {
+      read_more = false;
     }
+  }
+
   return true;
 }
 
-class SBuffer hibernateDeviceData(const struct Z_Device & device, bool log = false) {
+class SBuffer hibernateDeviceData(const struct Z_Device & device, bool mqtt = false) {
   SBuffer buf(192);
 
   // If we have zero information about the device, just skip ir
@@ -155,33 +179,48 @@ class SBuffer hibernateDeviceData(const struct Z_Device & device, bool log = fal
     // update overall length
     buf.set8(0, buf.len() - 1);
 
-    if (log) {
+    {
       size_t buf_len = buf.len() - 3;
       char hex[2*buf_len + 1];
       // skip first 3 bytes
       ToHex_P(buf.buf(3), buf_len, hex, sizeof(hex));
 
+      if (mqtt) {
         Response_P(PSTR("{\"" D_PRFX_ZB D_CMND_ZIGBEE_DATA "\":\"ZbData 0x%04X,%s\"}"), device.shortaddr, hex);
         MqttPublishPrefixTopicRulesProcess_P(RESULT_OR_STAT, PSTR(D_PRFX_ZB D_CMND_ZIGBEE_DATA));
+      } else {
+        AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData 0x%04X,%s"), device.shortaddr, hex);
+      }
     }
   }
 
   return buf;
 }
 
+/*********************************************************************************************\
+ * 
+ * Hibernate data to the EEPROM
+ * 
+\*********************************************************************************************/
 void hibernateAllData(void) {
 
+  if (!zigbee.eeprom_ready) { return; }
+
+  ZFS_Write_File write_data(ZIGB_DATA2);
   // first prefix is number of devices
   uint8_t device_num = zigbee_devices.devicesSize();
 
   for (const auto & device : zigbee_devices.getDevices()) {
     // allocte a buffer for a single device
-    SBuffer buf = hibernateDeviceData(device, true);    // log
+    SBuffer buf = hibernateDeviceData(device, false);    // simple log, no mqtt
     if (buf.len() > 0) {
-      // TODO store in EEPROM
+      write_data.addBytes(buf.getBuffer(), buf.len());
     }
   }
-
+  int32_t ret = write_data.close();
+#ifdef Z_EEPROM_DEBUG
+  AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData - %d bytes written to EEPROM"), ret);
+#endif
 }
 
 #endif // USE_ZIGBEE

tasmota/xdrv_23_zigbee_7_statemachine.ino

@@ -867,7 +867,9 @@ static const Zigbee_Instruction zb_prog[] PROGMEM = {
     ZI_MQTT_STATE(ZIGBEE_STATUS_OK, kStarted)
     ZI_LOG(LOG_LEVEL_INFO, kZigbeeStarted)
     ZI_CALL(&Z_State_Ready, 1)                    // Now accept incoming messages
+    ZI_CALL(&Z_Prepare_EEPROM, 0)
     ZI_CALL(&Z_Load_Devices, 0)
+    ZI_CALL(&Z_Load_Data, 0)
     ZI_CALL(&Z_Query_Bulbs, 0)
 
   ZI_LABEL(ZIGBEE_LABEL_MAIN_LOOP)

tasmota/xdrv_23_zigbee_8_parsers.ino

@@ -1850,6 +1850,14 @@ int32_t ZNP_Recv_Default(int32_t res, const class SBuffer &buf) {
  * Functions called by State Machine
 \*********************************************************************************************/
 
+//
+// Callback for loading preparing EEPROM, called by the state machine
+//
+int32_t Z_Prepare_EEPROM(uint8_t value) {
+  ZFS::initOrFormat();
+  return 0;                              // continue
+}
+
 //
 // Callback for loading Zigbee configuration from Flash, called by the state machine
 //
@@ -1859,6 +1867,14 @@ int32_t Z_Load_Devices(uint8_t value) {
   return 0;                              // continue
 }
 
+//
+// Callback for loading Zigbee data from EEPROM, called by the state machine
+//
+int32_t Z_Load_Data(uint8_t value) {
+  hydrateDevicesDataFromEEPROM();
+  return 0;                              // continue
+}
+
 //
 // Query the state of a bulb (light) if its type allows it
 //

tasmota/xdrv_23_zigbee_A_impl.ino

@@ -1250,9 +1250,16 @@ void CmndZbSave(void) {
     case -1:      // dump configuration
       loadZigbeeDevices(true);    // dump only
       break;
-    case -2:      // dump data
+    case -2:
-      dumpZigbeeDevicesData();
+      hydrateDevicesDataFromEEPROM();
       break;
+#ifdef Z_EEPROM_DEBUG
+    case -10:
+      { // reinit EEPROM
+      ZFS::erase();
+      break;
+      }
+#endif
     default:
       saveZigbeeDevices();
       break;
@@ -1456,7 +1463,14 @@ void CmndZbStatus(void) {
 //
 void CmndZbData(void) {
   if (zigbee.init_phase) { ResponseCmndChar_P(PSTR(D_ZIGBEE_NOT_STARTED)); return; }
+  RemoveSpace(XdrvMailbox.data);
 
+  if (strlen(XdrvMailbox.data) == 0) {
+    // if empty, log values for all devices
+    for (const auto & device : zigbee_devices.getDevices()) {
+      hibernateDeviceData(device, true);    // simple log, no mqtt
+    }
+  } else {
     // check if parameters contain a comma ','
     char *p;
     strtok_r(XdrvMailbox.data, ",", &p);
@@ -1473,7 +1487,8 @@ void CmndZbData(void) {
       // non-JSON, export current data
       // ZbData 0x1234
       // ZbData Device_Name
-    hibernateDeviceData(device, true);    // log
+      hibernateDeviceData(device, true);    // mqtt
+    }
   }
 
   ResponseCmndDone();
@@ -1839,6 +1854,11 @@ bool Xdrv23(uint8_t function)
       case FUNC_COMMAND:
         result = DecodeCommand(kZbCommands, ZigbeeCommand);
         break;
+#ifdef USE_ZIGBEE_EZSP
+      case FUNC_SAVE_BEFORE_RESTART:
+        hibernateAllData();
+        break;
+#endif  // USE_ZIGBEE_EZSP
     }
   }
   return result;