From ed01748b6d8178fed652831ffb14d7c215042a3b Mon Sep 17 00:00:00 2001 From: Stephan Hadinger Date: Thu, 22 Apr 2021 14:49:38 +0200 Subject: [PATCH] Zigbee refactored storage for devices and data --- tasmota/my_user_config.h | 1 + tasmota/tasmota_configurations.h | 1 + tasmota/tasmota_globals.h | 3 +- tasmota/xdrv_23_zigbee_1_headers.ino | 12 +- tasmota/xdrv_23_zigbee_4_persistence.ino | 449 ---------------- tasmota/xdrv_23_zigbee_4a_nano_fs.ino | 256 ++++++++- ..._eeprom.ino => xdrv_23_zigbee_4b_data.ino} | 266 ++++------ tasmota/xdrv_23_zigbee_4c_devices.ino | 497 ++++++++++++++++++ tasmota/xdrv_23_zigbee_7_0_statemachine.ino | 10 +- tasmota/xdrv_23_zigbee_8_parsers.ino | 12 +- tasmota/xdrv_23_zigbee_A_impl.ino | 12 +- 11 files changed, 891 insertions(+), 628 deletions(-) delete mode 100644 tasmota/xdrv_23_zigbee_4_persistence.ino rename tasmota/{xdrv_23_zigbee_4b_eeprom.ino => xdrv_23_zigbee_4b_data.ino} (53%) create mode 100644 tasmota/xdrv_23_zigbee_4c_devices.ino diff --git a/tasmota/my_user_config.h b/tasmota/my_user_config.h index e046703dd..b7147058b 100644 --- a/tasmota/my_user_config.h +++ b/tasmota/my_user_config.h @@ -784,6 +784,7 @@ #define USE_ZIGBEE_ZNP // Enable ZNP protocol, needed for CC2530 based devices // #define USE_ZIGBEE_EZSP // Enable EZSP protocol, needed for EFR32 EmberZNet based devices, like Sonoff Zigbee bridge // Note: USE_ZIGBEE_ZNP and USE_ZIGBEE_EZSP are mutually incompatible, you must select exactly one + // #define USE_ZIGBEE_EEPROM // Use the EEPROM from the Sonoff ZBBridge to save Zigbee configuration and data #define USE_ZIGBEE_CHANNEL 11 // Zigbee Channel (11-26) #define USE_ZIGBEE_TXRADIO_DBM 20 // Tx Radio power in dBm (only for EZSP, EFR32 can go up to 20 dBm) diff --git a/tasmota/tasmota_configurations.h b/tasmota/tasmota_configurations.h index 0ee06c35d..54c953c80 100644 --- a/tasmota/tasmota_configurations.h +++ b/tasmota/tasmota_configurations.h @@ -607,6 +607,7 @@ #define USE_ZIGBEE #undef USE_ZIGBEE_ZNP #define USE_ZIGBEE_EZSP +#define USE_ZIGBEE_EEPROM // EEPROM of Sonoff ZBBridge via I2C #define USE_TCP_BRIDGE #define USE_ZIGBEE_CHANNEL 11 // Zigbee Channel (11-26) #define USE_ZIGBEE_COALESCE_ATTR_TIMER 350 // timer to coalesce attribute values (in ms) diff --git a/tasmota/tasmota_globals.h b/tasmota/tasmota_globals.h index 2eb37309d..bc0f9cd74 100644 --- a/tasmota/tasmota_globals.h +++ b/tasmota/tasmota_globals.h @@ -261,7 +261,8 @@ const uint16_t LOG_BUFFER_SIZE = 4000; // Max number of characters in lo #define TASM_FILE_DRIVER "/.drvset%03d" #define TASM_FILE_SENSOR "/.snsset%03d" #define TASM_FILE_TLSKEY "/tlskey" // TLS private key -#define TASM_FILE_ZIGBEE "/zb" // Zigbee settings blob as used by CC2530 on ESP32 +#define TASM_FILE_ZIGBEE "/zb" // Zigbee devices information blob +#define TASM_FILE_ZIGBEE_DATA "/zbdata" // Zigbee last known values of devices #define TASM_FILE_AUTOEXEC "/autoexec.bat" // Commands executed after restart #define TASM_FILE_CONFIG "/config.sys" // Settings executed after restart diff --git a/tasmota/xdrv_23_zigbee_1_headers.ino b/tasmota/xdrv_23_zigbee_1_headers.ino index ba01ded30..3e541aa7d 100644 --- a/tasmota/xdrv_23_zigbee_1_headers.ino +++ b/tasmota/xdrv_23_zigbee_1_headers.ino @@ -19,9 +19,9 @@ #ifdef USE_ZIGBEE -#ifdef USE_ZIGBEE_EZSP +#ifdef USE_ZIGBEE_EEPROM #include "Eeprom24C512.h" -#endif // USE_ZIGBEE_EZSP +#endif // USE_ZIGBEE_EEPROM // channels numbers for Zigbee radio energy scan #define USE_ZIGBEE_CHANNEL_MIN 11 @@ -100,9 +100,9 @@ const uint8_t ZIGBEE_LABEL_UNSUPPORTED_VERSION = 98; // Unsupported ZNP versio class ZigbeeStatus { public: ZigbeeStatus() -#ifdef USE_ZIGBEE_EZSP +#ifdef USE_ZIGBEE_EEPROM : eeprom(USE_ZIGBEE_ZBBRIDGE_EEPROM) -#endif // USE_ZIGBEE_EZSP +#endif // USE_ZIGBEE_EEPROM {} bool active = true; // is Zigbee active for this device, i.e. GPIOs configured @@ -142,9 +142,9 @@ public: uint16_t ezsp_version = 0; #endif -#ifdef USE_ZIGBEE_EZSP +#ifdef USE_ZIGBEE_EEPROM Eeprom24C512 eeprom; // takes only 1 bytes of RAM -#endif // USE_ZIGBEE_EZSP +#endif // USE_ZIGBEE_EEPROM }; struct ZigbeeStatus zigbee; SBuffer *zigbee_buffer = nullptr; diff --git a/tasmota/xdrv_23_zigbee_4_persistence.ino b/tasmota/xdrv_23_zigbee_4_persistence.ino deleted file mode 100644 index 8e29cc61c..000000000 --- a/tasmota/xdrv_23_zigbee_4_persistence.ino +++ /dev/null @@ -1,449 +0,0 @@ -/* - xdrv_23_zigbee.ino - zigbee support for Tasmota - - Copyright (C) 2021 Theo Arends and Stephan Hadinger - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -#ifdef USE_ZIGBEE - -// Ensure persistence of devices into Flash -// -// Structure: -// (from file info): -// uint16 - start address in Flash (offset) -// uint16 - length in bytes (makes sure parsing stops) -// -// First byte: -// 0x00 - Empty or V3 format -// 0x01-0xFE - Legacy format -// 0xFF - invalid -// -// -// V1 Legacy -// ========= -// File structure: -// uint8 - number of devices, 0=none, 0xFF=invalid entry (probably Flash was erased) -// -// [Array of devices] -// [Offset = 2] -// uint8 - length of device record -// uint16 - short address -// uint64 - long IEEE address -// uint8 - number of endpoints -// [Array of endpoints] -// uint8 - endpoint number -// uint16 - profileID of the endpoint -// Array of uint8 - clusters In codes, 0xFF end marker -// Array of uint8 - clusters Out codes, 0xFF end marker -// -// str - ModelID (null terminated C string, 32 chars max) -// str - Manuf (null terminated C string, 32 chars max) -// str - FriendlyName (null terminated C string, 32 chars max) -// reserved for extensions -// -- V2 -- -// int8_t - zigbee profile of the device -// -// ======================= -// v3 with version number -// File structure: -// -// uint8 - number of devices, 0=none, 0xFF=invalid entry (probably Flash was erased) -// -// [Array of devices] -// [Offset = 2] -// uint8 - length of device record -// uint16 - short address -// uint64 - long IEEE address -// -// str - ModelID (null terminated C string, 32 chars max) -// str - Manuf (null terminated C string, 32 chars max) -// str - FriendlyName (null terminated C string, 32 chars max) -// -// [Array of endpoints] -// uint8 - endpoint number, 0xFF marks the end of endpoints -// uint8[] - list of configuration bytes, 0xFF marks the end -// i.e. 0xFF-0xFF marks the end of the array of endpoints -// - - -// Memory footprint -#ifdef ESP8266 -const static uint16_t z_spi_start_sector = 0xFF; // Force last bank of first MB -const static uint8_t* z_spi_start = (uint8_t*) 0x402FF000; // 0x402FF000 -const static uint8_t* z_dev_start = z_spi_start + 0x0800; // 0x402FF800 - 2KB -const static size_t z_spi_len = 0x1000; // 4kb blocks -const static size_t z_block_offset = 0x0800; -const static size_t z_block_len = 0x0800; // 2kb -#endif // ESP8266 -#ifdef ESP32 -uint8_t* z_dev_start; -const static size_t z_spi_len = 0x1000; // 4kb blocks -const static size_t z_block_offset = 0x0000; // No offset needed -const static size_t z_block_len = 0x1000; // 4kb -#endif // ESP32 - -// Each entry consumes 8 bytes -class Z_Flashentry { -public: - uint32_t name; // simple 4 letters name. Currently 'zig1', 'zig2'. 0xFFFFFFFF if not entry - uint16_t len; // len of object in bytes, 0xFFFF if no entry - uint16_t start; // address of start, 0xFFFF if empty, must be aligned on 128 bytes boundaries -}; - -class Z_Flashdirectory { -public: - // 8 bytes header - uint32_t magic; // magic value 'Tsmt' to check that the block is initialized - uint32_t clock; // clock vector to discard entries that are made before this one. This should be incremented by 1 for each new entry (future anti-weavering) - // entries, 14*8 = 112 bytes - Z_Flashentry entries[14]; - uint32_t name; // simple 4 letters name. Currently 'skey', 'crt ', 'crt1', 'crt2' - uint16_t len; // len of object - uint16_t reserved; // align on 4 bytes boundary - // link to next entry, none for now, but may be used for anti-weavering - uint16_t next_dir; // 0xFFFF if none - uint16_t reserved1; // must be 0xFFFF - uint32_t reserved2; // must be 0xFFFFFFFF -}; - - -const static uint32_t ZIGB_NAME1 = 0x3167697A; // 'zig1' little endian -const static uint32_t ZIGB_NAME2 = 0x3267697A; // 'zig2' little endian, v2 -const static uint32_t ZIGB_DATA2 = 0x32746164; // 'dat2' little endian, v2 -const static size_t Z_MAX_FLASH = z_block_len - sizeof(Z_Flashentry); // 2040 - -bool hibernateDeviceConfiguration(SBuffer & buf, const class Z_Data_Set & data, uint8_t endpoint) { - bool found = false; - for (auto & elt : data) { - if (endpoint == elt.getEndpoint()) { - buf.add8(elt.getConfigByte()); - found = true; - } - } - return found; -} - -SBuffer hibernateDevicev2(const struct Z_Device &device) { - SBuffer buf(128); - - buf.add8(0x00); // overall length, will be updated later - buf.add16(device.shortaddr); - buf.add64(device.longaddr); - - char *names[3] = { device.modelId, device.manufacturerId, device.friendlyName }; - - for (uint32_t i=0; i 32) { len = 32; } // max 32 chars - buf.addBuffer(p, len); - } - buf.add8(0x00); // end of string marker - } - - // check if we need to write fake endpoint 0x00 - buf.add8(0x00); - if (hibernateDeviceConfiguration(buf, device.data, 0)) { - buf.add8(0xFF); // end of configuration - } else { - buf.setLen(buf.len()-1); // remove 1 byte header - } - // scan endpoints - for (uint32_t i=0; i 48) { devices_size = 48; } // arbitrarily limit to 48 devices on ESP32, we will go beyond when we remove the limit of 2kb buffer -#else - if (devices_size > 32) { devices_size = 32; } // arbitrarily limit to 32 devices, for now -#endif - buf.add8(devices_size); // number of devices - - for (uint32_t i = 0; i < devices_size; i++) { - const Z_Device & device = zigbee_devices.devicesAt(i); - const SBuffer buf_device = hibernateDevicev2(device); - buf.addBuffer(buf_device); - } - - return buf; -} - -// parse a single string from the saved data -// if something wrong happens, returns nullptr to ignore the string -// Index d is incremented to just after the string -const char * hydrateSingleString(const SBuffer & buf, uint32_t *d) { - size_t s_len = buf.strlen(*d); - const char * ptr = s_len ? buf.charptr(*d) : ""; - *d += s_len + 1; - return ptr; -} - -void hydrateSingleDevice(const SBuffer & buf_d, uint32_t version = 2) { - uint32_t d = 1; // index in device buffer - uint16_t shortaddr = buf_d.get16(d); d += 2; - uint64_t longaddr = buf_d.get64(d); d += 8; - size_t buf_len = buf_d.len(); - Z_Device & device = zigbee_devices.updateDevice(shortaddr, longaddr); // update device's addresses - - if (1 == version) { - uint32_t endpoints = buf_d.get8(d++); - for (uint32_t j = 0; j < endpoints; j++) { - uint8_t ep = buf_d.get8(d++); - // uint16_t ep_profile = buf_d.get16(d); d += 2; - device.addEndpoint(ep); - - // in clusters - while (d < buf_len) { // safe guard against overflow - uint8_t ep_cluster = buf_d.get8(d++); - if (0xFF == ep_cluster) { break; } // end of block - // ignore - } - // out clusters - while (d < buf_len) { // safe guard against overflow - uint8_t ep_cluster = buf_d.get8(d++); - if (0xFF == ep_cluster) { break; } // end of block - // ignore - } - } - } - - // ModelId - device.setModelId(hydrateSingleString(buf_d, &d)); - - // ManufID - device.setManufId(hydrateSingleString(buf_d, &d)); - - // FriendlyName - device.setFriendlyName(hydrateSingleString(buf_d, &d)); - - if (d >= buf_len) { return; } - - // Hue bulbtype - if present - if (1 == version) { - device.setLightChannels(buf_d.get8(d)); - d++; - } else if (2 == version) { - // v2 parser - while (d < buf_len) { - uint8_t ep = buf_d.get8(d++); - if (0xFF == ep) { break; } // ep 0xFF marks the end of the endpoints - if (ep > 240) { ep = 0xFF; } // ep == 0xFF means ignore - device.addEndpoint(ep); // it will ignore invalid endpoints - while (d < buf_len) { - uint8_t config_type = buf_d.get8(d++); - if (0xFF == config_type) { break; } // 0xFF marks the end of congiguration - uint8_t config = config_type & 0x0F; - Z_Data_Type type = (Z_Data_Type) (config_type >> 4); - // set the configuration - if (ep != 0xFF) { - Z_Data & z_data = device.data.getByType(type, ep); - if (&z_data != nullptr) { - z_data.setConfig(config); - Z_Data_Set::updateData(z_data); - } - } - } - } - } -} - -void hydrateDevices(const SBuffer &buf, uint32_t version) { - uint32_t buf_len = buf.len(); - if (buf_len <= 10) { return; } - - uint32_t k = 0; // byte index in global buffer - uint32_t num_devices = buf.get8(k++); - for (uint32_t i = 0; (i < num_devices) && (k < buf_len); i++) { - uint32_t dev_record_len = buf.get8(k); - - SBuffer buf_d = buf.subBuffer(k, dev_record_len); - hydrateSingleDevice(buf_d, version); - - // next iteration - k += dev_record_len; - } -} - -// dump = true, only dump to logs, don't actually load -void loadZigbeeDevices(bool dump_only = false) { -#ifdef USE_ZIGBEE_EZSP - if (loadZigbeeDevicesFromEEPROM()) { - return; // we succesfully loaded from EEPROM, skip the read from Flash - } -#endif -#ifdef ESP32 - // first copy SPI buffer into ram - uint8_t *spi_buffer = (uint8_t*) malloc(z_spi_len); - if (!spi_buffer) { - AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Cannot allocate 4KB buffer")); - return; - } -#ifdef USE_UFILESYS - TfsLoadFile(TASM_FILE_ZIGBEE, spi_buffer, z_spi_len); -#endif - z_dev_start = spi_buffer; -#endif // ESP32 - Z_Flashentry flashdata; - memcpy_P(&flashdata, z_dev_start, sizeof(Z_Flashentry)); -// AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_ZIGBEE "Memory %d"), ESP_getFreeHeap()); - // AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_ZIGBEE "Zigbee signature in Flash: %08X - %d"), flashdata.name, flashdata.len); - - // Check the signature - if ( ((flashdata.name == ZIGB_NAME1) || (flashdata.name == ZIGB_NAME2)) - && (flashdata.len > 0)) { - uint16_t buf_len = flashdata.len; - uint32_t version = (flashdata.name == ZIGB_NAME2) ? 2 : 1; - // parse what seems to be a valid entry - SBuffer buf(buf_len); - buf.addBuffer(z_dev_start + sizeof(Z_Flashentry), buf_len); - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee device information in %s (%d bytes)"), PSTR("Flash"), buf_len); - - if (dump_only) { - size_t buf_len = buf.len(); - if (buf_len > 192) { buf_len = 192; } - AddLogBuffer(LOG_LEVEL_INFO, buf.getBuffer(), buf_len); - // Serial.printf(">> Buffer="); - // for (uint32_t i=0; i 2040) { - AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Buffer too big to fit in Flash (%d bytes)"), buf_len); - return; - } - - // first copy SPI buffer into ram - uint8_t *spi_buffer = (uint8_t*) malloc(z_spi_len); - if (!spi_buffer) { - AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Cannot allocate 4KB buffer")); - return; - } - // copy the flash into RAM to make local change, and write back the whole buffer -#ifdef ESP8266 - ESP.flashRead(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); -#endif // ESP8266 -#ifdef ESP32 -#ifdef USE_UFILESYS - TfsLoadFile(TASM_FILE_ZIGBEE, spi_buffer, z_spi_len); -#endif -#endif // ESP32 - - Z_Flashentry *flashdata = (Z_Flashentry*)(spi_buffer + z_block_offset); - flashdata->name = ZIGB_NAME2; // v2 - flashdata->len = buf_len; - flashdata->start = 0; - - memcpy(spi_buffer + z_block_offset + sizeof(Z_Flashentry), buf.getBuffer(), buf_len); - - // buffer is now ready, write it back -#ifdef ESP8266 - if (ESP.flashEraseSector(z_spi_start_sector)) { - ESP.flashWrite(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); - } - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data store in Flash (0x%08X - %d bytes)"), z_dev_start, buf_len); -#endif // ESP8266 -#ifdef ESP32 -#ifdef USE_UFILESYS - TfsSaveFile(TASM_FILE_ZIGBEE, spi_buffer, z_spi_len); -#endif - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data saved in %s (%d bytes)"), PSTR("Flash"), buf_len); -#endif // ESP32 - free(spi_buffer); -} - -// Erase the flash area containing the ZigbeeData -void eraseZigbeeDevices(void) { - zigbee_devices.clean(); // avoid writing data to flash after erase -#ifdef USE_ZIGBEE_EZSP - ZFS_Erase(); -#endif // USE_ZIGBEE_EZSP -#ifdef ESP8266 - // first copy SPI buffer into ram - uint8_t *spi_buffer = (uint8_t*) malloc(z_spi_len); - if (!spi_buffer) { - AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Cannot allocate 4KB buffer")); - return; - } - // copy the flash into RAM to make local change, and write back the whole buffer - ESP.flashRead(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); - - // Fill the Zigbee area with 0xFF - memset(spi_buffer + z_block_offset, 0xFF, z_block_len); - - // buffer is now ready, write it back - if (ESP.flashEraseSector(z_spi_start_sector)) { - ESP.flashWrite(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); - } - - free(spi_buffer); - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data erased in %s"), PSTR("Flash")); -#endif // ESP8266 -#ifdef ESP32 -#ifdef USE_UFILESYS - TfsInitFile(TASM_FILE_ZIGBEE, z_block_len, 0xFF); -#endif - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data erased (%d bytes)"), z_block_len); -#endif // ESP32 -} - -void restoreDumpAllDevices(void) { - for (const auto & device : zigbee_devices.getDevices()) { - const SBuffer buf = hibernateDevicev2(device); - if (buf.len() > 0) { - Response_P(PSTR("{\"" D_PRFX_ZB D_CMND_ZIGBEE_RESTORE "\":\"ZbRestore %_B\"}"), &buf); - MqttPublishPrefixTopicRulesProcess_P(RESULT_OR_STAT, PSTR(D_PRFX_ZB D_CMND_ZIGBEE_DATA)); - } - } -} - -#endif // USE_ZIGBEE diff --git a/tasmota/xdrv_23_zigbee_4a_nano_fs.ino b/tasmota/xdrv_23_zigbee_4a_nano_fs.ino index 54cbb3b54..488ba2fca 100644 --- a/tasmota/xdrv_23_zigbee_4a_nano_fs.ino +++ b/tasmota/xdrv_23_zigbee_4a_nano_fs.ino @@ -18,10 +18,23 @@ */ #ifdef USE_ZIGBEE -#ifdef USE_ZIGBEE_EZSP // #define Z_EEPROM_DEBUG + +const static uint32_t ZIGB_NAME1 = 0x3167697A; // 'zig1' little endian +const static uint32_t ZIGB_NAME2 = 0x3267697A; // 'zig2' little endian, v2 +const static uint32_t ZIGB_DATA2 = 0x32746164; // 'dat2' little endian, v2 +extern FS *dfsp; +extern "C" uint32_t _FS_end; +// Is it ok to write to bank 0x402FF000 +bool flash_valid(void) { + return (_FS_end > 0x40280000) && (_FS_end < 0x402FF000); +} + +void hydrateSingleDevice(const SBuffer & buf_d); + +#ifdef USE_ZIGBEE_EEPROM // The EEPROM is 64KB in size with individually writable bytes. // They are conveniently organized in pages of 128 bytes to accelerate // data transfer, but unlike flash memory, you don't need to erase an entire page. @@ -243,7 +256,11 @@ public: uint8_t blk_start; // if 0x00 then file does not exist uint8_t entry_idx; // entry number in the directory - ZFS_Write_File(uint32_t _name) : name(_name), cursor(0), length(0), blk_start(0) { findOrCreate(); } + ZFS_Write_File(void) : name(0), cursor(0), length(0), blk_start(0) {} + void init(uint32_t _name) { + name = _name; + findOrCreate(); + } inline bool valid(void) const { return blk_start != 0; } // does the file exist? @@ -453,5 +470,238 @@ int32_t ZFS_Write_File::close(void) { return length; } -#endif // USE_ZIGBEE_EZSP + +#endif // USE_ZIGBEE_EEPROM + + +/*********************************************************************************************\ + * + * Generic for Reading a file + * + * Can work in 3 modes: + * - if passed a filename, use the ZFS for EEPROM nano-fs + * - if passed a File* object, use this object + * - if passed a buffer, read from a binary buffer in RAM +\*********************************************************************************************/ + +class Univ_Read_File { +public: + // file info + uint16_t len = 0; + uint16_t cursor = 0; + bool is_valid = false; + + Univ_Read_File(void) {} + + // == EEPROM ================================================ +#ifdef USE_ZIGBEE_EEPROM + uint32_t eeprom_name = 0; + ZFS_File_Entry entry; + // uint16_t length; + // uint8_t blk_start; // if 0x00 then file does not exist + uint8_t entry_idx; // entry number in the directory + + void init(uint32_t _name) { + eeprom_name = _name; + if (ZFS::findFileEntry(eeprom_name, entry, &entry_idx)) { + len = ZFS::getLength(eeprom_name); + is_valid = (len > 0); + } + } +#endif // USE_ZIGBEE_EEPROM + + // == File ================================================ +#ifdef USE_UFILESYS + File * file = nullptr; + + void init(File * _file) { + file = _file; + is_valid = (bool) *file; + len = file->size(); + } +#endif + +#ifdef ESP8266 + // == Buffer ================================================ + // binary buffer + const uint8_t * buffer = nullptr; + void init(const uint8_t * buf, size_t buflen) { + buffer = buf; + len = buflen; + is_valid = (buffer != nullptr) && (len > 0); + } +#endif // ESP8266 + + + // ================================================== + inline bool valid(void) const { return is_valid; } // does the file exist? + + int32_t readBytes(uint8_t* buf, size_t buflen); + void close(void); +}; + +void Univ_Read_File::close(void) { +#ifdef USE_UFILESYS + if (file != nullptr) { + file->close(); + } +#endif // USE_UFILESYS + // don't do anything for ZFS read of buffer +} + +int32_t Univ_Read_File::readBytes(uint8_t* buf, size_t btr) { + if (!is_valid) { return -1; } +#ifdef USE_UFILESYS + if (file != nullptr) { + return file->read(buf, btr); + } +#endif // USE_UFILESYS +#ifdef USE_ZIGBEE_EEPROM + if (eeprom_name != 0) { + int32_t bytes_read = ZFS::readBytes(eeprom_name, buf, btr, cursor, btr); + if (bytes_read < 0) { return -1; } + cursor += bytes_read; + return bytes_read; + } +#endif // USE_ZIGBEE_EEPROM + +#ifdef ESP8266 + // binary buffer + if (buffer != nullptr) { + if (btr > len - cursor) { btr = len - cursor; } + memcpy_P(buf, buffer + cursor, btr); + cursor += btr; + return btr; + } +#endif // ESP8266 + + return -1; +} + + +/*********************************************************************************************\ + * + * Generic for Writing a file + * + * Can work in 3 modes: + * - if passed a filename, use the ZFS for EEPROM nano-fs + * - if passed a File* object, use this object + * - if passed a buffer, write to a binary buffer in RAM +\*********************************************************************************************/ + +class Univ_Write_File { +public: + // file info + bool is_valid = false; + + Univ_Write_File(void) {} + + // == EEPROM ================================================ +#ifdef USE_ZIGBEE_EEPROM + ZFS_Write_File eeprom_file; + + void init(uint32_t _name) { + eeprom_file.init(_name); + is_valid = eeprom_file.valid(); + } +#endif // USE_ZIGBEE_EEPROM + + // == File ================================================ +#ifdef USE_UFILESYS + File * file = nullptr; + + void init(File * _file) { + file = _file; + is_valid = (bool) *file; + } +#endif + +#ifdef ESP8266 + // == Buffer ================================================ + // binary buffer + size_t buflen = 0; + uint8_t * buffer = nullptr; + uint16_t cursor = 0; + void init(uint8_t * buf, size_t _buflen) { + buffer = buf; + buflen = _buflen; + is_valid = (buffer != nullptr) && (buflen > 0); + } +#endif // ESP8266 + + // ================================================== + inline bool valid(void) const { return is_valid; } // does the file exist? + + int32_t writeBytes(uint8_t* buf, size_t buflen); + int32_t getCursor(void); + void close(void); +}; + +void Univ_Write_File::close(void) { +#ifdef USE_UFILESYS + if (file != nullptr) { + file->close(); + } +#endif // USE_UFILESYS +#ifdef USE_ZIGBEE_EEPROM + if (eeprom_file.valid()) { + eeprom_file.close(); + } +#endif // USE_ZIGBEE_EEPROM + // binary buffer doesn't need a close +} + +int32_t Univ_Write_File::getCursor(void) { + if (!is_valid) { return -1; } + +#ifdef USE_UFILESYS + if (file != nullptr) { + return file->position(); + } +#endif // USE_UFILESYS +#ifdef USE_ZIGBEE_EEPROM + if (eeprom_file.valid()) { + return eeprom_file.length; + } +#endif // USE_ZIGBEE_EEPROM + +#ifdef ESP8266 + if (buffer != nullptr) { + return cursor; + } +#endif // ESP8266 + + return -1; +} + +int32_t Univ_Write_File::writeBytes(uint8_t* buf, size_t btw) { + if (!is_valid) { return -1; } + +#ifdef USE_UFILESYS + if (file != nullptr) { + return file->write(buf, btw); + } +#endif // USE_UFILESYS +#ifdef USE_ZIGBEE_EEPROM + if (eeprom_file.valid()) { + uint16_t length_before = eeprom_file.length; + eeprom_file.addBytes(buf, btw); + return eeprom_file.length - length_before; // compute the increase in size + } +#endif // USE_ZIGBEE_EEPROM + +#ifdef ESP8266 + if (buffer != nullptr) { + // binary buffer + if (btw > buflen - cursor) { btw = buflen - cursor; } + memcpy_P(buffer + cursor, buf, btw); + cursor += btw; + return btw; + } +#endif // ESP8266 + + return -1; +} + + #endif // USE_ZIGBEE diff --git a/tasmota/xdrv_23_zigbee_4b_eeprom.ino b/tasmota/xdrv_23_zigbee_4b_data.ino similarity index 53% rename from tasmota/xdrv_23_zigbee_4b_eeprom.ino rename to tasmota/xdrv_23_zigbee_4b_data.ino index b42b26a75..080afe8a2 100644 --- a/tasmota/xdrv_23_zigbee_4b_eeprom.ino +++ b/tasmota/xdrv_23_zigbee_4b_data.ino @@ -74,14 +74,14 @@ bool hydrateDeviceData(class Z_Device & device, const SBuffer & buf, size_t star // negative means error // positive is the segment length -int32_t hydrateSingleDevice(const SBuffer & buf, size_t start, size_t len) { - uint8_t segment_len = buf.get8(start); - if ((segment_len < 4) || (start + segment_len > len)) { +int32_t hydrateSingleDeviceData(const SBuffer & buf) { + uint8_t segment_len = buf.len(); + if (segment_len < 4) { AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "invalid segment_len=%d"), segment_len); return -1; } // read shortaddr - uint16_t shortaddr = buf.get16(start + 1); + uint16_t shortaddr = buf.get16(0); if (shortaddr >= 0xFFF0) { AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "invalid shortaddr=0x%04X"), shortaddr); return -1; @@ -89,7 +89,7 @@ int32_t hydrateSingleDevice(const SBuffer & buf, size_t start, size_t len) { #ifdef Z_EEPROM_DEBUG { if (segment_len > 3) { - AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData 0x%04X,%*_H"), shortaddr, segment_len+1-3, buf.buf(start+3)); + AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData 0x%04X,%*_H"), shortaddr, buf.buf(2), buf.len() - 2); } } #endif @@ -98,62 +98,14 @@ int32_t hydrateSingleDevice(const SBuffer & buf, size_t start, size_t len) { if (&device != nullptr) { // parse the rest - bool ret = hydrateDeviceData(device, buf, start + 3, segment_len - 3); + bool ret = hydrateDeviceData(device, buf, 2, segment_len - 2); if (!ret) { return -1; } } - return segment_len + 1; + return segment_len; } -/*********************************************************************************************\ - * - * Hydrate data from the EEPROM - * -\*********************************************************************************************/ -// Parse the entire blob -// return true if ok -bool hydrateDevicesDataFromEEPROM(void) { -#ifdef USE_ZIGBEE_EZSP - if (!zigbee.eeprom_ready) { return false; } - int32_t file_length = ZFS::getLength(ZIGB_DATA2); - if (file_length > 0) { - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee device data in EEPROM (%d bytes)"), file_length); - } else { - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No Zigbee device data in EEPROM")); - return false; - } - - const uint16_t READ_BUFFER = 192; - uint16_t cursor = 0x0000; // cursor in the file - bool read_more = true; - - SBuffer buf(READ_BUFFER); - while (read_more) { - buf.setLen(buf.size()); // set to max size and fill with zeros - int32_t bytes_read = ZFS::readBytes(ZIGB_DATA2, buf.getBuffer(), buf.size(), cursor, READ_BUFFER); -// #ifdef Z_EEPROM_DEBUG -// AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "readBytes buffer_len=%d, read_start=%d, read_len=%d, actual_read=%d"), buf.size(), cursor, length, bytes_read); -// #endif - if (bytes_read > 0) { - buf.setLen(bytes_read); // adjust to actual size - int32_t segment_len = hydrateSingleDevice(buf, 0, buf.len()); -// #ifdef Z_EEPROM_DEBUG -// AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "hydrateSingleDevice segment_len=%d"), segment_len); -// #endif - if (segment_len <= 0) { return false; } - - cursor += segment_len; - } else { - read_more = false; - } - } - return true; -#else // USE_ZIGBEE_EZSP - return false; -#endif // USE_ZIGBEE_EZSP -} - -SBuffer hibernateDeviceData(const struct Z_Device & device, bool mqtt = false) { +SBuffer hibernateDeviceData(const struct Z_Device & device) { SBuffer buf(192); // If we have zero information about the device, just skip ir @@ -183,45 +135,128 @@ SBuffer hibernateDeviceData(const struct Z_Device & device, bool mqtt = false) { { // skip first 3 bytes size_t buf_len = buf.len() - 3; - - if (mqtt) { - Response_P(PSTR("{\"" D_PRFX_ZB D_CMND_ZIGBEE_DATA "\":\"ZbData 0x%04X,%*_H\"}"), device.shortaddr, buf_len, buf.buf(3)); - MqttPublishPrefixTopicRulesProcess_P(RESULT_OR_STAT, PSTR(D_PRFX_ZB D_CMND_ZIGBEE_DATA)); - } else { - AddLog_P(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData 0x%04X,%*_H"), device.shortaddr, buf_len, buf.buf(3)); - } + Response_P(PSTR("{\"" D_PRFX_ZB D_CMND_ZIGBEE_DATA "\":\"ZbData 0x%04X,%*_H\"}"), device.shortaddr, buf_len, buf.buf(3)); + MqttPublishPrefixTopicRulesProcess_P(RESULT_OR_STAT, PSTR(D_PRFX_ZB D_CMND_ZIGBEE_DATA)); } } return buf; } +/*********************************************************************************************\ + * + * Hydrate data from the EEPROM + * +\*********************************************************************************************/ +// Parse the entire blob +// return true if ok +bool hydrateDevicesData(void) { + Univ_Read_File f; // universal reader + const char * storage_class = PSTR(""); +#ifdef USE_ZIGBEE_EEPROM + if (zigbee.eeprom_ready) { + f.init(ZIGB_DATA2); + storage_class = PSTR("EEPROM"); + } +#endif // USE_ZIGBEE_EEPROM + +#ifdef USE_UFILESYS + File file; + if (!f.valid() && dfsp) { + file = dfsp->open(TASM_FILE_ZIGBEE_DATA, "r"); + if (file) { + f.init(&file); + storage_class = PSTR("File System"); + } + } +#endif // USE_UFILESYS + + if (!f.valid() || f.len <= 0) { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No Zigbee device data")); + return false; + } + + uint32_t file_len = f.len; + + if (file_len > 0) { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee device data in %s (%d bytes)"), storage_class, file_len); + } else { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No Zigbee device data in %s"), storage_class); + f.close(); + return false; + } + + while (1) { + uint8_t dev_record_len = 0; + int32_t ret = f.readBytes(&dev_record_len, sizeof(dev_record_len)); + if (ret <= 0) { + break; // finished + } + + SBuffer buf(dev_record_len); + buf.setLen(dev_record_len); + + ret = f.readBytes(buf.getBuffer(), dev_record_len); + if (ret != dev_record_len) { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Invalid device data information, aborting")); + f.close(); + return false; + } + int32_t segment_len = hydrateSingleDeviceData(buf); + if (segment_len <= 0) { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Invalid device data information, aborting")); + f.close(); + return false; + } + } + f.close(); + return true; +} + /*********************************************************************************************\ * * Hibernate data to the EEPROM * \*********************************************************************************************/ void hibernateAllData(void) { -#ifdef USE_ZIGBEE_EZSP if (Rtc.utc_time < START_VALID_TIME) { return; } - if (!zigbee.eeprom_ready) { return; } + Univ_Write_File f; + const char * storage_class = PSTR(""); - ZFS_Write_File write_data(ZIGB_DATA2); - // first prefix is number of devices - uint8_t device_num = zigbee_devices.devicesSize(); +#ifdef USE_ZIGBEE_EEPROM + if (!f.valid() && zigbee.eeprom_ready) { + f.init(ZIGB_DATA2); + storage_class = PSTR("EEPROM"); + } +#endif - for (const auto & device : zigbee_devices.getDevices()) { - // allocte a buffer for a single device - SBuffer buf = hibernateDeviceData(device, false); // simple log, no mqtt - if (buf.len() > 0) { - write_data.addBytes(buf.getBuffer(), buf.len()); +#ifdef USE_UFILESYS + File file; + if (!f.valid() && dfsp) { + file = dfsp->open(TASM_FILE_ZIGBEE_DATA, "w"); + if (file) { + f.init(&file); + storage_class = PSTR("File System"); } } - int32_t ret = write_data.close(); -#ifdef Z_EEPROM_DEBUG - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData - %d bytes written to EEPROM"), ret); #endif -#endif // USE_ZIGBEE_EZSP + + if (f.valid()) { + // first prefix is number of devices + uint8_t device_num = zigbee_devices.devicesSize(); + + for (const auto & device : zigbee_devices.getDevices()) { + // allocte a buffer for a single device + SBuffer buf = hibernateDeviceData(device); + if (buf.len() > 0) { + f.writeBytes(buf.getBuffer(), buf.len()); + } + } + size_t buf_len = f.getCursor(); + f.close(); + + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "ZbData - %d bytes written to %s"), buf_len, storage_class); + } } /*********************************************************************************************\ @@ -232,86 +267,17 @@ const uint32_t Z_SAVE_DATA_TIMER = 60 * 60 * 1000; // save data every 60 m // // Callback for setting the timer to save Zigbee Data in x seconds // -int32_t Z_Set_Save_Data_Timer_EEPROM(uint8_t value) { +int32_t Z_Set_Save_Data_Timer(uint8_t value) { zigbee_devices.setTimer(0x0000, 0, Z_SAVE_DATA_TIMER, 0, 0, Z_CAT_ALWAYS, 0 /* value */, &Z_SaveDataTimer); return 0; // continue } void Z_SaveDataTimer(uint16_t shortaddr, uint16_t groupaddr, uint16_t cluster, uint8_t endpoint, uint32_t value) { hibernateAllData(); - Z_Set_Save_Data_Timer_EEPROM(0); // set a new timer -} - -#ifdef USE_ZIGBEE_EZSP -/*********************************************************************************************\ - * Write Devices in EEPROM -\*********************************************************************************************/ -// EEPROM variant that writes one item at a time and is not limited to 2KB -bool hibernateDevicesInEEPROM(void) { - if (Rtc.utc_time < START_VALID_TIME) { return false; } - if (!zigbee.eeprom_ready) { return false; } - - ZFS_Write_File write_data(ZIGB_NAME2); - - // first prefix is number of devices - uint8_t devices_size = zigbee_devices.devicesSize(); - if (devices_size > 64) { devices_size = 64; } // arbitrarily limit to 64 devices in EEPROM instead of 32 in Flash - write_data.addBytes(&devices_size, sizeof(devices_size)); - - for (const auto & device : zigbee_devices.getDevices()) { - const SBuffer buf = hibernateDevicev2(device); - if (buf.len() > 0) { - write_data.addBytes(buf.getBuffer(), buf.len()); - } - } - int32_t ret = write_data.close(); - - if (ret < 0) { - AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Error writing Devices to EEPROM")); - return false; - } else { - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data saved in %s (%d bytes)"), PSTR("EEPROM"), ret); - } - return true; -} - - -// dump = true, only dump to logs, don't actually load -bool loadZigbeeDevicesFromEEPROM(void) { - if (!zigbee.eeprom_ready) { return false; } - uint16_t file_len = ZFS::getLength(ZIGB_NAME2); - - uint8_t num_devices = 0; - ZFS::readBytes(ZIGB_NAME2, &num_devices, sizeof(num_devices), 0, sizeof(num_devices)); - - if ((file_len < 10) || (num_devices == 0x00) || (num_devices == 0xFF)) { // No data - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No Zigbee device information in %s"), PSTR("EEPROM")); - return false; - } - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee device information in %s (%d bytes)"), PSTR("EEPROM"), file_len); - - uint32_t k = 1; // byte index in global buffer - for (uint32_t i = 0; (i < num_devices) && (k < file_len); i++) { - uint8_t dev_record_len = 0; - int32_t ret = ZFS::readBytes(ZIGB_NAME2, &dev_record_len, 1, k, 1); - SBuffer buf(dev_record_len); - buf.setLen(dev_record_len); - ret = ZFS::readBytes(ZIGB_NAME2, buf.getBuffer(), dev_record_len, k, dev_record_len); - if (ret != dev_record_len) { - AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "File too short when reading EEPROM")); - return false; - } - - hydrateSingleDevice(buf, 2); - - // next iteration - k += dev_record_len; - } - - zigbee_devices.clean(); // don't write back to Flash what we just loaded - return true; + Z_Set_Save_Data_Timer(0); // set a new timer } +#ifdef USE_ZIGBEE_EEPROM void ZFS_Erase(void) { if (zigbee.eeprom_present) { ZFS::erase(); @@ -319,6 +285,6 @@ void ZFS_Erase(void) { } } -#endif // USE_ZIGBEE_EZSP +#endif // USE_ZIGBEE_EEPROM #endif // USE_ZIGBEE diff --git a/tasmota/xdrv_23_zigbee_4c_devices.ino b/tasmota/xdrv_23_zigbee_4c_devices.ino new file mode 100644 index 000000000..b87f4e0e1 --- /dev/null +++ b/tasmota/xdrv_23_zigbee_4c_devices.ino @@ -0,0 +1,497 @@ +/* + xdrv_23_zigbee.ino - zigbee support for Tasmota + + Copyright (C) 2021 Theo Arends and Stephan Hadinger + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#ifdef USE_ZIGBEE + +// Ensure persistence of devices into Flash +// +// Structure: +// (from file info): +// uint16 - start address in Flash (offset) +// uint16 - length in bytes (makes sure parsing stops) +// +// First byte: +// 0x00 - Empty or V3 format +// 0x01-0xFE - Legacy format +// 0xFF - invalid +// +// +// V1 Legacy +// ========= +// File structure: +// uint8 - number of devices, 0=none, 0xFF=invalid entry (probably Flash was erased) +// +// [Array of devices] +// [Offset = 2] +// uint8 - length of device record +// uint16 - short address +// uint64 - long IEEE address +// uint8 - number of endpoints +// [Array of endpoints] +// uint8 - endpoint number +// uint16 - profileID of the endpoint +// Array of uint8 - clusters In codes, 0xFF end marker +// Array of uint8 - clusters Out codes, 0xFF end marker +// +// str - ModelID (null terminated C string, 32 chars max) +// str - Manuf (null terminated C string, 32 chars max) +// str - FriendlyName (null terminated C string, 32 chars max) +// reserved for extensions +// -- V2 -- +// int8_t - zigbee profile of the device +// +// ======================= +// v3 with version number +// File structure: +// +// uint8 - number of devices, 0=none, 0xFF=invalid entry (probably Flash was erased) +// +// [Array of devices] +// [Offset = 2] +// uint8 - length of device record +// uint16 - short address +// uint64 - long IEEE address +// +// str - ModelID (null terminated C string, 32 chars max) +// str - Manuf (null terminated C string, 32 chars max) +// str - FriendlyName (null terminated C string, 32 chars max) +// +// [Array of endpoints] +// uint8 - endpoint number, 0xFF marks the end of endpoints +// uint8[] - list of configuration bytes, 0xFF marks the end +// i.e. 0xFF-0xFF marks the end of the array of endpoints +// + + +// Memory footprint +#ifdef ESP8266 +const static uint16_t z_spi_start_sector = 0xFF; // Force last bank of first MB +const static uint8_t* z_spi_start = (uint8_t*) 0x402FF000; // 0x402FF000 +const static uint8_t* z_dev_start = z_spi_start + 0x0800; // 0x402FF800 - 2KB +const static size_t z_spi_len = 0x1000; // 4kb blocks +const static size_t z_block_offset = 0x0800; +const static size_t z_block_len = 0x0800; // 2kb +#endif // ESP8266 +#ifdef ESP32 +uint8_t* z_dev_start; +const static size_t z_spi_len = 0x1000; // 4kb blocks +const static size_t z_block_offset = 0x0000; // No offset needed +const static size_t z_block_len = 0x1000; // 4kb +#endif // ESP32 + +// Each entry consumes 8 bytes +class Z_Flashentry { +public: + uint32_t name; // simple 4 letters name. Currently 'zig1', 'zig2'. 0xFFFFFFFF if not entry + uint16_t len; // len of object in bytes, 0xFFFF if no entry + uint16_t start; // address of start, 0xFFFF if empty, must be aligned on 128 bytes boundaries +}; + +class Z_Flashdirectory { +public: + // 8 bytes header + uint32_t magic; // magic value 'Tsmt' to check that the block is initialized + uint32_t clock; // clock vector to discard entries that are made before this one. This should be incremented by 1 for each new entry (future anti-weavering) + // entries, 14*8 = 112 bytes + Z_Flashentry entries[14]; + uint32_t name; // simple 4 letters name. Currently 'skey', 'crt ', 'crt1', 'crt2' + uint16_t len; // len of object + uint16_t reserved; // align on 4 bytes boundary + // link to next entry, none for now, but may be used for anti-weavering + uint16_t next_dir; // 0xFFFF if none + uint16_t reserved1; // must be 0xFFFF + uint32_t reserved2; // must be 0xFFFFFFFF +}; + +const static size_t Z_MAX_FLASH = z_block_len - sizeof(Z_Flashentry); // 2040 + +bool hibernateDeviceConfiguration(SBuffer & buf, const class Z_Data_Set & data, uint8_t endpoint) { + bool found = false; + for (auto & elt : data) { + if (endpoint == elt.getEndpoint()) { + buf.add8(elt.getConfigByte()); + found = true; + } + } + return found; +} + +/*********************************************************************************************\ + * hibernateDevice + * + * Transforms a single device into a SBuffer binary representation. + * Only supports v2 (not the legacy old one long forgotten) +\*********************************************************************************************/ +SBuffer hibernateDevice(const struct Z_Device &device) { + SBuffer buf(128); + + buf.add8(0x00); // overall length, will be updated later + buf.add16(device.shortaddr); + buf.add64(device.longaddr); + + char *names[3] = { device.modelId, device.manufacturerId, device.friendlyName }; + + for (uint32_t i=0; i 32) { len = 32; } // max 32 chars + buf.addBuffer(p, len); + } + buf.add8(0x00); // end of string marker + } + + // check if we need to write fake endpoint 0x00 + buf.add8(0x00); + if (hibernateDeviceConfiguration(buf, device.data, 0)) { + buf.add8(0xFF); // end of configuration + } else { + buf.setLen(buf.len()-1); // remove 1 byte header + } + // scan endpoints + for (uint32_t i=0; i 250) { devices_size = 250; } // arbitrarily limit to 250 devices in EEPROM instead of 32 in Flash + write_data.writeBytes(&devices_size, sizeof(devices_size)); + + for (const auto & device : zigbee_devices.getDevices()) { + const SBuffer buf = hibernateDevice(device); + if (buf.len() > 0) { + int32_t ret = write_data.writeBytes(buf.getBuffer(), buf.len()); + if (ret != buf.len()) { + AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Error writing Devices, written = %d, expected = %d"), ret, buf.len()); + return false; + } + } + } + return true; +} + +// parse a single string from the saved data +// if something wrong happens, returns nullptr to ignore the string +// Index d is incremented to just after the string +const char * hydrateSingleString(const SBuffer & buf, uint32_t *d) { + size_t s_len = buf.strlen(*d); + const char * ptr = s_len ? buf.charptr(*d) : ""; + *d += s_len + 1; + return ptr; +} + +/*********************************************************************************************\ + * hydrateSingleDevice + * + * Transforms a binary representation to a Zigbee device + * Supports only v2 +\*********************************************************************************************/ +void hydrateSingleDevice(const SBuffer & buf_d) { + uint32_t d = 1; // index in device buffer + uint16_t shortaddr = buf_d.get16(d); d += 2; + uint64_t longaddr = buf_d.get64(d); d += 8; + size_t buf_len = buf_d.len(); + Z_Device & device = zigbee_devices.updateDevice(shortaddr, longaddr); // update device's addresses + + // ModelId + device.setModelId(hydrateSingleString(buf_d, &d)); + + // ManufID + device.setManufId(hydrateSingleString(buf_d, &d)); + + // FriendlyName + device.setFriendlyName(hydrateSingleString(buf_d, &d)); + + if (d >= buf_len) { return; } + + // Hue bulbtype - if present + while (d < buf_len) { + uint8_t ep = buf_d.get8(d++); + if (0xFF == ep) { break; } // ep 0xFF marks the end of the endpoints + if (ep > 240) { ep = 0xFF; } // ep == 0xFF means ignore + device.addEndpoint(ep); // it will ignore invalid endpoints + while (d < buf_len) { + uint8_t config_type = buf_d.get8(d++); + if (0xFF == config_type) { break; } // 0xFF marks the end of congiguration + uint8_t config = config_type & 0x0F; + Z_Data_Type type = (Z_Data_Type) (config_type >> 4); + // set the configuration + if (ep != 0xFF) { + Z_Data & z_data = device.data.getByType(type, ep); + if (&z_data != nullptr) { + z_data.setConfig(config); + Z_Data_Set::updateData(z_data); + } + } + } + } +} + +/*********************************************************************************************\ + * loadZigbeeDevices + * + * Load device configuration from storage. + * Order of storage for loading is: 1/ EEPROM 2/ File system 3/ Flash (ESP8266 only) +\*********************************************************************************************/ +// dump = true, only dump to logs, don't actually load +bool loadZigbeeDevices(void) { + Univ_Read_File f; // universal reader + const char * storage_class = PSTR(""); + +#ifdef USE_ZIGBEE_EEPROM + if (zigbee.eeprom_ready) { + f.init(ZIGB_NAME2); + storage_class = PSTR("EEPROM"); + } +#endif // USE_ZIGBEE_EEPROM + +#ifdef USE_UFILESYS + File file; + if (!f.valid() && dfsp) { + file = dfsp->open(TASM_FILE_ZIGBEE, "r"); + if (file) { + f.init(&file); + storage_class = PSTR("File System"); + } + } +#endif // USE_UFILESYS + +#ifdef ESP8266 + if (!f.valid() && flash_valid()) { + // Read binary data from Flash + + Z_Flashentry flashdata; + memcpy_P(&flashdata, z_dev_start, sizeof(Z_Flashentry)); + // AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_ZIGBEE "z_dev_start %p"), z_dev_start); + AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_ZIGBEE "Zigbee signature in Flash: %08X - %d"), flashdata.name, flashdata.len); + + // Check the signature + if ( ((flashdata.name == ZIGB_NAME1) || (flashdata.name == ZIGB_NAME2)) + && (flashdata.len > 0)) { + uint16_t buf_len = flashdata.len; + // uint32_t version = (flashdata.name == ZIGB_NAME2) ? 2 : 1; + f.init(z_dev_start + sizeof(Z_Flashentry), buf_len); + storage_class = PSTR("Flash"); + } + } +#endif // ESP8266 + + uint32_t file_len = 0; + uint8_t num_devices = 0; + if (f.valid()) { + file_len = f.len; + f.readBytes(&num_devices, sizeof(num_devices)); + } + if ((file_len < 10) || (num_devices == 0x00) || (num_devices == 0xFF)) { // No data + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "No Zigbee device information")); + return false; + } + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee device information found in %s (%d devices - %d bytes)"), storage_class, num_devices, file_len); + + uint32_t k = 1; // byte index in global buffer + for (uint32_t i = 0; (i < num_devices) && (k < file_len); i++) { + uint8_t dev_record_len = 0; + f.readBytes(&dev_record_len, sizeof(dev_record_len)); + // int32_t ret = ZFS::readBytes(ZIGB_NAME2, &dev_record_len, 1, k, 1); + if (dev_record_len == 0) { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Invalid device information, aborting")); + zigbee_devices.clean(); // don't write back to Flash what we just loaded + return false; + } + SBuffer buf(dev_record_len); + buf.setLen(dev_record_len); + buf.set8(0, dev_record_len); // push the first byte (len including this first byte) + int32_t ret = f.readBytes(buf.buf(1), dev_record_len - 1); + // ret = ZFS::readBytes(ZIGB_NAME2, buf.getBuffer(), dev_record_len, k, dev_record_len); + if (ret != dev_record_len - 1) { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Invalid device information, aborting")); + zigbee_devices.clean(); // don't write back to Flash what we just loaded + return false; + } + + hydrateSingleDevice(buf); + + // next iteration + k += dev_record_len; + } + + zigbee_devices.clean(); // don't write back to Flash what we just loaded + return true; +} + +/*********************************************************************************************\ + * saveZigbeeDevices + * + * Save device configuration from storage. + * Order of storage for saving is: 1/ EEPROM 2/ File system 3/ Flash (ESP8266 only) +\*********************************************************************************************/ +void saveZigbeeDevices(void) { + Univ_Write_File f; + const char * storage_class = PSTR(""); + +#ifdef USE_ZIGBEE_EEPROM + if (!f.valid() && zigbee.eeprom_ready) { + f.init(ZIGB_NAME2); + storage_class = PSTR("EEPROM"); + } +#endif + +#ifdef USE_UFILESYS + File file; + if (!f.valid() && dfsp) { + file = dfsp->open(TASM_FILE_ZIGBEE, "w"); + if (file) { + f.init(&file); + storage_class = PSTR("File System"); + } + } +#endif + +#if defined(ESP8266) + uint8_t *sbuffer = nullptr; + static const size_t max_flash_size = 2040; + if (!f.valid() && flash_valid()) { + sbuffer = (uint8_t*) malloc(max_flash_size); + f.init(sbuffer, max_flash_size); + storage_class = PSTR("Flash"); + } +#endif // defined(ESP8266) + + bool written = false; + size_t buf_len = 0; + if (f.valid()) { + written = hibernateDevices(f); + + buf_len = f.getCursor(); + f.close(); + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data saved in %s (%d bytes)"), storage_class, buf_len); + } + +#if defined(ESP8266) + if (written && sbuffer != nullptr) { + // first copy SPI buffer into ram + uint8_t *spi_buffer = (uint8_t*) malloc(z_spi_len); + if (!spi_buffer) { + AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Cannot allocate 4KB buffer")); + free(sbuffer); + return; + } + ESP.flashRead(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); + + Z_Flashentry *flashdata = (Z_Flashentry*)(spi_buffer + z_block_offset); + flashdata->name = ZIGB_NAME2; // v2 + flashdata->len = buf_len; + flashdata->start = 0; + + memcpy(spi_buffer + z_block_offset + sizeof(Z_Flashentry), sbuffer, buf_len); + + // buffer is now ready, write it back + if (ESP.flashEraseSector(z_spi_start_sector)) { + ESP.flashWrite(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); + } + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data store in Flash (0x%08X - %d bytes)"), z_dev_start, buf_len); + free(spi_buffer); + free(sbuffer); + } +#endif // defined(ESP8266) + +} + + +/*********************************************************************************************\ + * eraseZigbeeDevices + * + * Erase all storage locations: 1/ EEPROM, 2/ File system 3/ Flash (ESP8266 only if no filesystem) +\*********************************************************************************************/ +// Erase the flash area containing the ZigbeeData +void eraseZigbeeDevices(void) { + zigbee_devices.clean(); // avoid writing data to flash after erase +#ifdef USE_ZIGBEE_EEPROM + ZFS_Erase(); +#endif // USE_ZIGBEE_EEPROM + +#if defined(ESP8266) && !defined(USE_UFILESYS) + // first copy SPI buffer into ram + uint8_t *spi_buffer = (uint8_t*) malloc(z_spi_len); + if (!spi_buffer) { + AddLog(LOG_LEVEL_ERROR, PSTR(D_LOG_ZIGBEE "Cannot allocate 4KB buffer")); + return; + } + // copy the flash into RAM to make local change, and write back the whole buffer + ESP.flashRead(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); + + // Fill the Zigbee area with 0xFF + memset(spi_buffer + z_block_offset, 0xFF, z_block_len); + + // buffer is now ready, write it back + if (ESP.flashEraseSector(z_spi_start_sector)) { + ESP.flashWrite(z_spi_start_sector * SPI_FLASH_SEC_SIZE, (uint32_t*) spi_buffer, SPI_FLASH_SEC_SIZE); + } + + free(spi_buffer); + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data erased in %s"), PSTR("Flash")); +#endif // defined(ESP8266) && !defined(USE_UFILESYS) + +#ifdef USE_UFILESYS + if (TfsDeleteFile(TASM_FILE_ZIGBEE)) { + AddLog(LOG_LEVEL_INFO, PSTR(D_LOG_ZIGBEE "Zigbee Devices Data erased")); + } +#endif // USE_UFILESYS +} + +/*********************************************************************************************\ + * restoreDumpAllDevices + * + * Dump all devices in `ZbRestore ` format ready to copy/paste +\*********************************************************************************************/ +void restoreDumpAllDevices(void) { + for (const auto & device : zigbee_devices.getDevices()) { + const SBuffer buf = hibernateDevice(device); + if (buf.len() > 0) { + Response_P(PSTR("{\"" D_PRFX_ZB D_CMND_ZIGBEE_RESTORE "\":\"ZbRestore %_B\"}"), &buf); + MqttPublishPrefixTopicRulesProcess_P(RESULT_OR_STAT, PSTR(D_PRFX_ZB D_CMND_ZIGBEE_DATA)); + } + } +} + +#endif // USE_ZIGBEE diff --git a/tasmota/xdrv_23_zigbee_7_0_statemachine.ino b/tasmota/xdrv_23_zigbee_7_0_statemachine.ino index a9afb9764..89bff56dd 100644 --- a/tasmota/xdrv_23_zigbee_7_0_statemachine.ino +++ b/tasmota/xdrv_23_zigbee_7_0_statemachine.ino @@ -481,8 +481,12 @@ static const Zigbee_Instruction zb_prog[] PROGMEM = { ZI_MQTT_STATE(ZIGBEE_STATUS_OK, kStarted) ZI_LOG(LOG_LEVEL_INFO, kZigbeeStarted) ZI_CALL(&Z_State_Ready, 1) // Now accept incoming messages + ZI_CALL(&Z_Prepare_Storage, 0) ZI_CALL(&Z_Load_Devices, 0) + ZI_CALL(&Z_Load_Data, 0) + ZI_CALL(&Z_Set_Save_Data_Timer, 0) ZI_CALL(&Z_Query_Bulbs, 0) + ZI_LABEL(ZIGBEE_LABEL_MAIN_LOOP) ZI_WAIT_FOREVER() ZI_GOTO(ZIGBEE_LABEL_READY) @@ -907,10 +911,10 @@ static const Zigbee_Instruction zb_prog[] PROGMEM = { ZI_MQTT_STATE(ZIGBEE_STATUS_OK, kStarted) ZI_LOG(LOG_LEVEL_INFO, kZigbeeStarted) ZI_CALL(&Z_State_Ready, 1) // Now accept incoming messages - ZI_CALL(&Z_Prepare_EEPROM, 0) + ZI_CALL(&Z_Prepare_Storage, 0) ZI_CALL(&Z_Load_Devices, 0) - ZI_CALL(&Z_Load_Data_EEPROM, 0) - ZI_CALL(&Z_Set_Save_Data_Timer_EEPROM, 0) + ZI_CALL(&Z_Load_Data, 0) + ZI_CALL(&Z_Set_Save_Data_Timer, 0) ZI_CALL(&Z_Query_Bulbs, 0) ZI_LABEL(ZIGBEE_LABEL_MAIN_LOOP) diff --git a/tasmota/xdrv_23_zigbee_8_parsers.ino b/tasmota/xdrv_23_zigbee_8_parsers.ino index b6206a0f7..a738d5dc4 100644 --- a/tasmota/xdrv_23_zigbee_8_parsers.ino +++ b/tasmota/xdrv_23_zigbee_8_parsers.ino @@ -2017,15 +2017,15 @@ int32_t ZNP_Recv_Default(int32_t res, const SBuffer &buf) { // // Callback for loading preparing EEPROM, called by the state machine // -#ifdef USE_ZIGBEE_EZSP -int32_t Z_Prepare_EEPROM(uint8_t value) { +int32_t Z_Prepare_Storage(uint8_t value) { +#ifdef USE_ZIGBEE_EEPROM ZFS::initOrFormat(); +#endif return 0; // continue } -#endif // USE_ZIGBEE_EZSP // -// Callback for loading Zigbee configuration from Flash, called by the state machine +// Callback for loading Zigbee configuration, called by the state machine // int32_t Z_Load_Devices(uint8_t value) { // try to hidrate from known devices @@ -2036,8 +2036,8 @@ int32_t Z_Load_Devices(uint8_t value) { // // Callback for loading Zigbee data from EEPROM, called by the state machine // -int32_t Z_Load_Data_EEPROM(uint8_t value) { - hydrateDevicesDataFromEEPROM(); +int32_t Z_Load_Data(uint8_t value) { + hydrateDevicesData(); return 0; // continue } diff --git a/tasmota/xdrv_23_zigbee_A_impl.ino b/tasmota/xdrv_23_zigbee_A_impl.ino index 00309085b..d1eb6d8bc 100644 --- a/tasmota/xdrv_23_zigbee_A_impl.ino +++ b/tasmota/xdrv_23_zigbee_A_impl.ino @@ -1291,12 +1291,6 @@ void CmndZbSave(void) { case 2: // save only data hibernateAllData(); break; - case -1: // dump configuration - loadZigbeeDevices(true); // dump only - break; - case -2: - hydrateDevicesDataFromEEPROM(); - break; #ifdef Z_EEPROM_DEBUG case -10: { // reinit EEPROM @@ -1569,7 +1563,7 @@ void CmndZbData(void) { if (strlen(XdrvMailbox.data) == 0) { // if empty, log values for all devices for (const auto & device : zigbee_devices.getDevices()) { - hibernateDeviceData(device, true); // simple log, no mqtt + hibernateDeviceData(device); } } else { // check if parameters contain a comma ',' @@ -1588,7 +1582,7 @@ void CmndZbData(void) { // non-JSON, export current data // ZbData 0x1234 // ZbData Device_Name - hibernateDeviceData(device, true); // mqtt + hibernateDeviceData(device); } } @@ -2183,9 +2177,7 @@ bool Xdrv23(uint8_t function) result = DecodeCommand(kZbCommands, ZigbeeCommand, kZbSynonyms); break; case FUNC_SAVE_BEFORE_RESTART: -#ifdef USE_ZIGBEE_EZSP hibernateAllData(); -#endif // USE_ZIGBEE_EZSP restoreDumpAllDevices(); break; }