From 2e9cd6a57d76c0272481117f0e65c8d179ef1976 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 29 Jul 2023 19:48:43 +0200 Subject: [PATCH] Auto detect flash size and adjust FS (#349) (#19215) --- pio-tools/get_flash_size.py | 43 ++++++++++++++++++++++++++++ pio-tools/post_esp32.py | 46 +++++++++++++++++++++++++++--- platformio_tasmota32.ini | 1 + platformio_tasmota_cenv_sample.ini | 7 ----- 4 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 pio-tools/get_flash_size.py diff --git a/pio-tools/get_flash_size.py b/pio-tools/get_flash_size.py new file mode 100644 index 000000000..7d1bf09f5 --- /dev/null +++ b/pio-tools/get_flash_size.py @@ -0,0 +1,43 @@ +from os.path import join +import subprocess +from SCons.Script import COMMAND_LINE_TARGETS + +Import("env") +env = DefaultEnvironment() +platform = env.PioPlatform() + +if "upload" in COMMAND_LINE_TARGETS: + + def esp32_detect_flashsize(): + esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") + esptoolpy_flags = ["flash_id"] + esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags + try: + output = subprocess.run(esptoolpy_cmd, capture_output=True).stdout.splitlines() + for l in output: + if l.decode().startswith("Detected flash size: "): + size = (l.decode().split(": ")[1]) + print("Did get flash size:",size) + return size, True + return "4MB",False + except subprocess.CalledProcessError as exc: + print("Did get chip info failed with " + str(exc)) + return "4MB",False + + size, overridden = esp32_detect_flashsize() + old_flash_size = env.BoardConfig().get("upload.flash_size") + old_maximum_size = env.BoardConfig().get("upload.maximum_size") + new_maximum_size = int(size.split("MB")[0]) * 0x100000 + + extra_flags = env.BoardConfig().get("build.extra_flags").split(" ") + new_flags = "" + for flag in extra_flags: + if flag.startswith("-DESP32"): + flag = f"-DESP32_{size}" + new_flags += flag + " " + env.BoardConfig().update("build.extra_flags",new_flags) + env.BoardConfig().update("upload.flash_size",size) + + env.BoardConfig().update("upload.maximum_size", new_maximum_size) + + env.Replace(TASMOTA_flash_size=size) \ No newline at end of file diff --git a/pio-tools/post_esp32.py b/pio-tools/post_esp32.py index 9f8a2f637..33e9ffa7e 100644 --- a/pio-tools/post_esp32.py +++ b/pio-tools/post_esp32.py @@ -1,6 +1,8 @@ # From: https://github.com/letscontrolit/ESPEasy/blob/mega/tools/pio/post_esp32.py # Thanks TD-er :) +# Thanks @staars for safeboot and auto resizing LittleFS code and enhancements + # Combines separate bin files with their respective offsets into a single file # This single file must then be flashed to an ESP32 node with 0 offset. # @@ -29,6 +31,8 @@ import csv import requests import shutil import subprocess +import codecs +from SCons.Script import COMMAND_LINE_TARGETS sys.path.append(join(platform.get_package_dir("tool-esptoolpy"))) import esptool @@ -52,6 +56,29 @@ else: variants_dir = join(FRAMEWORK_DIR, "variants", "tasmota") +def patch_partitions_bin(size_string): + partition_bin_path = join(env.subst("$BUILD_DIR"),"partitions.bin") + with open(partition_bin_path, 'r+b') as file: + binary_data = file.read(0xb0) + import hashlib + bin_list = list(binary_data) + size = codecs.decode(size_string[2:], 'hex_codec') # 0xc50000 -> [c5,00,00] + bin_list[0x8a] = size[0] + bin_list[0x8b] = size[1] + result = hashlib.md5(bytes(bin_list[0:0xa0])) + partition_data = bytes(bin_list) + result.digest() + file.seek(0) + file.write(partition_data) + print("New partition hash:",result.digest().hex()) + +def esp32_detect_flashsize(): + if not "upload" in COMMAND_LINE_TARGETS: + return "4MB",False + try: + return env.get("TASMOTA_flash_size"),True + except: + return "4MB",False + def esp32_create_chip_string(chip): tasmota_platform = env.subst("$BUILD_DIR").split(os.path.sep)[-1] tasmota_platform = tasmota_platform.split('-')[0] @@ -120,6 +147,8 @@ def esp32_create_combined_bin(source, target, env): # factory_offset = -1 # error code value - currently unused app_offset = 0x10000 # default value for "old" scheme fs_offset = -1 # error code value + flash_size_from_esp, flash_size_was_overridden = esp32_detect_flashsize() + with open(env.BoardConfig().get("build.partitions")) as csv_file: print("Read partitions from ",env.BoardConfig().get("build.partitions")) csv_reader = csv.reader(csv_file, delimiter=',') @@ -136,7 +165,13 @@ def esp32_create_combined_bin(source, target, env): # elif(row[0] == 'factory'): # factory_offset = int(row[3],base=16) elif(row[0] == 'spiffs'): - if esp32_build_filesystem(row[4]): + partition_size = row[4] + if flash_size_was_overridden: + print(f"Will override fixed FS partition size from {env.BoardConfig().get('build.partitions')}: {partition_size} ...") + partition_size = hex(int(flash_size_from_esp.split("MB")[0]) * 0x100000 - int(row[3],base=16)) + print(f"... with computed maximum size from connected {env.get('BOARD_MCU')}: {partition_size}") + patch_partitions_bin(partition_size) + if esp32_build_filesystem(partition_size): fs_offset = int(row[3],base=16) @@ -147,16 +182,19 @@ def esp32_create_combined_bin(source, target, env): tasmota_platform = esp32_create_chip_string(chip) if "-DUSE_USB_CDC_CONSOLE" in env.BoardConfig().get("build.extra_flags") and "cdc" not in tasmota_platform: - tasmota_platform += "cdc" + tasmota_platform += "cdc" print("WARNING: board definition uses CDC configuration, but environment name does not -> changing tasmota safeboot binary to:", tasmota_platform + "-safeboot.bin") - + if not os.path.exists(variants_dir): os.makedirs(variants_dir) if("safeboot" in firmware_name): esp32_copy_new_safeboot_bin(tasmota_platform,firmware_name) else: esp32_fetch_safeboot_bin(tasmota_platform) + flash_size = env.BoardConfig().get("upload.flash_size", "4MB") + if flash_size_was_overridden: + flash_size = flash_size_from_esp flash_freq = env.BoardConfig().get("build.f_flash", "40000000L") flash_freq = str(flash_freq).replace("L", "") flash_freq = str(int(int(flash_freq) / 1000000)) + "m" @@ -225,4 +263,4 @@ def esp32_create_combined_bin(source, target, env): esptool.main(cmd) -env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin) +env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin) \ No newline at end of file diff --git a/platformio_tasmota32.ini b/platformio_tasmota32.ini index 814af20a0..30f42a99a 100644 --- a/platformio_tasmota32.ini +++ b/platformio_tasmota32.ini @@ -37,6 +37,7 @@ build_flags = ${esp_defaults.build_flags} -Wl,--wrap=_Z11analogWritehi ; `analogWrite(unsigned char, int)` use the Tasmota version of analogWrite for deeper integration and phase control -Wl,--wrap=ledcReadFreq ; `uint32_t ledcReadFreq(uint8_t chan)` extra_scripts = pre:pio-tools/add_c_flags.py + pre:pio-tools/get_flash_size.py pre:pio-tools/gen-berry-structures.py post:pio-tools/post_esp32.py ${esp_defaults.extra_scripts} diff --git a/platformio_tasmota_cenv_sample.ini b/platformio_tasmota_cenv_sample.ini index 2f987d809..ed427a6ed 100644 --- a/platformio_tasmota_cenv_sample.ini +++ b/platformio_tasmota_cenv_sample.ini @@ -22,13 +22,6 @@ board_build.f_flash = 80000000L build_flags = ${env:tasmota32_base.build_flags} -DFIRMWARE_TASMOTA32 -DOTA_URL='"http://ota.tasmota.com/tasmota32/release/tasmota32s3.bin"' -; !!! Real flash size needed, avoid autoresize since it is formating FS !!! -board_upload.flash_size = 8MB -board_upload.maximum_size = 8388608 -; Without autoresize a partition scheme is needed which does fit to flash size -board_build.partitions = partitions/esp32_partition_app2944k_fs2M.csv -; Dont use safeboot, not used in this partition scheme -> an empty entry needed to overwrite the default setting -board_upload.arduino.flash_extra_images = ; Example for custom file upload in Tasmota Filesystem custom_files_upload = ${env:tasmota32_base.custom_files_upload} tasmota/berry/modules/Partition_Wizard.tapp