Keep FS intact when over flashing with VSC (#403) (#19816)

pio-tools/get_flash_size.py

@@ -1,45 +0,0 @@
-from os.path import join
-import subprocess
-from SCons.Script import COMMAND_LINE_TARGETS
-
-Import("env")
-env = DefaultEnvironment()
-platform = env.PioPlatform()
-
-if "upload" in COMMAND_LINE_TARGETS:
-
-    def esp32_detect_flashsize():
-        esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py")
-        esptoolpy_flags = ["flash_id"]
-        esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
-        try:
-            output = subprocess.run(esptoolpy_cmd, capture_output=True).stdout.splitlines()
-            for l in output:
-                if l.decode().startswith("Detected flash size: "):
-                    size = (l.decode().split(": ")[1])
-                    print("Did get flash size:",size)
-                    return size, True
-            return "4MB",False
-        except subprocess.CalledProcessError as exc:
-            print("Did get chip info failed with " + str(exc))
-            return "4MB",False
-
-    size, overridden = esp32_detect_flashsize()
-    old_flash_size = env.BoardConfig().get("upload.flash_size")
-    old_maximum_size = env.BoardConfig().get("upload.maximum_size")
-    new_maximum_size = int(size.split("MB")[0]) * 0x100000
-
-    if new_maximum_size > old_maximum_size:
-
-        extra_flags = env.BoardConfig().get("build.extra_flags").split(" ")
-        new_flags = ""
-        for flag in extra_flags:
-            if flag.startswith("-DESP32"):
-                flag = f"-DESP32_{size}"
-            new_flags += flag + " "
-        env.BoardConfig().update("build.extra_flags",new_flags)
-        env.BoardConfig().update("upload.flash_size",size)
-
-        env.BoardConfig().update("upload.maximum_size", new_maximum_size)
-
-        env.Replace(TASMOTA_flash_size=size)

pio-tools/name-firmware.py

@@ -14,12 +14,14 @@ def bin_map_copy(source, target, env):
     map_file = tasmotapiolib.get_final_map_path(env)
     bin_file = tasmotapiolib.get_final_bin_path(env)
     one_bin_file = bin_file
+    firmware_name = env.subst("$BUILD_DIR/${PROGNAME}.bin")
 
     if env["PIOPLATFORM"] == "espressif32":
-        factory_tmp = pathlib.Path(firsttarget).with_suffix("")
+        if("safeboot" not in firmware_name):
-        factory = factory_tmp.with_suffix(factory_tmp.suffix + ".factory.bin")
+            factory_tmp = pathlib.Path(firsttarget).with_suffix("")
-        one_bin_tmp = pathlib.Path(bin_file).with_suffix("")
+            factory = factory_tmp.with_suffix(factory_tmp.suffix + ".factory.bin")
-        one_bin_file = one_bin_tmp.with_suffix(one_bin_tmp.suffix + ".factory.bin")
+            one_bin_tmp = pathlib.Path(bin_file).with_suffix("")
+            one_bin_file = one_bin_tmp.with_suffix(one_bin_tmp.suffix + ".factory.bin")
 
     # check if new target files exist and remove if necessary
     for f in [map_file, bin_file, one_bin_file]:
@@ -31,7 +33,8 @@ def bin_map_copy(source, target, env):
     if env["PIOPLATFORM"] == "espressif32":
         # the map file is needed later for fimrmware-metrics.py
         shutil.copy(tasmotapiolib.get_source_map_path(env), map_file)
-        shutil.copy(factory, one_bin_file)
+        if("safeboot" not in firmware_name):
+            shutil.copy(factory, one_bin_file)
     else:
         map_firm = join(env.subst("$BUILD_DIR")) + os.sep + "firmware.map"
         shutil.copy(tasmotapiolib.get_source_map_path(env), map_firm)

pio-tools/post_esp32.py

@@ -56,6 +56,35 @@ else:
 
 variants_dir = join(FRAMEWORK_DIR, "variants", "tasmota")
 
+def esp32_detect_flashsize():
+    uploader = env.subst("$UPLOADER")
+    if not "upload" in COMMAND_LINE_TARGETS:
+        return "4MB",False
+    if not "esptool" in uploader:
+        return "4MB",False
+    else:
+        esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py")
+        esptoolpy_flags = ["flash_id"]
+        esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
+        try:
+            output = subprocess.run(esptoolpy_cmd, capture_output=True).stdout.splitlines()
+            for l in output:
+                if l.decode().startswith("Detected flash size: "):
+                    size = (l.decode().split(": ")[1])
+                    print("Did get flash size:", size)
+                    stored_flash_size_mb = env.BoardConfig().get("upload.flash_size")
+                    stored_flash_size = int(stored_flash_size_mb.split("MB")[0]) * 0x100000
+                    detected_flash_size = int(size.split("MB")[0]) * 0x100000
+                    if detected_flash_size > stored_flash_size:
+                        env.BoardConfig().update("upload.flash_size", size)
+                        return size, True
+            return "4MB",False
+        except subprocess.CalledProcessError as exc:
+            print("Did get chip info failed with " + str(exc))
+            return "4MB",False
+
+flash_size_from_esp, flash_size_was_overridden = esp32_detect_flashsize()
+
 def patch_partitions_bin(size_string):
     partition_bin_path = join(env.subst("$BUILD_DIR"),"partitions.bin")
     with open(partition_bin_path, 'r+b') as file:
@@ -71,15 +100,6 @@ def patch_partitions_bin(size_string):
         file.write(partition_data)
         print("New partition hash:",result.digest().hex())
 
-def esp32_detect_flashsize():
-    if not "upload" in COMMAND_LINE_TARGETS:
-        return "4MB",False
-    size = env.get("TASMOTA_flash_size")
-    if size == None:
-        return "4MB",False
-    else:
-        return size,True
-
 def esp32_create_chip_string(chip):
     tasmota_platform = env.subst("$BUILD_DIR").split(os.path.sep)[-1]
     tasmota_platform = tasmota_platform.split('-')[0]
@@ -142,13 +162,11 @@ def esp32_copy_new_safeboot_bin(tasmota_platform,new_local_safeboot_fw):
 
 def esp32_create_combined_bin(source, target, env):
     #print("Generating combined binary for serial flashing")
-
     # The offset from begin of the file where the app0 partition starts
     # This is defined in the partition .csv file
     # factory_offset = -1      # error code value - currently unused
     app_offset = 0x10000     # default value for "old" scheme
     fs_offset = -1           # error code value
-    flash_size_from_esp, flash_size_was_overridden = esp32_detect_flashsize()
 
     with open(env.BoardConfig().get("build.partitions")) as csv_file:
         print("Read partitions from ",env.BoardConfig().get("build.partitions"))
@@ -188,14 +206,12 @@ def esp32_create_combined_bin(source, target, env):
 
     if not os.path.exists(variants_dir):
         os.makedirs(variants_dir)
-    if("safeboot" in firmware_name):
+    if "safeboot" in firmware_name:
         esp32_copy_new_safeboot_bin(tasmota_platform,firmware_name)
     else:
         esp32_fetch_safeboot_bin(tasmota_platform)
 
     flash_size = env.BoardConfig().get("upload.flash_size", "4MB")
-    if flash_size_was_overridden:
-        flash_size = flash_size_from_esp
     flash_freq = env.BoardConfig().get("build.f_flash", "40000000L")
     flash_freq = str(flash_freq).replace("L", "")
     flash_freq = str(int(int(flash_freq) / 1000000)) + "m"
@@ -227,16 +243,15 @@ def esp32_create_combined_bin(source, target, env):
         cmd += [sect_adr, sect_file]
 
     # "main" firmware to app0 - mandatory, except we just built a new safeboot bin locally
-    if("safeboot" not in firmware_name):
+    if "safeboot" not in firmware_name:
         print(f" - {hex(app_offset)} | {firmware_name}")
         cmd += [hex(app_offset), firmware_name]
 
     else:
         print("Upload new safeboot binary only")
 
-#    if(fs_offset != -1):
+    upload_protocol = env.subst("$UPLOAD_PROTOCOL")
-    upload_port = env.subst("$UPLOAD_PORT")
+    if(upload_protocol == "esptool") and (fs_offset != -1):
-    if("upload-tasmota.php" not in upload_port) and (fs_offset != -1):
         fs_bin = join(env.subst("$BUILD_DIR"),"littlefs.bin")
         if exists(fs_bin):
             before_reset = env.BoardConfig().get("upload.before_reset", "default_reset")
@@ -259,9 +274,9 @@ def esp32_create_combined_bin(source, target, env):
             )
             print("Will use custom upload command for flashing operation to add file system defined for this build target.")
 
-    # print('Using esptool.py arguments: %s' % ' '.join(cmd))
+    #print('Using esptool.py arguments: %s' % ' '.join(cmd))
-
+    if("safeboot" not in firmware_name):
-    esptool.main(cmd)
+        esptool.main(cmd)
 
 
-env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)

platformio_tasmota32.ini

@@ -47,7 +47,6 @@ lib_ignore                  =
                               ArduinoOTA
                               ESP32-HomeKit
 extra_scripts               = pre:pio-tools/add_c_flags.py
-                            ;   pre:pio-tools/get_flash_size.py
                               pre:pio-tools/gen-berry-structures.py
                               post:pio-tools/post_esp32.py
                               ${esp_defaults.extra_scripts}
@@ -79,7 +78,7 @@ lib_ignore                  = ${esp32_defaults.lib_ignore}
                               ccronexpr
 
 [core32]
-platform                    = https://github.com/tasmota/platform-espressif32/releases/download/2023.10.04/platform-espressif32.zip
+platform                    = https://github.com/tasmota/platform-espressif32/releases/download/2023.10.05/platform-espressif32.zip
 platform_packages           =
 build_unflags               = ${esp32_defaults.build_unflags}
 build_flags                 = ${esp32_defaults.build_flags}