mirrors
/
Tasmota
mirror of https://github.com/arendst/Tasmota.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor Pio filesystem download script (#20544) 

* refactor FS download Pio script
This commit is contained in:
Jason2866 2024-01-19 18:36:46 +01:00 committed by GitHub
parent 1eb1025c18
commit 0ccd958c6c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 24 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
pio-tools/custom_target.py
View File

@ -1,9 +1,11 @@
# Written by Maximilian Gerhardt <maximilian.gerhardt@rub.de> # Written by Maximilian Gerhardt <maximilian.gerhardt@rub.de>
# 29th December 2020 # 29th December 2020
# and Christian Baars, Johann Obermeier
# 2023 / 2024
# License: Apache # License: Apache
# Expanded from functionality provided by PlatformIO's espressif32 and espressif8266 platforms, credited below. # Expanded from functionality provided by PlatformIO's espressif32 and espressif8266 platforms, credited below.
# This script provides functions to download the filesystem (SPIFFS or LittleFS) from a running ESP32 / ESP8266 # This script provides functions to download the filesystem (LittleFS) from a running ESP32 / ESP8266
# over the serial bootloader using esptool.py, and mklittlefs / mkspiffs for extracting. # over the serial bootloader using esptool.py, and mklittlefs for extracting.
# run by either using the VSCode task "Custom" -> "Download Filesystem" # run by either using the VSCode task "Custom" -> "Download Filesystem"
# or by doing 'pio run -t downloadfs' (with optional '-e <environment>') from the commandline. # or by doing 'pio run -t downloadfs' (with optional '-e <environment>') from the commandline.
# output will be saved, by default, in the "unpacked_fs" of the project. # output will be saved, by default, in the "unpacked_fs" of the project.
@ -22,10 +24,6 @@ Import("env")
platform = env.PioPlatform() platform = env.PioPlatform()
board = env.BoardConfig() board = env.BoardConfig()
mcu = board.get("build.mcu", "esp32") mcu = board.get("build.mcu", "esp32")
# Hack for using mklittlefs instead of mkspiffs -> needed since littlefs is not supported with this for ESP32
if env["PIOPLATFORM"] == "espressif32":
#print("Replace MKSPIFFSTOOL with mklittlefs")
env.Replace( MKSPIFFSTOOL=platform.get_package_dir("tool-mklittlefs") + '/mklittlefs' )
class FSType(Enum): class FSType(Enum):
@ -48,30 +46,22 @@ class FSInfo:
class LittleFSInfo(FSInfo): class LittleFSInfo(FSInfo):
def __init__(self, start, length, page_size, block_size): def __init__(self, start, length, page_size, block_size):
if env["PIOPLATFORM"] == "espressif32": self.tool = env["MKFSTOOL"]
#for ESP32: retrieve and evaluate, e.g. to mkspiffs_espressif32_arduino
self.tool = env.subst(env["MKSPIFFSTOOL"])
else:
self.tool = env["MKFSTOOL"] # from mkspiffs package
self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool) self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
super().__init__(FSType.LITTLEFS, start, length, page_size, block_size) super().__init__(FSType.LITTLEFS, start, length, page_size, block_size)
def __repr__(self): def __repr__(self):
return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}" return f"{self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
def get_extract_cmd(self, input_file, output_dir): def get_extract_cmd(self, input_file, output_dir):
return [self.tool, "-b", str(self.block_size), "-p", str(self.page_size), "--unpack", output_dir, input_file] return [self.tool, "-b", str(self.block_size), "-s", str(self.length), "-p", str(self.page_size), "--unpack", output_dir, input_file]
class SPIFFSInfo(FSInfo): class SPIFFSInfo(FSInfo):
def __init__(self, start, length, page_size, block_size): def __init__(self, start, length, page_size, block_size):
if env["PIOPLATFORM"] == "espressif32": self.tool = env["MKFSTOOL"]
#for ESP32: retrieve and evaluate, e.g. to mkspiffs_espressif32_arduino self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
self.tool = env.subst(env["MKSPIFFSTOOL"])
else:
self.tool = env["MKFSTOOL"] # from mkspiffs package
self.tool = join(platform.get_package_dir("tool-mkspiffs"), self.tool)
super().__init__(FSType.SPIFFS, start, length, page_size, block_size) super().__init__(FSType.SPIFFS, start, length, page_size, block_size)
def __repr__(self): def __repr__(self):
return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}" return f"{self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
def get_extract_cmd(self, input_file, output_dir): def get_extract_cmd(self, input_file, output_dir):
return f'"{self.tool}" -b {self.block_size} -p {self.page_size} --unpack "{output_dir}" "{input_file}"' return f'"{self.tool}" -b {self.block_size} -p {self.page_size} --unpack "{output_dir}" "{input_file}"'
@ -169,15 +159,15 @@ def esp8266_get_esptoolpy_reset_flags(resetmethod):
## Script interface functions ## Script interface functions
def parse_partition_table(content): def parse_partition_table(content):
entries = [e for e in content.split(b'\xaaP') if len(e) > 0] entries = [e for e in content.split(b'\xaaP') if len(e) > 0]
print("Partition data:") #print("Partition data:")
for entry in entries: for entry in entries:
type = entry[1] type = entry[1]
if type in [0x82,0x83]: # SPIFFS or LITTLEFS if type in [0x82,0x83]: # SPIFFS or LITTLEFS
offset = int.from_bytes(entry[2:5], byteorder='little', signed=False) offset = int.from_bytes(entry[2:5], byteorder='little', signed=False)
size = int.from_bytes(entry[6:9], byteorder='little', signed=False) size = int.from_bytes(entry[6:9], byteorder='little', signed=False)
print("type:",hex(type)) #print("type:",hex(type))
print("address:",hex(offset)) #print("address:",hex(offset))
print("size:",hex(size)) #print("size:",hex(size))
env["SPIFFS_START"] = offset env["SPIFFS_START"] = offset
env["SPIFFS_SIZE"] = size env["SPIFFS_SIZE"] = size
env["SPIFFS_PAGE"] = int("0x100", 16) env["SPIFFS_PAGE"] = int("0x100", 16)
@ -203,8 +193,8 @@ def get_partition_table():
fs_file fs_file
] ]
esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
print("Executing flash download command to read partition table.") #print("Executing flash download command to read partition table.")
print(esptoolpy_cmd) #print(esptoolpy_cmd)
try: try:
returncode = subprocess.call(esptoolpy_cmd, shell=False) returncode = subprocess.call(esptoolpy_cmd, shell=False)
except subprocess.CalledProcessError as exc: except subprocess.CalledProcessError as exc:
@ -217,14 +207,14 @@ def get_fs_type_start_and_length():
platform = env["PIOPLATFORM"] platform = env["PIOPLATFORM"]
if platform == "espressif32": if platform == "espressif32":
print(f"Retrieving filesystem info for {mcu}.") print(f"Retrieving filesystem info for {mcu}.")
print("Partition file: " + str(env.subst("$PARTITIONS_TABLE_CSV"))) #print("Partition file: " + str(env.subst("$PARTITIONS_TABLE_CSV")))
# esp32_fetch_spiffs_size(env) # esp32_fetch_spiffs_size(env)
get_partition_table() get_partition_table()
return SPIFFSInfo(env["SPIFFS_START"], env["SPIFFS_SIZE"], env["SPIFFS_PAGE"], env["SPIFFS_BLOCK"]) return SPIFFSInfo(env["SPIFFS_START"], env["SPIFFS_SIZE"], env["SPIFFS_PAGE"], env["SPIFFS_BLOCK"])
elif platform == "espressif8266": elif platform == "espressif8266":
print("Retrieving filesystem info for ESP8266.") print("Retrieving filesystem info for ESP8266.")
filesystem = board.get("build.filesystem", "spiffs") filesystem = board.get("build.filesystem", "littlefs")
if filesystem not in ("spiffs", "littlefs"): if filesystem not in ("littlefs"):
print("Unrecognized board_build.filesystem option '" + str(filesystem) + "'.") print("Unrecognized board_build.filesystem option '" + str(filesystem) + "'.")
env.Exit(1) env.Exit(1)
# fetching sizes is the same for all filesystems # fetching sizes is the same for all filesystems
@ -233,10 +223,7 @@ def get_fs_type_start_and_length():
print("FS_END: " + hex(env["FS_END"])) print("FS_END: " + hex(env["FS_END"]))
print("FS_PAGE: " + hex(env["FS_PAGE"])) print("FS_PAGE: " + hex(env["FS_PAGE"]))
print("FS_BLOCK: " + hex(env["FS_BLOCK"])) print("FS_BLOCK: " + hex(env["FS_BLOCK"]))
if filesystem == "spiffs": if filesystem == "littlefs":
print("Recognized SPIFFS filesystem.")
return SPIFFSInfo(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"])
elif filesystem == "littlefs":
print("Recognized LittleFS filesystem.") print("Recognized LittleFS filesystem.")
return LittleFSInfo(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"]) return LittleFSInfo(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"])
else: else:
@ -264,8 +251,8 @@ def download_fs(fs_info: FSInfo):
fs_file fs_file
] ]
esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
print("Executing flash download command.") print("Download filesystem image")
print(esptoolpy_cmd) #print(esptoolpy_cmd)
try: try:
returncode = subprocess.call(esptoolpy_cmd, shell=False) returncode = subprocess.call(esptoolpy_cmd, shell=False)
# print("Launched download of filesystem binary.") # print("Launched download of filesystem binary.")
@ -291,10 +278,10 @@ def unpack_fs(fs_info: FSInfo, downloaded_file: str):
os.makedirs(unpack_dir) os.makedirs(unpack_dir)
cmd = fs_info.get_extract_cmd(downloaded_file, unpack_dir) cmd = fs_info.get_extract_cmd(downloaded_file, unpack_dir)
print("Executing extraction command: " + str(cmd)) print("Unpack files from filesystem image")
#print("Extraction command: " + str(cmd))
try: try:
returncode = subprocess.call(cmd, shell=True) returncode = subprocess.call(cmd, shell=True)
print("Unpacked filesystem.")
return (True, unpack_dir) return (True, unpack_dir)
except subprocess.CalledProcessError as exc: except subprocess.CalledProcessError as exc:
print("Unpacking filesystem failed with " + str(exc)) print("Unpacking filesystem failed with " + str(exc))
@ -307,11 +294,8 @@ def display_fs(extracted_dir):
print("Extracted " + str(file_count) + " file(s) from filesystem.") print("Extracted " + str(file_count) + " file(s) from filesystem.")
def command_download_fs(*args, **kwargs): def command_download_fs(*args, **kwargs):
#print("Entrypoint")
#print(env.Dump())
get_partition_table() get_partition_table()
info = get_fs_type_start_and_length() info = get_fs_type_start_and_length()
print("Parsed FS info: " + str(info))
download_ok, downloaded_file = download_fs(info) download_ok, downloaded_file = download_fs(info)
# print("Download was okay: " + str(download_ok) + ". File at: "+ str(downloaded_file)) # this is wrong # print("Download was okay: " + str(download_ok) + ". File at: "+ str(downloaded_file)) # this is wrong
unpack_ok, unpacked_dir = unpack_fs(info, downloaded_file) unpack_ok, unpacked_dir = unpack_fs(info, downloaded_file)
@ -358,4 +342,3 @@ env.AddCustomTarget(
title="Flash factory", title="Flash factory",
description="Flash factory firmware" description="Flash factory firmware"
) )