2022-01-02 14:14:54 +00:00
# From: https://github.com/letscontrolit/ESPEasy/blob/mega/tools/pio/post_esp32.py
# Thanks TD-er :)
2023-07-29 18:48:43 +01:00
# Thanks @staars for safeboot and auto resizing LittleFS code and enhancements
2022-01-06 17:38:02 +00:00
# Combines separate bin files with their respective offsets into a single file
# This single file must then be flashed to an ESP32 node with 0 offset.
#
# Original implementation: Bartłomiej Zimoń (@uzi18)
#
# Special thanks to @Jason2866 for helping debug flashing to >4MB flash
# Thanks @jesserockz (esphome) for adapting to use esptool.py with merge_bin
#
# Typical layout of the generated file:
# Offset | File
# - 0x1000 | ~\.platformio\packages\framework-arduinoespressif32\tools\sdk\esp32\bin\bootloader_dout_40m.bin
# - 0x8000 | ~\Tasmota\.pio\build\<env name>\partitions.bin
# - 0xe000 | ~\.platformio\packages\framework-arduinoespressif32\tools\partitions\boot_app0.bin
2023-10-28 12:15:31 +01:00
# - 0x10000 | ~\Tasmota\<variants_dir>/<env name>-safeboot.bin
2022-05-11 17:30:20 +01:00
# - 0xe0000 | ~\Tasmota\.pio\build\<env name>/firmware.bin
# - 0x3b0000| ~\Tasmota\.pio\build\<env name>/littlefs.bin
2022-01-05 16:50:36 +00:00
2022-05-02 22:24:52 +01:00
env = DefaultEnvironment ( )
2022-01-07 10:07:36 +00:00
platform = env . PioPlatform ( )
2022-01-06 22:02:59 +00:00
2022-05-02 22:24:52 +01:00
from genericpath import exists
import os
2022-01-07 10:07:36 +00:00
import sys
from os . path import join
2022-04-30 12:57:54 +01:00
import csv
2022-05-02 22:24:52 +01:00
import requests
import shutil
2022-05-11 17:30:20 +01:00
import subprocess
2023-07-29 18:48:43 +01:00
import codecs
from SCons . Script import COMMAND_LINE_TARGETS
2022-01-06 22:02:59 +00:00
2022-01-07 10:07:36 +00:00
sys . path . append ( join ( platform . get_package_dir ( " tool-esptoolpy " ) ) )
2022-01-06 17:38:02 +00:00
import esptool
2022-01-05 16:50:36 +00:00
2023-10-28 12:15:31 +01:00
variants_dir = env . BoardConfig ( ) . get ( " build.variants_dir " , " " )
variant = env . BoardConfig ( ) . get ( " build.variant " , " " )
sections = env . subst ( env . get ( " FLASH_EXTRA_IMAGES " ) )
chip = env . get ( " BOARD_MCU " )
mcu_build_variant = env . BoardConfig ( ) . get ( " build.variant " , " " ) . lower ( )
# Copy safeboots firmwares in place when running in Github
2022-10-26 14:50:44 +01:00
github_actions = os . getenv ( ' GITHUB_ACTIONS ' )
2022-09-27 13:30:57 +01:00
extra_flags = ' ' . join ( [ element . replace ( " -D " , " " ) for element in env . BoardConfig ( ) . get ( " build.extra_flags " , " " ) ] )
build_flags = ' ' . join ( [ element . replace ( " -D " , " " ) for element in env . GetProjectOption ( " build_flags " ) ] )
2022-09-24 11:05:00 +01:00
if " CORE32SOLO1 " in extra_flags or " FRAMEWORK_ARDUINO_SOLO1 " in build_flags :
FRAMEWORK_DIR = platform . get_package_dir ( " framework-arduino-solo1 " )
2022-10-26 14:50:44 +01:00
if github_actions and os . path . exists ( " ./firmware/firmware " ) :
shutil . copytree ( " ./firmware/firmware " , " /home/runner/.platformio/packages/framework-arduino-solo1/variants/tasmota " )
2023-10-28 12:15:31 +01:00
if variants_dir :
shutil . copytree ( " ./firmware/firmware " , variants_dir , dirs_exist_ok = True )
2022-09-27 13:30:57 +01:00
elif " CORE32ITEAD " in extra_flags or " FRAMEWORK_ARDUINO_ITEAD " in build_flags :
2022-09-24 11:05:00 +01:00
FRAMEWORK_DIR = platform . get_package_dir ( " framework-arduino-ITEAD " )
2022-10-26 14:50:44 +01:00
if github_actions and os . path . exists ( " ./firmware/firmware " ) :
shutil . copytree ( " ./firmware/firmware " , " /home/runner/.platformio/packages/framework-arduino-ITEAD/variants/tasmota " )
2023-10-28 12:15:31 +01:00
if variants_dir :
shutil . copytree ( " ./firmware/firmware " , variants_dir , dirs_exist_ok = True )
2022-09-27 13:30:57 +01:00
else :
FRAMEWORK_DIR = platform . get_package_dir ( " framework-arduinoespressif32 " )
2022-10-26 14:50:44 +01:00
if github_actions and os . path . exists ( " ./firmware/firmware " ) :
shutil . copytree ( " ./firmware/firmware " , " /home/runner/.platformio/packages/framework-arduinoespressif32/variants/tasmota " )
2023-10-28 12:15:31 +01:00
if variants_dir :
shutil . copytree ( " ./firmware/firmware " , variants_dir , dirs_exist_ok = True )
# Copy pins_arduino.h to variants folder
if variants_dir :
mcu_build_variant_path = join ( FRAMEWORK_DIR , " variants " , mcu_build_variant , " pins_arduino.h " )
custom_variant_build = join ( env . subst ( " $PROJECT_DIR " ) , variants_dir , mcu_build_variant , " pins_arduino.h " )
os . makedirs ( join ( env . subst ( " $PROJECT_DIR " ) , variants_dir , mcu_build_variant ) , exist_ok = True )
shutil . copy ( mcu_build_variant_path , custom_variant_build )
2022-09-24 11:05:00 +01:00
2023-10-28 12:15:31 +01:00
if not variants_dir :
variants_dir = join ( FRAMEWORK_DIR , " variants " , " tasmota " )
env . BoardConfig ( ) . update ( " build.variants_dir " , variants_dir )
2022-05-02 22:24:52 +01:00
2023-10-22 16:10:59 +01:00
def esp32_detect_flashsize ( ) :
uploader = env . subst ( " $UPLOADER " )
if not " upload " in COMMAND_LINE_TARGETS :
return " 4MB " , False
if not " esptool " in uploader :
return " 4MB " , False
else :
esptoolpy = join ( platform . get_package_dir ( " tool-esptoolpy " ) or " " , " esptool.py " )
esptoolpy_flags = [ " flash_id " ]
esptoolpy_cmd = [ env [ " PYTHONEXE " ] , esptoolpy ] + esptoolpy_flags
try :
output = subprocess . run ( esptoolpy_cmd , capture_output = True ) . stdout . splitlines ( )
for l in output :
if l . decode ( ) . startswith ( " Detected flash size: " ) :
size = ( l . decode ( ) . split ( " : " ) [ 1 ] )
print ( " Did get flash size: " , size )
stored_flash_size_mb = env . BoardConfig ( ) . get ( " upload.flash_size " )
stored_flash_size = int ( stored_flash_size_mb . split ( " MB " ) [ 0 ] ) * 0x100000
detected_flash_size = int ( size . split ( " MB " ) [ 0 ] ) * 0x100000
if detected_flash_size > stored_flash_size :
env . BoardConfig ( ) . update ( " upload.flash_size " , size )
return size , True
return " 4MB " , False
except subprocess . CalledProcessError as exc :
print ( " Did get chip info failed with " + str ( exc ) )
return " 4MB " , False
flash_size_from_esp , flash_size_was_overridden = esp32_detect_flashsize ( )
2023-07-29 18:48:43 +01:00
def patch_partitions_bin ( size_string ) :
partition_bin_path = join ( env . subst ( " $BUILD_DIR " ) , " partitions.bin " )
with open ( partition_bin_path , ' r+b ' ) as file :
binary_data = file . read ( 0xb0 )
import hashlib
bin_list = list ( binary_data )
2023-10-30 17:07:24 +00:00
size_string = int ( size_string [ 2 : ] , 16 )
size_string = f " { size_string : 08X } "
size = codecs . decode ( size_string , ' hex_codec ' ) # 0xc50000 -> [00,c5,00,00]
bin_list [ 0x89 ] = size [ 2 ]
bin_list [ 0x8a ] = size [ 1 ]
bin_list [ 0x8b ] = size [ 0 ]
2023-07-29 18:48:43 +01:00
result = hashlib . md5 ( bytes ( bin_list [ 0 : 0xa0 ] ) )
partition_data = bytes ( bin_list ) + result . digest ( )
file . seek ( 0 )
file . write ( partition_data )
print ( " New partition hash: " , result . digest ( ) . hex ( ) )
2022-05-12 12:13:39 +01:00
def esp32_create_chip_string ( chip ) :
2022-05-20 16:00:56 +01:00
tasmota_platform = env . subst ( " $BUILD_DIR " ) . split ( os . path . sep ) [ - 1 ]
2022-05-12 12:13:39 +01:00
tasmota_platform = tasmota_platform . split ( ' - ' ) [ 0 ]
2022-07-06 19:08:16 +01:00
if ' tasmota ' + chip [ 3 : ] not in tasmota_platform : # quick check for a valid name like 'tasmota' + '32c3'
2022-05-12 12:13:39 +01:00
print ( ' Unexpected naming conventions in this build environment -> Undefined behavior for further build process!! ' )
2022-07-06 19:08:16 +01:00
print ( " Expected build environment name like ' tasmota32-whatever-you-want ' " )
2022-05-12 12:13:39 +01:00
return tasmota_platform
2022-05-11 17:30:20 +01:00
def esp32_build_filesystem ( fs_size ) :
files = env . GetProjectOption ( " custom_files_upload " ) . splitlines ( )
filesystem_dir = join ( env . subst ( " $BUILD_DIR " ) , " littlefs_data " )
if not os . path . exists ( filesystem_dir ) :
os . makedirs ( filesystem_dir )
print ( " Creating filesystem with content: " )
for file in files :
2022-05-12 09:58:36 +01:00
if " no_files " in file :
continue
2022-05-27 18:45:09 +01:00
if " http " and " :// " in file :
2023-06-25 20:21:27 +01:00
response = requests . get ( file . split ( " " ) [ 0 ] )
2022-05-27 18:45:09 +01:00
if response . ok :
target = join ( filesystem_dir , file . split ( os . path . sep ) [ - 1 ] )
2023-06-25 20:21:27 +01:00
if len ( file . split ( " " ) ) > 1 :
target = join ( filesystem_dir , file . split ( " " ) [ 1 ] )
print ( " Renaming " , ( file . split ( os . path . sep ) [ - 1 ] ) . split ( " " ) [ 0 ] , " to " , file . split ( " " ) [ 1 ] )
2022-05-27 18:45:09 +01:00
open ( target , " wb " ) . write ( response . content )
else :
print ( " Failed to download: " , file )
continue
2023-06-21 17:32:42 +01:00
if os . path . isdir ( file ) :
shutil . copytree ( file , filesystem_dir , dirs_exist_ok = True )
else :
shutil . copy ( file , filesystem_dir )
2022-05-12 09:58:36 +01:00
if not os . listdir ( filesystem_dir ) :
print ( " No files added -> will NOT create littlefs.bin and NOT overwrite fs partition! " )
return False
2022-05-11 17:30:20 +01:00
env . Replace ( MKSPIFFSTOOL = platform . get_package_dir ( " tool-mklittlefs " ) + ' /mklittlefs ' )
tool = env . subst ( env [ " MKSPIFFSTOOL " ] )
cmd = ( tool , " -c " , filesystem_dir , " -s " , fs_size , join ( env . subst ( " $BUILD_DIR " ) , " littlefs.bin " ) )
returncode = subprocess . call ( cmd , shell = False )
2022-05-12 12:13:39 +01:00
# print(returncode)
2022-05-12 09:58:36 +01:00
return True
2022-05-11 17:30:20 +01:00
2022-05-12 12:13:39 +01:00
def esp32_fetch_safeboot_bin ( tasmota_platform ) :
2022-06-22 17:50:02 +01:00
safeboot_fw_url = " http://ota.tasmota.com/tasmota32/release/ " + tasmota_platform + " -safeboot.bin "
2022-05-12 12:13:39 +01:00
safeboot_fw_name = join ( variants_dir , tasmota_platform + " -safeboot.bin " )
2022-05-05 10:19:39 +01:00
if ( exists ( safeboot_fw_name ) ) :
print ( " safeboot binary already in place. " )
2022-05-03 21:25:23 +01:00
return
2022-05-05 10:19:39 +01:00
print ( " Will download safeboot binary from URL: " )
print ( safeboot_fw_url )
response = requests . get ( safeboot_fw_url )
open ( safeboot_fw_name , " wb " ) . write ( response . content )
print ( " safeboot binary written to variants dir. " )
2022-05-12 12:13:39 +01:00
def esp32_copy_new_safeboot_bin ( tasmota_platform , new_local_safeboot_fw ) :
2022-05-05 10:19:39 +01:00
print ( " Copy new local safeboot firmware to variants dir -> using it for further flashing operations " )
2022-05-12 12:13:39 +01:00
safeboot_fw_name = join ( variants_dir , tasmota_platform + " -safeboot.bin " )
2022-05-03 21:25:23 +01:00
if os . path . exists ( variants_dir ) :
2022-05-05 10:19:39 +01:00
shutil . copy ( new_local_safeboot_fw , safeboot_fw_name )
2022-05-02 22:24:52 +01:00
2022-01-06 17:38:02 +00:00
def esp32_create_combined_bin ( source , target , env ) :
#print("Generating combined binary for serial flashing")
2022-01-05 16:50:36 +00:00
# The offset from begin of the file where the app0 partition starts
# This is defined in the partition .csv file
2022-05-12 09:58:36 +01:00
# factory_offset = -1 # error code value - currently unused
2022-05-02 22:24:52 +01:00
app_offset = 0x10000 # default value for "old" scheme
2022-05-12 09:58:36 +01:00
fs_offset = - 1 # error code value
2023-07-29 18:48:43 +01:00
2022-04-30 12:57:54 +01:00
with open ( env . BoardConfig ( ) . get ( " build.partitions " ) ) as csv_file :
print ( " Read partitions from " , env . BoardConfig ( ) . get ( " build.partitions " ) )
csv_reader = csv . reader ( csv_file , delimiter = ' , ' )
line_count = 0
for row in csv_reader :
if line_count == 0 :
print ( f ' { " , " . join ( row ) } ' )
line_count + = 1
else :
print ( f ' { row [ 0 ] } { row [ 1 ] } { row [ 2 ] } { row [ 3 ] } { row [ 4 ] } ' )
line_count + = 1
if ( row [ 0 ] == ' app0 ' ) :
app_offset = int ( row [ 3 ] , base = 16 )
2022-05-03 21:25:23 +01:00
# elif(row[0] == 'factory'):
# factory_offset = int(row[3],base=16)
2022-05-20 16:00:56 +01:00
elif ( row [ 0 ] == ' spiffs ' ) :
2023-07-29 18:48:43 +01:00
partition_size = row [ 4 ]
if flash_size_was_overridden :
print ( f " Will override fixed FS partition size from { env . BoardConfig ( ) . get ( ' build.partitions ' ) } : { partition_size } ... " )
partition_size = hex ( int ( flash_size_from_esp . split ( " MB " ) [ 0 ] ) * 0x100000 - int ( row [ 3 ] , base = 16 ) )
print ( f " ... with computed maximum size from connected { env . get ( ' BOARD_MCU ' ) } : { partition_size } " )
patch_partitions_bin ( partition_size )
if esp32_build_filesystem ( partition_size ) :
2022-05-12 09:58:36 +01:00
fs_offset = int ( row [ 3 ] , base = 16 )
2022-05-03 21:25:23 +01:00
2022-01-05 16:50:36 +00:00
2022-01-05 22:14:49 +00:00
new_file_name = env . subst ( " $BUILD_DIR/$ {PROGNAME} .factory.bin " )
2022-01-05 16:50:36 +00:00
firmware_name = env . subst ( " $BUILD_DIR/$ {PROGNAME} .bin " )
2022-05-12 12:13:39 +01:00
tasmota_platform = esp32_create_chip_string ( chip )
2022-10-29 09:56:53 +01:00
if " -DUSE_USB_CDC_CONSOLE " in env . BoardConfig ( ) . get ( " build.extra_flags " ) and " cdc " not in tasmota_platform :
2023-07-29 18:48:43 +01:00
tasmota_platform + = " cdc "
2022-10-29 09:56:53 +01:00
print ( " WARNING: board definition uses CDC configuration, but environment name does not -> changing tasmota safeboot binary to: " , tasmota_platform + " -safeboot.bin " )
2023-07-29 18:48:43 +01:00
2022-05-03 21:25:23 +01:00
if not os . path . exists ( variants_dir ) :
os . makedirs ( variants_dir )
2023-10-22 16:10:59 +01:00
if " safeboot " in firmware_name :
2022-05-12 12:13:39 +01:00
esp32_copy_new_safeboot_bin ( tasmota_platform , firmware_name )
2022-05-03 21:25:23 +01:00
else :
2022-05-12 12:13:39 +01:00
esp32_fetch_safeboot_bin ( tasmota_platform )
2023-07-29 18:48:43 +01:00
2022-05-10 07:24:56 +01:00
flash_size = env . BoardConfig ( ) . get ( " upload.flash_size " , " 4MB " )
2022-07-13 13:33:13 +01:00
flash_freq = env . BoardConfig ( ) . get ( " build.f_flash " , " 40000000L " )
flash_freq = str ( flash_freq ) . replace ( " L " , " " )
flash_freq = str ( int ( int ( flash_freq ) / 1000000 ) ) + " m "
2022-08-26 15:17:52 +01:00
flash_mode = env . BoardConfig ( ) . get ( " build.flash_mode " , " dio " )
2022-09-04 14:20:21 +01:00
memory_type = env . BoardConfig ( ) . get ( " build.arduino.memory_type " , " qio_qspi " )
if flash_mode == " qio " or flash_mode == " qout " :
2022-07-13 13:33:13 +01:00
flash_mode = " dio "
2022-09-04 14:20:21 +01:00
if memory_type == " opi_opi " or memory_type == " opi_qspi " :
2022-07-13 13:33:13 +01:00
flash_mode = " dout "
2022-01-06 17:38:02 +00:00
cmd = [
" --chip " ,
chip ,
" merge_bin " ,
" -o " ,
new_file_name ,
2022-07-12 12:27:17 +01:00
" --flash_mode " ,
flash_mode ,
" --flash_freq " ,
flash_freq ,
2022-01-06 17:38:02 +00:00
" --flash_size " ,
flash_size ,
]
print ( " Offset | File " )
2022-01-02 14:14:54 +00:00
for section in sections :
2022-01-06 17:38:02 +00:00
sect_adr , sect_file = section . split ( " " , 1 )
print ( f " - { sect_adr } | { sect_file } " )
cmd + = [ sect_adr , sect_file ]
2022-01-05 16:50:36 +00:00
2022-05-05 10:19:39 +01:00
# "main" firmware to app0 - mandatory, except we just built a new safeboot bin locally
2023-10-22 16:10:59 +01:00
if " safeboot " not in firmware_name :
2022-05-03 21:25:23 +01:00
print ( f " - { hex ( app_offset ) } | { firmware_name } " )
cmd + = [ hex ( app_offset ) , firmware_name ]
2022-05-11 17:30:20 +01:00
2022-05-03 21:25:23 +01:00
else :
2022-05-05 10:19:39 +01:00
print ( " Upload new safeboot binary only " )
2022-01-05 16:50:36 +00:00
2023-10-22 16:10:59 +01:00
upload_protocol = env . subst ( " $UPLOAD_PROTOCOL " )
if ( upload_protocol == " esptool " ) and ( fs_offset != - 1 ) :
2022-05-11 17:30:20 +01:00
fs_bin = join ( env . subst ( " $BUILD_DIR " ) , " littlefs.bin " )
if exists ( fs_bin ) :
2022-08-10 13:11:47 +01:00
before_reset = env . BoardConfig ( ) . get ( " upload.before_reset " , " default_reset " )
after_reset = env . BoardConfig ( ) . get ( " upload.after_reset " , " hard_reset " )
2022-05-11 17:30:20 +01:00
print ( f " - { hex ( fs_offset ) } | { fs_bin } " )
cmd + = [ hex ( fs_offset ) , fs_bin ]
2022-05-20 16:00:56 +01:00
env . Replace (
2022-05-11 17:30:20 +01:00
UPLOADERFLAGS = [
" --chip " , chip ,
" --port " , ' " $UPLOAD_PORT " ' ,
" --baud " , " $UPLOAD_SPEED " ,
2022-08-10 13:11:47 +01:00
" --before " , before_reset ,
" --after " , after_reset ,
2022-05-11 17:30:20 +01:00
" write_flash " , " -z " ,
" --flash_mode " , " $ { __get_board_flash_mode(__env__)} " ,
" --flash_freq " , " $ { __get_board_f_flash(__env__)} " ,
" --flash_size " , flash_size
] ,
UPLOADCMD = ' " $PYTHONEXE " " $UPLOADER " $UPLOADERFLAGS ' + " " . join ( cmd [ 7 : ] )
)
print ( " Will use custom upload command for flashing operation to add file system defined for this build target. " )
2023-10-22 16:10:59 +01:00
if ( " safeboot " not in firmware_name ) :
2023-10-28 12:15:31 +01:00
#print('Using esptool.py arguments: %s' % ' '.join(cmd))
2023-10-22 16:10:59 +01:00
esptool . main ( cmd )
2022-01-05 16:50:36 +00:00
2022-01-02 14:14:54 +00:00
2023-10-22 16:10:59 +01:00
env . AddPostAction ( " $BUILD_DIR/$ {PROGNAME} .bin " , esp32_create_combined_bin )