auto compress updated html (#22738)

This commit is contained in:
Ryan Castellucci 2024-12-29 21:27:10 +00:00 committed by GitHub
parent a1aef6b39f
commit dc531de02b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 101 additions and 31 deletions

View File

@ -0,0 +1,15 @@
Import("env")
import sys
from pathlib import Path
base_dir = env['PROJECT_DIR']
unishox_dir = Path(base_dir, 'tools', 'unishox')
sys.path.append(str(unishox_dir.resolve()))
sys.dont_write_bytecode = True
compress_dir = __import__('compress-html-uncompressed').compress_dir
path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed')
path_compressed = Path(base_dir, 'tasmota', 'html_compressed')
compress_dir(path_uncompressed, path_compressed)

View File

@ -73,6 +73,7 @@ build_flags =
extra_scripts = pre:pio-tools/pre_source_dir.py
pre:pio-tools/set_partition_table.py
pre:pio-tools/override_copy.py
pre:pio-tools/compress-html.py
post:pio-tools/strip-flags.py
[esp_defaults]

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: f5150873f8737621b7fd8bab0e471d73c893429c79c045a1b20c26c5fee1d3fa
/////////////////////////////////////////////////////////////////////
const size_t HTTP_GV_PAGE_SIZE = 463;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: 3417a3f0d32f6537d82e4638a9cd462098b9da4f641160355e0c8d9334aed1d4
/////////////////////////////////////////////////////////////////////
const size_t HTTP_HEADER1_SIZE = 683;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: e0fcb684eb9f29a5fd0f678262046d9d05d055466b1752a58a35da2648ecd0e7
/////////////////////////////////////////////////////////////////////
const size_t HTTP_HEAD_LAST_SCRIPT_SIZE = 355;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: 996873eafc91c9d084e3f68b60d32a37a27e91dd1acd0863fc1096119d8de6a2
/////////////////////////////////////////////////////////////////////
const size_t HTTP_HEAD_LAST_SCRIPT32_SIZE = 1013;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: 8c22c19284fa41f8eb66b1f50cb94cc3fe14369f900031e791107fe56d583c2f
/////////////////////////////////////////////////////////////////////
const size_t HTTP_HEAD_STYLE1_SIZE = 591;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: cff4350b756f01fb7866cbbffa2d169d4fe9eaca6ba45634f368ca1d714cd582
/////////////////////////////////////////////////////////////////////
const size_t HTTP_HEAD_STYLE2_SIZE = 496;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: ee7e68972559c2ad3df6a6594445bfdfeb14a0a37dec2389ec20a197c26c9311
/////////////////////////////////////////////////////////////////////
const size_t HTTP_HEAD_STYLE_ZIGBEE_SIZE = 363;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: 960fbc2354bc4029cbc93953e54edc74024a46ca58902af175f5ecf8839eb0a8
/////////////////////////////////////////////////////////////////////
const size_t HTTP_SCRIPT_CONSOL_SIZE = 985;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: 92e33b521e56657da8f50887c7e1431219de0a8f19048247211f3b6d4c9b68ba
/////////////////////////////////////////////////////////////////////
const size_t HTTP_SCRIPT_MODULE_TEMPLATE_SIZE = 589;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: 6e369f0e06cba0656d3c7187ac833b999a54f84edfd9385b4cf44ba8643e01d8
/////////////////////////////////////////////////////////////////////
const size_t HTTP_SCRIPT_ROOT_SIZE = 499;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: c85b76468eb793a235944c16a2d986bce127bd4fc1b0690499b3b5b88ab4f70f
/////////////////////////////////////////////////////////////////////
const size_t HTTP_SCRIPT_ROOT_PART2_SIZE = 222;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: b42b87eb23e656d5ae799709721147c847ae381f0d1f0cb9f86bd55e9509bf51
/////////////////////////////////////////////////////////////////////
const size_t HTTP_SCRIPT_ROOT_SIZE = 434;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: c137f990d750da7e1e51a6ec80baad5445853da492c12c4dad87c95724c50441
/////////////////////////////////////////////////////////////////////
const size_t HTTP_SCRIPT_ROOT_SIZE = 872;

View File

@ -1,5 +1,6 @@
/////////////////////////////////////////////////////////////////////
// compressed by tools/unishox/compress-html-uncompressed.py
// input sha256: 464453a8f35b349965adc050d3e4f968239a974f171cfa64efc665bafe3ba3f4
/////////////////////////////////////////////////////////////////////
const size_t HTTP_SCRIPT_TEMPLATE_SIZE = 288;

View File

@ -20,9 +20,13 @@
###############################################################
import unishox
from os import listdir
from os import path
from sys import argv
from datetime import datetime
from pathlib import Path
from hashlib import sha256
self_dir = Path(__file__).absolute().parent
base_dir = self_dir.parent.parent
def extract_c_string(s: str) -> str:
state = 0
@ -48,18 +52,24 @@ def extract_c_string(s: str) -> str:
out += c
return out
path_compressed = path.join('..','..','tasmota','html_compressed')
path_uncompressed = path.join('..','..','tasmota','html_uncompressed')
def compress_html(source, target, argv=None, verbose=False):
if argv is None: argv = []
files = listdir(path_uncompressed)
with open(source, "r") as f:
text = f.read()
totalIn = 0
totalSaved = 0
src_sha, old_sha = sha256(text.encode()).hexdigest(), None
for file in files:
f = open(path_uncompressed + path.sep + file, "r")
text = f.read()
f.close()
if not ('--force' in argv):
with open(target, "r") as f:
for line in f:
prefix = line[:17]
if prefix == '// input sha256: ':
old_sha = line[17:17+64]
break
if src_sha == old_sha:
return (0, 0)
#text = Tk().clipboard_get()
# print(text)
@ -109,8 +119,9 @@ for file in files:
# print(text[lastel+1:pos:])
lastel = pos
print("####### Parsing input from " + path_uncompressed + path.sep + file)
print(" Const char name: "+const_name)
if verbose:
print("####### Parsing input from " + str(source.relative_to(base_dir)))
print(" Const char name: "+const_name)
#print('####### Cleaned input:')
#print(input)
@ -128,20 +139,27 @@ for file in files:
UNISHOX = unishox.Unishox()
out_len = UNISHOX.compress(in_bytes, len(in_bytes), out_bytes, len(out_bytes))
print(" ####### Compression result:")
print(" Compressed from {i} to {o}, -{p:.1f}%".format(i=in_len, o=out_len, p=(100-(float(out_len)/float(in_len)*100))))
if verbose:
print(" ####### Compression result:")
reduction = 100-(float(out_len)/float(in_len)*100)
print(f" Compressed from {in_len} to {out_len}, -{reduction:.1f}%")
out_bytes = out_bytes[:out_len] # truncate to right size
#PROGMEM is growing in steps 0,8,24,40,56,... bytes of data resulting in size of 0,16,32,48,64,... bytes
for in_real in range(8,in_len+16,16):
if in_real>=in_len:
print(" Old real PROGMEM-size:"+str(in_real+8)+"(unused bytes:"+str(in_real-in_len)+")")
if verbose:
print(f" Old real PROGMEM-size:{in_real+8}(unused bytes:{in_real-in_len})")
break
for out_real in range(8,out_len+16,16):
if out_real>=out_len:
print(" New real PROGMEM-size:"+str(out_real+8)+"(unused bytes:"+str(out_real-out_len)+")")
if verbose:
print(f" New real PROGMEM-size:{out_real+8}(unused bytes:{out_real-out_len})")
break
print(" the optimal case would be raw bytes + 8, real difference: "+str(in_real - out_real)+ "bytes")
if verbose:
print(f" the optimal case would be raw bytes + 8, real difference: {in_real - out_real}bytes")
# https://www.geeksforgeeks.org/break-list-chunks-size-n-python/
def chunked(my_list, n):
return [my_list[i * n:(i + 1) * n] for i in range((len(my_list) + n - 1) // n )]
@ -150,27 +168,49 @@ for file in files:
chunks = chunked(out_bytes, 20)
lines_raw = [ "\"\\x" + "\\x".join( [ '{:02X}'.format(b) for b in chunk ] ) + "\"" for chunk in chunks ]
line_complete = "const char " + const_name + "_COMPRESSED" +"[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";"
lines = "\nconst size_t " + const_name +"_SIZE = {size};\n{lines}\n\n".format(size=in_len, lines=line_complete)
line_complete = f"const char {const_name}_COMPRESSED[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";"
lines = f"\nconst size_t {const_name}_SIZE = {in_len};\n{line_complete}\n\n"
#print('####### Final output:')
#print(lines)
definition = "#define " + const_name + " Decompress(" + const_name + "_COMPRESSED" + "," + const_name +"_SIZE" + ").c_str()"
definition = f"#define {const_name} Decompress({const_name}_COMPRESSED,{const_name}_SIZE).c_str()"
#print(definition)
now = datetime.now() # current date and time
percent = int((float(out_real)/float(in_real))*100.0)
saving = in_real - out_real
totalIn = totalIn + in_real
totalSaved = totalSaved + saving
comment = "/////////////////////////////////////////////////////////////////////\n"
comment = comment + "// compressed by tools/unishox/compress-html-uncompressed.py\n"
comment = comment + "/////////////////////////////////////////////////////////////////////\n"
#totalIn = totalIn + in_real
#totalSaved = totalSaved + saving
comment = "/////////////////////////////////////////////////////////////////////\n"
comment += "// compressed by tools/unishox/compress-html-uncompressed.py\n"
comment += f"// input sha256: {src_sha}\n"
comment += "/////////////////////////////////////////////////////////////////////\n"
f = open(path_compressed + path.sep + file, "w")
f.write(comment + lines + definition)
f.close()
print("####### Wrote output to " + path_compressed + path.sep + file)
with open(target, "w") as f:
f.write(comment + lines + definition)
print("If all files are in use, total saving was "+str(totalSaved)+" out of "+str(totalIn))
return (in_real, saving)
if verbose:
print("####### Wrote output to " + str(target.relative_to(base_dir)))
def compress_dir(source_dir, target_dir, argv=None, verbose=False):
totalIn, totalSaved = 0, 0
for source in source_dir.iterdir():
target = Path(target_dir, source.stem + ".h")
bytesIn, bytesSaved = compress_html(source, target, argv, verbose)
totalIn += bytesIn
totalSaved += bytesSaved
return (totalIn, totalSaved)
if __name__ == '__main__':
path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed')
path_compressed = Path(base_dir, 'tasmota', 'html_compressed')
totalIn, totalSaved = compress_dir(path_uncompressed, path_compressed, argv, True)
if totalSaved > 0:
print(f"If all files are in use, total saving was {totalSaved} out of {totalIn}")