mirror of https://github.com/arendst/Tasmota.git
auto compress updated html (#22738)
This commit is contained in:
parent
a1aef6b39f
commit
dc531de02b
|
@ -0,0 +1,15 @@
|
|||
Import("env")
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
base_dir = env['PROJECT_DIR']
|
||||
unishox_dir = Path(base_dir, 'tools', 'unishox')
|
||||
sys.path.append(str(unishox_dir.resolve()))
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
compress_dir = __import__('compress-html-uncompressed').compress_dir
|
||||
|
||||
path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed')
|
||||
path_compressed = Path(base_dir, 'tasmota', 'html_compressed')
|
||||
compress_dir(path_uncompressed, path_compressed)
|
|
@ -73,6 +73,7 @@ build_flags =
|
|||
extra_scripts = pre:pio-tools/pre_source_dir.py
|
||||
pre:pio-tools/set_partition_table.py
|
||||
pre:pio-tools/override_copy.py
|
||||
pre:pio-tools/compress-html.py
|
||||
post:pio-tools/strip-flags.py
|
||||
|
||||
[esp_defaults]
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: f5150873f8737621b7fd8bab0e471d73c893429c79c045a1b20c26c5fee1d3fa
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_GV_PAGE_SIZE = 463;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: 3417a3f0d32f6537d82e4638a9cd462098b9da4f641160355e0c8d9334aed1d4
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_HEADER1_SIZE = 683;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: e0fcb684eb9f29a5fd0f678262046d9d05d055466b1752a58a35da2648ecd0e7
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_HEAD_LAST_SCRIPT_SIZE = 355;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: 996873eafc91c9d084e3f68b60d32a37a27e91dd1acd0863fc1096119d8de6a2
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_HEAD_LAST_SCRIPT32_SIZE = 1013;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: 8c22c19284fa41f8eb66b1f50cb94cc3fe14369f900031e791107fe56d583c2f
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_HEAD_STYLE1_SIZE = 591;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: cff4350b756f01fb7866cbbffa2d169d4fe9eaca6ba45634f368ca1d714cd582
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_HEAD_STYLE2_SIZE = 496;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: ee7e68972559c2ad3df6a6594445bfdfeb14a0a37dec2389ec20a197c26c9311
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_HEAD_STYLE_ZIGBEE_SIZE = 363;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: 960fbc2354bc4029cbc93953e54edc74024a46ca58902af175f5ecf8839eb0a8
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_SCRIPT_CONSOL_SIZE = 985;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: 92e33b521e56657da8f50887c7e1431219de0a8f19048247211f3b6d4c9b68ba
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_SCRIPT_MODULE_TEMPLATE_SIZE = 589;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: 6e369f0e06cba0656d3c7187ac833b999a54f84edfd9385b4cf44ba8643e01d8
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_SCRIPT_ROOT_SIZE = 499;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: c85b76468eb793a235944c16a2d986bce127bd4fc1b0690499b3b5b88ab4f70f
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_SCRIPT_ROOT_PART2_SIZE = 222;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: b42b87eb23e656d5ae799709721147c847ae381f0d1f0cb9f86bd55e9509bf51
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_SCRIPT_ROOT_SIZE = 434;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: c137f990d750da7e1e51a6ec80baad5445853da492c12c4dad87c95724c50441
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_SCRIPT_ROOT_SIZE = 872;
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
/////////////////////////////////////////////////////////////////////
|
||||
// compressed by tools/unishox/compress-html-uncompressed.py
|
||||
// input sha256: 464453a8f35b349965adc050d3e4f968239a974f171cfa64efc665bafe3ba3f4
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
const size_t HTTP_SCRIPT_TEMPLATE_SIZE = 288;
|
||||
|
|
|
@ -20,9 +20,13 @@
|
|||
###############################################################
|
||||
|
||||
import unishox
|
||||
from os import listdir
|
||||
from os import path
|
||||
from sys import argv
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from hashlib import sha256
|
||||
|
||||
self_dir = Path(__file__).absolute().parent
|
||||
base_dir = self_dir.parent.parent
|
||||
|
||||
def extract_c_string(s: str) -> str:
|
||||
state = 0
|
||||
|
@ -48,18 +52,24 @@ def extract_c_string(s: str) -> str:
|
|||
out += c
|
||||
return out
|
||||
|
||||
path_compressed = path.join('..','..','tasmota','html_compressed')
|
||||
path_uncompressed = path.join('..','..','tasmota','html_uncompressed')
|
||||
def compress_html(source, target, argv=None, verbose=False):
|
||||
if argv is None: argv = []
|
||||
|
||||
files = listdir(path_uncompressed)
|
||||
with open(source, "r") as f:
|
||||
text = f.read()
|
||||
|
||||
totalIn = 0
|
||||
totalSaved = 0
|
||||
src_sha, old_sha = sha256(text.encode()).hexdigest(), None
|
||||
|
||||
for file in files:
|
||||
f = open(path_uncompressed + path.sep + file, "r")
|
||||
text = f.read()
|
||||
f.close()
|
||||
if not ('--force' in argv):
|
||||
with open(target, "r") as f:
|
||||
for line in f:
|
||||
prefix = line[:17]
|
||||
if prefix == '// input sha256: ':
|
||||
old_sha = line[17:17+64]
|
||||
break
|
||||
|
||||
if src_sha == old_sha:
|
||||
return (0, 0)
|
||||
|
||||
#text = Tk().clipboard_get()
|
||||
# print(text)
|
||||
|
@ -109,8 +119,9 @@ for file in files:
|
|||
# print(text[lastel+1:pos:])
|
||||
lastel = pos
|
||||
|
||||
print("####### Parsing input from " + path_uncompressed + path.sep + file)
|
||||
print(" Const char name: "+const_name)
|
||||
if verbose:
|
||||
print("####### Parsing input from " + str(source.relative_to(base_dir)))
|
||||
print(" Const char name: "+const_name)
|
||||
#print('####### Cleaned input:')
|
||||
#print(input)
|
||||
|
||||
|
@ -128,20 +139,27 @@ for file in files:
|
|||
|
||||
UNISHOX = unishox.Unishox()
|
||||
out_len = UNISHOX.compress(in_bytes, len(in_bytes), out_bytes, len(out_bytes))
|
||||
print(" ####### Compression result:")
|
||||
print(" Compressed from {i} to {o}, -{p:.1f}%".format(i=in_len, o=out_len, p=(100-(float(out_len)/float(in_len)*100))))
|
||||
if verbose:
|
||||
print(" ####### Compression result:")
|
||||
reduction = 100-(float(out_len)/float(in_len)*100)
|
||||
print(f" Compressed from {in_len} to {out_len}, -{reduction:.1f}%")
|
||||
out_bytes = out_bytes[:out_len] # truncate to right size
|
||||
|
||||
#PROGMEM is growing in steps 0,8,24,40,56,... bytes of data resulting in size of 0,16,32,48,64,... bytes
|
||||
for in_real in range(8,in_len+16,16):
|
||||
if in_real>=in_len:
|
||||
print(" Old real PROGMEM-size:"+str(in_real+8)+"(unused bytes:"+str(in_real-in_len)+")")
|
||||
if verbose:
|
||||
print(f" Old real PROGMEM-size:{in_real+8}(unused bytes:{in_real-in_len})")
|
||||
break
|
||||
for out_real in range(8,out_len+16,16):
|
||||
if out_real>=out_len:
|
||||
print(" New real PROGMEM-size:"+str(out_real+8)+"(unused bytes:"+str(out_real-out_len)+")")
|
||||
if verbose:
|
||||
print(f" New real PROGMEM-size:{out_real+8}(unused bytes:{out_real-out_len})")
|
||||
break
|
||||
print(" the optimal case would be raw bytes + 8, real difference: "+str(in_real - out_real)+ "bytes")
|
||||
|
||||
if verbose:
|
||||
print(f" the optimal case would be raw bytes + 8, real difference: {in_real - out_real}bytes")
|
||||
|
||||
# https://www.geeksforgeeks.org/break-list-chunks-size-n-python/
|
||||
def chunked(my_list, n):
|
||||
return [my_list[i * n:(i + 1) * n] for i in range((len(my_list) + n - 1) // n )]
|
||||
|
@ -150,27 +168,49 @@ for file in files:
|
|||
chunks = chunked(out_bytes, 20)
|
||||
|
||||
lines_raw = [ "\"\\x" + "\\x".join( [ '{:02X}'.format(b) for b in chunk ] ) + "\"" for chunk in chunks ]
|
||||
line_complete = "const char " + const_name + "_COMPRESSED" +"[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";"
|
||||
lines = "\nconst size_t " + const_name +"_SIZE = {size};\n{lines}\n\n".format(size=in_len, lines=line_complete)
|
||||
line_complete = f"const char {const_name}_COMPRESSED[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";"
|
||||
lines = f"\nconst size_t {const_name}_SIZE = {in_len};\n{line_complete}\n\n"
|
||||
|
||||
#print('####### Final output:')
|
||||
#print(lines)
|
||||
|
||||
definition = "#define " + const_name + " Decompress(" + const_name + "_COMPRESSED" + "," + const_name +"_SIZE" + ").c_str()"
|
||||
definition = f"#define {const_name} Decompress({const_name}_COMPRESSED,{const_name}_SIZE).c_str()"
|
||||
#print(definition)
|
||||
|
||||
now = datetime.now() # current date and time
|
||||
percent = int((float(out_real)/float(in_real))*100.0)
|
||||
saving = in_real - out_real
|
||||
totalIn = totalIn + in_real
|
||||
totalSaved = totalSaved + saving
|
||||
comment = "/////////////////////////////////////////////////////////////////////\n"
|
||||
comment = comment + "// compressed by tools/unishox/compress-html-uncompressed.py\n"
|
||||
comment = comment + "/////////////////////////////////////////////////////////////////////\n"
|
||||
#totalIn = totalIn + in_real
|
||||
#totalSaved = totalSaved + saving
|
||||
comment = "/////////////////////////////////////////////////////////////////////\n"
|
||||
comment += "// compressed by tools/unishox/compress-html-uncompressed.py\n"
|
||||
comment += f"// input sha256: {src_sha}\n"
|
||||
comment += "/////////////////////////////////////////////////////////////////////\n"
|
||||
|
||||
f = open(path_compressed + path.sep + file, "w")
|
||||
f.write(comment + lines + definition)
|
||||
f.close()
|
||||
print("####### Wrote output to " + path_compressed + path.sep + file)
|
||||
with open(target, "w") as f:
|
||||
f.write(comment + lines + definition)
|
||||
|
||||
print("If all files are in use, total saving was "+str(totalSaved)+" out of "+str(totalIn))
|
||||
return (in_real, saving)
|
||||
|
||||
if verbose:
|
||||
print("####### Wrote output to " + str(target.relative_to(base_dir)))
|
||||
|
||||
def compress_dir(source_dir, target_dir, argv=None, verbose=False):
|
||||
totalIn, totalSaved = 0, 0
|
||||
|
||||
for source in source_dir.iterdir():
|
||||
target = Path(target_dir, source.stem + ".h")
|
||||
bytesIn, bytesSaved = compress_html(source, target, argv, verbose)
|
||||
totalIn += bytesIn
|
||||
totalSaved += bytesSaved
|
||||
|
||||
return (totalIn, totalSaved)
|
||||
|
||||
if __name__ == '__main__':
|
||||
path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed')
|
||||
path_compressed = Path(base_dir, 'tasmota', 'html_compressed')
|
||||
|
||||
totalIn, totalSaved = compress_dir(path_uncompressed, path_compressed, argv, True)
|
||||
|
||||
if totalSaved > 0:
|
||||
print(f"If all files are in use, total saving was {totalSaved} out of {totalIn}")
|
||||
|
|
Loading…
Reference in New Issue