From dc531de02b5f8303d22c1e3165c9459a05b75c4e Mon Sep 17 00:00:00 2001 From: Ryan Castellucci Date: Sun, 29 Dec 2024 21:27:10 +0000 Subject: [PATCH] auto compress updated html (#22738) --- pio-tools/compress-html.py | 15 +++ platformio.ini | 1 + tasmota/html_compressed/HTTP_GV_PAGE.h | 1 + tasmota/html_compressed/HTTP_HEADER1_ES6.h | 1 + .../html_compressed/HTTP_HEAD_LAST_SCRIPT.h | 1 + .../html_compressed/HTTP_HEAD_LAST_SCRIPT32.h | 1 + tasmota/html_compressed/HTTP_HEAD_STYLE1.h | 1 + tasmota/html_compressed/HTTP_HEAD_STYLE2.h | 1 + .../html_compressed/HTTP_HEAD_STYLE_ZIGBEE.h | 1 + tasmota/html_compressed/HTTP_SCRIPT_CONSOL.h | 1 + .../HTTP_SCRIPT_MODULE_TEMPLATE.h | 1 + .../HTTP_SCRIPT_ROOT_NO_WEB_DISPLAY.h | 1 + .../html_compressed/HTTP_SCRIPT_ROOT_PART2.h | 1 + .../HTTP_SCRIPT_ROOT_SSE_NO_WEB_DISPLAY.h | 1 + .../HTTP_SCRIPT_ROOT_WEB_DISPLAY.h | 1 + .../html_compressed/HTTP_SCRIPT_TEMPLATE.h | 1 + tools/unishox/compress-html-uncompressed.py | 102 ++++++++++++------ 17 files changed, 101 insertions(+), 31 deletions(-) create mode 100644 pio-tools/compress-html.py diff --git a/pio-tools/compress-html.py b/pio-tools/compress-html.py new file mode 100644 index 000000000..bdfed1568 --- /dev/null +++ b/pio-tools/compress-html.py @@ -0,0 +1,15 @@ +Import("env") + +import sys +from pathlib import Path + +base_dir = env['PROJECT_DIR'] +unishox_dir = Path(base_dir, 'tools', 'unishox') +sys.path.append(str(unishox_dir.resolve())) +sys.dont_write_bytecode = True + +compress_dir = __import__('compress-html-uncompressed').compress_dir + +path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed') +path_compressed = Path(base_dir, 'tasmota', 'html_compressed') +compress_dir(path_uncompressed, path_compressed) diff --git a/platformio.ini b/platformio.ini index 1d1c9c0bf..859beec14 100644 --- a/platformio.ini +++ b/platformio.ini @@ -73,6 +73,7 @@ build_flags = extra_scripts = pre:pio-tools/pre_source_dir.py pre:pio-tools/set_partition_table.py pre:pio-tools/override_copy.py + pre:pio-tools/compress-html.py post:pio-tools/strip-flags.py [esp_defaults] diff --git a/tasmota/html_compressed/HTTP_GV_PAGE.h b/tasmota/html_compressed/HTTP_GV_PAGE.h index eb6da793d..4aaba697c 100644 --- a/tasmota/html_compressed/HTTP_GV_PAGE.h +++ b/tasmota/html_compressed/HTTP_GV_PAGE.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: f5150873f8737621b7fd8bab0e471d73c893429c79c045a1b20c26c5fee1d3fa ///////////////////////////////////////////////////////////////////// const size_t HTTP_GV_PAGE_SIZE = 463; diff --git a/tasmota/html_compressed/HTTP_HEADER1_ES6.h b/tasmota/html_compressed/HTTP_HEADER1_ES6.h index b92414770..2530349c4 100644 --- a/tasmota/html_compressed/HTTP_HEADER1_ES6.h +++ b/tasmota/html_compressed/HTTP_HEADER1_ES6.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: 3417a3f0d32f6537d82e4638a9cd462098b9da4f641160355e0c8d9334aed1d4 ///////////////////////////////////////////////////////////////////// const size_t HTTP_HEADER1_SIZE = 683; diff --git a/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT.h b/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT.h index 41ed93443..0597b9e4d 100644 --- a/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT.h +++ b/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: e0fcb684eb9f29a5fd0f678262046d9d05d055466b1752a58a35da2648ecd0e7 ///////////////////////////////////////////////////////////////////// const size_t HTTP_HEAD_LAST_SCRIPT_SIZE = 355; diff --git a/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT32.h b/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT32.h index 3c1303950..a23c18ca4 100644 --- a/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT32.h +++ b/tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT32.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: 996873eafc91c9d084e3f68b60d32a37a27e91dd1acd0863fc1096119d8de6a2 ///////////////////////////////////////////////////////////////////// const size_t HTTP_HEAD_LAST_SCRIPT32_SIZE = 1013; diff --git a/tasmota/html_compressed/HTTP_HEAD_STYLE1.h b/tasmota/html_compressed/HTTP_HEAD_STYLE1.h index 52440cd82..fe20e27db 100644 --- a/tasmota/html_compressed/HTTP_HEAD_STYLE1.h +++ b/tasmota/html_compressed/HTTP_HEAD_STYLE1.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: 8c22c19284fa41f8eb66b1f50cb94cc3fe14369f900031e791107fe56d583c2f ///////////////////////////////////////////////////////////////////// const size_t HTTP_HEAD_STYLE1_SIZE = 591; diff --git a/tasmota/html_compressed/HTTP_HEAD_STYLE2.h b/tasmota/html_compressed/HTTP_HEAD_STYLE2.h index 0103a484f..37e52fedb 100644 --- a/tasmota/html_compressed/HTTP_HEAD_STYLE2.h +++ b/tasmota/html_compressed/HTTP_HEAD_STYLE2.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: cff4350b756f01fb7866cbbffa2d169d4fe9eaca6ba45634f368ca1d714cd582 ///////////////////////////////////////////////////////////////////// const size_t HTTP_HEAD_STYLE2_SIZE = 496; diff --git a/tasmota/html_compressed/HTTP_HEAD_STYLE_ZIGBEE.h b/tasmota/html_compressed/HTTP_HEAD_STYLE_ZIGBEE.h index 29269d0d0..f98dade27 100644 --- a/tasmota/html_compressed/HTTP_HEAD_STYLE_ZIGBEE.h +++ b/tasmota/html_compressed/HTTP_HEAD_STYLE_ZIGBEE.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: ee7e68972559c2ad3df6a6594445bfdfeb14a0a37dec2389ec20a197c26c9311 ///////////////////////////////////////////////////////////////////// const size_t HTTP_HEAD_STYLE_ZIGBEE_SIZE = 363; diff --git a/tasmota/html_compressed/HTTP_SCRIPT_CONSOL.h b/tasmota/html_compressed/HTTP_SCRIPT_CONSOL.h index 792f58f84..071388f55 100644 --- a/tasmota/html_compressed/HTTP_SCRIPT_CONSOL.h +++ b/tasmota/html_compressed/HTTP_SCRIPT_CONSOL.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: 960fbc2354bc4029cbc93953e54edc74024a46ca58902af175f5ecf8839eb0a8 ///////////////////////////////////////////////////////////////////// const size_t HTTP_SCRIPT_CONSOL_SIZE = 985; diff --git a/tasmota/html_compressed/HTTP_SCRIPT_MODULE_TEMPLATE.h b/tasmota/html_compressed/HTTP_SCRIPT_MODULE_TEMPLATE.h index 99c40b1ac..afe052fde 100644 --- a/tasmota/html_compressed/HTTP_SCRIPT_MODULE_TEMPLATE.h +++ b/tasmota/html_compressed/HTTP_SCRIPT_MODULE_TEMPLATE.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: 92e33b521e56657da8f50887c7e1431219de0a8f19048247211f3b6d4c9b68ba ///////////////////////////////////////////////////////////////////// const size_t HTTP_SCRIPT_MODULE_TEMPLATE_SIZE = 589; diff --git a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_NO_WEB_DISPLAY.h b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_NO_WEB_DISPLAY.h index abf07e30d..56005b282 100644 --- a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_NO_WEB_DISPLAY.h +++ b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_NO_WEB_DISPLAY.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: 6e369f0e06cba0656d3c7187ac833b999a54f84edfd9385b4cf44ba8643e01d8 ///////////////////////////////////////////////////////////////////// const size_t HTTP_SCRIPT_ROOT_SIZE = 499; diff --git a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_PART2.h b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_PART2.h index 17af45427..2fcb4c7d5 100644 --- a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_PART2.h +++ b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_PART2.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: c85b76468eb793a235944c16a2d986bce127bd4fc1b0690499b3b5b88ab4f70f ///////////////////////////////////////////////////////////////////// const size_t HTTP_SCRIPT_ROOT_PART2_SIZE = 222; diff --git a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_SSE_NO_WEB_DISPLAY.h b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_SSE_NO_WEB_DISPLAY.h index 0cb9e3616..2cd66bfcc 100644 --- a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_SSE_NO_WEB_DISPLAY.h +++ b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_SSE_NO_WEB_DISPLAY.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: b42b87eb23e656d5ae799709721147c847ae381f0d1f0cb9f86bd55e9509bf51 ///////////////////////////////////////////////////////////////////// const size_t HTTP_SCRIPT_ROOT_SIZE = 434; diff --git a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_WEB_DISPLAY.h b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_WEB_DISPLAY.h index dd84f501d..eb0471941 100644 --- a/tasmota/html_compressed/HTTP_SCRIPT_ROOT_WEB_DISPLAY.h +++ b/tasmota/html_compressed/HTTP_SCRIPT_ROOT_WEB_DISPLAY.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: c137f990d750da7e1e51a6ec80baad5445853da492c12c4dad87c95724c50441 ///////////////////////////////////////////////////////////////////// const size_t HTTP_SCRIPT_ROOT_SIZE = 872; diff --git a/tasmota/html_compressed/HTTP_SCRIPT_TEMPLATE.h b/tasmota/html_compressed/HTTP_SCRIPT_TEMPLATE.h index 35da2d8bd..8bf7b1f64 100644 --- a/tasmota/html_compressed/HTTP_SCRIPT_TEMPLATE.h +++ b/tasmota/html_compressed/HTTP_SCRIPT_TEMPLATE.h @@ -1,5 +1,6 @@ ///////////////////////////////////////////////////////////////////// // compressed by tools/unishox/compress-html-uncompressed.py +// input sha256: 464453a8f35b349965adc050d3e4f968239a974f171cfa64efc665bafe3ba3f4 ///////////////////////////////////////////////////////////////////// const size_t HTTP_SCRIPT_TEMPLATE_SIZE = 288; diff --git a/tools/unishox/compress-html-uncompressed.py b/tools/unishox/compress-html-uncompressed.py index 5a58528fe..c950a5b22 100644 --- a/tools/unishox/compress-html-uncompressed.py +++ b/tools/unishox/compress-html-uncompressed.py @@ -20,9 +20,13 @@ ############################################################### import unishox -from os import listdir -from os import path +from sys import argv from datetime import datetime +from pathlib import Path +from hashlib import sha256 + +self_dir = Path(__file__).absolute().parent +base_dir = self_dir.parent.parent def extract_c_string(s: str) -> str: state = 0 @@ -48,18 +52,24 @@ def extract_c_string(s: str) -> str: out += c return out -path_compressed = path.join('..','..','tasmota','html_compressed') -path_uncompressed = path.join('..','..','tasmota','html_uncompressed') +def compress_html(source, target, argv=None, verbose=False): + if argv is None: argv = [] -files = listdir(path_uncompressed) + with open(source, "r") as f: + text = f.read() -totalIn = 0 -totalSaved = 0 + src_sha, old_sha = sha256(text.encode()).hexdigest(), None -for file in files: - f = open(path_uncompressed + path.sep + file, "r") - text = f.read() - f.close() + if not ('--force' in argv): + with open(target, "r") as f: + for line in f: + prefix = line[:17] + if prefix == '// input sha256: ': + old_sha = line[17:17+64] + break + + if src_sha == old_sha: + return (0, 0) #text = Tk().clipboard_get() # print(text) @@ -109,8 +119,9 @@ for file in files: # print(text[lastel+1:pos:]) lastel = pos - print("####### Parsing input from " + path_uncompressed + path.sep + file) - print(" Const char name: "+const_name) + if verbose: + print("####### Parsing input from " + str(source.relative_to(base_dir))) + print(" Const char name: "+const_name) #print('####### Cleaned input:') #print(input) @@ -128,20 +139,27 @@ for file in files: UNISHOX = unishox.Unishox() out_len = UNISHOX.compress(in_bytes, len(in_bytes), out_bytes, len(out_bytes)) - print(" ####### Compression result:") - print(" Compressed from {i} to {o}, -{p:.1f}%".format(i=in_len, o=out_len, p=(100-(float(out_len)/float(in_len)*100)))) + if verbose: + print(" ####### Compression result:") + reduction = 100-(float(out_len)/float(in_len)*100) + print(f" Compressed from {in_len} to {out_len}, -{reduction:.1f}%") out_bytes = out_bytes[:out_len] # truncate to right size #PROGMEM is growing in steps 0,8,24,40,56,... bytes of data resulting in size of 0,16,32,48,64,... bytes for in_real in range(8,in_len+16,16): if in_real>=in_len: - print(" Old real PROGMEM-size:"+str(in_real+8)+"(unused bytes:"+str(in_real-in_len)+")") + if verbose: + print(f" Old real PROGMEM-size:{in_real+8}(unused bytes:{in_real-in_len})") break for out_real in range(8,out_len+16,16): if out_real>=out_len: - print(" New real PROGMEM-size:"+str(out_real+8)+"(unused bytes:"+str(out_real-out_len)+")") + if verbose: + print(f" New real PROGMEM-size:{out_real+8}(unused bytes:{out_real-out_len})") break - print(" the optimal case would be raw bytes + 8, real difference: "+str(in_real - out_real)+ "bytes") + + if verbose: + print(f" the optimal case would be raw bytes + 8, real difference: {in_real - out_real}bytes") + # https://www.geeksforgeeks.org/break-list-chunks-size-n-python/ def chunked(my_list, n): return [my_list[i * n:(i + 1) * n] for i in range((len(my_list) + n - 1) // n )] @@ -150,27 +168,49 @@ for file in files: chunks = chunked(out_bytes, 20) lines_raw = [ "\"\\x" + "\\x".join( [ '{:02X}'.format(b) for b in chunk ] ) + "\"" for chunk in chunks ] - line_complete = "const char " + const_name + "_COMPRESSED" +"[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";" - lines = "\nconst size_t " + const_name +"_SIZE = {size};\n{lines}\n\n".format(size=in_len, lines=line_complete) + line_complete = f"const char {const_name}_COMPRESSED[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";" + lines = f"\nconst size_t {const_name}_SIZE = {in_len};\n{line_complete}\n\n" #print('####### Final output:') #print(lines) - definition = "#define " + const_name + " Decompress(" + const_name + "_COMPRESSED" + "," + const_name +"_SIZE" + ").c_str()" + definition = f"#define {const_name} Decompress({const_name}_COMPRESSED,{const_name}_SIZE).c_str()" #print(definition) now = datetime.now() # current date and time percent = int((float(out_real)/float(in_real))*100.0) saving = in_real - out_real - totalIn = totalIn + in_real - totalSaved = totalSaved + saving - comment = "/////////////////////////////////////////////////////////////////////\n" - comment = comment + "// compressed by tools/unishox/compress-html-uncompressed.py\n" - comment = comment + "/////////////////////////////////////////////////////////////////////\n" + #totalIn = totalIn + in_real + #totalSaved = totalSaved + saving + comment = "/////////////////////////////////////////////////////////////////////\n" + comment += "// compressed by tools/unishox/compress-html-uncompressed.py\n" + comment += f"// input sha256: {src_sha}\n" + comment += "/////////////////////////////////////////////////////////////////////\n" - f = open(path_compressed + path.sep + file, "w") - f.write(comment + lines + definition) - f.close() - print("####### Wrote output to " + path_compressed + path.sep + file) + with open(target, "w") as f: + f.write(comment + lines + definition) -print("If all files are in use, total saving was "+str(totalSaved)+" out of "+str(totalIn)) + return (in_real, saving) + + if verbose: + print("####### Wrote output to " + str(target.relative_to(base_dir))) + +def compress_dir(source_dir, target_dir, argv=None, verbose=False): + totalIn, totalSaved = 0, 0 + + for source in source_dir.iterdir(): + target = Path(target_dir, source.stem + ".h") + bytesIn, bytesSaved = compress_html(source, target, argv, verbose) + totalIn += bytesIn + totalSaved += bytesSaved + + return (totalIn, totalSaved) + +if __name__ == '__main__': + path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed') + path_compressed = Path(base_dir, 'tasmota', 'html_compressed') + + totalIn, totalSaved = compress_dir(path_uncompressed, path_compressed, argv, True) + + if totalSaved > 0: + print(f"If all files are in use, total saving was {totalSaved} out of {totalIn}")