auto compress updated html (#22738)

pio-tools/compress-html.py

@@ -0,0 +1,15 @@
+Import("env")
+
+import sys
+from pathlib import Path
+
+base_dir = env['PROJECT_DIR']
+unishox_dir = Path(base_dir, 'tools', 'unishox')
+sys.path.append(str(unishox_dir.resolve()))
+sys.dont_write_bytecode = True
+
+compress_dir = __import__('compress-html-uncompressed').compress_dir
+
+path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed')
+path_compressed   = Path(base_dir, 'tasmota', 'html_compressed')
+compress_dir(path_uncompressed, path_compressed)

platformio.ini

@@ -73,6 +73,7 @@ build_flags                 =
 extra_scripts               = pre:pio-tools/pre_source_dir.py
                               pre:pio-tools/set_partition_table.py
                               pre:pio-tools/override_copy.py
+                              pre:pio-tools/compress-html.py
                               post:pio-tools/strip-flags.py
 
 [esp_defaults]

tasmota/html_compressed/HTTP_GV_PAGE.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: f5150873f8737621b7fd8bab0e471d73c893429c79c045a1b20c26c5fee1d3fa
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_GV_PAGE_SIZE = 463;

tasmota/html_compressed/HTTP_HEADER1_ES6.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: 3417a3f0d32f6537d82e4638a9cd462098b9da4f641160355e0c8d9334aed1d4
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_HEADER1_SIZE = 683;

tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: e0fcb684eb9f29a5fd0f678262046d9d05d055466b1752a58a35da2648ecd0e7
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_HEAD_LAST_SCRIPT_SIZE = 355;

tasmota/html_compressed/HTTP_HEAD_LAST_SCRIPT32.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: 996873eafc91c9d084e3f68b60d32a37a27e91dd1acd0863fc1096119d8de6a2
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_HEAD_LAST_SCRIPT32_SIZE = 1013;

tasmota/html_compressed/HTTP_HEAD_STYLE1.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: 8c22c19284fa41f8eb66b1f50cb94cc3fe14369f900031e791107fe56d583c2f
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_HEAD_STYLE1_SIZE = 591;

tasmota/html_compressed/HTTP_HEAD_STYLE2.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: cff4350b756f01fb7866cbbffa2d169d4fe9eaca6ba45634f368ca1d714cd582
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_HEAD_STYLE2_SIZE = 496;

tasmota/html_compressed/HTTP_HEAD_STYLE_ZIGBEE.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: ee7e68972559c2ad3df6a6594445bfdfeb14a0a37dec2389ec20a197c26c9311
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_HEAD_STYLE_ZIGBEE_SIZE = 363;

tasmota/html_compressed/HTTP_SCRIPT_CONSOL.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: 960fbc2354bc4029cbc93953e54edc74024a46ca58902af175f5ecf8839eb0a8
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_SCRIPT_CONSOL_SIZE = 985;

tasmota/html_compressed/HTTP_SCRIPT_MODULE_TEMPLATE.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: 92e33b521e56657da8f50887c7e1431219de0a8f19048247211f3b6d4c9b68ba
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_SCRIPT_MODULE_TEMPLATE_SIZE = 589;

tasmota/html_compressed/HTTP_SCRIPT_ROOT_NO_WEB_DISPLAY.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: 6e369f0e06cba0656d3c7187ac833b999a54f84edfd9385b4cf44ba8643e01d8
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_SCRIPT_ROOT_SIZE = 499;

tasmota/html_compressed/HTTP_SCRIPT_ROOT_PART2.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: c85b76468eb793a235944c16a2d986bce127bd4fc1b0690499b3b5b88ab4f70f
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_SCRIPT_ROOT_PART2_SIZE = 222;

tasmota/html_compressed/HTTP_SCRIPT_ROOT_SSE_NO_WEB_DISPLAY.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: b42b87eb23e656d5ae799709721147c847ae381f0d1f0cb9f86bd55e9509bf51
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_SCRIPT_ROOT_SIZE = 434;

tasmota/html_compressed/HTTP_SCRIPT_ROOT_WEB_DISPLAY.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: c137f990d750da7e1e51a6ec80baad5445853da492c12c4dad87c95724c50441
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_SCRIPT_ROOT_SIZE = 872;

tasmota/html_compressed/HTTP_SCRIPT_TEMPLATE.h

@@ -1,5 +1,6 @@
 /////////////////////////////////////////////////////////////////////
 // compressed by tools/unishox/compress-html-uncompressed.py
+// input sha256: 464453a8f35b349965adc050d3e4f968239a974f171cfa64efc665bafe3ba3f4
 /////////////////////////////////////////////////////////////////////
 
 const size_t HTTP_SCRIPT_TEMPLATE_SIZE = 288;

tools/unishox/compress-html-uncompressed.py

@@ -20,9 +20,13 @@
 ###############################################################
 
 import unishox
-from os import listdir
+from sys import argv
-from os import path
 from datetime import datetime
+from pathlib import Path
+from hashlib import sha256
+
+self_dir = Path(__file__).absolute().parent
+base_dir = self_dir.parent.parent
 
 def extract_c_string(s: str) -> str:
   state = 0
@@ -48,18 +52,24 @@ def extract_c_string(s: str) -> str:
         out += c
   return out
 
-path_compressed   = path.join('..','..','tasmota','html_compressed')
+def compress_html(source, target, argv=None, verbose=False):
-path_uncompressed = path.join('..','..','tasmota','html_uncompressed')
+  if argv is None: argv = []
 
-files = listdir(path_uncompressed)
+  with open(source, "r") as f:
-
-totalIn = 0
-totalSaved = 0
-
-for file in files:
-  f = open(path_uncompressed + path.sep + file, "r")
     text = f.read()
-  f.close()
+
+  src_sha, old_sha = sha256(text.encode()).hexdigest(), None
+
+  if not ('--force' in argv):
+    with open(target, "r") as f:
+      for line in f:
+        prefix = line[:17]
+        if prefix == '// input sha256: ':
+          old_sha = line[17:17+64]
+          break
+
+    if src_sha == old_sha:
+      return (0, 0)
 
   #text = Tk().clipboard_get()
   # print(text)
@@ -109,7 +119,8 @@ for file in files:
           # print(text[lastel+1:pos:])
       lastel = pos
 
-  print("####### Parsing input from " + path_uncompressed + path.sep + file)
+  if verbose:
+    print("####### Parsing input from " + str(source.relative_to(base_dir)))
     print("  Const char name: "+const_name)
   #print('####### Cleaned input:')
   #print(input)
@@ -128,20 +139,27 @@ for file in files:
 
   UNISHOX = unishox.Unishox()
   out_len = UNISHOX.compress(in_bytes, len(in_bytes), out_bytes, len(out_bytes))
+  if verbose:
     print("  ####### Compression result:")
-  print("  Compressed from {i} to {o}, -{p:.1f}%".format(i=in_len, o=out_len, p=(100-(float(out_len)/float(in_len)*100))))
+    reduction = 100-(float(out_len)/float(in_len)*100)
+    print(f"  Compressed from {in_len} to {out_len}, -{reduction:.1f}%")
   out_bytes = out_bytes[:out_len]     # truncate to right size
 
   #PROGMEM is growing in steps 0,8,24,40,56,... bytes of data resulting in size of 0,16,32,48,64,... bytes
   for in_real in range(8,in_len+16,16):
       if in_real>=in_len:
-        print("  Old real PROGMEM-size:"+str(in_real+8)+"(unused bytes:"+str(in_real-in_len)+")")
+        if verbose:
+          print(f"  Old real PROGMEM-size:{in_real+8}(unused bytes:{in_real-in_len})")
         break
   for out_real in range(8,out_len+16,16):
       if out_real>=out_len:
-        print("  New real PROGMEM-size:"+str(out_real+8)+"(unused bytes:"+str(out_real-out_len)+")")
+        if verbose:
+          print(f"  New real PROGMEM-size:{out_real+8}(unused bytes:{out_real-out_len})")
         break
-  print("  the optimal case would be raw bytes + 8, real difference: "+str(in_real - out_real)+ "bytes")
+
+  if verbose:
+    print(f"  the optimal case would be raw bytes + 8, real difference: {in_real - out_real}bytes")
+
   # https://www.geeksforgeeks.org/break-list-chunks-size-n-python/
   def chunked(my_list, n):
       return [my_list[i * n:(i + 1) * n] for i in range((len(my_list) + n - 1) // n )]
@@ -150,27 +168,49 @@ for file in files:
   chunks = chunked(out_bytes, 20)
 
   lines_raw = [ "\"\\x" + "\\x".join( [ '{:02X}'.format(b) for b in chunk ] ) + "\"" for chunk in chunks ]
-  line_complete = "const char " + const_name + "_COMPRESSED" +"[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";"
+  line_complete = f"const char {const_name}_COMPRESSED[] PROGMEM = " + ("\n" + " "*29).join(lines_raw) + ";"
-  lines = "\nconst size_t " + const_name +"_SIZE = {size};\n{lines}\n\n".format(size=in_len, lines=line_complete)
+  lines = f"\nconst size_t {const_name}_SIZE = {in_len};\n{line_complete}\n\n"
 
   #print('####### Final output:')
   #print(lines)
 
-  definition = "#define  " + const_name +  "       Decompress(" + const_name + "_COMPRESSED" + "," + const_name +"_SIZE" + ").c_str()"
+  definition = f"#define  {const_name}       Decompress({const_name}_COMPRESSED,{const_name}_SIZE).c_str()"
   #print(definition)
 
   now = datetime.now() # current date and time
   percent = int((float(out_real)/float(in_real))*100.0)
   saving = in_real - out_real
-  totalIn = totalIn + in_real
+  #totalIn = totalIn + in_real
-  totalSaved = totalSaved + saving
+  #totalSaved = totalSaved + saving
   comment  = "/////////////////////////////////////////////////////////////////////\n"
-  comment = comment + "// compressed by tools/unishox/compress-html-uncompressed.py\n"
+  comment += "// compressed by tools/unishox/compress-html-uncompressed.py\n"
-  comment = comment + "/////////////////////////////////////////////////////////////////////\n"
+  comment += f"// input sha256: {src_sha}\n"
+  comment += "/////////////////////////////////////////////////////////////////////\n"
 
-  f = open(path_compressed + path.sep + file, "w")
+  with open(target, "w") as f:
       f.write(comment + lines + definition)
-  f.close()
-  print("####### Wrote output to " + path_compressed + path.sep + file)
 
-print("If all files are in use, total saving was "+str(totalSaved)+" out of "+str(totalIn))
+  return (in_real, saving)
+
+  if verbose:
+    print("####### Wrote output to " + str(target.relative_to(base_dir)))
+
+def compress_dir(source_dir, target_dir, argv=None, verbose=False):
+  totalIn, totalSaved = 0, 0
+
+  for source in source_dir.iterdir():
+    target = Path(target_dir, source.stem + ".h")
+    bytesIn, bytesSaved = compress_html(source, target, argv, verbose)
+    totalIn += bytesIn
+    totalSaved += bytesSaved
+
+  return (totalIn, totalSaved)
+
+if __name__ == '__main__':
+  path_uncompressed = Path(base_dir, 'tasmota', 'html_uncompressed')
+  path_compressed   = Path(base_dir, 'tasmota', 'html_compressed')
+
+  totalIn, totalSaved = compress_dir(path_uncompressed, path_compressed, argv, True)
+
+  if totalSaved > 0:
+    print(f"If all files are in use, total saving was {totalSaved} out of {totalIn}")