Berry support for unishox compression

lib/default/Unishox-1.0-shadinger/src/unishox.cpp

@@ -76,13 +76,6 @@ static char sets[][11] PROGMEM =
                    {'.', ',', '-', '/', '?', '+', ' ', '(', ')', '$', '@'},
                    {';', '#', ':', '<', '^', '*', '"', '{', '}', '[', ']'},
                    {'=', '%', '\'', '>', '&', '_', '!', '\\', '|', '~', '`'}};
-                  // {{  0, ' ', 'e',   0, 't', 'a', 'o', 'i', 'n', 's', 'r'},
-                  //  {  0, 'l', 'c', 'd', 'h', 'u', 'p', 'm', 'b', 'g', 'w'},
-                  //  {'f', 'y', 'v', 'k', 'q', 'j', 'x', 'z',   0,   0,   0},
-                  //  {  0, '9', '0', '1', '2', '3', '4', '5', '6', '7', '8'},
-                  //  {'.', ',', '-', '/', '=', '+', ' ', '(', ')', '$', '%'},
-                  //  {'&', ';', ':', '<', '>', '*', '"', '{', '}', '[', ']'},
-                  //  {'@', '?', '\'', '^', '#', '_', '!', '\\', '|', '~', '`'}};
 
 // Decoder is designed for using less memory, not speed
 // Decode lookup table for code index and length
@@ -117,28 +110,16 @@ static const uint16_t DICT_CODE = 0x0000;
 static const uint16_t DICT_CODE_LEN = 5;
 static const uint16_t DICT_OTHER_CODE = 0x0000; // not used
 static const uint16_t DICT_OTHER_CODE_LEN = 6;
-// const uint16_t RPT_CODE = 0x2370;
-// const uint16_t RPT_CODE_LEN = 13;
 static const uint16_t RPT_CODE_TASMOTA = 0x3780;
 static const uint16_t RPT_CODE_TASMOTA_LEN = 10;
 static const uint16_t BACK2_STATE1_CODE = 0x2000;    // 0010 = back to lower case
 static const uint16_t BACK2_STATE1_CODE_LEN = 4;
 static const uint16_t BACK_FROM_UNI_CODE = 0xFE00;
 static const uint16_t BACK_FROM_UNI_CODE_LEN = 8;
-// const uint16_t CRLF_CODE = 0x3780;
-// const uint16_t CRLF_CODE_LEN = 10;
 static const uint16_t LF_CODE = 0x3700;
 static const uint16_t LF_CODE_LEN = 9;
 static const uint16_t TAB_CODE = 0x2400;
 static const uint16_t TAB_CODE_LEN = 7;
-// const uint16_t UNI_CODE = 0x8000;        // Unicode disabled
-// const uint16_t UNI_CODE_LEN = 3;
-// const uint16_t UNI_STATE_SPL_CODE = 0xF800;
-// const uint16_t UNI_STATE_SPL_CODE_LEN = 5;
-// const uint16_t UNI_STATE_DICT_CODE = 0xFC00;
-// const uint16_t UNI_STATE_DICT_CODE_LEN = 7;
-// const uint16_t CONT_UNI_CODE = 0x2800;
-// const uint16_t CONT_UNI_CODE_LEN = 7;
 static const uint16_t ALL_UPPER_CODE = 0x2200;
 static const uint16_t ALL_UPPER_CODE_LEN = 8;
 static const uint16_t SW2_STATE2_CODE = 0x3800;
@@ -147,8 +128,6 @@ static const uint16_t ST2_SPC_CODE = 0x3B80;
 static const uint16_t ST2_SPC_CODE_LEN = 11;
 static const uint16_t BIN_CODE_TASMOTA = 0x8000;
 static const uint16_t BIN_CODE_TASMOTA_LEN = 3;
-// const uint16_t BIN_CODE = 0x2000;
-// const uint16_t BIN_CODE_LEN = 9;
 
 #define NICE_LEN 5
 
@@ -467,9 +446,9 @@ void Unishox::decodeRepeat(void) {
   uint32_t dict_len = readCount() + NICE_LEN;
   uint32_t dist = readCount() + NICE_LEN - 1;
   if (ol + dict_len <= len_out) {
-  memcpy(out + ol, out + ol - dist, dict_len);
+    memcpy(out + ol, out + ol - dist, dict_len);
-  ol += dict_len;
+    ol += dict_len;
-}
+  }
 }
 
 int32_t Unishox::unishox_decompress(const char *p_in, size_t p_len, char *p_out, size_t p_len_out) {
@@ -485,10 +464,10 @@ int32_t Unishox::unishox_decompress(const char *p_in, size_t p_len, char *p_out,
   dstate = SHX_SET1;
   is_all_upper = 0;
 
-  out[ol] = 0;
+  if (out) out[ol] = 0;
   // while ((byte_no << 3) + bit_no - 8 < len) {
   while (!in_eof) {
-    if (ol >= len_out) {
+    if (out && ol >= len_out) {
       break;
     }
     int32_t h, v;
@@ -535,7 +514,8 @@ int32_t Unishox::unishox_decompress(const char *p_in, size_t p_len, char *p_out,
 
     if (v == 0 && h == SHX_SET1A) {
       if (is_upper) {
-        out[ol++] = 255 - readCount();    // binary
+        if (out) out[ol] = 255 - readCount();    // binary
+        ol++;
       } else {
         decodeRepeat();   // dist
       }
@@ -544,7 +524,8 @@ int32_t Unishox::unishox_decompress(const char *p_in, size_t p_len, char *p_out,
 
     if (h == SHX_SET1 && v == 3) {
       // was Unicode, will do Binary instead
-      out[ol++] = 255 - readCount();    // binary
+      if (out) out[ol] = 255 - readCount();    // binary
+      ol++;
       continue;
     }
     if (h < 7 && v < 11)     // TODO: are these the actual limits? Not 11x7 ?
@@ -557,17 +538,22 @@ int32_t Unishox::unishox_decompress(const char *p_in, size_t p_len, char *p_out,
         c = '\t';     // If UpperCase Space, change to TAB
       if (h == SHX_SET1B) {
         if (8 == v) {   // was LF or RPT, now only LF
-          out[ol++] = '\n';
+          if (out) out[ol] = '\n';
+          ol++;
           continue;
         }
         if (9 == v) {           // was CRLF, now RPT
           uint32_t count = readCount() + 4;
-          if (ol + count >= len_out) {
+          if (out && ol + count >= len_out) {
             return -1;        // overflow
           }
-          char rpt_c = out[ol - 1];
+          if (out) {
-          while (count--)
+            char rpt_c = out[ol - 1];
-            out[ol++] = rpt_c;
+            while (count--)
+              out[ol++] = rpt_c;
+          } else {
+            ol += count;
+          }
           continue;
         }
         if (10 == v) {
@@ -576,10 +562,11 @@ int32_t Unishox::unishox_decompress(const char *p_in, size_t p_len, char *p_out,
       }
     }
     // Serial.printf(">>>>>>>>>>>>>>>>>>>>>> Out = %c\n", c);
-    out[ol++] = c;
+    if (out) out[ol] = c;
+    ol++;
   }
 
-  if (ol > len_out) {
+  if (out && ol > len_out) {
     return -1;    // overflow
   } else {
     return ol;

lib/libesp32/Berry/default/be_modtab.c

@@ -32,6 +32,7 @@ be_extern_native_module(energy);
 be_extern_native_module(webserver);
 be_extern_native_module(flash);
 be_extern_native_module(path);
+be_extern_native_module(unishox);
 #ifdef USE_LVGL
 be_extern_native_module(lv);
 #endif // USE_LVGL
@@ -89,6 +90,10 @@ BERRY_LOCAL const bntvmodule* const be_module_table[] = {
     &be_native_module(light),
 #endif
 
+#ifdef USE_UNISHOX_COMPRESSION
+    &be_native_module(unishox),
+#endif // USE_UNISHOX_COMPRESSION
+
 #ifdef USE_LVGL
     &be_native_module(lv),
 #endif // USE_LVGL

lib/libesp32/Berry/default/be_unishox_lib.c

@@ -0,0 +1,28 @@
+/********************************************************************
+ * Berry module `unishox`
+ * 
+ * To use: `import unishox`
+ * 
+ * Allows to respond to HTTP request
+ *******************************************************************/
+#include "be_constobj.h"
+
+#ifdef USE_UNISHOX_COMPRESSION
+
+extern int be_unishox_compress(bvm *vm);
+extern int be_unishox_decompress(bvm *vm);
+
+/********************************************************************
+** Solidified module: unishox
+********************************************************************/
+be_local_module(unishox,
+    "unishox",
+    be_nested_map(2,
+    ( (struct bmapnode*) &(const bmapnode[]) {
+        { be_nested_key("decompress", -1407935646, 10, -1), be_const_func(be_unishox_decompress) },
+        { be_nested_key("compress", -1476883059, 8, -1), be_const_func(be_unishox_compress) },
+    }))
+);
+BE_EXPORT_VARIABLE be_define_const_native_module(unishox);
+
+#endif // USE_UNISHOX_COMPRESSION

tasmota/xdrv_52_3_berry_unishox.ino

@@ -0,0 +1,94 @@
+/*
+  xdrv_52_3_berry_unishox.ino - Berry scripting language, native fucnctions
+
+  Copyright (C) 2021 Stephan Hadinger, Berry language by Guan Wenliang https://github.com/Skiars/berry
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#ifdef USE_BERRY
+
+#include <berry.h>
+
+#ifdef USE_UNISHOX_COMPRESSION
+
+extern Unishox compressor;
+
+/*********************************************************************************************\
+ * Native functions mapped to Berry functions
+ * 
+ * import unishox
+ * 
+ * 
+\*********************************************************************************************/
+extern "C" {
+
+  int be_unishox_compress(bvm *vm);
+  int be_unishox_compress(bvm *vm) {
+    int32_t argc = be_top(vm); // Get the number of arguments
+    if (argc == 1 && be_isstring(vm, 1)) {
+      const char * s = be_tostring(vm, 1);
+      // do a dry-run to know the compressed size
+      int32_t compressed_size = compressor.unishox_compress(s, strlen(s), (char*) nullptr, 0);
+      if (compressed_size < 0) {
+        be_raise(vm, "internal_error", nullptr);
+      }
+      void * buf = be_pushbytes(vm, NULL, compressed_size);
+      if (compressed_size > 0) {
+        int32_t ret = compressor.unishox_compress(s, strlen(s), (char*) buf, compressed_size+5);  // We expand by 4 the buffer size to avoid an error, but we are sure it will not overflow (see unishox implementation)
+        if (ret < 0 || ret != compressed_size) {
+          be_raisef(vm, "internal_error", "unishox size=%i ret=%i", compressed_size, ret);
+        }
+      }
+      be_return(vm);
+    }
+    be_raise(vm, kTypeError, nullptr);
+  }
+
+  int be_unishox_decompress(bvm *vm);
+  int be_unishox_decompress(bvm *vm) {
+    int32_t argc = be_top(vm); // Get the number of arguments
+    if (argc == 1 && be_isbytes(vm, 1)) {
+      size_t len;
+      const void * buf = be_tobytes(vm, 1, &len);
+      if (len == 0) {
+        be_pushstring(vm, "");
+      } else {
+        int32_t decomp_size = compressor.unishox_decompress((const char*)buf, len, (char*) nullptr, 0);
+        if (decomp_size < 0) {
+          be_raise(vm, "internal_error", nullptr);
+        }
+        if (decomp_size == 0) {
+          be_pushstring(vm, "");
+        } else {
+          void * buf_out = be_pushbuffer(vm, decomp_size);
+          int32_t ret = compressor.unishox_decompress((const char*)buf, len, (char*) buf_out, decomp_size);
+          if (ret < 0 || ret != decomp_size) {
+            be_raisef(vm, "internal_error", "unishox size=%i ret=%i", decomp_size, ret);
+          }
+          be_pushnstring(vm, (const char*) buf_out, decomp_size);
+        }
+      }
+      be_return(vm);
+    }
+    be_raise(vm, kTypeError, nullptr);
+  }
+
+}
+
+
+#endif // USE_UNISHOX_COMPRESSION
+
+#endif  // USE_BERRY