Berry add solidification of strings longer than 255 bytes (#20529)

CHANGELOG.md

@@ -14,6 +14,7 @@ All notable changes to this project will be documented in this file.
 - LVGL option to add `lv.keyboard` extra widget (#20496)
 - GUI sensor separators (#20495)
 - Command ``TimedPower<index> <milliseconds>[,ON|OFF|TOGGLE|BLINK]`` executes ``Power<index> [ON|OFF|TOGGLE|BLINK] `` and after <millisecond> executes ``Power<index> [OFF|ON|TOGGLE|OFF]``
+- Berry add solidification of strings longer than 255 bytes
 
 ### Breaking Changed
 

lib/libesp32/berry/src/be_constobj.h

@@ -262,6 +262,13 @@ const bntvmodule_t be_native_module(_module) = {                  \
     BE_STRING                                                   \
   }
 
+/* variant for long strings that does not trigger strtab */
+#define be_nested_str_long(_name_)                              \
+  {                                                             \
+    { .s=((bstring*)&be_const_str_##_name_) },                  \
+    BE_STRING                                                   \
+  }
+
 #define be_nested_str_literal(_name_)                           \
   {                                                             \
     { .s=(be_nested_const_str(_name_, _hash, sizeof(_name_)-1 ))\

lib/libesp32/berry/src/be_solidifylib.c

@@ -16,6 +16,7 @@
 #include "be_vm.h"
 #include "be_decoder.h"
 #include "be_sys.h"
+#include "be_mem.h"
 #include <string.h>
 #include <stdio.h>
 #include <ctype.h>
@@ -45,6 +46,16 @@ extern const bclass be_class_map;
         }                                               \
     } while (0)
 
+/* minimal version without formatting and without size limit */
+#define lognofmt(__s)                                   \
+    do {                                                \
+        if (fout) {                                     \
+            be_fwrite(fout, __s, strlen(__s));          \
+        } else {                                        \
+            be_writestring(__s);                        \
+        }                                               \
+    } while (0)
+
 /********************************************************************\
  * Encode string to identifiers
  * 
@@ -200,17 +211,29 @@ static void m_solidify_bvalue(bvm *vm, bbool str_literal, bvalue * value, const
         {
             const char * str = str(var_tostr(value));
             size_t len = strlen(str);
-            if (len >= 255) {
-                be_raise(vm, "internal_error", "Strings greater than 255 chars not supported yet");
-            }
             size_t id_len = toidentifier_length(str);
-            char id_buf[id_len];
+            char id_buf_stack[64];
+            char *id_buf = id_buf_stack;
+            if (id_len >= 64) {
+                id_buf = be_os_malloc(id_len);
+                if (!id_buf) {
+                    be_raise(vm, "memory_error", "could not allocated buffer");
+                }
+            }
             toidentifier(id_buf, str);
-            if (!str_literal) {
+            if (len >= 255) {
+                /* decompose to avoid any size limit */
+                lognofmt("be_nested_str_long(");
+                lognofmt(id_buf);
+                lognofmt(")");
+            } else if (!str_literal) {
                 logfmt("be_nested_str(%s)", id_buf);
             } else {
                 logfmt("be_nested_str_weak(%s)", id_buf);
             }
+            if (id_buf != id_buf_stack) {
+                be_os_free(id_buf);
+            }
         }
         break;
     case BE_CLOSURE:

lib/libesp32/berry/src/be_string.c

@@ -27,6 +27,17 @@
         .s = _s                                                    \
     }
 
+#define be_define_const_str_long(_name, _s, _len)                  \
+    BERRY_LOCAL const bclstring be_const_str_##_name = {           \
+        .next = (bgcobject *)NULL,                                 \
+        .type = BE_STRING,                                         \
+        .marked = GC_CONST,                                        \
+        .extra = 0,                                                \
+        .slen = 255,                                               \
+        .llen = _len,                                              \
+        .s = _s                                                    \
+    }
+
 /* const string table */
 struct bconststrtab {
     const bstring* const *table;
@@ -279,7 +290,7 @@ void be_gcstrtab(bvm *vm)
 
 uint32_t be_strhash(const bstring *s)
 {
-    if (gc_isconst(s)) {
+    if (gc_isconst(s) && (s->slen != 255)) {
         bcstring* cs = cast(bcstring*, s);
         if (cs->hash) {  /* if hash is null we need to compute it */
             return cs->hash;
@@ -298,12 +309,12 @@ uint32_t be_strhash(const bstring *s)
 const char* be_str2cstr(const bstring *s)
 {
     be_assert(cast_str(s) != NULL);
-    if (gc_isconst(s)) {
-        return cstr(s);
-    }
     if (s->slen == 255) {
         return lstr(s);
     }
+    if (gc_isconst(s)) {
+        return cstr(s);
+    }
     return sstr(s);
 }
 

lib/libesp32/berry/src/be_string.h

@@ -26,6 +26,12 @@ typedef struct {
     /* char s[]; */
 } blstring;
 
+typedef struct {        /* const long string */
+    bstring_header;
+    int llen;
+    char s[];
+} bclstring;
+
 typedef struct {
     bstring_header;
     uint32_t hash;

lib/libesp32/berry/tools/coc/block_builder.py

@@ -23,6 +23,7 @@ class block_builder:
         self.block = block()
         self.strtab = []
         self.strtab_weak = []
+        self.strtab_long = []
 
         self.block.name = obj.name
         if depend(obj, macro):

lib/libesp32/berry/tools/coc/coc

@@ -19,6 +19,7 @@ class builder:
         self.macro = None
         self.strmap = {}
         self.strmap_weak = {}
+        self.strmap_long = {}
 
         self.macro = macro_table()
         for path in self.config:
@@ -27,7 +28,7 @@ class builder:
         for d in self.input:
             self.scandir(d)
         
-        sb = str_build(self.strmap, self.strmap_weak)
+        sb = str_build(self.strmap, self.strmap_weak, self.strmap_long)
         sb.build(self.output)
     
     def parse_file(self, filename):
@@ -42,12 +43,16 @@ class builder:
                 self.strmap[s] = 0
             for s in parser.strtab_weak:
                 self.strmap_weak[s] = 0
+            for s in parser.strtab_long:
+                self.strmap_long[s] = 0
             for obj in parser.objects:
                 builder = block_builder(obj, self.macro)
                 for s in builder.strtab:
                     self.strmap[s] = 0
                 for s in builder.strtab_weak:
                     self.strmap_weak[s] = 0
+                for s in builder.strtab_long:
+                    self.strmap_long[s] = 0
                 builder.dumpfile(self.output)
 
     def scandir(self, srcpath):

lib/libesp32/berry/tools/coc/coc_parser.py

@@ -22,6 +22,7 @@ class coc_parser:
         self.objects = []
         self.strtab = set()
         self.strtab_weak = set()
+        self.strtab_long = set()
         self.text = text
         self.parsers = {
             "@const_object_info_begin": self.parse_object,
@@ -30,6 +31,7 @@ class coc_parser:
             "be_nested_str(": self.parse_string,
             "be_const_key_weak(": self.parse_string_weak,
             "be_nested_str_weak(": self.parse_string_weak,
+            "be_nested_str_long(": self.parse_string_long,
             "be_str_weak(": self.parse_string_weak,
         }
 
@@ -131,6 +133,13 @@ class coc_parser:
             self.strtab_weak.add(literal)
             # print(f"str '{ident}' -> {literal}")
 
+    def parse_string_long(self):
+        if not self.text[0].isalnum() and self.text[0] != '_': return      # do not proceed, maybe false positive in solidify
+        ident = self.parse_word()
+        literal = unescape_operator(ident)
+        if not literal in self.strtab:
+            self.strtab_long.add(literal)
+
     #################################################################################
     # Parse a block of definition like module, class...
     #################################################################################

lib/libesp32/berry/tools/coc/str_build.py

@@ -22,9 +22,10 @@ class str_info:
         self.extra = 0
 
 class str_build:
-    def __init__(self, map, map_weak):
+    def __init__(self, map, map_weak, map_long):
         self.map = map.copy()
         self.str_weak = []
+        self.str_long = []
 
         size = int(len(self.map) / 2)         # voluntarily reduce hash size to half
         if size < 4: size = 4
@@ -44,6 +45,11 @@ class str_build:
             if not k in self.map:
                 self.str_weak.append(k)
 
+        # handle long strings
+        for k in sorted(map_long.keys()):
+            if not k in self.map:
+                self.str_long.append(k)
+
     def build(self, path):
         prefix = path + "/be_const_strtab"
         self.writefile(prefix + "_def.h", self.build_table_def())
@@ -121,6 +127,10 @@ class str_build:
             ostr += escape_operator(k) + ", " + escape_c(k) + ", "
             ostr += "0u, 0, " + str(len(k)) + ", NULL);\n"
 
+        for k in self.str_long:
+            ostr += "be_define_const_str_long("
+            ostr += escape_operator(k) + ", " + escape_c(k) + ", " + str(len(k)) + ");\n"
+
         ostr += "\n"
         ostr += "static const bstring* const m_string_table[] = {\n"
 
@@ -156,4 +166,6 @@ class str_build:
         ostr += "\n/* weak strings */\n"
         for s in self.str_weak:
             ostr += "extern const bcstring be_const_str_" + escape_operator(s) + ";\n"
+        for s in self.str_long:
+            ostr += "extern const bclstring be_const_str_" + escape_operator(s) + ";\n"
         return ostr