diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b76fb816..ff7287194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ All notable changes to this project will be documented in this file. - LVGL option to add `lv.keyboard` extra widget (#20496) - GUI sensor separators (#20495) - Command ``TimedPower [,ON|OFF|TOGGLE|BLINK]`` executes ``Power [ON|OFF|TOGGLE|BLINK] `` and after executes ``Power [OFF|ON|TOGGLE|OFF]`` +- Berry add solidification of strings longer than 255 bytes ### Breaking Changed diff --git a/lib/libesp32/berry/src/be_constobj.h b/lib/libesp32/berry/src/be_constobj.h index 485bf85e6..34245050a 100644 --- a/lib/libesp32/berry/src/be_constobj.h +++ b/lib/libesp32/berry/src/be_constobj.h @@ -262,6 +262,13 @@ const bntvmodule_t be_native_module(_module) = { \ BE_STRING \ } +/* variant for long strings that does not trigger strtab */ +#define be_nested_str_long(_name_) \ + { \ + { .s=((bstring*)&be_const_str_##_name_) }, \ + BE_STRING \ + } + #define be_nested_str_literal(_name_) \ { \ { .s=(be_nested_const_str(_name_, _hash, sizeof(_name_)-1 ))\ diff --git a/lib/libesp32/berry/src/be_solidifylib.c b/lib/libesp32/berry/src/be_solidifylib.c index b0164ddba..0cafe2a73 100644 --- a/lib/libesp32/berry/src/be_solidifylib.c +++ b/lib/libesp32/berry/src/be_solidifylib.c @@ -16,6 +16,7 @@ #include "be_vm.h" #include "be_decoder.h" #include "be_sys.h" +#include "be_mem.h" #include #include #include @@ -45,6 +46,16 @@ extern const bclass be_class_map; } \ } while (0) +/* minimal version without formatting and without size limit */ +#define lognofmt(__s) \ + do { \ + if (fout) { \ + be_fwrite(fout, __s, strlen(__s)); \ + } else { \ + be_writestring(__s); \ + } \ + } while (0) + /********************************************************************\ * Encode string to identifiers * @@ -200,17 +211,29 @@ static void m_solidify_bvalue(bvm *vm, bbool str_literal, bvalue * value, const { const char * str = str(var_tostr(value)); size_t len = strlen(str); - if (len >= 255) { - be_raise(vm, "internal_error", "Strings greater than 255 chars not supported yet"); - } size_t id_len = toidentifier_length(str); - char id_buf[id_len]; + char id_buf_stack[64]; + char *id_buf = id_buf_stack; + if (id_len >= 64) { + id_buf = be_os_malloc(id_len); + if (!id_buf) { + be_raise(vm, "memory_error", "could not allocated buffer"); + } + } toidentifier(id_buf, str); - if (!str_literal) { + if (len >= 255) { + /* decompose to avoid any size limit */ + lognofmt("be_nested_str_long("); + lognofmt(id_buf); + lognofmt(")"); + } else if (!str_literal) { logfmt("be_nested_str(%s)", id_buf); } else { logfmt("be_nested_str_weak(%s)", id_buf); } + if (id_buf != id_buf_stack) { + be_os_free(id_buf); + } } break; case BE_CLOSURE: diff --git a/lib/libesp32/berry/src/be_string.c b/lib/libesp32/berry/src/be_string.c index c6cb093c2..806e37ff0 100644 --- a/lib/libesp32/berry/src/be_string.c +++ b/lib/libesp32/berry/src/be_string.c @@ -27,6 +27,17 @@ .s = _s \ } +#define be_define_const_str_long(_name, _s, _len) \ + BERRY_LOCAL const bclstring be_const_str_##_name = { \ + .next = (bgcobject *)NULL, \ + .type = BE_STRING, \ + .marked = GC_CONST, \ + .extra = 0, \ + .slen = 255, \ + .llen = _len, \ + .s = _s \ + } + /* const string table */ struct bconststrtab { const bstring* const *table; @@ -279,7 +290,7 @@ void be_gcstrtab(bvm *vm) uint32_t be_strhash(const bstring *s) { - if (gc_isconst(s)) { + if (gc_isconst(s) && (s->slen != 255)) { bcstring* cs = cast(bcstring*, s); if (cs->hash) { /* if hash is null we need to compute it */ return cs->hash; @@ -298,12 +309,12 @@ uint32_t be_strhash(const bstring *s) const char* be_str2cstr(const bstring *s) { be_assert(cast_str(s) != NULL); - if (gc_isconst(s)) { - return cstr(s); - } if (s->slen == 255) { return lstr(s); } + if (gc_isconst(s)) { + return cstr(s); + } return sstr(s); } diff --git a/lib/libesp32/berry/src/be_string.h b/lib/libesp32/berry/src/be_string.h index 51b3fd016..f39ae6c4f 100644 --- a/lib/libesp32/berry/src/be_string.h +++ b/lib/libesp32/berry/src/be_string.h @@ -26,6 +26,12 @@ typedef struct { /* char s[]; */ } blstring; +typedef struct { /* const long string */ + bstring_header; + int llen; + char s[]; +} bclstring; + typedef struct { bstring_header; uint32_t hash; diff --git a/lib/libesp32/berry/tools/coc/block_builder.py b/lib/libesp32/berry/tools/coc/block_builder.py index 6535fd40b..a65ddb720 100644 --- a/lib/libesp32/berry/tools/coc/block_builder.py +++ b/lib/libesp32/berry/tools/coc/block_builder.py @@ -23,6 +23,7 @@ class block_builder: self.block = block() self.strtab = [] self.strtab_weak = [] + self.strtab_long = [] self.block.name = obj.name if depend(obj, macro): diff --git a/lib/libesp32/berry/tools/coc/coc b/lib/libesp32/berry/tools/coc/coc index 1a12c01e6..b80d0ac08 100755 --- a/lib/libesp32/berry/tools/coc/coc +++ b/lib/libesp32/berry/tools/coc/coc @@ -19,6 +19,7 @@ class builder: self.macro = None self.strmap = {} self.strmap_weak = {} + self.strmap_long = {} self.macro = macro_table() for path in self.config: @@ -27,7 +28,7 @@ class builder: for d in self.input: self.scandir(d) - sb = str_build(self.strmap, self.strmap_weak) + sb = str_build(self.strmap, self.strmap_weak, self.strmap_long) sb.build(self.output) def parse_file(self, filename): @@ -42,12 +43,16 @@ class builder: self.strmap[s] = 0 for s in parser.strtab_weak: self.strmap_weak[s] = 0 + for s in parser.strtab_long: + self.strmap_long[s] = 0 for obj in parser.objects: builder = block_builder(obj, self.macro) for s in builder.strtab: self.strmap[s] = 0 for s in builder.strtab_weak: self.strmap_weak[s] = 0 + for s in builder.strtab_long: + self.strmap_long[s] = 0 builder.dumpfile(self.output) def scandir(self, srcpath): diff --git a/lib/libesp32/berry/tools/coc/coc_parser.py b/lib/libesp32/berry/tools/coc/coc_parser.py index c5603ebb7..68c81a2ba 100644 --- a/lib/libesp32/berry/tools/coc/coc_parser.py +++ b/lib/libesp32/berry/tools/coc/coc_parser.py @@ -22,6 +22,7 @@ class coc_parser: self.objects = [] self.strtab = set() self.strtab_weak = set() + self.strtab_long = set() self.text = text self.parsers = { "@const_object_info_begin": self.parse_object, @@ -30,6 +31,7 @@ class coc_parser: "be_nested_str(": self.parse_string, "be_const_key_weak(": self.parse_string_weak, "be_nested_str_weak(": self.parse_string_weak, + "be_nested_str_long(": self.parse_string_long, "be_str_weak(": self.parse_string_weak, } @@ -131,6 +133,13 @@ class coc_parser: self.strtab_weak.add(literal) # print(f"str '{ident}' -> {literal}") + def parse_string_long(self): + if not self.text[0].isalnum() and self.text[0] != '_': return # do not proceed, maybe false positive in solidify + ident = self.parse_word() + literal = unescape_operator(ident) + if not literal in self.strtab: + self.strtab_long.add(literal) + ################################################################################# # Parse a block of definition like module, class... ################################################################################# diff --git a/lib/libesp32/berry/tools/coc/str_build.py b/lib/libesp32/berry/tools/coc/str_build.py index 31caba1d3..b8380bf57 100644 --- a/lib/libesp32/berry/tools/coc/str_build.py +++ b/lib/libesp32/berry/tools/coc/str_build.py @@ -22,9 +22,10 @@ class str_info: self.extra = 0 class str_build: - def __init__(self, map, map_weak): + def __init__(self, map, map_weak, map_long): self.map = map.copy() self.str_weak = [] + self.str_long = [] size = int(len(self.map) / 2) # voluntarily reduce hash size to half if size < 4: size = 4 @@ -44,6 +45,11 @@ class str_build: if not k in self.map: self.str_weak.append(k) + # handle long strings + for k in sorted(map_long.keys()): + if not k in self.map: + self.str_long.append(k) + def build(self, path): prefix = path + "/be_const_strtab" self.writefile(prefix + "_def.h", self.build_table_def()) @@ -121,6 +127,10 @@ class str_build: ostr += escape_operator(k) + ", " + escape_c(k) + ", " ostr += "0u, 0, " + str(len(k)) + ", NULL);\n" + for k in self.str_long: + ostr += "be_define_const_str_long(" + ostr += escape_operator(k) + ", " + escape_c(k) + ", " + str(len(k)) + ");\n" + ostr += "\n" ostr += "static const bstring* const m_string_table[] = {\n" @@ -156,4 +166,6 @@ class str_build: ostr += "\n/* weak strings */\n" for s in self.str_weak: ostr += "extern const bcstring be_const_str_" + escape_operator(s) + ";\n" + for s in self.str_long: + ostr += "extern const bclstring be_const_str_" + escape_operator(s) + ";\n" return ostr