Build system: allow easy solidification of external Berry (#21430)

* custom solidification

* solidify-from-url

* forgot folders

---------

Co-authored-by: Radio Loge <radiologe@MacBook-Pro-von-Radio.local>

lib/libesp32/berry/default/be_modtab.c

@@ -6,6 +6,7 @@
 ** https://github.com/Skiars/berry/blob/master/LICENSE
 ********************************************************************/
 #include "berry.h"
+#include "../../berry_custom/src/modules.h"
 
 /* this file contains the declaration of the module table. */
 
@@ -189,6 +190,7 @@ BERRY_LOCAL const bntvmodule_t* const be_module_table[] = {
     &be_native_module(matter),
 #endif // USE_MATTER_DEVICE
 #endif // TASMOTA
+    CUSTOM_NATIVE_MODULES
     /* user-defined modules register end */
     NULL /* do not remove */
 };
@@ -313,6 +315,7 @@ BERRY_LOCAL bclass_array be_class_table = {
 #if defined(USE_BERRY_INT64) || defined(USE_MATTER_DEVICE)
     &be_native_class(int64),
 #endif
+    CUSTOM_NATIVE_CLASSES
     NULL, /* do not remove */
 };
 

lib/libesp32/berry_custom/library.json

@@ -0,0 +1,17 @@
+{
+  "name": "Berry custom template",
+  "version": "0.1",
+  "description": "Berry library to solidify external scripts in the build process",
+  "license": "MIT",
+  "homepage": "https://github.com/arendst/Tasmota",
+  "frameworks": "arduino",
+  "platforms": "espressif32",
+  "authors":
+  {
+    "name": "Christian Baars",
+    "maintainer": true
+  },
+  "build": {
+    "flags": [ "-I$PROJECT_DIR/include", "-includetasmota_options.h" ]
+  }
+}

lib/libesp32/berry_custom/path.be

@@ -0,0 +1,2 @@
+# empty module
+# allows stand-alone `import path`

lib/libesp32/berry_custom/solidify_all.be

@@ -0,0 +1,99 @@
+#!/usr/bin/env -S ../berry/berry -s -g
+#
+# Berry solidify files
+
+import os
+import global
+import solidify
+import string as string2
+import re
+
+import sys
+sys.path().push('src/embedded')   # allow to import from src/embedded
+
+# globals that need to exist to make compilation succeed
+var globs = "path,ctypes_bytes_dyn,tasmota,ccronexpr,gpio,light,webclient,load,MD5,lv,light_state,udp,tcpclientasync,"
+            "lv_clock,lv_clock_icon,lv_signal_arcs,lv_signal_bars,lv_wifi_arcs_icon,lv_wifi_arcs,"
+            "lv_wifi_bars_icon,lv_wifi_bars,"
+            "_lvgl,"
+            "int64"
+
+for g:string2.split(globs, ",")
+  global.(g) = nil
+end
+
+var prefix_dir = "src/embedded/"
+var prefix_out = "src/solidify/"
+
+def sort(l)
+  # insertion sort
+  for i:1..size(l)-1
+    var k = l[i]
+    var j = i
+    while (j > 0) && (l[j-1] > k)
+      l[j] = l[j-1]
+      j -= 1
+    end
+    l[j] = k
+  end
+  return l
+end
+
+def clean_directory(dir)
+  var file_list = os.listdir(dir)
+  for f : file_list
+    if f[0] == '.'  continue end    # ignore files starting with `.`
+    os.remove(dir + f)
+  end
+end
+
+var pattern = "#@\\s*solidify:([A-Za-z0-9_.,]+)"
+
+def parse_file(fname, prefix_out)
+  print("Parsing: ", fname)
+  var f = open(prefix_dir + fname)
+  var src = f.read()
+  f.close()
+  # try to compile
+  var compiled = compile(src)
+  compiled()      # run the compile code to instanciate the classes and modules
+  # output solidified
+  var fname_h = string2.split(fname, '.be')[0] + '.h'  # take whatever is before the first '.be'
+  var fout = open(prefix_out + "solidified_" + fname_h, "w")
+  fout.write(f"/* Solidification of {fname_h} */\n")
+  fout.write("/********************************************************************\\\n")
+  fout.write("* Generated code, don't edit                                         *\n")
+  fout.write("\\********************************************************************/\n")
+  fout.write('#include "be_constobj.h"\n')
+
+  var directives = re.searchall(pattern, src)
+  # print(directives)
+
+  for directive : directives
+    var object_list = string2.split(directive[1], ',')
+    var object_name = object_list[0]
+    var weak = (object_list.find('weak') != nil)          # do we solidify with weak strings?
+    var o = global
+    var cl_name = nil
+    var obj_name = nil
+    for subname : string2.split(object_name, '.')
+      o = o.(subname)
+      cl_name = obj_name
+      obj_name = subname
+    end
+    solidify.dump(o, weak, fout, cl_name)
+  end
+
+  fout.write("/********************************************************************/\n")
+  fout.write("/* End of solidification */\n")
+  fout.close()
+end
+
+clean_directory(prefix_out)
+
+var src_file_list = os.listdir(prefix_dir)
+src_file_list = sort(src_file_list)
+for src_file : src_file_list
+  if src_file[0] == '.'  continue end
+  parse_file(src_file, prefix_out)
+end

lib/libesp32/berry_custom/src/.gitignore

@@ -0,0 +1 @@
+_temp*

lib/libesp32/berry_custom/src/be_custom_module.c

@@ -0,0 +1,31 @@
+/*
+  be_custom_module.c - allows solidification of external Berry files
+
+  Copyright (C) 2023  Stephan Hadinger, Christian Baars & Theo Arends
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*******************************************************************\
+ * `custom` modules and classes
+\*******************************************************************/
+
+#ifdef USE_BERRY
+#include "be_constobj.h"
+#include "be_mapping.h"
+
+/*solidify*/
+
+
+#endif // USE_BERRY

lib/libesp32/berry_custom/src/berry_custom.h

@@ -0,0 +1,8 @@
+// force compilation by including this file
+
+#ifndef __BERRY_CUSTOM__
+#define __BERRY_CUSTOM__
+
+
+
+#endif // __BERRY_CUSTOM__

lib/libesp32/berry_custom/src/embedded/.keep

@@ -0,0 +1 @@
+*

lib/libesp32/berry_custom/src/modules.h

@@ -0,0 +1,2 @@
+#define CUSTOM_NATIVE_MODULES
+#define CUSTOM_NATIVE_CLASSES

lib/libesp32/berry_custom/src/solidify/.keep

@@ -0,0 +1 @@
+*

pio-tools/gen-berry-structures.py

@@ -16,6 +16,6 @@ for filePath in fileList:
         # print("Deleting file : ", filePath)
     except:
         print("Error while deleting file : ", filePath)
-cmd = (env["PYTHONEXE"],join("tools","coc","coc"),"-o","generate","src","default",join("..","berry_tasmota","src"),join("..","berry_matter","src","solidify"),join("..","berry_matter","src"),join("..","berry_animate","src","solidify"),join("..","berry_animate","src"),join("..","berry_tasmota","src","solidify"),join("..","berry_mapping","src"),join("..","berry_int64","src"),join("..","..","libesp32_lvgl","lv_binding_berry","src"),join("..","..","libesp32_lvgl","lv_binding_berry","src","solidify"),join("..","..","libesp32_lvgl","lv_binding_berry","generate"),join("..","..","libesp32_lvgl","lv_haspmota","src","solidify"),"-c",join("default","berry_conf.h"))
+cmd = (env["PYTHONEXE"],join("tools","coc","coc"),"-o","generate","src","default",join("..","berry_tasmota","src"),join("..","berry_matter","src","solidify"),join("..","berry_matter","src"),join("..","berry_custom","src","solidify"),join("..","berry_custom","src"),join("..","berry_animate","src","solidify"),join("..","berry_animate","src"),join("..","berry_tasmota","src","solidify"),join("..","berry_mapping","src"),join("..","berry_int64","src"),join("..","..","libesp32_lvgl","lv_binding_berry","src"),join("..","..","libesp32_lvgl","lv_binding_berry","src","solidify"),join("..","..","libesp32_lvgl","lv_binding_berry","generate"),join("..","..","libesp32_lvgl","lv_haspmota","src","solidify"),"-c",join("default","berry_conf.h"))
 returncode = subprocess.call(cmd, shell=False)
 os.chdir(CURRENT_DIR)

pio-tools/solidify-from-url.py

@@ -0,0 +1,152 @@
+# Little convenience script to solidify external berry files as embedded
+
+Import("env")
+
+import os
+import sys
+from genericpath import exists
+from os.path import join
+import subprocess
+from colorama import Fore, Back, Style
+import requests
+import re
+
+IS_WINDOWS = sys.platform.startswith("win")
+
+def ensureBerry():
+    BERRY_GEN_DIR = join(env.subst("$PROJECT_DIR"), "lib", "libesp32","berry")
+    os.chdir(BERRY_GEN_DIR)
+    BERRY_EXECUTABLE = join(BERRY_GEN_DIR,"berry")
+    if IS_WINDOWS:
+        berry_executable = join(BERRY_GEN_DIR,"berry.exe")
+    else:
+        if os.path.exists(BERRY_EXECUTABLE) == False:
+            print("Will compile Berry executable")
+            make_cmd = "make"
+            subprocess.call(make_cmd, shell=False)
+    
+    if os.path.exists(BERRY_EXECUTABLE):
+        return BERRY_EXECUTABLE
+    else:
+        return Null
+
+def cleanFolder():
+    with open(HEADER_FILE_PATH, 'w') as file:
+        code = "#define CUSTOM_NATIVE_MODULES\n#define CUSTOM_NATIVE_CLASSES"
+        file.write(code)
+    tempfiles = [f for f in os.listdir(join(BERRY_SOLIDIFY_DIR,"src")) if re.match(r'_temp', f)]
+    for file in tempfiles:
+        os.remove(join(BERRY_SOLIDIFY_DIR,"src",file))
+    tempfiles = [f for f in os.listdir(join(BERRY_SOLIDIFY_DIR,"src","embedded")) if ".gitignore" not in f]
+    for file in tempfiles:
+        os.remove(join(BERRY_SOLIDIFY_DIR,"src","embedded",file))
+    tempfiles = [f for f in os.listdir(join(BERRY_SOLIDIFY_DIR,"src","solidify")) if ".gitignore" not in f]
+    for file in tempfiles:
+        os.remove(join(BERRY_SOLIDIFY_DIR,"src","solidify",file))
+    
+
+def addEntryToModtab(source): 
+    code = source.decode("utf-8")
+    class_name = None
+    is_module = False
+
+
+    pattern = (r'''(?<=module\()[^"].*''')  # module??
+    result =  re.findall(pattern,code)
+    if len(result) > 0:
+        class_name = result[0].replace("'","").replace('"','').replace(")","")
+        print(class_name+" is a module")
+        is_module = True
+    else: # just a class
+        pattern = (r'(?<=#@ solidify:).*')
+        result =  re.findall(pattern,code)
+        if len(result) > 0:
+            class_name = result[0].split(",")[0]
+        if class_name == None:
+            print("Could not find class name - is '#@ solidify:' used in Berry file??")
+            print(Fore.RED + "Aborting build process!!")
+            quit()
+    MODTAB_PATH = join(env.subst("$PROJECT_DIR"), "lib", "libesp32","berry","default","be_modtab.c")
+    with open(HEADER_FILE_PATH, 'r') as file:
+        code = file.read() # reuse code var for modtab file
+    if is_module:
+        nmodule = f"&be_native_module({class_name}),"
+        if code.find(nmodule) == -1:
+            code = code.replace(
+                "CUSTOM_NATIVE_MODULES",
+                f'CUSTOM_NATIVE_MODULES {nmodule}'
+            )
+        enmodule = f"be_extern_native_module({class_name});"
+        if code.find(enmodule) == -1:
+            code += f'\n{enmodule}'
+    else:
+        nclass = f"&be_native_class({class_name}),"
+        if code.find(nclass) == -1:
+            code = code.replace(
+                "CUSTOM_NATIVE_CLASSES",
+                f'CUSTOM_NATIVE_CLASSES {nclass}'
+            )
+        enclass = f"be_extern_native_class({class_name});"
+        if code.find(enclass) == -1:
+            code += f'\n{enclass}'
+
+    with open(HEADER_FILE_PATH, 'w') as file:
+        file.write(code)
+
+ 
+def addHeaderFile(name):
+    print("Will solidify ",name)
+    name = name.split(".")[0]
+    data = f"""
+/********************************************************************
+* {name} module
+* 
+*******************************************************************/
+#include "solidify/solidified_{name}.h"
+"""
+    file_name = f"_temp_be_{name}_lib.c"
+    file_path = join(BERRY_SOLIDIFY_DIR,"src",file_name)
+    open(file_path,"w").write(data)
+
+def prepareBerryFiles(files):
+    embedded_dir = join("src","embedded")
+    for file in files:
+        if "http" and "://" in file:
+            response = requests.get(file.split(" ")[0])
+            if response.ok:
+                target = join(embedded_dir,file.split(os.path.sep)[-1])
+                if len(file.split(" ")) > 1:
+                    target = join(embedded_dir,file.split(" ")[1])
+                    print("Renaming",(file.split(os.path.sep)[-1]).split(" ")[0],"to",file.split(" ")[1])
+                open(target, "wb").write(response.content)
+                addHeaderFile(file.split(os.path.sep)[-1])
+                addEntryToModtab(response.content)
+            else:
+                print(Fore.RED + "Failed to download: ",file)
+            continue
+        # maybe later ...
+        # if os.path.isdir(file):
+        #     continue
+        # else:
+        #     shutil.copy(file, embedded_dir)
+    return True
+
+
+BERRY_SOLIDIFY_DIR = join(env.subst("$PROJECT_DIR"), "lib", "libesp32","berry_custom")
+HEADER_FILE_PATH = join(BERRY_SOLIDIFY_DIR,"src","modules.h")
+cleanFolder() # always clean up this folder
+try:
+    files = env.GetProjectOption("custom_berry_solidify")
+except:
+    print("Nothing more to solidify")
+else:
+    if env.IsCleanTarget() == False:
+        BERRY_EXECUTABLE = ensureBerry()
+        
+        os.chdir(BERRY_SOLIDIFY_DIR)
+
+        if prepareBerryFiles(files.splitlines()):
+            solidify_command = BERRY_EXECUTABLE
+            solidify_flags = " -s -g solidify_all.be"
+            print("Start solidification for 'berry_custom':")
+            subprocess.call(solidify_command + solidify_flags, shell=True)

platformio_tasmota32.ini

@@ -46,6 +46,7 @@ lib_ignore                  =
                               Preferences
                               ArduinoOTA
 extra_scripts               = pre:pio-tools/add_c_flags.py
+                              pre:pio-tools/solidify-from-url.py
                               pre:pio-tools/gen-berry-structures.py
                               post:pio-tools/post_esp32.py
                               ${esp_defaults.extra_scripts}

tasmota/tasmota_xdrv_driver/xdrv_52_9_berry.ino

@@ -37,6 +37,7 @@ extern "C" {
 #include "be_vm.h"
 #include "ZipReadFS.h"
 #include "ccronexpr.h"
+#include "berry_custom.h"
 
 extern "C" {
   extern void be_load_custom_libs(bvm *vm);