2022-10-21 21:13:00

24 files changed, 196 insertions, 270 deletions

diff --git a/source/luametatex/CMakeLists.txt b/source/luametatex/CMakeLists.txt
index d972a0312..a5021595f 100644
--- a/source/luametatex/CMakeLists.txt
+++ b/source/luametatex/CMakeLists.txt
@@ -205,8 +205,8 @@ if (DEFINED LMT_OPTIMIZE)
         # set_property(TARGET luametatex PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
         #
         # mingw64: 2865664, nocona: 2819584, lto: 2835968 (around 1% gain on manual)
-
-       set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+        #
+        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
         #
     else()
         # No message needed, just accept the fact.
diff --git a/source/luametatex/source/lua/lmtinterface.h b/source/luametatex/source/lua/lmtinterface.h
index 3fd87db2f..e761252a8 100644
--- a/source/luametatex/source/lua/lmtinterface.h
+++ b/source/luametatex/source/lua/lmtinterface.h
@@ -865,9 +865,7 @@ make_lua_key(L, match);\
 make_lua_key(L, math);\
 make_lua_key(L, mathspec);\
 make_lua_key(L, math_accent);\
-make_lua_key(L, math_char_given);\
 make_lua_key(L, math_char_number);\
-make_lua_key(L, math_char_xgiven);\
 make_lua_key(L, math_choice);\
 make_lua_key(L, math_component);\
 make_lua_key(L, math_fence);\
diff --git a/source/luametatex/source/lua/lmtnodelib.c b/source/luametatex/source/lua/lmtnodelib.c
index 7d9d7bcbe..eb29611e2 100644
--- a/source/luametatex/source/lua/lmtnodelib.c
+++ b/source/luametatex/source/lua/lmtnodelib.c
@@ -2997,7 +2997,7 @@ static int nodelib_direct_getdata(lua_State *L)
                 }
             case mark_node:
                 if (lua_toboolean(L, 2)) {
-                    lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0);
+                    lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0, 0);
                 } else {
                     lmt_token_list_to_lua(L, mark_ptr(n));
                 }
@@ -6555,7 +6555,7 @@ static int nodelib_common_getfield(lua_State *L, int direct, halfword n)
                                 lua_pushinteger(L, mark_index(n));
                             } else if (lua_key_eq(s, data) || lua_key_eq(s, mark)) {
                                 if (lua_toboolean(L, 3)) {
-                                    lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0);
+                                    lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0, 0);
                                 } else {
                                     lmt_token_list_to_lua(L, mark_ptr(n));
                                 }
diff --git a/source/luametatex/source/lua/lmttexlib.c b/source/luametatex/source/lua/lmttexlib.c
index d44127330..afafb068b 100644
--- a/source/luametatex/source/lua/lmttexlib.c
+++ b/source/luametatex/source/lua/lmttexlib.c
@@ -1715,10 +1715,9 @@ static int texlib_gettoks(lua_State *L)
         if (lua_toboolean(L, slot)) {
             lmt_token_register_to_lua(L, state ? toks_parameter(index) : toks_register(index));
         } else {
+
             strnumber value = tex_get_tex_toks_register(index, state);
-            char *s = tex_makecstring(value);
-            lua_pushstring(L, s);
-            lmt_memory_free(s);
+            lua_pushstring(L, tex_to_cstring(value));
             tex_flush_str(value);
         }
     } else {
@@ -1754,7 +1753,7 @@ static int texlib_getmark(lua_State *L)
             if (num >= 0 && num <= lmt_mark_state.mark_data.ptr) {
                 halfword ptr = tex_get_some_mark(mrk, num);
                 if (ptr) {
-                    char *str = tex_tokenlist_to_tstring(ptr, 1, NULL, 0, 0, 0);
+                    char *str = tex_tokenlist_to_tstring(ptr, 1, NULL, 0, 0, 0, 0);
                     if (str) {
                         lua_pushstring(L, str);
                     } else {
@@ -1783,7 +1782,6 @@ int lmt_get_box_id(lua_State *L, int i, int report)
                 int cmd = eq_type(cs);
                 switch (cmd) {
                     case char_given_cmd:
-                 // case math_char_given_cmd:
                     case integer_cmd:
                         index = eq_value(cs);
                         break;
@@ -2597,7 +2595,6 @@ static int texlib_newindex(lua_State *L)
 static int texlib_aux_convert(lua_State *L, int cur_code)
 {
     int i = -1;
-    char *str = NULL;
     switch (cur_code) {
         /* ignored (yet) */
         case insert_progress_code:     /* arg <register int> */
@@ -2632,19 +2629,22 @@ static int texlib_aux_convert(lua_State *L, int cur_code)
             if (cur_code < 32) {
                 int texstr = tex_the_convert_string(cur_code, i);
                 if (texstr) {
-                    str = tex_makecstring(texstr);
+                    lua_pushstring(L, tex_to_cstring(texstr));
                     tex_flush_str(texstr);
+
+                 // int allocated = 0;
+                 // char *str = tex_makecstring(texstr, &allocated);
+                 // lua_pushstring(L, str);
+                 // if (allocated) {
+                 //     lmt_memory_free(str);
+                 // }
+                 // tex_flush_str(texstr);
+                    return 1; 
                 }
             }
             break;
     }
-    /* end */
-    if (str) {
-        lua_pushstring(L, str);
-        lmt_memory_free(str);
-    } else {
-        lua_pushnil(L);
-    }
+    lua_pushnil(L);
     return 1;
 }
 
@@ -2687,10 +2687,9 @@ static int texlib_aux_scan_internal(lua_State *L, int cmd, int code, int values)
         default:
             {
                 int texstr = tex_the_scanned_result();
-                char *str = tex_makecstring(texstr);
+                char *str = tex_to_cstring(texstr);
                 if (str) {
                     lua_pushstring(L, str);
-                    lmt_memory_free(str);
                 } else {
                     lua_pushnil(L);
                 }
@@ -3013,7 +3012,6 @@ static int texlib_get_internal(lua_State *L, int index, int all)
                     case set_auxiliary_cmd:
                     case set_page_property_cmd:
                     case char_given_cmd:
-                 // case math_char_given_cmd:
                     case integer_cmd:
                     case dimension_cmd:
                     case gluespec_cmd:
@@ -3407,26 +3405,22 @@ static int texlib_hashtokens(lua_State *L)
     int nx = 0;
     int all = lua_toboolean(L, 1);
     lua_createtable(L, hash_size, 0);
+    /* todo: check active characters as these have three bogus bytes in front */
     if (all) {
         while (cs <= hash_size) {
-            /* because strings never get freed we can as well directly access |s|. */
             strnumber s = cs_text(cs);
             if (s > 0) {
                 halfword n = cs_next(cs);
-                char *ss = tex_makecstring(s);
                 if (n) {
                     int mt = 0;
                     lua_createtable(L, 2, 0);
-                    lua_pushstring(L, ss);
-                    lmt_memory_free(ss);
+                    lua_pushstring(L, tex_to_cstring(s));
                     ++nt;
                     lua_rawseti(L, -2, ++mt);
                     while (n) {
                         s = cs_text(n);
                         if (s) {
-                            ss = tex_makecstring(s);
-                            lua_pushstring(L, ss);
-                            lmt_memory_free(ss);
+                            lua_pushstring(L, tex_to_cstring(s));
                             lua_rawseti(L, -2, ++mt);
                             ++nt;
                             ++nx;
@@ -3434,8 +3428,7 @@ static int texlib_hashtokens(lua_State *L)
                         n = cs_next(n);
                     }
                 } else {
-                    lua_pushstring(L, ss);
-                    lmt_memory_free(ss);
+                    lua_pushstring(L, tex_to_cstring(s));
                     ++nt;
                 }
             } else {
@@ -3449,16 +3442,12 @@ static int texlib_hashtokens(lua_State *L)
             strnumber s = cs_text(cs);
             if (s > 0) {
                 halfword n = cs_next(cs);
-                char *ss = tex_makecstring(s);
-                lua_pushstring(L, ss);
-                lmt_memory_free(ss);
+                lua_pushstring(L, tex_to_cstring(s));
                 lua_rawseti(L, -2, ++nt);
                 while (n) {
                     s = cs_text(n);
                     if (s) {
-                        ss = tex_makecstring(s);
-                        lua_pushstring(L, ss);
-                        lmt_memory_free(ss);
+                        lua_pushstring(L, tex_to_cstring(s));
                         lua_rawseti(L, -2, ++nt);
                         ++nx;
                     }
@@ -3482,9 +3471,7 @@ static int texlib_primitives(lua_State *L)
     while (cs < prim_size) {
         strnumber s = get_prim_text(cs);
         if (s > 0 && (get_prim_origin(cs) != no_command)) {
-            char *ss = tex_makecstring(s);
-            lua_pushstring(L, ss);
-            lmt_memory_free(ss);
+            lua_pushstring(L, tex_to_cstring(s));
             lua_rawseti(L, -2, ++nt);
         }
         cs++;
@@ -3518,9 +3505,7 @@ static int texlib_extraprimitives(lua_State *L)
     while (cs < prim_size) {
         strnumber s = get_prim_text(cs);
         if (s > 0 && (get_prim_origin(cs) & mask)) {
-            char *ss = tex_makecstring(s);
-            lua_pushstring(L, ss);
-            lmt_memory_free(ss);
+            lua_pushstring(L, tex_to_cstring(s));
             lua_rawseti(L, -2, ++nt);
         }
         cs++;
@@ -3531,7 +3516,7 @@ static int texlib_extraprimitives(lua_State *L)
 static void texlib_aux_enableprimitive(const char *pre, size_t prel, const char *prm)
 {
     strnumber s = tex_maketexstring(prm);
-    halfword prm_val = tex_prim_lookup(s);
+    halfword prm_val = tex_prim_lookup(s); /* todo: no need for tex string */
     tex_flush_str(s);
     if (prm_val != undefined_primitive && get_prim_origin(prm_val) != no_command) {
         char *newprm;
@@ -3594,10 +3579,8 @@ static int texlib_enableprimitives(lua_State *L)
                     for (int cs = 0; cs < prim_size; cs++) {
                         strnumber s = get_prim_text(cs);
                         if (s > 0) {
-                            /* there is actually no need to copy */
-                            char *prm = tex_makecstring(s);
+                            char *prm = tex_to_cstring(s);
                             texlib_aux_enableprimitive(pre, lpre, prm);
-                            lmt_memory_free(prm);
                         }
                     }
                 }
@@ -4561,8 +4544,6 @@ static int texlib_mathchardef(lua_State *L)
             d.index = lmt_optinteger(L, 8, 0);
             if (class_in_range(m.class_value) && family_in_range(m.family_value) && character_in_range(m.character_value)) {
                 tex_define(flags, cs, mathspec_cmd, tex_new_math_dict_spec(d, m, umath_mathcode));
-             // halfword code = math_packed_character(m.class_value, m.family_value, m.character_value);
-             // tex_define(flags, cs, (quarterword) math_char_xgiven_cmd, code);
             } else {
                 tex_normal_error("lua", "mathchardef needs proper class, family and character codes");
             }
diff --git a/source/luametatex/source/lua/lmttokenlib.c b/source/luametatex/source/lua/lmttokenlib.c
index ac69241ba..feac7cd46 100644
--- a/source/luametatex/source/lua/lmttokenlib.c
+++ b/source/luametatex/source/lua/lmttokenlib.c
@@ -156,8 +156,6 @@ void lmt_tokenlib_initialize(void)
     lmt_interface.command_names[math_shift_cs_cmd]                = (command_item) { .id = math_shift_cs_cmd,                  .lua = lua_key_index(math_shift_cs),                .name = lua_key(math_shift_cs),                .kind = regular_command_item,   .min = 0,                         .max = last_math_shift_cs_code,      .base = 0,                       .fixedvalue = 0            }; /* a bit too tolerant */
     lmt_interface.command_names[end_cs_name_cmd]                  = (command_item) { .id = end_cs_name_cmd,                    .lua = lua_key_index(end_cs_name),                  .name = lua_key(end_cs_name),                  .kind = regular_command_item,   .min = 0,                         .max = 0,                            .base = 0,                       .fixedvalue = 0            };
     lmt_interface.command_names[char_given_cmd]                   = (command_item) { .id = char_given_cmd,                     .lua = lua_key_index(char_given),                   .name = lua_key(char_given),                   .kind = character_command_item, .min = 0,                         .max = max_character_code,           .base = 0,                       .fixedvalue = 0            };
- // lmt_interface.command_names[math_char_given_cmd]              = (command_item) { .id = math_char_given_cmd,                .lua = lua_key_index(math_char_given),              .name = lua_key(math_char_given),              .kind = character_command_item, .min = 0,                         .max = max_character_code,           .base = 0,                       .fixedvalue = 0            };
- // lmt_interface.command_names[math_char_xgiven_cmd]             = (command_item) { .id = math_char_xgiven_cmd,               .lua = lua_key_index(math_char_xgiven),             .name = lua_key(math_char_xgiven),             .kind = character_command_item, .min = 0,                         .max = max_character_code,           .base = 0,                       .fixedvalue = 0            };
     lmt_interface.command_names[some_item_cmd]                    = (command_item) { .id = some_item_cmd,                      .lua = lua_key_index(some_item),                    .name = lua_key(some_item),                    .kind = regular_command_item,   .min = 0,                         .max = last_some_item_code,          .base = 0,                       .fixedvalue = 0            };
     lmt_interface.command_names[internal_toks_cmd]                = (command_item) { .id = internal_toks_cmd,                  .lua = lua_key_index(internal_toks),                .name = lua_key(internal_toks),                .kind = internal_command_item,  .min = first_toks_code,           .max = last_toks_code,               .base = internal_toks_base,      .fixedvalue = 0            };
     lmt_interface.command_names[register_toks_cmd]                = (command_item) { .id = register_toks_cmd,                  .lua = lua_key_index(register_toks),                .name = lua_key(register_toks),                .kind = register_command_item,  .min = 0,                         .max = biggest_reg,                  .base = register_toks_base,      .fixedvalue = 0            };
@@ -433,10 +431,10 @@ void lmt_token_list_to_lua(lua_State *L, halfword p)
     }
 }
 
-void lmt_token_list_to_luastring(lua_State *L, halfword p, int nospace, int strip)
+void lmt_token_list_to_luastring(lua_State *L, halfword p, int nospace, int strip, int wipe)
 {
     int l;
-    char *s = tex_tokenlist_to_tstring(p, 1, &l, 0, nospace, strip); /* nasty ... preambles or not, could have been endmatchtoken  */
+    char *s = tex_tokenlist_to_tstring(p, 1, &l, 0, nospace, strip, wipe); /* nasty ... preambles or not, could have been endmatchtoken  */
     if (l) {
         lua_pushlstring(L, s, (size_t) l);
     } else {
@@ -511,16 +509,19 @@ halfword lmt_token_code_from_lua(lua_State *L, int slot)
 
 # define is_active_string(s) (strlen(s) > 3 && *s == 0xEF && *(s+1) == 0xBF && *(s+2) == 0xBF)
 
-static unsigned char *tokenlib_aux_get_cs_text(int cs)
+static unsigned char *tokenlib_aux_get_cs_text(int cs, int *allocated)
 {
     if (cs == null_cs) {
-        return (unsigned char *) lmt_memory_strdup("\\csname\\endcsname");
+        return (unsigned char *) "\\csname\\endcsname";
     } else if ((cs_text(cs) < 0) || (cs_text(cs) >= lmt_string_pool_state.string_pool_data.ptr)) {
-        return (unsigned char *) lmt_memory_strdup("");
-    } else if (tex_is_active_cs(cs_text(cs))) {
-        return (unsigned char *) tex_makecstring(cs_text(cs));
+        return (unsigned char *) "";
+ // } else {
+ //     return (unsigned char *) tex_makecstring(cs_text(cs));
+    } else if (cs_text(cs) < cs_offset_value) {
+        *allocated = 1;
+        return (unsigned char *) aux_uni2str((unsigned) cs_text(cs));
     } else {
-        return (unsigned char *) tex_makecstring(cs_text(cs));
+        return (unsigned char *) (str_length(cs_text(cs)) > 0 ?  (unsigned char *) str_string(cs_text(cs)) :  (unsigned char *) "");
     }
 }
 
@@ -871,14 +872,17 @@ static int tokenlib_scan_csname(lua_State *L)
     }
     t = cur_cs ? cs_token_flag + cur_cs : token_val (cur_cmd, cur_chr);
     if (t >= cs_token_flag) {
-        unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag);
+        int allocated = 0;
+        unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag, &allocated);
         if (s) {
             if (tex_is_active_cs(cs_text(t - cs_token_flag))) {
                 lua_pushstring(L, (char *) (s + 3));
             } else {
                 lua_pushstring(L, (char *) s);
             }
-            lmt_memory_free(s);
+            if (allocated) {
+                lmt_memory_free(s);
+            }
         } else {
             lua_pushnil(L);
         }
@@ -1408,8 +1412,7 @@ static int tokenlib_scan_string(lua_State *L)
             {
                 halfword defref = lmt_input_state.def_ref;
                 halfword result = tex_scan_toks_expand(1, NULL, 0);
-                lmt_token_list_to_luastring(L, result, 0, 0);
-                tex_flush_token_list(result);
+                lmt_token_list_to_luastring(L, result, 0, 0, 1);
                 lmt_input_state.def_ref = defref;
                 break;
             }
@@ -1421,8 +1424,7 @@ static int tokenlib_scan_string(lua_State *L)
         case tolerant_semi_protected_call_cmd:
             {
                 halfword t = token_link(cur_chr);
-                lmt_token_list_to_luastring(L, t, 0, 0);
-                tex_flush_token_list(t);
+                lmt_token_list_to_luastring(L, t, 0, 0, 1);
                 break;
             }
         case letter_cmd:
@@ -1463,8 +1465,7 @@ static int tokenlib_scan_argument(lua_State *L)
                 halfword defref = lmt_input_state.def_ref;
                 int expand = lua_type(L, 1) == LUA_TBOOLEAN ? lua_toboolean(L, 1) : 1;
                 halfword result = expand ? tex_scan_toks_expand(1, NULL, 0) : tex_scan_toks_normal(1, NULL);
-                lmt_token_list_to_luastring(L, result, 0, 0);
-                tex_flush_token_list(result);
+                lmt_token_list_to_luastring(L, result, 0, 0, 1);
                 lmt_input_state.def_ref = defref;
                 break;
             }
@@ -1485,8 +1486,7 @@ static int tokenlib_scan_argument(lua_State *L)
                       tex_back_input(cur_tok);
                       result = tex_scan_toks_expand(1, NULL, 0);
                   }
-                  lmt_token_list_to_luastring(L, result, 0, 0);
-                  tex_flush_token_list(result);
+                  lmt_token_list_to_luastring(L, result, 0, 0, 1);
                   lmt_input_state.def_ref = defref;
                   break;
               }
@@ -1685,8 +1685,7 @@ static int tokenlib_scan_delimited(lua_State *L)
       FINALYDONE:
         /* */
         lmt_input_state.def_ref = defref;
-        lmt_token_list_to_luastring(L, result, 0, 0);
-        tex_flush_token_list(result);
+        lmt_token_list_to_luastring(L, result, 0, 0, 1);
     } else {
         tex_back_input(cur_tok);
         lua_pushnil(L);
@@ -1781,8 +1780,7 @@ static int tokenlib_grab_until(lua_State *L)
         }
         tokenlib_aux_unsave_tex_scanner(texstate);
         lmt_input_state.def_ref = defref;
-        lmt_token_list_to_luastring(L, result, nospace, strip);
-        tex_flush_token_list(result);
+        lmt_token_list_to_luastring(L, result, nospace, strip, 1);
     } else { 
         lua_pushnil(L);
     }
@@ -2013,8 +2011,7 @@ static int tokenlib_scan_value(lua_State *L)
                 halfword defref = lmt_input_state.def_ref;
                 result = tex_scan_toks_expand(1, NULL, 0);
                 lmt_input_state.def_ref = defref;
-                lmt_token_list_to_luastring(L, result, 0, 0);
-                tex_flush_token_list(result);
+                lmt_token_list_to_luastring(L, result, 0, 0, 1);
             }
             break;
         /*
@@ -2052,8 +2049,7 @@ static int tokenlib_scan_value(lua_State *L)
                                 halfword defref = lmt_input_state.def_ref;
                                 result = tex_scan_toks_expand(1, NULL, 0);
                                 lmt_input_state.def_ref = defref;
-                                lmt_token_list_to_luastring(L, result, 0, 0);
-                                tex_flush_token_list(result);
+                                lmt_token_list_to_luastring(L, result, 0, 0, 1);
                                 luaL_addchar(&b, '{');
                                 luaL_addvalue(&b);
                                 luaL_addchar(&b, '}');
@@ -2069,7 +2065,8 @@ static int tokenlib_scan_value(lua_State *L)
                                 /*tex We need to add a space. */
                                 halfword t = cur_cs ? cs_token_flag + cur_cs : token_val(cur_cmd, cur_chr);
                                 if (t >= cs_token_flag) {
-                                    unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag);
+                                    int allocated = 0;
+                                    unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag, &allocated);
                                     if (s) {
                                         if (tex_is_active_cs(cs_text(t - cs_token_flag))) {
                                             lua_pushstring(L, (char *) (s + 3));
@@ -2080,7 +2077,9 @@ static int tokenlib_scan_value(lua_State *L)
                                             luaL_addvalue(&b);
                                             luaL_addchar(&b, ' ');
                                         }
-                                        lmt_memory_free(s);
+                                        if (allocated) {
+                                            lmt_memory_free(s);
+                                        }
                                     }
                                 }
                             }
@@ -2377,7 +2376,7 @@ static int tokenlib_getprimitives(lua_State *L)
     while (cs < prim_size) {
         strnumber s = get_prim_text(cs);
         if (s > 0 && (get_prim_origin(cs) != no_command)) {
-            char *ss = tex_makecstring(s);
+            char *ss = tex_to_cstring(s);
             int cmd = prim_eq_type(cs);
             int chr = prim_equiv(cs);
             if (! raw) {
@@ -2393,7 +2392,6 @@ static int tokenlib_getprimitives(lua_State *L)
             lua_pushinteger(L, prim_origin(cs));
             lua_rawseti(L, -2, 4);
             lua_rawseti(L, -2, ++nt);
-            lmt_memory_free(ss);
         }
         cs++;
     }
@@ -2508,14 +2506,17 @@ inline static int tokenlib_get_csname(lua_State *L)
     lua_token *n = tokenlib_aux_check_istoken(L, 1);
     halfword tok = token_info(n->token);
     if (tok >= cs_token_flag) {
-        unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag);
+        int allocated = 0;
+        unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag, &allocated);
         if (s) {
             if (tex_is_active_cs(cs_text(tok - cs_token_flag))) {
                 lua_pushstring(L, (char *) (s + 3));
             } else {
                 lua_pushstring(L, (char *) s);
             }
-            lmt_memory_free(s);
+            if (allocated) {
+                lmt_memory_free(s);
+            }
             return 1;
         }
     }
@@ -2544,10 +2545,13 @@ inline static int tokenlib_get_active(lua_State *L)
     halfword tok = token_info(n->token);
     int result = 0;
     if (tok >= cs_token_flag) {
-        unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag);
+        int allocated = 0;
+        unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag, &allocated);
         if (s) {
             result = tex_is_active_cs(cs_text(tok - cs_token_flag));
-            lmt_memory_free(s);
+            if (allocated) {
+                lmt_memory_free(s);
+            }
         }
     }
     lua_pushboolean(L, result);
@@ -2753,7 +2757,8 @@ static int tokenlib_get_fields(lua_State *L)
                 lua_createtable(L, 0, onlyflags ? 0 : 5);
                 if (tok >= cs_token_flag) {
                     int t = tok - cs_token_flag;
-                    unsigned char* str = tokenlib_aux_get_cs_text(t);
+                    int allocated = 0;
+                    unsigned char* str = tokenlib_aux_get_cs_text(t, &allocated);
                     if (str) {
                         if (! onlyflags) {
                             lua_push_key(csname);
@@ -2767,7 +2772,9 @@ static int tokenlib_get_fields(lua_State *L)
                             }
                             lua_rawset(L, -3);
                         }
-                        lmt_memory_free(str);
+                        if (allocated) {
+                            lmt_memory_free(str);
+                        }
                     }
                     cmd = eq_type(t);
                     chr = eq_value(t);
@@ -2836,9 +2843,12 @@ static int tokenlib_tostring(lua_State* L)
         unsigned char* csp = NULL;
         const char* cmn = NULL;
         if (tok >= cs_token_flag) {
+            int allocated = 0;
             tok -= cs_token_flag;
-            csn = tokenlib_aux_get_cs_text(tok);
-            csp = csn;
+            csn = tokenlib_aux_get_cs_text(tok, &allocated);
+            if (allocated) {
+                csp = csn;
+            }
             if (csn && tex_is_active_cs(cs_text(tok))) {
                 csn += 3;
             }
@@ -2974,7 +2984,7 @@ static int tokenlib_get_meaning(lua_State *L)
                     lmt_token_register_to_lua(L, chr);
                 }
             } else {
-                char *str = tex_tokenlist_to_tstring(chr, 1, NULL, 0, 0, 0);
+                char *str = tex_tokenlist_to_tstring(chr, 1, NULL, 0, 0, 0, 0);
                 lua_pushstring(L, str ? str : "");
             }
             return 1;
@@ -3041,10 +3051,9 @@ static int tokenlib_get_macro(lua_State *L)
             char *str = NULL;
             if (lua_toboolean(L, 2)) {
                 tokenlib_aux_expand_macros_in_tokenlist(chr); // todo: use return value instead of def_ref
-                str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0);
-                tex_flush_token_list(lmt_input_state.def_ref);
+                str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0, 1);
             } else {
-                str = tex_tokenlist_to_tstring(chr, 1, NULL, 1, 0, 0);
+                str = tex_tokenlist_to_tstring(chr, 1, NULL, 1, 0, 0, 0);
             }
             lua_pushstring(L, str ? str : "");
             return 1;
@@ -3091,8 +3100,7 @@ char *lmt_get_expansion(halfword head, int *len)
     halfword ref = get_reference_token();
     set_token_link(ref, head);
     tokenlib_aux_expand_macros_in_tokenlist(ref); // todo: use return value instead of def_ref
-    str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, len, 0, 0, 0);
-    tex_flush_token_list(lmt_input_state.def_ref);
+    str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, len, 0, 0, 0, 1);
     tex_flush_token_list(ref);
     return str;
 }
@@ -3113,12 +3121,9 @@ static int tokenlib_get_expansion(lua_State* L)
         char *s;
         int l;
         tex_parse_str_to_tok(h, &t, ct, str, len, 2); /* ignore unknown */
-
         tokenlib_aux_expand_macros_in_tokenlist(h); // todo: use return value instead of def_ref
-        s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, &l, 0, 0, 0);
-        tex_flush_token_list(lmt_input_state.def_ref);
+        s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, &l, 0, 0, 0, 1);
         tex_flush_token_list(h);
-
         if (l > 0) {
             lua_pushlstring(L, (const char *) s, (size_t) l);
             return 1;
@@ -3472,9 +3477,8 @@ static int tokenlib_serialize(lua_State *L)
         halfword t = n->token;
         char *s;
         tokenlib_aux_expand_macros_in_tokenlist(t); // todo: use return value instead of def_ref
-        s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0);
+        s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0, 1);
         lua_pushstring(L, s ? s : "");
-        tex_flush_token_list(lmt_input_state.def_ref);
     } else {
         lua_pushnil(L);
     }
@@ -3656,7 +3660,7 @@ void lmt_token_call(int p) /*tex The \TEX\ pointer to the token list. */
 {
     LoadS ls;
     int l = 0;
-    ls.s = tex_tokenlist_to_tstring(p, 1, &l, 0, 0, 0);
+    ls.s = tex_tokenlist_to_tstring(p, 1, &l, 0, 0, 0, 0);
     ls.size = (size_t) l;
     if (ls.size > 0) {
         lua_State *L = lmt_lua_state.lua_instance;
diff --git a/source/luametatex/source/lua/lmttokenlib.h b/source/luametatex/source/lua/lmttokenlib.h
index 5339a80fe..450c6173a 100644
--- a/source/luametatex/source/lua/lmttokenlib.h
+++ b/source/luametatex/source/lua/lmttokenlib.h
@@ -28,7 +28,7 @@ typedef enum command_item_types {
 } command_item_types;
 
 extern void     lmt_token_list_to_lua         (lua_State *L, halfword p);
-extern void     lmt_token_list_to_luastring   (lua_State *L, halfword p, int nospace, int strip);
+extern void     lmt_token_list_to_luastring   (lua_State *L, halfword p, int nospace, int strip, int wipe);
 extern halfword lmt_token_list_from_lua       (lua_State *L, int slot);
 extern halfword lmt_token_code_from_lua       (lua_State *L, int slot);
 
diff --git a/source/luametatex/source/luacore/lua54/src/ltable.c b/source/luametatex/source/luacore/lua54/src/ltable.c
index 1b1cd2415..d03e74865 100644
--- a/source/luametatex/source/luacore/lua54/src/ltable.c
+++ b/source/luametatex/source/luacore/lua54/src/ltable.c
@@ -107,7 +107,7 @@ static const TValue absentkey = {ABSTKEYCONSTANT};
 */
 static Node *hashint (const Table *t, lua_Integer i) {
   lua_Unsigned ui = l_castS2U(i);
-  if (ui <= (unsigned int)INT_MAX)
+  if (ui <= cast_uint(INT_MAX))
     return hashmod(t, cast_int(ui));
   else
     return hashmod(t, ui);
@@ -488,7 +488,7 @@ static void setnodevector (lua_State *L, Table *t, unsigned int size) {
       luaG_runerror(L, "table overflow");
     size = twoto(lsize);
     t->node = luaM_newvector(L, size, Node);
-    for (i = 0; i < (int)size; i++) {
+    for (i = 0; i < cast_int(size); i++) {
       Node *n = gnode(t, i);
       gnext(n) = 0;
       setnilkey(n);
diff --git a/source/luametatex/source/luametatex.h b/source/luametatex/source/luametatex.h
index 32fd78f6d..736d21173 100644
--- a/source/luametatex/source/luametatex.h
+++ b/source/luametatex/source/luametatex.h
@@ -87,9 +87,9 @@
 # include "tex/textypes.h"
 
 # define luametatex_version          210
-# define luametatex_revision         01
-# define luametatex_version_string   "2.10.01"
-# define luametatex_development_id   20221014
+# define luametatex_revision         02
+# define luametatex_version_string   "2.10.02"
+# define luametatex_development_id   20221019
 
 # define luametatex_name_camelcase   "LuaMetaTeX"
 # define luametatex_name_lowercase   "luametatex"
diff --git a/source/luametatex/source/tex/texequivalents.c b/source/luametatex/source/tex/texequivalents.c
index 291126fff..aa434d8fd 100644
--- a/source/luametatex/source/tex/texequivalents.c
+++ b/source/luametatex/source/tex/texequivalents.c
@@ -1437,7 +1437,7 @@ void tex_forced_word_define(int g, halfword p, singleword f, halfword w)
 
 void tex_save_for_after_group(halfword t)
 {
-    if (cur_level > level_one && tex_room_on_save_stack()) {
+    if (t && cur_level > level_one && tex_room_on_save_stack()) {
         save_type(lmt_save_state.save_stack_data.ptr) = insert_tokens_save_type;
         save_level(lmt_save_state.save_stack_data.ptr) = level_zero;
         save_value(lmt_save_state.save_stack_data.ptr) = t;
diff --git a/source/luametatex/source/tex/texexpand.c b/source/luametatex/source/tex/texexpand.c
index 25dcccdf3..a0416d118 100644
--- a/source/luametatex/source/tex/texexpand.c
+++ b/source/luametatex/source/tex/texexpand.c
@@ -684,7 +684,7 @@ inline static halfword tex_aux_get_cs_name(void)
     if (tex_aux_collect_cs_tokens(&p, &n)) {
         /*tex Look up the characters of list |r| in the hash table, and set |cur_cs|. */
         int siz;
-        char *s = tex_tokenlist_to_tstring(h, 1, &siz, 0, 0, 0);
+        char *s = tex_tokenlist_to_tstring(h, 1, &siz, 0, 0, 0, 0);
         cur_cs = (siz > 0) ? tex_string_locate((char *) s, siz, 1) : null_cs;
     } else {
         tex_aux_complain_missing_csname();
diff --git a/source/luametatex/source/tex/texfileio.c b/source/luametatex/source/tex/texfileio.c
index 4f712401b..c29e7cece 100644
--- a/source/luametatex/source/tex/texfileio.c
+++ b/source/luametatex/source/tex/texfileio.c
@@ -875,10 +875,8 @@ char *tex_read_file_name(int optionalequal, const char * name, const char* ext)
         }
     }
   DONE:
-    s = tex_tokenlist_to_tstring(result, 1, &l, 0, 0, 0);
+    s = tex_tokenlist_to_tstring(result, 1, &l, 0, 0, 0, 1);
     fn = s ? tex_aux_pack_file_name(s, l, name, ext) : NULL;
-    /*tex Shouldn't we also free |result| ? */
-    tex_flush_token_list(result);
     return fn;
 }
 
diff --git a/source/luametatex/source/tex/texlanguage.c b/source/luametatex/source/tex/texlanguage.c
index 6f3460c22..22627ee58 100644
--- a/source/luametatex/source/tex/texlanguage.c
+++ b/source/luametatex/source/tex/texlanguage.c
@@ -364,7 +364,7 @@ void tex_clear_patterns(struct tex_language *lang)
 
 void tex_load_tex_patterns(halfword curlang, halfword head)
 {
-    char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0);
+    char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0, 0);
     if (s) {
         tex_load_patterns(tex_get_language(curlang), (unsigned char *) s);
     }
@@ -539,7 +539,7 @@ void tex_clear_hyphenation(struct tex_language *lang)
 
 void tex_load_tex_hyphenation(halfword curlang, halfword head)
 {
-    char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0);
+    char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0, 0);
     if (s) {
         tex_load_hyphenation(tex_get_language(curlang), (unsigned char *) s);
     }
diff --git a/source/luametatex/source/tex/texlinebreak.c b/source/luametatex/source/tex/texlinebreak.c
index 2172277ec..2e0c945a7 100644
--- a/source/luametatex/source/tex/texlinebreak.c
+++ b/source/luametatex/source/tex/texlinebreak.c
@@ -3145,7 +3145,7 @@ static void tex_aux_post_line_break(const line_break_properties *properties, hal
                     n = node_next(n);
                 }
             }
-            if (! n) {
+            if (! n && normalize_line_mode_par) {
                 /*tex For the moment: */
                 tex_normal_warning("tex", "right parinit skip is gone");
             }
@@ -3163,7 +3163,7 @@ static void tex_aux_post_line_break(const line_break_properties *properties, hal
                     n = node_prev(n);
                 }
             }
-            if (! n) {
+            if (! n && normalize_line_mode_par) {
                 /*tex For the moment: */
                 tex_normal_warning("tex", "left parfill skip is gone");
             }
diff --git a/source/luametatex/source/tex/texmaincontrol.c b/source/luametatex/source/tex/texmaincontrol.c
index a44aa1167..5e0bb2f92 100644
--- a/source/luametatex/source/tex/texmaincontrol.c
+++ b/source/luametatex/source/tex/texmaincontrol.c
@@ -1372,11 +1372,13 @@ static void tex_aux_run_after_something(void) {
                 } while (cur_cmd == spacer_cmd);
                 if (cur_cmd == left_brace_cmd) {
                     halfword source = tex_scan_toks_normal(1, NULL);
-                    if (source) {
-                        tex_save_for_after_group(token_link(source));
-                        token_link(source) = null;
+                    if (source) { 
+                        if (token_link(source)) {
+                            tex_save_for_after_group(token_link(source));
+                            token_link(source) = null;
+                        }
+                        tex_put_available_token(source);
                     }
-                    tex_flush_token_list(source);
                 } else {
                     tex_handle_error(
                         normal_error_type,
@@ -1394,10 +1396,11 @@ static void tex_aux_run_after_something(void) {
                 if (cur_cmd == left_brace_cmd) {
                     halfword source = tex_scan_toks_normal(1, NULL);
                     if (source) {
+                        /*tex Always, also when empty. */
                         lmt_main_control_state.after_tokens = token_link(source);
                         token_link(source) = null;
+                        tex_put_available_token(source);
                     }
-                    tex_flush_token_list(source);
                 } else {
                     tex_handle_error(
                         normal_error_type,
@@ -1422,7 +1425,7 @@ static void tex_aux_run_after_something(void) {
                             }
                             token_link(p) = token_link(source);
                             token_link(source) = null;
-                            tex_flush_token_list(source);
+                            tex_put_available_token(source);
                         } else {
                             update_tex_end_of_group(source);
                         }
@@ -4118,7 +4121,6 @@ static void tex_aux_set_shorthand_def(int a, int force)
                 {
                     mathcodeval mval = tex_scan_mathchar(tex_mathcode);
                     tex_define(a, p, mathspec_cmd, tex_new_math_spec(mval, tex_mathcode));
-                 // tex_define(a, p, math_char_given_cmd, math_old_packed_character(mval.class_value,mval.family_value,mval.character_value));
                     break;
                 }
             case math_dchar_def_code:
@@ -4126,14 +4128,12 @@ static void tex_aux_set_shorthand_def(int a, int force)
                     mathdictval dval = tex_scan_mathdict();
                     mathcodeval mval = tex_scan_mathchar(umath_mathcode);
                     tex_define(a, p, mathspec_cmd, tex_new_math_dict_spec(dval, mval, umath_mathcode));
-                 // tex_define(a, p, math_char_xgiven_cmd, math_packed_character(mval.class_value,mval.family_value,mval.character_value));
                     break;
                 }
             case math_xchar_def_code:
                 {
                     mathcodeval mval = tex_scan_mathchar(umath_mathcode);
                     tex_define(a, p, mathspec_cmd, tex_new_math_spec(mval, umath_mathcode));
-                 // tex_define(a, p, math_char_xgiven_cmd, math_packed_character(mval.class_value,mval.family_value,mval.character_value));
                     break;
                 }
             case count_def_code:
@@ -5854,14 +5854,13 @@ static void tex_aux_run_message(void)
                 strnumber s = tex_aux_scan_string();
                 if (error_help_par) {
                     strnumber helpinfo = tex_tokens_to_string(error_help_par);
-                    char *h = tex_makecstring(helpinfo);
+                    char *h = tex_to_cstring(helpinfo);
                     tex_handle_error(
                         normal_error_type,
                         "%T",
                         s,
                         h
                     );
-                    lmt_memory_free(h);
                     tex_flush_str(helpinfo);
                 } else if (lmt_error_state.long_help_seen) {
                     tex_handle_error(
@@ -6298,8 +6297,6 @@ inline static void tex_aux_big_switch(int mode, int cmd)
 
         register_runner(italic_correction_cmd,  tex_aux_run_illegal_case,          tex_aux_run_text_italic_correction, tex_run_math_italic_correction);
         register_runner(math_char_number_cmd,   tex_aux_run_math_non_math,         tex_run_text_math_char_number,      tex_run_math_math_char_number);
-     // register_runner(math_char_given_cmd,    tex_aux_run_math_non_math,         tex_run_text_math_char_given,       tex_run_math_math_char_given);
-     // register_runner(math_char_xgiven_cmd,   tex_aux_run_math_non_math,         tex_run_text_math_char_xgiven,      tex_run_math_math_char_xgiven);
         register_runner(mathspec_cmd,           tex_aux_run_math_non_math,         tex_run_text_math_spec,             tex_run_math_math_spec);
         register_runner(vadjust_cmd,            tex_aux_run_illegal_case,          tex_run_vadjust,                    tex_run_vadjust);
 
diff --git a/source/luametatex/source/tex/texmath.c b/source/luametatex/source/tex/texmath.c
index 52e200657..3f452b1ff 100644
--- a/source/luametatex/source/tex/texmath.c
+++ b/source/luametatex/source/tex/texmath.c
@@ -1713,12 +1713,6 @@ static int tex_aux_scan_math(halfword target, halfword style, int usetextfont, h
             mval = tex_get_math_spec(cur_chr);
             dval = tex_get_math_dict(cur_chr);
             break;
-     // case math_char_given_cmd:
-     //     mval = tex_mathchar_from_integer(cur_chr, tex_mathcode);
-     //     break;
-     // case math_char_xgiven_cmd:
-     //     mval = tex_mathchar_from_integer(cur_chr, umath_mathcode);
-     //     break;
         case delimiter_number_cmd:
             switch (cur_chr) {
                 case math_delimiter_code:
@@ -1962,12 +1956,6 @@ int tex_scan_math_cmd_val(mathcodeval *mval, mathdictval *dval)
         tex_get_x_token();
     } while (cur_cmd == spacer_cmd);
     switch (cur_cmd) {
-     // case math_char_given_cmd:
-     //     *mval = tex_mathchar_from_integer(cur_chr, tex_mathcode);
-     //     break;
-     // case math_char_xgiven_cmd:
-     //     *mval = tex_mathchar_from_integer(cur_chr, umath_mathcode);
-     //     break;
         case mathspec_cmd:
             *mval = tex_get_math_spec(cur_chr);
             break;
@@ -2050,26 +2038,6 @@ void tex_run_math_math_char_number(void) {
     }
 }
 
-/*tex We build up an argument to |append_math_char|: */
-
-// void tex_run_text_math_char_given(void) {
-//     tex_aux_append_math_char_in_text(tex_mathchar_from_integer(cur_chr, tex_mathcode));
-// }
-//
-// void tex_run_math_math_char_given(void) {
-//     tex_aux_append_math_char(tex_mathchar_from_integer(cur_chr, tex_mathcode));
-// }
-
-/*tex We build up an argument to |append_math_char| the \LUATEX\ way: */
-
-// void tex_run_text_math_char_xgiven(void) {
-//     tex_aux_append_math_char_in_text(tex_mathchar_from_integer(cur_chr, umath_mathcode));
-// }
-//
-// void tex_run_math_math_char_xgiven(void) {
-//     tex_aux_append_math_char(tex_mathchar_from_integer(cur_chr, umath_mathcode));
-// }
-
 void tex_run_math_delimiter_number(void) {
     switch (cur_chr) {
         case math_delimiter_code:
diff --git a/source/luametatex/source/tex/texmath.h b/source/luametatex/source/tex/texmath.h
index e024a6ba2..cc7aa7e7f 100644
--- a/source/luametatex/source/tex/texmath.h
+++ b/source/luametatex/source/tex/texmath.h
@@ -642,10 +642,6 @@ extern void     tex_run_math_math_char_number  (void);
 extern void     tex_run_text_math_char_number  (void);
 extern void     tex_run_math_char_number       (void);
 extern void     tex_run_math_delimiter_number  (void);
-//     void     tex_run_math_math_char_given   (void);
-//     void     tex_run_text_math_char_given   (void);
-//     void     tex_run_math_math_char_xgiven  (void);
-//     void     tex_run_text_math_char_xgiven  (void);
 extern void     tex_run_math_equation_number   (void);
 extern void     tex_run_math_shift             (void);
 extern void     tex_run_math_italic_correction (void);
diff --git a/source/luametatex/source/tex/texprimitive.c b/source/luametatex/source/tex/texprimitive.c
index bbeea1bc0..42344af15 100644
--- a/source/luametatex/source/tex/texprimitive.c
+++ b/source/luametatex/source/tex/texprimitive.c
@@ -185,7 +185,7 @@ static int tex_aux_room_in_hash(void)
 inline static halfword tex_aux_compute_hash(const char *j, int l)
 {
     halfword h = (unsigned const char) j[0];
-    for (int k = 1; k < l; k++) {
+    for (unsigned k = 1; k < l; k++) {
         h = (h + h + (unsigned const char) j[k]) % hash_prime;
     }
     return h;
@@ -765,18 +765,6 @@ void tex_print_cmd_chr(singleword cmd, halfword chr)
             tex_print_str_esc("char");
             tex_print_qhex(chr);
             break;
-     // case math_char_given_cmd:
-     //     /*tex
-     //         Okay, it's better for old macro packages that mess with meaning to report a more
-     //         traditional value. A compromise. But, this might be dropped.
-     //     */
-     //     tex_print_str_esc("mathchar");
-     //     tex_show_mathcode_value(tex_mathchar_from_integer(chr, tex_mathcode), tex_mathcode);
-     //     break;
-     // case math_char_xgiven_cmd:
-     //     tex_print_str_esc("Umathchar");
-     //     tex_show_mathcode_value(tex_mathchar_from_integer(chr, umath_mathcode), umath_mathcode);
-     //     break;
         case lua_call_cmd:
             tex_aux_show_lua_call("luacall", chr);
             break;
diff --git a/source/luametatex/source/tex/texprinting.c b/source/luametatex/source/tex/texprinting.c
index 9a7c1d729..deabb4b72 100644
--- a/source/luametatex/source/tex/texprinting.c
+++ b/source/luametatex/source/tex/texprinting.c
@@ -217,9 +217,9 @@ void tex_print_char(int s)
 
     The first 256 entries above the 17th unicode plane are used for a special trick: when \TEX\ has
     to print items in that range, it will instead print the character that results from substracting
-    0x110000 from that value. This allows byte-oriented output to things like |\specials|.
-
-    This feature will disappear.
+    0x110000 from that value. This allows byte-oriented output to things like |\specials|. We dropped 
+    this feature because it was never used (we used it as part of experiments with \LUATEX). The old 
+    code branches can be found in the repository. 
 
 */
 
@@ -245,13 +245,6 @@ static void tex_aux_uprint(int s)
         tex_print_char(0xE0 + (s / 0x1000));
         tex_print_char(0x80 + ((s % 0x1000) / 0x40));
         tex_print_char(0x80 + ((s % 0x1000) % 0x40));
-    } else if (s >= 0x110000) {
-        int c = s - 0x110000;
-        if (c >= 256) {
-            tex_formatted_warning("print", "bad raw byte to print (c=%d), skipped",c);
-        } else {
-            tex_print_char(c);
-        }
     } else {
         tex_print_char(0xF0 + (s / 0x40000));
         tex_print_char(0x80 + ((s % 0x40000) / 0x1000));
@@ -260,24 +253,6 @@ static void tex_aux_uprint(int s)
     }
 }
 
-static void tex_aux_lprint(lstring *ss) {
-    /*tex current character code position */
-    unsigned char *j = ss->s;
-    unsigned char *l = j + ss->l;
-    while (j < l) {
-        /*tex We don't bother checking the last two bytes explicitly */
-        /* 0x110000 in utf=8: 0xF4 0x90 0x80 0x80 */
-        if ((j < l - 4) && (*j == 0xF4) && (*(j + 1) == 0x90)) {
-            int c = (*(j + 2) - 128) * 64 + (*(j + 3) - 128);
-            tex_print_char(c);
-            j = j + 4;
-        } else {
-            tex_print_char(*j);
-            ++j;
-        }
-    }
-}
-
 void tex_print_tex_str(int s)
 {
     if (s >= lmt_string_pool_state.string_pool_data.ptr) {
@@ -291,7 +266,10 @@ void tex_print_tex_str(int s)
     } else if (lmt_print_state.selector == new_string_selector_code) {
         tex_append_string(str_string(s), (unsigned) str_length(s));
     } else {
-        tex_aux_lprint(&str_lstring(s));
+        unsigned char *j = str_string(s);
+        for (unsigned i = 0; i < str_length(s); i++) {
+            tex_print_char(j[i]);
+        }
     }
 }
 
@@ -453,7 +431,7 @@ void tex_print_tex_str_esc(strnumber s)
 {
     /*tex Set variable |c| to the current escape character: */
     int c = escape_char_par;
-    if (c >= 0 && c < 0x110000) {
+    if (c >= 0) {
         tex_print_tex_str(c);
     }
     if (s) {
@@ -467,7 +445,7 @@ void tex_print_str_esc(const char *s)
 {
     /*tex Set variable |c| to the current escape character: */
     int c = escape_char_par;
-    if (c >= 0 && c < 0x110000) {
+    if (c >= 0) {
         tex_print_tex_str(c);
     }
     if (s) {
diff --git a/source/luametatex/source/tex/texscanning.c b/source/luametatex/source/tex/texscanning.c
index 5dd34b3b6..fec04356b 100644
--- a/source/luametatex/source/tex/texscanning.c
+++ b/source/luametatex/source/tex/texscanning.c
@@ -1088,18 +1088,13 @@ void tex_scan_something_simple(halfword cmd, halfword chr)
     switch (cmd) {
         /* begin of tex_aux_short_scan_something_internal */
         case char_given_cmd:
-     // case math_char_given_cmd:
-     // case math_char_xgiven_cmd:
             cur_val = chr;
             cur_val_level = int_val_level;
             break;
-
-case mathspec_cmd:
-    cur_val = (chr && node_subtype(chr) == tex_mathcode) ? math_spec_value(chr) : 0;
-    cur_val_level = int_val_level;
-    break;
-
-
+        case mathspec_cmd:
+            cur_val = (chr && node_subtype(chr) == tex_mathcode) ? math_spec_value(chr) : 0;
+            cur_val_level = int_val_level;
+            break;
         case iterator_value_cmd:
             cur_val = chr > 0x100000 ? - (chr - 0x100000) : chr;
             cur_val_level = int_val_level;
@@ -1254,8 +1249,6 @@ static halfword tex_aux_scan_something_internal(halfword cmd, halfword chr, int
     switch (cmd) {
         /* begin of tex_aux_short_scan_something_internal */
         case char_given_cmd:
-     // case math_char_given_cmd:
-     // case math_char_xgiven_cmd:
             cur_val = chr;
             cur_val_level = int_val_level;
             break;
diff --git a/source/luametatex/source/tex/texstringpool.c b/source/luametatex/source/tex/texstringpool.c
index 8367447da..fd93758b5 100644
--- a/source/luametatex/source/tex/texstringpool.c
+++ b/source/luametatex/source/tex/texstringpool.c
@@ -435,6 +435,7 @@ char *tex_makeclstring(int s, size_t *len)
     }
 }
 
+/* 
 char *tex_makecstring(int s)
 {
     if (s < cs_offset_value) {
@@ -443,6 +444,23 @@ char *tex_makecstring(int s)
         return lmt_memory_strdup((str_length(s) > 0) ? (const char *) str_string(s) : "");
     }
 }
+*/
+
+/*tex 
+    I might eventually replace this because in qite some calls we know that we knwo that we have
+    a pointer in string space. We can kin dof predict in what cases we are below |cs_offset_value|
+    anyway. 
+*/
+
+char *tex_makecstring(int s, int *allocated)
+{
+    *allocated = s < cs_offset_value;
+    if (*allocated) {
+        return (char *) aux_uni2str((unsigned) s);
+    } else {
+        return str_length(s) > 0 ? (char *) str_string(s) : "";
+    }
+}
 
 /*tex
 
diff --git a/source/luametatex/source/tex/texstringpool.h b/source/luametatex/source/tex/texstringpool.h
index b3924a0fe..7302e7fb4 100644
--- a/source/luametatex/source/tex/texstringpool.h
+++ b/source/luametatex/source/tex/texstringpool.h
@@ -21,7 +21,7 @@ typedef struct lstring {
         unsigned char *s;
         const    char *c;
     };
-    size_t         l; /* could be int, but this way we padd */
+    size_t l; /* could be int, but this way we padd */
 } lstring;
 
 typedef struct string_pool_info {
@@ -76,35 +76,37 @@ extern string_pool_info lmt_string_pool_state;
 
 /*tex Forget the last character in the pool. */
 
-inline void        tex_flush_char(void)       { --lmt_string_pool_state.string_temp_top; }
-
-extern strnumber   tex_make_string            (void);
-extern strnumber   tex_push_string            (const unsigned char *s, int l);
-extern char       *tex_take_string            (int *len);
-extern int         tex_str_eq_buf             (strnumber s, int k, int n);
-extern int         tex_str_eq_str             (strnumber s, strnumber t);
-extern int         tex_str_eq_cstr            (strnumber s, const char *, size_t);
-extern int         tex_get_strings_started    (void);
-extern void        tex_reset_cur_string       (void);
-/*     strnumber   tex_search_string          (strnumber search); */
-/*     int         tex_used_strings           (void); */
-extern strnumber   tex_maketexstring          (const char *s);
-extern strnumber   tex_maketexlstring         (const char *s, size_t);
-extern void        tex_append_char            (unsigned char c);
-extern void        tex_append_string          (const unsigned char *s, unsigned l);
-extern char       *tex_makecstring            (int s);
-extern char       *tex_makeclstring           (int s, size_t *len);
-extern void        tex_dump_string_pool       (dumpstream f);
-extern void        tex_undump_string_pool     (dumpstream f);
-extern void        tex_initialize_string_pool (void);
-extern void        tex_initialize_string_mem  (void);
-extern void        tex_flush_str              (strnumber s);
-extern strnumber   tex_save_cur_string        (void);
-extern void        tex_restore_cur_string     (strnumber u);
-
-/*     void        tex_increment_pool_string  (int n); */
-/*     void        tex_decrement_pool_string  (int n); */
-
-extern void        tex_compact_string_pool    (void);
+inline void       tex_flush_char(void)       { --lmt_string_pool_state.string_temp_top; }
+
+extern strnumber  tex_make_string            (void);
+extern strnumber  tex_push_string            (const unsigned char *s, int l);
+extern char      *tex_take_string            (int *len);
+extern int        tex_str_eq_buf             (strnumber s, int k, int n);
+extern int        tex_str_eq_str             (strnumber s, strnumber t);
+extern int        tex_str_eq_cstr            (strnumber s, const char *, size_t);
+extern int        tex_get_strings_started    (void);
+extern void       tex_reset_cur_string       (void);
+/*     strnumber  tex_search_string          (strnumber search); */
+/*     int        tex_used_strings           (void); */
+extern strnumber  tex_maketexstring          (const char *s);
+extern strnumber  tex_maketexlstring         (const char *s, size_t);
+extern void       tex_append_char            (unsigned char c);
+extern void       tex_append_string          (const unsigned char *s, unsigned l);
+extern char      *tex_makecstring            (int s, int *allocated);
+extern char      *tex_makeclstring           (int s, size_t *len);
+extern void       tex_dump_string_pool       (dumpstream f);
+extern void       tex_undump_string_pool     (dumpstream f);
+extern void       tex_initialize_string_pool (void);
+extern void       tex_initialize_string_mem  (void);
+extern void       tex_flush_str              (strnumber s);
+extern strnumber  tex_save_cur_string        (void);
+extern void       tex_restore_cur_string     (strnumber u);
+
+/*     void       tex_increment_pool_string  (int n); */
+/*     void       tex_decrement_pool_string  (int n); */
+
+extern void       tex_compact_string_pool    (void);
+
+inline char      *tex_to_cstring             (int s) { return str_length(s) > 0 ? (char *) str_string(s) : ""; }
 
 # endif
diff --git a/source/luametatex/source/tex/textoken.c b/source/luametatex/source/tex/textoken.c
index 0bb0f3e45..93bf3913a 100644
--- a/source/luametatex/source/tex/textoken.c
+++ b/source/luametatex/source/tex/textoken.c
@@ -2820,7 +2820,7 @@ void tex_run_convert_tokens(halfword code)
              /* halfword result = tex_scan_toks_expand(0, NULL, code == lua_token_string_code); */
                 lmt_token_state.in_lua_escape = 1;
                 escape_char_par = '\\';
-                str.s = (unsigned char *) tex_tokenlist_to_tstring(result, 0, &length, 0, 0, 0);
+                str.s = (unsigned char *) tex_tokenlist_to_tstring(result, 0, &length, 0, 0, 0, 0);
                 str.l = (unsigned) length;
                 lmt_token_state.in_lua_escape = saved_in_lua_escape;
                 escape_char_par = saved_escape_char;
@@ -3034,8 +3034,6 @@ static void tex_aux_append_uchar_to_buffer(int s)
         lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0xE0 +  (s / 0x1000));
         lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x1000) / 0x40));
         lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x1000) % 0x40));
-    } else if (s >= 0x110000) {
-        lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (s - 0x11000);
     } else {
         lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0xF0 +   (s / 0x40000));
         lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 +  ((s % 0x40000) / 0x1000));
@@ -3078,7 +3076,7 @@ static void tex_aux_append_esc_to_buffer(const char *s)
 
 /* make two versions: macro and not */
 
-char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamble, int nospace, int strip)
+char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamble, int nospace, int strip, int wipe)
 {
     if (pp) {
         /*tex We need to go beyond the reference. */
@@ -3100,6 +3098,8 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
             int min = 0;
             int max = lmt_token_memory_state.tokens_data.top;
             int skip = 0;
+            int tail = p; 
+            int count = 0;
             if (skippreamble) {
                 skip = get_token_parameters(pp);
             }
@@ -3210,7 +3210,8 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
                             if (txt  < 0 || txt  >= lmt_string_pool_state.string_pool_data.ptr) {
                                 tex_aux_append_str_to_buffer(error_string_nonexistent(36));
                             } else {
-                                char *sh = tex_makecstring(txt);
+                                int allocated = 0;
+                                char *sh = tex_makecstring(txt, &allocated);
                                 char *s = sh;
                                 if (tex_is_active_cs(txt)) {
                                     s = s + 3;
@@ -3219,7 +3220,7 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
                                         s++;
                                     }
                                 } else {
-                                    if (e >= 0 && e < 0x110000) {
+                                    if (e >= 0) {
                                         tex_aux_append_uchar_to_buffer(e);
                                     }
                                     while (*s) {
@@ -3230,10 +3231,14 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
                                         tex_aux_append_char_to_buffer(' ');
                                     }
                                 }
-                                lmt_memory_free(sh);
+                                if (allocated) {
+                                    lmt_memory_free(sh);    
+                                }
                             }
                         }
                     }
+                    tail = p; 
+                    ++count;
                     p = token_link(p);
                 }
             }
@@ -3251,7 +3256,14 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
             if (siz) {
                 *siz = lmt_token_state.bufloc;
             }
+            if (wipe) { 
+                tex_flush_token_list_head_tail(pp, tail, count);
+            }
             return lmt_token_state.buffer;
+        } else { 
+            if (wipe) {
+                 tex_put_available_token(pp);
+            }
         }
     }
     if (siz) {
diff --git a/source/luametatex/source/tex/textoken.h b/source/luametatex/source/tex/textoken.h
index 1996f351c..ad67dfcb5 100644
--- a/source/luametatex/source/tex/textoken.h
+++ b/source/luametatex/source/tex/textoken.h
@@ -366,8 +366,7 @@ extern void       tex_run_combine_the_toks        (void);
 extern void       tex_run_convert_tokens          (halfword code);
 extern strnumber  tex_the_convert_string          (halfword c, int i);
 extern strnumber  tex_tokens_to_string            (halfword p);
-/*     char      *tex_tokenlist_to_cstring        (int p, int inhibit_par, int *siz); */
-extern char      *tex_tokenlist_to_tstring        (int p, int inhibit_par, int *siz, int skip, int nospace, int strip);
+extern char      *tex_tokenlist_to_tstring        (int p, int inhibit_par, int *siz, int skip, int nospace, int strip, int wipe);
 
 extern halfword   tex_get_tex_dimen_register      (int j, int internal);
 extern halfword   tex_get_tex_skip_register       (int j, int internal);
@@ -391,9 +390,6 @@ extern halfword   tex_copy_token_list             (halfword h, halfword *t);
 
 extern halfword   tex_parse_str_to_tok            (halfword head, halfword *tail, halfword ct, const char *str, size_t lstr, int option);
 
-inline int tex_valid_token(int t)
-{
-    return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top));
-}
+inline int        tex_valid_token                 (int t) { return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top)); }
 
 # endif
diff --git a/source/luametatex/source/utilities/auxunistring.c b/source/luametatex/source/utilities/auxunistring.c
index e95854a93..30dc518e6 100644
--- a/source/luametatex/source/utilities/auxunistring.c
+++ b/source/luametatex/source/utilities/auxunistring.c
@@ -52,15 +52,12 @@ unsigned char *aux_uni2str(unsigned unic)
             buf[0] = (unsigned char) (0xc0 | (unic >> 6));
             buf[1] = (unsigned char) (0x80 | (unic & 0x3f));
             buf[2] = '\0';
-        } else if (unic >= 0x110000) {
-            buf[0] = (unsigned char) (unic - 0x110000);
-            buf[1] = '\0';
         } else if (unic < 0x10000) {
             buf[0] = (unsigned char) (0xe0 | (unic >> 12));
             buf[1] = (unsigned char) (0x80 | ((unic >> 6) & 0x3f));
             buf[2] = (unsigned char) (0x80 | (unic & 0x3f));
             buf[3] = '\0';
-        } else {
+        } else if (unic < 0x110000) {
             unic -= 0x10000;
             int u = (int) (((unic & 0xf0000) >> 16) + 1);
             buf[0] = (unsigned char) (0xf0 | (u >> 2));