summaryrefslogtreecommitdiff
path: root/source
diff options
context:
space:
mode:
Diffstat (limited to 'source')
-rw-r--r--source/luametatex/CMakeLists.txt4
-rw-r--r--source/luametatex/source/lua/lmtinterface.h2
-rw-r--r--source/luametatex/source/lua/lmtnodelib.c4
-rw-r--r--source/luametatex/source/lua/lmttexlib.c69
-rw-r--r--source/luametatex/source/lua/lmttokenlib.c110
-rw-r--r--source/luametatex/source/lua/lmttokenlib.h2
-rw-r--r--source/luametatex/source/luacore/lua54/src/ltable.c4
-rw-r--r--source/luametatex/source/luametatex.h6
-rw-r--r--source/luametatex/source/tex/texequivalents.c2
-rw-r--r--source/luametatex/source/tex/texexpand.c2
-rw-r--r--source/luametatex/source/tex/texfileio.c4
-rw-r--r--source/luametatex/source/tex/texlanguage.c4
-rw-r--r--source/luametatex/source/tex/texlinebreak.c4
-rw-r--r--source/luametatex/source/tex/texmaincontrol.c23
-rw-r--r--source/luametatex/source/tex/texmath.c32
-rw-r--r--source/luametatex/source/tex/texmath.h4
-rw-r--r--source/luametatex/source/tex/texprimitive.c14
-rw-r--r--source/luametatex/source/tex/texprinting.c40
-rw-r--r--source/luametatex/source/tex/texscanning.c15
-rw-r--r--source/luametatex/source/tex/texstringpool.c18
-rw-r--r--source/luametatex/source/tex/texstringpool.h64
-rw-r--r--source/luametatex/source/tex/textoken.c26
-rw-r--r--source/luametatex/source/tex/textoken.h8
-rw-r--r--source/luametatex/source/utilities/auxunistring.c5
24 files changed, 196 insertions, 270 deletions
diff --git a/source/luametatex/CMakeLists.txt b/source/luametatex/CMakeLists.txt
index d972a0312..a5021595f 100644
--- a/source/luametatex/CMakeLists.txt
+++ b/source/luametatex/CMakeLists.txt
@@ -205,8 +205,8 @@ if (DEFINED LMT_OPTIMIZE)
# set_property(TARGET luametatex PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
#
# mingw64: 2865664, nocona: 2819584, lto: 2835968 (around 1% gain on manual)
-
- set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+ #
+ set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
#
else()
# No message needed, just accept the fact.
diff --git a/source/luametatex/source/lua/lmtinterface.h b/source/luametatex/source/lua/lmtinterface.h
index 3fd87db2f..e761252a8 100644
--- a/source/luametatex/source/lua/lmtinterface.h
+++ b/source/luametatex/source/lua/lmtinterface.h
@@ -865,9 +865,7 @@ make_lua_key(L, match);\
make_lua_key(L, math);\
make_lua_key(L, mathspec);\
make_lua_key(L, math_accent);\
-make_lua_key(L, math_char_given);\
make_lua_key(L, math_char_number);\
-make_lua_key(L, math_char_xgiven);\
make_lua_key(L, math_choice);\
make_lua_key(L, math_component);\
make_lua_key(L, math_fence);\
diff --git a/source/luametatex/source/lua/lmtnodelib.c b/source/luametatex/source/lua/lmtnodelib.c
index 7d9d7bcbe..eb29611e2 100644
--- a/source/luametatex/source/lua/lmtnodelib.c
+++ b/source/luametatex/source/lua/lmtnodelib.c
@@ -2997,7 +2997,7 @@ static int nodelib_direct_getdata(lua_State *L)
}
case mark_node:
if (lua_toboolean(L, 2)) {
- lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0);
+ lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0, 0);
} else {
lmt_token_list_to_lua(L, mark_ptr(n));
}
@@ -6555,7 +6555,7 @@ static int nodelib_common_getfield(lua_State *L, int direct, halfword n)
lua_pushinteger(L, mark_index(n));
} else if (lua_key_eq(s, data) || lua_key_eq(s, mark)) {
if (lua_toboolean(L, 3)) {
- lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0);
+ lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0, 0);
} else {
lmt_token_list_to_lua(L, mark_ptr(n));
}
diff --git a/source/luametatex/source/lua/lmttexlib.c b/source/luametatex/source/lua/lmttexlib.c
index d44127330..afafb068b 100644
--- a/source/luametatex/source/lua/lmttexlib.c
+++ b/source/luametatex/source/lua/lmttexlib.c
@@ -1715,10 +1715,9 @@ static int texlib_gettoks(lua_State *L)
if (lua_toboolean(L, slot)) {
lmt_token_register_to_lua(L, state ? toks_parameter(index) : toks_register(index));
} else {
+
strnumber value = tex_get_tex_toks_register(index, state);
- char *s = tex_makecstring(value);
- lua_pushstring(L, s);
- lmt_memory_free(s);
+ lua_pushstring(L, tex_to_cstring(value));
tex_flush_str(value);
}
} else {
@@ -1754,7 +1753,7 @@ static int texlib_getmark(lua_State *L)
if (num >= 0 && num <= lmt_mark_state.mark_data.ptr) {
halfword ptr = tex_get_some_mark(mrk, num);
if (ptr) {
- char *str = tex_tokenlist_to_tstring(ptr, 1, NULL, 0, 0, 0);
+ char *str = tex_tokenlist_to_tstring(ptr, 1, NULL, 0, 0, 0, 0);
if (str) {
lua_pushstring(L, str);
} else {
@@ -1783,7 +1782,6 @@ int lmt_get_box_id(lua_State *L, int i, int report)
int cmd = eq_type(cs);
switch (cmd) {
case char_given_cmd:
- // case math_char_given_cmd:
case integer_cmd:
index = eq_value(cs);
break;
@@ -2597,7 +2595,6 @@ static int texlib_newindex(lua_State *L)
static int texlib_aux_convert(lua_State *L, int cur_code)
{
int i = -1;
- char *str = NULL;
switch (cur_code) {
/* ignored (yet) */
case insert_progress_code: /* arg <register int> */
@@ -2632,19 +2629,22 @@ static int texlib_aux_convert(lua_State *L, int cur_code)
if (cur_code < 32) {
int texstr = tex_the_convert_string(cur_code, i);
if (texstr) {
- str = tex_makecstring(texstr);
+ lua_pushstring(L, tex_to_cstring(texstr));
tex_flush_str(texstr);
+
+ // int allocated = 0;
+ // char *str = tex_makecstring(texstr, &allocated);
+ // lua_pushstring(L, str);
+ // if (allocated) {
+ // lmt_memory_free(str);
+ // }
+ // tex_flush_str(texstr);
+ return 1;
}
}
break;
}
- /* end */
- if (str) {
- lua_pushstring(L, str);
- lmt_memory_free(str);
- } else {
- lua_pushnil(L);
- }
+ lua_pushnil(L);
return 1;
}
@@ -2687,10 +2687,9 @@ static int texlib_aux_scan_internal(lua_State *L, int cmd, int code, int values)
default:
{
int texstr = tex_the_scanned_result();
- char *str = tex_makecstring(texstr);
+ char *str = tex_to_cstring(texstr);
if (str) {
lua_pushstring(L, str);
- lmt_memory_free(str);
} else {
lua_pushnil(L);
}
@@ -3013,7 +3012,6 @@ static int texlib_get_internal(lua_State *L, int index, int all)
case set_auxiliary_cmd:
case set_page_property_cmd:
case char_given_cmd:
- // case math_char_given_cmd:
case integer_cmd:
case dimension_cmd:
case gluespec_cmd:
@@ -3407,26 +3405,22 @@ static int texlib_hashtokens(lua_State *L)
int nx = 0;
int all = lua_toboolean(L, 1);
lua_createtable(L, hash_size, 0);
+ /* todo: check active characters as these have three bogus bytes in front */
if (all) {
while (cs <= hash_size) {
- /* because strings never get freed we can as well directly access |s|. */
strnumber s = cs_text(cs);
if (s > 0) {
halfword n = cs_next(cs);
- char *ss = tex_makecstring(s);
if (n) {
int mt = 0;
lua_createtable(L, 2, 0);
- lua_pushstring(L, ss);
- lmt_memory_free(ss);
+ lua_pushstring(L, tex_to_cstring(s));
++nt;
lua_rawseti(L, -2, ++mt);
while (n) {
s = cs_text(n);
if (s) {
- ss = tex_makecstring(s);
- lua_pushstring(L, ss);
- lmt_memory_free(ss);
+ lua_pushstring(L, tex_to_cstring(s));
lua_rawseti(L, -2, ++mt);
++nt;
++nx;
@@ -3434,8 +3428,7 @@ static int texlib_hashtokens(lua_State *L)
n = cs_next(n);
}
} else {
- lua_pushstring(L, ss);
- lmt_memory_free(ss);
+ lua_pushstring(L, tex_to_cstring(s));
++nt;
}
} else {
@@ -3449,16 +3442,12 @@ static int texlib_hashtokens(lua_State *L)
strnumber s = cs_text(cs);
if (s > 0) {
halfword n = cs_next(cs);
- char *ss = tex_makecstring(s);
- lua_pushstring(L, ss);
- lmt_memory_free(ss);
+ lua_pushstring(L, tex_to_cstring(s));
lua_rawseti(L, -2, ++nt);
while (n) {
s = cs_text(n);
if (s) {
- ss = tex_makecstring(s);
- lua_pushstring(L, ss);
- lmt_memory_free(ss);
+ lua_pushstring(L, tex_to_cstring(s));
lua_rawseti(L, -2, ++nt);
++nx;
}
@@ -3482,9 +3471,7 @@ static int texlib_primitives(lua_State *L)
while (cs < prim_size) {
strnumber s = get_prim_text(cs);
if (s > 0 && (get_prim_origin(cs) != no_command)) {
- char *ss = tex_makecstring(s);
- lua_pushstring(L, ss);
- lmt_memory_free(ss);
+ lua_pushstring(L, tex_to_cstring(s));
lua_rawseti(L, -2, ++nt);
}
cs++;
@@ -3518,9 +3505,7 @@ static int texlib_extraprimitives(lua_State *L)
while (cs < prim_size) {
strnumber s = get_prim_text(cs);
if (s > 0 && (get_prim_origin(cs) & mask)) {
- char *ss = tex_makecstring(s);
- lua_pushstring(L, ss);
- lmt_memory_free(ss);
+ lua_pushstring(L, tex_to_cstring(s));
lua_rawseti(L, -2, ++nt);
}
cs++;
@@ -3531,7 +3516,7 @@ static int texlib_extraprimitives(lua_State *L)
static void texlib_aux_enableprimitive(const char *pre, size_t prel, const char *prm)
{
strnumber s = tex_maketexstring(prm);
- halfword prm_val = tex_prim_lookup(s);
+ halfword prm_val = tex_prim_lookup(s); /* todo: no need for tex string */
tex_flush_str(s);
if (prm_val != undefined_primitive && get_prim_origin(prm_val) != no_command) {
char *newprm;
@@ -3594,10 +3579,8 @@ static int texlib_enableprimitives(lua_State *L)
for (int cs = 0; cs < prim_size; cs++) {
strnumber s = get_prim_text(cs);
if (s > 0) {
- /* there is actually no need to copy */
- char *prm = tex_makecstring(s);
+ char *prm = tex_to_cstring(s);
texlib_aux_enableprimitive(pre, lpre, prm);
- lmt_memory_free(prm);
}
}
}
@@ -4561,8 +4544,6 @@ static int texlib_mathchardef(lua_State *L)
d.index = lmt_optinteger(L, 8, 0);
if (class_in_range(m.class_value) && family_in_range(m.family_value) && character_in_range(m.character_value)) {
tex_define(flags, cs, mathspec_cmd, tex_new_math_dict_spec(d, m, umath_mathcode));
- // halfword code = math_packed_character(m.class_value, m.family_value, m.character_value);
- // tex_define(flags, cs, (quarterword) math_char_xgiven_cmd, code);
} else {
tex_normal_error("lua", "mathchardef needs proper class, family and character codes");
}
diff --git a/source/luametatex/source/lua/lmttokenlib.c b/source/luametatex/source/lua/lmttokenlib.c
index ac69241ba..feac7cd46 100644
--- a/source/luametatex/source/lua/lmttokenlib.c
+++ b/source/luametatex/source/lua/lmttokenlib.c
@@ -156,8 +156,6 @@ void lmt_tokenlib_initialize(void)
lmt_interface.command_names[math_shift_cs_cmd] = (command_item) { .id = math_shift_cs_cmd, .lua = lua_key_index(math_shift_cs), .name = lua_key(math_shift_cs), .kind = regular_command_item, .min = 0, .max = last_math_shift_cs_code, .base = 0, .fixedvalue = 0 }; /* a bit too tolerant */
lmt_interface.command_names[end_cs_name_cmd] = (command_item) { .id = end_cs_name_cmd, .lua = lua_key_index(end_cs_name), .name = lua_key(end_cs_name), .kind = regular_command_item, .min = 0, .max = 0, .base = 0, .fixedvalue = 0 };
lmt_interface.command_names[char_given_cmd] = (command_item) { .id = char_given_cmd, .lua = lua_key_index(char_given), .name = lua_key(char_given), .kind = character_command_item, .min = 0, .max = max_character_code, .base = 0, .fixedvalue = 0 };
- // lmt_interface.command_names[math_char_given_cmd] = (command_item) { .id = math_char_given_cmd, .lua = lua_key_index(math_char_given), .name = lua_key(math_char_given), .kind = character_command_item, .min = 0, .max = max_character_code, .base = 0, .fixedvalue = 0 };
- // lmt_interface.command_names[math_char_xgiven_cmd] = (command_item) { .id = math_char_xgiven_cmd, .lua = lua_key_index(math_char_xgiven), .name = lua_key(math_char_xgiven), .kind = character_command_item, .min = 0, .max = max_character_code, .base = 0, .fixedvalue = 0 };
lmt_interface.command_names[some_item_cmd] = (command_item) { .id = some_item_cmd, .lua = lua_key_index(some_item), .name = lua_key(some_item), .kind = regular_command_item, .min = 0, .max = last_some_item_code, .base = 0, .fixedvalue = 0 };
lmt_interface.command_names[internal_toks_cmd] = (command_item) { .id = internal_toks_cmd, .lua = lua_key_index(internal_toks), .name = lua_key(internal_toks), .kind = internal_command_item, .min = first_toks_code, .max = last_toks_code, .base = internal_toks_base, .fixedvalue = 0 };
lmt_interface.command_names[register_toks_cmd] = (command_item) { .id = register_toks_cmd, .lua = lua_key_index(register_toks), .name = lua_key(register_toks), .kind = register_command_item, .min = 0, .max = biggest_reg, .base = register_toks_base, .fixedvalue = 0 };
@@ -433,10 +431,10 @@ void lmt_token_list_to_lua(lua_State *L, halfword p)
}
}
-void lmt_token_list_to_luastring(lua_State *L, halfword p, int nospace, int strip)
+void lmt_token_list_to_luastring(lua_State *L, halfword p, int nospace, int strip, int wipe)
{
int l;
- char *s = tex_tokenlist_to_tstring(p, 1, &l, 0, nospace, strip); /* nasty ... preambles or not, could have been endmatchtoken */
+ char *s = tex_tokenlist_to_tstring(p, 1, &l, 0, nospace, strip, wipe); /* nasty ... preambles or not, could have been endmatchtoken */
if (l) {
lua_pushlstring(L, s, (size_t) l);
} else {
@@ -511,16 +509,19 @@ halfword lmt_token_code_from_lua(lua_State *L, int slot)
# define is_active_string(s) (strlen(s) > 3 && *s == 0xEF && *(s+1) == 0xBF && *(s+2) == 0xBF)
-static unsigned char *tokenlib_aux_get_cs_text(int cs)
+static unsigned char *tokenlib_aux_get_cs_text(int cs, int *allocated)
{
if (cs == null_cs) {
- return (unsigned char *) lmt_memory_strdup("\\csname\\endcsname");
+ return (unsigned char *) "\\csname\\endcsname";
} else if ((cs_text(cs) < 0) || (cs_text(cs) >= lmt_string_pool_state.string_pool_data.ptr)) {
- return (unsigned char *) lmt_memory_strdup("");
- } else if (tex_is_active_cs(cs_text(cs))) {
- return (unsigned char *) tex_makecstring(cs_text(cs));
+ return (unsigned char *) "";
+ // } else {
+ // return (unsigned char *) tex_makecstring(cs_text(cs));
+ } else if (cs_text(cs) < cs_offset_value) {
+ *allocated = 1;
+ return (unsigned char *) aux_uni2str((unsigned) cs_text(cs));
} else {
- return (unsigned char *) tex_makecstring(cs_text(cs));
+ return (unsigned char *) (str_length(cs_text(cs)) > 0 ? (unsigned char *) str_string(cs_text(cs)) : (unsigned char *) "");
}
}
@@ -871,14 +872,17 @@ static int tokenlib_scan_csname(lua_State *L)
}
t = cur_cs ? cs_token_flag + cur_cs : token_val (cur_cmd, cur_chr);
if (t >= cs_token_flag) {
- unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag);
+ int allocated = 0;
+ unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag, &allocated);
if (s) {
if (tex_is_active_cs(cs_text(t - cs_token_flag))) {
lua_pushstring(L, (char *) (s + 3));
} else {
lua_pushstring(L, (char *) s);
}
- lmt_memory_free(s);
+ if (allocated) {
+ lmt_memory_free(s);
+ }
} else {
lua_pushnil(L);
}
@@ -1408,8 +1412,7 @@ static int tokenlib_scan_string(lua_State *L)
{
halfword defref = lmt_input_state.def_ref;
halfword result = tex_scan_toks_expand(1, NULL, 0);
- lmt_token_list_to_luastring(L, result, 0, 0);
- tex_flush_token_list(result);
+ lmt_token_list_to_luastring(L, result, 0, 0, 1);
lmt_input_state.def_ref = defref;
break;
}
@@ -1421,8 +1424,7 @@ static int tokenlib_scan_string(lua_State *L)
case tolerant_semi_protected_call_cmd:
{
halfword t = token_link(cur_chr);
- lmt_token_list_to_luastring(L, t, 0, 0);
- tex_flush_token_list(t);
+ lmt_token_list_to_luastring(L, t, 0, 0, 1);
break;
}
case letter_cmd:
@@ -1463,8 +1465,7 @@ static int tokenlib_scan_argument(lua_State *L)
halfword defref = lmt_input_state.def_ref;
int expand = lua_type(L, 1) == LUA_TBOOLEAN ? lua_toboolean(L, 1) : 1;
halfword result = expand ? tex_scan_toks_expand(1, NULL, 0) : tex_scan_toks_normal(1, NULL);
- lmt_token_list_to_luastring(L, result, 0, 0);
- tex_flush_token_list(result);
+ lmt_token_list_to_luastring(L, result, 0, 0, 1);
lmt_input_state.def_ref = defref;
break;
}
@@ -1485,8 +1486,7 @@ static int tokenlib_scan_argument(lua_State *L)
tex_back_input(cur_tok);
result = tex_scan_toks_expand(1, NULL, 0);
}
- lmt_token_list_to_luastring(L, result, 0, 0);
- tex_flush_token_list(result);
+ lmt_token_list_to_luastring(L, result, 0, 0, 1);
lmt_input_state.def_ref = defref;
break;
}
@@ -1685,8 +1685,7 @@ static int tokenlib_scan_delimited(lua_State *L)
FINALYDONE:
/* */
lmt_input_state.def_ref = defref;
- lmt_token_list_to_luastring(L, result, 0, 0);
- tex_flush_token_list(result);
+ lmt_token_list_to_luastring(L, result, 0, 0, 1);
} else {
tex_back_input(cur_tok);
lua_pushnil(L);
@@ -1781,8 +1780,7 @@ static int tokenlib_grab_until(lua_State *L)
}
tokenlib_aux_unsave_tex_scanner(texstate);
lmt_input_state.def_ref = defref;
- lmt_token_list_to_luastring(L, result, nospace, strip);
- tex_flush_token_list(result);
+ lmt_token_list_to_luastring(L, result, nospace, strip, 1);
} else {
lua_pushnil(L);
}
@@ -2013,8 +2011,7 @@ static int tokenlib_scan_value(lua_State *L)
halfword defref = lmt_input_state.def_ref;
result = tex_scan_toks_expand(1, NULL, 0);
lmt_input_state.def_ref = defref;
- lmt_token_list_to_luastring(L, result, 0, 0);
- tex_flush_token_list(result);
+ lmt_token_list_to_luastring(L, result, 0, 0, 1);
}
break;
/*
@@ -2052,8 +2049,7 @@ static int tokenlib_scan_value(lua_State *L)
halfword defref = lmt_input_state.def_ref;
result = tex_scan_toks_expand(1, NULL, 0);
lmt_input_state.def_ref = defref;
- lmt_token_list_to_luastring(L, result, 0, 0);
- tex_flush_token_list(result);
+ lmt_token_list_to_luastring(L, result, 0, 0, 1);
luaL_addchar(&b, '{');
luaL_addvalue(&b);
luaL_addchar(&b, '}');
@@ -2069,7 +2065,8 @@ static int tokenlib_scan_value(lua_State *L)
/*tex We need to add a space. */
halfword t = cur_cs ? cs_token_flag + cur_cs : token_val(cur_cmd, cur_chr);
if (t >= cs_token_flag) {
- unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag);
+ int allocated = 0;
+ unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag, &allocated);
if (s) {
if (tex_is_active_cs(cs_text(t - cs_token_flag))) {
lua_pushstring(L, (char *) (s + 3));
@@ -2080,7 +2077,9 @@ static int tokenlib_scan_value(lua_State *L)
luaL_addvalue(&b);
luaL_addchar(&b, ' ');
}
- lmt_memory_free(s);
+ if (allocated) {
+ lmt_memory_free(s);
+ }
}
}
}
@@ -2377,7 +2376,7 @@ static int tokenlib_getprimitives(lua_State *L)
while (cs < prim_size) {
strnumber s = get_prim_text(cs);
if (s > 0 && (get_prim_origin(cs) != no_command)) {
- char *ss = tex_makecstring(s);
+ char *ss = tex_to_cstring(s);
int cmd = prim_eq_type(cs);
int chr = prim_equiv(cs);
if (! raw) {
@@ -2393,7 +2392,6 @@ static int tokenlib_getprimitives(lua_State *L)
lua_pushinteger(L, prim_origin(cs));
lua_rawseti(L, -2, 4);
lua_rawseti(L, -2, ++nt);
- lmt_memory_free(ss);
}
cs++;
}
@@ -2508,14 +2506,17 @@ inline static int tokenlib_get_csname(lua_State *L)
lua_token *n = tokenlib_aux_check_istoken(L, 1);
halfword tok = token_info(n->token);
if (tok >= cs_token_flag) {
- unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag);
+ int allocated = 0;
+ unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag, &allocated);
if (s) {
if (tex_is_active_cs(cs_text(tok - cs_token_flag))) {
lua_pushstring(L, (char *) (s + 3));
} else {
lua_pushstring(L, (char *) s);
}
- lmt_memory_free(s);
+ if (allocated) {
+ lmt_memory_free(s);
+ }
return 1;
}
}
@@ -2544,10 +2545,13 @@ inline static int tokenlib_get_active(lua_State *L)
halfword tok = token_info(n->token);
int result = 0;
if (tok >= cs_token_flag) {
- unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag);
+ int allocated = 0;
+ unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag, &allocated);
if (s) {
result = tex_is_active_cs(cs_text(tok - cs_token_flag));
- lmt_memory_free(s);
+ if (allocated) {
+ lmt_memory_free(s);
+ }
}
}
lua_pushboolean(L, result);
@@ -2753,7 +2757,8 @@ static int tokenlib_get_fields(lua_State *L)
lua_createtable(L, 0, onlyflags ? 0 : 5);
if (tok >= cs_token_flag) {
int t = tok - cs_token_flag;
- unsigned char* str = tokenlib_aux_get_cs_text(t);
+ int allocated = 0;
+ unsigned char* str = tokenlib_aux_get_cs_text(t, &allocated);
if (str) {
if (! onlyflags) {
lua_push_key(csname);
@@ -2767,7 +2772,9 @@ static int tokenlib_get_fields(lua_State *L)
}
lua_rawset(L, -3);
}
- lmt_memory_free(str);
+ if (allocated) {
+ lmt_memory_free(str);
+ }
}
cmd = eq_type(t);
chr = eq_value(t);
@@ -2836,9 +2843,12 @@ static int tokenlib_tostring(lua_State* L)
unsigned char* csp = NULL;
const char* cmn = NULL;
if (tok >= cs_token_flag) {
+ int allocated = 0;
tok -= cs_token_flag;
- csn = tokenlib_aux_get_cs_text(tok);
- csp = csn;
+ csn = tokenlib_aux_get_cs_text(tok, &allocated);
+ if (allocated) {
+ csp = csn;
+ }
if (csn && tex_is_active_cs(cs_text(tok))) {
csn += 3;
}
@@ -2974,7 +2984,7 @@ static int tokenlib_get_meaning(lua_State *L)
lmt_token_register_to_lua(L, chr);
}
} else {
- char *str = tex_tokenlist_to_tstring(chr, 1, NULL, 0, 0, 0);
+ char *str = tex_tokenlist_to_tstring(chr, 1, NULL, 0, 0, 0, 0);
lua_pushstring(L, str ? str : "");
}
return 1;
@@ -3041,10 +3051,9 @@ static int tokenlib_get_macro(lua_State *L)
char *str = NULL;
if (lua_toboolean(L, 2)) {
tokenlib_aux_expand_macros_in_tokenlist(chr); // todo: use return value instead of def_ref
- str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0);
- tex_flush_token_list(lmt_input_state.def_ref);
+ str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0, 1);
} else {
- str = tex_tokenlist_to_tstring(chr, 1, NULL, 1, 0, 0);
+ str = tex_tokenlist_to_tstring(chr, 1, NULL, 1, 0, 0, 0);
}
lua_pushstring(L, str ? str : "");
return 1;
@@ -3091,8 +3100,7 @@ char *lmt_get_expansion(halfword head, int *len)
halfword ref = get_reference_token();
set_token_link(ref, head);
tokenlib_aux_expand_macros_in_tokenlist(ref); // todo: use return value instead of def_ref
- str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, len, 0, 0, 0);
- tex_flush_token_list(lmt_input_state.def_ref);
+ str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, len, 0, 0, 0, 1);
tex_flush_token_list(ref);
return str;
}
@@ -3113,12 +3121,9 @@ static int tokenlib_get_expansion(lua_State* L)
char *s;
int l;
tex_parse_str_to_tok(h, &t, ct, str, len, 2); /* ignore unknown */
-
tokenlib_aux_expand_macros_in_tokenlist(h); // todo: use return value instead of def_ref
- s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, &l, 0, 0, 0);
- tex_flush_token_list(lmt_input_state.def_ref);
+ s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, &l, 0, 0, 0, 1);
tex_flush_token_list(h);
-
if (l > 0) {
lua_pushlstring(L, (const char *) s, (size_t) l);
return 1;
@@ -3472,9 +3477,8 @@ static int tokenlib_serialize(lua_State *L)
halfword t = n->token;
char *s;
tokenlib_aux_expand_macros_in_tokenlist(t); // todo: use return value instead of def_ref
- s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0);
+ s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0, 1);
lua_pushstring(L, s ? s : "");
- tex_flush_token_list(lmt_input_state.def_ref);
} else {
lua_pushnil(L);
}
@@ -3656,7 +3660,7 @@ void lmt_token_call(int p) /*tex The \TEX\ pointer to the token list. */
{
LoadS ls;
int l = 0;
- ls.s = tex_tokenlist_to_tstring(p, 1, &l, 0, 0, 0);
+ ls.s = tex_tokenlist_to_tstring(p, 1, &l, 0, 0, 0, 0);
ls.size = (size_t) l;
if (ls.size > 0) {
lua_State *L = lmt_lua_state.lua_instance;
diff --git a/source/luametatex/source/lua/lmttokenlib.h b/source/luametatex/source/lua/lmttokenlib.h
index 5339a80fe..450c6173a 100644
--- a/source/luametatex/source/lua/lmttokenlib.h
+++ b/source/luametatex/source/lua/lmttokenlib.h
@@ -28,7 +28,7 @@ typedef enum command_item_types {
} command_item_types;
extern void lmt_token_list_to_lua (lua_State *L, halfword p);
-extern void lmt_token_list_to_luastring (lua_State *L, halfword p, int nospace, int strip);
+extern void lmt_token_list_to_luastring (lua_State *L, halfword p, int nospace, int strip, int wipe);
extern halfword lmt_token_list_from_lua (lua_State *L, int slot);
extern halfword lmt_token_code_from_lua (lua_State *L, int slot);
diff --git a/source/luametatex/source/luacore/lua54/src/ltable.c b/source/luametatex/source/luacore/lua54/src/ltable.c
index 1b1cd2415..d03e74865 100644
--- a/source/luametatex/source/luacore/lua54/src/ltable.c
+++ b/source/luametatex/source/luacore/lua54/src/ltable.c
@@ -107,7 +107,7 @@ static const TValue absentkey = {ABSTKEYCONSTANT};
*/
static Node *hashint (const Table *t, lua_Integer i) {
lua_Unsigned ui = l_castS2U(i);
- if (ui <= (unsigned int)INT_MAX)
+ if (ui <= cast_uint(INT_MAX))
return hashmod(t, cast_int(ui));
else
return hashmod(t, ui);
@@ -488,7 +488,7 @@ static void setnodevector (lua_State *L, Table *t, unsigned int size) {
luaG_runerror(L, "table overflow");
size = twoto(lsize);
t->node = luaM_newvector(L, size, Node);
- for (i = 0; i < (int)size; i++) {
+ for (i = 0; i < cast_int(size); i++) {
Node *n = gnode(t, i);
gnext(n) = 0;
setnilkey(n);
diff --git a/source/luametatex/source/luametatex.h b/source/luametatex/source/luametatex.h
index 32fd78f6d..736d21173 100644
--- a/source/luametatex/source/luametatex.h
+++ b/source/luametatex/source/luametatex.h
@@ -87,9 +87,9 @@
# include "tex/textypes.h"
# define luametatex_version 210
-# define luametatex_revision 01
-# define luametatex_version_string "2.10.01"
-# define luametatex_development_id 20221014
+# define luametatex_revision 02
+# define luametatex_version_string "2.10.02"
+# define luametatex_development_id 20221019
# define luametatex_name_camelcase "LuaMetaTeX"
# define luametatex_name_lowercase "luametatex"
diff --git a/source/luametatex/source/tex/texequivalents.c b/source/luametatex/source/tex/texequivalents.c
index 291126fff..aa434d8fd 100644
--- a/source/luametatex/source/tex/texequivalents.c
+++ b/source/luametatex/source/tex/texequivalents.c
@@ -1437,7 +1437,7 @@ void tex_forced_word_define(int g, halfword p, singleword f, halfword w)
void tex_save_for_after_group(halfword t)
{
- if (cur_level > level_one && tex_room_on_save_stack()) {
+ if (t && cur_level > level_one && tex_room_on_save_stack()) {
save_type(lmt_save_state.save_stack_data.ptr) = insert_tokens_save_type;
save_level(lmt_save_state.save_stack_data.ptr) = level_zero;
save_value(lmt_save_state.save_stack_data.ptr) = t;
diff --git a/source/luametatex/source/tex/texexpand.c b/source/luametatex/source/tex/texexpand.c
index 25dcccdf3..a0416d118 100644
--- a/source/luametatex/source/tex/texexpand.c
+++ b/source/luametatex/source/tex/texexpand.c
@@ -684,7 +684,7 @@ inline static halfword tex_aux_get_cs_name(void)
if (tex_aux_collect_cs_tokens(&p, &n)) {
/*tex Look up the characters of list |r| in the hash table, and set |cur_cs|. */
int siz;
- char *s = tex_tokenlist_to_tstring(h, 1, &siz, 0, 0, 0);
+ char *s = tex_tokenlist_to_tstring(h, 1, &siz, 0, 0, 0, 0);
cur_cs = (siz > 0) ? tex_string_locate((char *) s, siz, 1) : null_cs;
} else {
tex_aux_complain_missing_csname();
diff --git a/source/luametatex/source/tex/texfileio.c b/source/luametatex/source/tex/texfileio.c
index 4f712401b..c29e7cece 100644
--- a/source/luametatex/source/tex/texfileio.c
+++ b/source/luametatex/source/tex/texfileio.c
@@ -875,10 +875,8 @@ char *tex_read_file_name(int optionalequal, const char * name, const char* ext)
}
}
DONE:
- s = tex_tokenlist_to_tstring(result, 1, &l, 0, 0, 0);
+ s = tex_tokenlist_to_tstring(result, 1, &l, 0, 0, 0, 1);
fn = s ? tex_aux_pack_file_name(s, l, name, ext) : NULL;
- /*tex Shouldn't we also free |result| ? */
- tex_flush_token_list(result);
return fn;
}
diff --git a/source/luametatex/source/tex/texlanguage.c b/source/luametatex/source/tex/texlanguage.c
index 6f3460c22..22627ee58 100644
--- a/source/luametatex/source/tex/texlanguage.c
+++ b/source/luametatex/source/tex/texlanguage.c
@@ -364,7 +364,7 @@ void tex_clear_patterns(struct tex_language *lang)
void tex_load_tex_patterns(halfword curlang, halfword head)
{
- char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0);
+ char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0, 0);
if (s) {
tex_load_patterns(tex_get_language(curlang), (unsigned char *) s);
}
@@ -539,7 +539,7 @@ void tex_clear_hyphenation(struct tex_language *lang)
void tex_load_tex_hyphenation(halfword curlang, halfword head)
{
- char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0);
+ char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0, 0);
if (s) {
tex_load_hyphenation(tex_get_language(curlang), (unsigned char *) s);
}
diff --git a/source/luametatex/source/tex/texlinebreak.c b/source/luametatex/source/tex/texlinebreak.c
index 2172277ec..2e0c945a7 100644
--- a/source/luametatex/source/tex/texlinebreak.c
+++ b/source/luametatex/source/tex/texlinebreak.c
@@ -3145,7 +3145,7 @@ static void tex_aux_post_line_break(const line_break_properties *properties, hal
n = node_next(n);
}
}
- if (! n) {
+ if (! n && normalize_line_mode_par) {
/*tex For the moment: */
tex_normal_warning("tex", "right parinit skip is gone");
}
@@ -3163,7 +3163,7 @@ static void tex_aux_post_line_break(const line_break_properties *properties, hal
n = node_prev(n);
}
}
- if (! n) {
+ if (! n && normalize_line_mode_par) {
/*tex For the moment: */
tex_normal_warning("tex", "left parfill skip is gone");
}
diff --git a/source/luametatex/source/tex/texmaincontrol.c b/source/luametatex/source/tex/texmaincontrol.c
index a44aa1167..5e0bb2f92 100644
--- a/source/luametatex/source/tex/texmaincontrol.c
+++ b/source/luametatex/source/tex/texmaincontrol.c
@@ -1372,11 +1372,13 @@ static void tex_aux_run_after_something(void) {
} while (cur_cmd == spacer_cmd);
if (cur_cmd == left_brace_cmd) {
halfword source = tex_scan_toks_normal(1, NULL);
- if (source) {
- tex_save_for_after_group(token_link(source));
- token_link(source) = null;
+ if (source) {
+ if (token_link(source)) {
+ tex_save_for_after_group(token_link(source));
+ token_link(source) = null;
+ }
+ tex_put_available_token(source);
}
- tex_flush_token_list(source);
} else {
tex_handle_error(
normal_error_type,
@@ -1394,10 +1396,11 @@ static void tex_aux_run_after_something(void) {
if (cur_cmd == left_brace_cmd) {
halfword source = tex_scan_toks_normal(1, NULL);
if (source) {
+ /*tex Always, also when empty. */
lmt_main_control_state.after_tokens = token_link(source);
token_link(source) = null;
+ tex_put_available_token(source);
}
- tex_flush_token_list(source);
} else {
tex_handle_error(
normal_error_type,
@@ -1422,7 +1425,7 @@ static void tex_aux_run_after_something(void) {
}
token_link(p) = token_link(source);
token_link(source) = null;
- tex_flush_token_list(source);
+ tex_put_available_token(source);
} else {
update_tex_end_of_group(source);
}
@@ -4118,7 +4121,6 @@ static void tex_aux_set_shorthand_def(int a, int force)
{
mathcodeval mval = tex_scan_mathchar(tex_mathcode);
tex_define(a, p, mathspec_cmd, tex_new_math_spec(mval, tex_mathcode));
- // tex_define(a, p, math_char_given_cmd, math_old_packed_character(mval.class_value,mval.family_value,mval.character_value));
break;
}
case math_dchar_def_code:
@@ -4126,14 +4128,12 @@ static void tex_aux_set_shorthand_def(int a, int force)
mathdictval dval = tex_scan_mathdict();
mathcodeval mval = tex_scan_mathchar(umath_mathcode);
tex_define(a, p, mathspec_cmd, tex_new_math_dict_spec(dval, mval, umath_mathcode));
- // tex_define(a, p, math_char_xgiven_cmd, math_packed_character(mval.class_value,mval.family_value,mval.character_value));
break;
}
case math_xchar_def_code:
{
mathcodeval mval = tex_scan_mathchar(umath_mathcode);
tex_define(a, p, mathspec_cmd, tex_new_math_spec(mval, umath_mathcode));
- // tex_define(a, p, math_char_xgiven_cmd, math_packed_character(mval.class_value,mval.family_value,mval.character_value));
break;
}
case count_def_code:
@@ -5854,14 +5854,13 @@ static void tex_aux_run_message(void)
strnumber s = tex_aux_scan_string();
if (error_help_par) {
strnumber helpinfo = tex_tokens_to_string(error_help_par);
- char *h = tex_makecstring(helpinfo);
+ char *h = tex_to_cstring(helpinfo);
tex_handle_error(
normal_error_type,
"%T",
s,
h
);
- lmt_memory_free(h);
tex_flush_str(helpinfo);
} else if (lmt_error_state.long_help_seen) {
tex_handle_error(
@@ -6298,8 +6297,6 @@ inline static void tex_aux_big_switch(int mode, int cmd)
register_runner(italic_correction_cmd, tex_aux_run_illegal_case, tex_aux_run_text_italic_correction, tex_run_math_italic_correction);
register_runner(math_char_number_cmd, tex_aux_run_math_non_math, tex_run_text_math_char_number, tex_run_math_math_char_number);
- // register_runner(math_char_given_cmd, tex_aux_run_math_non_math, tex_run_text_math_char_given, tex_run_math_math_char_given);
- // register_runner(math_char_xgiven_cmd, tex_aux_run_math_non_math, tex_run_text_math_char_xgiven, tex_run_math_math_char_xgiven);
register_runner(mathspec_cmd, tex_aux_run_math_non_math, tex_run_text_math_spec, tex_run_math_math_spec);
register_runner(vadjust_cmd, tex_aux_run_illegal_case, tex_run_vadjust, tex_run_vadjust);
diff --git a/source/luametatex/source/tex/texmath.c b/source/luametatex/source/tex/texmath.c
index 52e200657..3f452b1ff 100644
--- a/source/luametatex/source/tex/texmath.c
+++ b/source/luametatex/source/tex/texmath.c
@@ -1713,12 +1713,6 @@ static int tex_aux_scan_math(halfword target, halfword style, int usetextfont, h
mval = tex_get_math_spec(cur_chr);
dval = tex_get_math_dict(cur_chr);
break;
- // case math_char_given_cmd:
- // mval = tex_mathchar_from_integer(cur_chr, tex_mathcode);
- // break;
- // case math_char_xgiven_cmd:
- // mval = tex_mathchar_from_integer(cur_chr, umath_mathcode);
- // break;
case delimiter_number_cmd:
switch (cur_chr) {
case math_delimiter_code:
@@ -1962,12 +1956,6 @@ int tex_scan_math_cmd_val(mathcodeval *mval, mathdictval *dval)
tex_get_x_token();
} while (cur_cmd == spacer_cmd);
switch (cur_cmd) {
- // case math_char_given_cmd:
- // *mval = tex_mathchar_from_integer(cur_chr, tex_mathcode);
- // break;
- // case math_char_xgiven_cmd:
- // *mval = tex_mathchar_from_integer(cur_chr, umath_mathcode);
- // break;
case mathspec_cmd:
*mval = tex_get_math_spec(cur_chr);
break;
@@ -2050,26 +2038,6 @@ void tex_run_math_math_char_number(void) {
}
}
-/*tex We build up an argument to |append_math_char|: */
-
-// void tex_run_text_math_char_given(void) {
-// tex_aux_append_math_char_in_text(tex_mathchar_from_integer(cur_chr, tex_mathcode));
-// }
-//
-// void tex_run_math_math_char_given(void) {
-// tex_aux_append_math_char(tex_mathchar_from_integer(cur_chr, tex_mathcode));
-// }
-
-/*tex We build up an argument to |append_math_char| the \LUATEX\ way: */
-
-// void tex_run_text_math_char_xgiven(void) {
-// tex_aux_append_math_char_in_text(tex_mathchar_from_integer(cur_chr, umath_mathcode));
-// }
-//
-// void tex_run_math_math_char_xgiven(void) {
-// tex_aux_append_math_char(tex_mathchar_from_integer(cur_chr, umath_mathcode));
-// }
-
void tex_run_math_delimiter_number(void) {
switch (cur_chr) {
case math_delimiter_code:
diff --git a/source/luametatex/source/tex/texmath.h b/source/luametatex/source/tex/texmath.h
index e024a6ba2..cc7aa7e7f 100644
--- a/source/luametatex/source/tex/texmath.h
+++ b/source/luametatex/source/tex/texmath.h
@@ -642,10 +642,6 @@ extern void tex_run_math_math_char_number (void);
extern void tex_run_text_math_char_number (void);
extern void tex_run_math_char_number (void);
extern void tex_run_math_delimiter_number (void);
-// void tex_run_math_math_char_given (void);
-// void tex_run_text_math_char_given (void);
-// void tex_run_math_math_char_xgiven (void);
-// void tex_run_text_math_char_xgiven (void);
extern void tex_run_math_equation_number (void);
extern void tex_run_math_shift (void);
extern void tex_run_math_italic_correction (void);
diff --git a/source/luametatex/source/tex/texprimitive.c b/source/luametatex/source/tex/texprimitive.c
index bbeea1bc0..42344af15 100644
--- a/source/luametatex/source/tex/texprimitive.c
+++ b/source/luametatex/source/tex/texprimitive.c
@@ -185,7 +185,7 @@ static int tex_aux_room_in_hash(void)
inline static halfword tex_aux_compute_hash(const char *j, int l)
{
halfword h = (unsigned const char) j[0];
- for (int k = 1; k < l; k++) {
+ for (unsigned k = 1; k < l; k++) {
h = (h + h + (unsigned const char) j[k]) % hash_prime;
}
return h;
@@ -765,18 +765,6 @@ void tex_print_cmd_chr(singleword cmd, halfword chr)
tex_print_str_esc("char");
tex_print_qhex(chr);
break;
- // case math_char_given_cmd:
- // /*tex
- // Okay, it's better for old macro packages that mess with meaning to report a more
- // traditional value. A compromise. But, this might be dropped.
- // */
- // tex_print_str_esc("mathchar");
- // tex_show_mathcode_value(tex_mathchar_from_integer(chr, tex_mathcode), tex_mathcode);
- // break;
- // case math_char_xgiven_cmd:
- // tex_print_str_esc("Umathchar");
- // tex_show_mathcode_value(tex_mathchar_from_integer(chr, umath_mathcode), umath_mathcode);
- // break;
case lua_call_cmd:
tex_aux_show_lua_call("luacall", chr);
break;
diff --git a/source/luametatex/source/tex/texprinting.c b/source/luametatex/source/tex/texprinting.c
index 9a7c1d729..deabb4b72 100644
--- a/source/luametatex/source/tex/texprinting.c
+++ b/source/luametatex/source/tex/texprinting.c
@@ -217,9 +217,9 @@ void tex_print_char(int s)
The first 256 entries above the 17th unicode plane are used for a special trick: when \TEX\ has
to print items in that range, it will instead print the character that results from substracting
- 0x110000 from that value. This allows byte-oriented output to things like |\specials|.
-
- This feature will disappear.
+ 0x110000 from that value. This allows byte-oriented output to things like |\specials|. We dropped
+ this feature because it was never used (we used it as part of experiments with \LUATEX). The old
+ code branches can be found in the repository.
*/
@@ -245,13 +245,6 @@ static void tex_aux_uprint(int s)
tex_print_char(0xE0 + (s / 0x1000));
tex_print_char(0x80 + ((s % 0x1000) / 0x40));
tex_print_char(0x80 + ((s % 0x1000) % 0x40));
- } else if (s >= 0x110000) {
- int c = s - 0x110000;
- if (c >= 256) {
- tex_formatted_warning("print", "bad raw byte to print (c=%d), skipped",c);
- } else {
- tex_print_char(c);
- }
} else {
tex_print_char(0xF0 + (s / 0x40000));
tex_print_char(0x80 + ((s % 0x40000) / 0x1000));
@@ -260,24 +253,6 @@ static void tex_aux_uprint(int s)
}
}
-static void tex_aux_lprint(lstring *ss) {
- /*tex current character code position */
- unsigned char *j = ss->s;
- unsigned char *l = j + ss->l;
- while (j < l) {
- /*tex We don't bother checking the last two bytes explicitly */
- /* 0x110000 in utf=8: 0xF4 0x90 0x80 0x80 */
- if ((j < l - 4) && (*j == 0xF4) && (*(j + 1) == 0x90)) {
- int c = (*(j + 2) - 128) * 64 + (*(j + 3) - 128);
- tex_print_char(c);
- j = j + 4;
- } else {
- tex_print_char(*j);
- ++j;
- }
- }
-}
-
void tex_print_tex_str(int s)
{
if (s >= lmt_string_pool_state.string_pool_data.ptr) {
@@ -291,7 +266,10 @@ void tex_print_tex_str(int s)
} else if (lmt_print_state.selector == new_string_selector_code) {
tex_append_string(str_string(s), (unsigned) str_length(s));
} else {
- tex_aux_lprint(&str_lstring(s));
+ unsigned char *j = str_string(s);
+ for (unsigned i = 0; i < str_length(s); i++) {
+ tex_print_char(j[i]);
+ }
}
}
@@ -453,7 +431,7 @@ void tex_print_tex_str_esc(strnumber s)
{
/*tex Set variable |c| to the current escape character: */
int c = escape_char_par;
- if (c >= 0 && c < 0x110000) {
+ if (c >= 0) {
tex_print_tex_str(c);
}
if (s) {
@@ -467,7 +445,7 @@ void tex_print_str_esc(const char *s)
{
/*tex Set variable |c| to the current escape character: */
int c = escape_char_par;
- if (c >= 0 && c < 0x110000) {
+ if (c >= 0) {
tex_print_tex_str(c);
}
if (s) {
diff --git a/source/luametatex/source/tex/texscanning.c b/source/luametatex/source/tex/texscanning.c
index 5dd34b3b6..fec04356b 100644
--- a/source/luametatex/source/tex/texscanning.c
+++ b/source/luametatex/source/tex/texscanning.c
@@ -1088,18 +1088,13 @@ void tex_scan_something_simple(halfword cmd, halfword chr)
switch (cmd) {
/* begin of tex_aux_short_scan_something_internal */
case char_given_cmd:
- // case math_char_given_cmd:
- // case math_char_xgiven_cmd:
cur_val = chr;
cur_val_level = int_val_level;
break;
-
-case mathspec_cmd:
- cur_val = (chr && node_subtype(chr) == tex_mathcode) ? math_spec_value(chr) : 0;
- cur_val_level = int_val_level;
- break;
-
-
+ case mathspec_cmd:
+ cur_val = (chr && node_subtype(chr) == tex_mathcode) ? math_spec_value(chr) : 0;
+ cur_val_level = int_val_level;
+ break;
case iterator_value_cmd:
cur_val = chr > 0x100000 ? - (chr - 0x100000) : chr;
cur_val_level = int_val_level;
@@ -1254,8 +1249,6 @@ static halfword tex_aux_scan_something_internal(halfword cmd, halfword chr, int
switch (cmd) {
/* begin of tex_aux_short_scan_something_internal */
case char_given_cmd:
- // case math_char_given_cmd:
- // case math_char_xgiven_cmd:
cur_val = chr;
cur_val_level = int_val_level;
break;
diff --git a/source/luametatex/source/tex/texstringpool.c b/source/luametatex/source/tex/texstringpool.c
index 8367447da..fd93758b5 100644
--- a/source/luametatex/source/tex/texstringpool.c
+++ b/source/luametatex/source/tex/texstringpool.c
@@ -435,6 +435,7 @@ char *tex_makeclstring(int s, size_t *len)
}
}
+/*
char *tex_makecstring(int s)
{
if (s < cs_offset_value) {
@@ -443,6 +444,23 @@ char *tex_makecstring(int s)
return lmt_memory_strdup((str_length(s) > 0) ? (const char *) str_string(s) : "");
}
}
+*/
+
+/*tex
+ I might eventually replace this because in qite some calls we know that we knwo that we have
+ a pointer in string space. We can kin dof predict in what cases we are below |cs_offset_value|
+ anyway.
+*/
+
+char *tex_makecstring(int s, int *allocated)
+{
+ *allocated = s < cs_offset_value;
+ if (*allocated) {
+ return (char *) aux_uni2str((unsigned) s);
+ } else {
+ return str_length(s) > 0 ? (char *) str_string(s) : "";
+ }
+}
/*tex
diff --git a/source/luametatex/source/tex/texstringpool.h b/source/luametatex/source/tex/texstringpool.h
index b3924a0fe..7302e7fb4 100644
--- a/source/luametatex/source/tex/texstringpool.h
+++ b/source/luametatex/source/tex/texstringpool.h
@@ -21,7 +21,7 @@ typedef struct lstring {
unsigned char *s;
const char *c;
};
- size_t l; /* could be int, but this way we padd */
+ size_t l; /* could be int, but this way we padd */
} lstring;
typedef struct string_pool_info {
@@ -76,35 +76,37 @@ extern string_pool_info lmt_string_pool_state;
/*tex Forget the last character in the pool. */
-inline void tex_flush_char(void) { --lmt_string_pool_state.string_temp_top; }
-
-extern strnumber tex_make_string (void);
-extern strnumber tex_push_string (const unsigned char *s, int l);
-extern char *tex_take_string (int *len);
-extern int tex_str_eq_buf (strnumber s, int k, int n);
-extern int tex_str_eq_str (strnumber s, strnumber t);
-extern int tex_str_eq_cstr (strnumber s, const char *, size_t);
-extern int tex_get_strings_started (void);
-extern void tex_reset_cur_string (void);
-/* strnumber tex_search_string (strnumber search); */
-/* int tex_used_strings (void); */
-extern strnumber tex_maketexstring (const char *s);
-extern strnumber tex_maketexlstring (const char *s, size_t);
-extern void tex_append_char (unsigned char c);
-extern void tex_append_string (const unsigned char *s, unsigned l);
-extern char *tex_makecstring (int s);
-extern char *tex_makeclstring (int s, size_t *len);
-extern void tex_dump_string_pool (dumpstream f);
-extern void tex_undump_string_pool (dumpstream f);
-extern void tex_initialize_string_pool (void);
-extern void tex_initialize_string_mem (void);
-extern void tex_flush_str (strnumber s);
-extern strnumber tex_save_cur_string (void);
-extern void tex_restore_cur_string (strnumber u);
-
-/* void tex_increment_pool_string (int n); */
-/* void tex_decrement_pool_string (int n); */
-
-extern void tex_compact_string_pool (void);
+inline void tex_flush_char(void) { --lmt_string_pool_state.string_temp_top; }
+
+extern strnumber tex_make_string (void);
+extern strnumber tex_push_string (const unsigned char *s, int l);
+extern char *tex_take_string (int *len);
+extern int tex_str_eq_buf (strnumber s, int k, int n);
+extern int tex_str_eq_str (strnumber s, strnumber t);
+extern int tex_str_eq_cstr (strnumber s, const char *, size_t);
+extern int tex_get_strings_started (void);
+extern void tex_reset_cur_string (void);
+/* strnumber tex_search_string (strnumber search); */
+/* int tex_used_strings (void); */
+extern strnumber tex_maketexstring (const char *s);
+extern strnumber tex_maketexlstring (const char *s, size_t);
+extern void tex_append_char (unsigned char c);
+extern void tex_append_string (const unsigned char *s, unsigned l);
+extern char *tex_makecstring (int s, int *allocated);
+extern char *tex_makeclstring (int s, size_t *len);
+extern void tex_dump_string_pool (dumpstream f);
+extern void tex_undump_string_pool (dumpstream f);
+extern void tex_initialize_string_pool (void);
+extern void tex_initialize_string_mem (void);
+extern void tex_flush_str (strnumber s);
+extern strnumber tex_save_cur_string (void);
+extern void tex_restore_cur_string (strnumber u);
+
+/* void tex_increment_pool_string (int n); */
+/* void tex_decrement_pool_string (int n); */
+
+extern void tex_compact_string_pool (void);
+
+inline char *tex_to_cstring (int s) { return str_length(s) > 0 ? (char *) str_string(s) : ""; }
# endif
diff --git a/source/luametatex/source/tex/textoken.c b/source/luametatex/source/tex/textoken.c
index 0bb0f3e45..93bf3913a 100644
--- a/source/luametatex/source/tex/textoken.c
+++ b/source/luametatex/source/tex/textoken.c
@@ -2820,7 +2820,7 @@ void tex_run_convert_tokens(halfword code)
/* halfword result = tex_scan_toks_expand(0, NULL, code == lua_token_string_code); */
lmt_token_state.in_lua_escape = 1;
escape_char_par = '\\';
- str.s = (unsigned char *) tex_tokenlist_to_tstring(result, 0, &length, 0, 0, 0);
+ str.s = (unsigned char *) tex_tokenlist_to_tstring(result, 0, &length, 0, 0, 0, 0);
str.l = (unsigned) length;
lmt_token_state.in_lua_escape = saved_in_lua_escape;
escape_char_par = saved_escape_char;
@@ -3034,8 +3034,6 @@ static void tex_aux_append_uchar_to_buffer(int s)
lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0xE0 + (s / 0x1000));
lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x1000) / 0x40));
lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x1000) % 0x40));
- } else if (s >= 0x110000) {
- lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (s - 0x11000);
} else {
lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0xF0 + (s / 0x40000));
lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x40000) / 0x1000));
@@ -3078,7 +3076,7 @@ static void tex_aux_append_esc_to_buffer(const char *s)
/* make two versions: macro and not */
-char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamble, int nospace, int strip)
+char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamble, int nospace, int strip, int wipe)
{
if (pp) {
/*tex We need to go beyond the reference. */
@@ -3100,6 +3098,8 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
int min = 0;
int max = lmt_token_memory_state.tokens_data.top;
int skip = 0;
+ int tail = p;
+ int count = 0;
if (skippreamble) {
skip = get_token_parameters(pp);
}
@@ -3210,7 +3210,8 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
if (txt < 0 || txt >= lmt_string_pool_state.string_pool_data.ptr) {
tex_aux_append_str_to_buffer(error_string_nonexistent(36));
} else {
- char *sh = tex_makecstring(txt);
+ int allocated = 0;
+ char *sh = tex_makecstring(txt, &allocated);
char *s = sh;
if (tex_is_active_cs(txt)) {
s = s + 3;
@@ -3219,7 +3220,7 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
s++;
}
} else {
- if (e >= 0 && e < 0x110000) {
+ if (e >= 0) {
tex_aux_append_uchar_to_buffer(e);
}
while (*s) {
@@ -3230,10 +3231,14 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
tex_aux_append_char_to_buffer(' ');
}
}
- lmt_memory_free(sh);
+ if (allocated) {
+ lmt_memory_free(sh);
+ }
}
}
}
+ tail = p;
+ ++count;
p = token_link(p);
}
}
@@ -3251,7 +3256,14 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb
if (siz) {
*siz = lmt_token_state.bufloc;
}
+ if (wipe) {
+ tex_flush_token_list_head_tail(pp, tail, count);
+ }
return lmt_token_state.buffer;
+ } else {
+ if (wipe) {
+ tex_put_available_token(pp);
+ }
}
}
if (siz) {
diff --git a/source/luametatex/source/tex/textoken.h b/source/luametatex/source/tex/textoken.h
index 1996f351c..ad67dfcb5 100644
--- a/source/luametatex/source/tex/textoken.h
+++ b/source/luametatex/source/tex/textoken.h
@@ -366,8 +366,7 @@ extern void tex_run_combine_the_toks (void);
extern void tex_run_convert_tokens (halfword code);
extern strnumber tex_the_convert_string (halfword c, int i);
extern strnumber tex_tokens_to_string (halfword p);
-/* char *tex_tokenlist_to_cstring (int p, int inhibit_par, int *siz); */
-extern char *tex_tokenlist_to_tstring (int p, int inhibit_par, int *siz, int skip, int nospace, int strip);
+extern char *tex_tokenlist_to_tstring (int p, int inhibit_par, int *siz, int skip, int nospace, int strip, int wipe);
extern halfword tex_get_tex_dimen_register (int j, int internal);
extern halfword tex_get_tex_skip_register (int j, int internal);
@@ -391,9 +390,6 @@ extern halfword tex_copy_token_list (halfword h, halfword *t);
extern halfword tex_parse_str_to_tok (halfword head, halfword *tail, halfword ct, const char *str, size_t lstr, int option);
-inline int tex_valid_token(int t)
-{
- return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top));
-}
+inline int tex_valid_token (int t) { return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top)); }
# endif
diff --git a/source/luametatex/source/utilities/auxunistring.c b/source/luametatex/source/utilities/auxunistring.c
index e95854a93..30dc518e6 100644
--- a/source/luametatex/source/utilities/auxunistring.c
+++ b/source/luametatex/source/utilities/auxunistring.c
@@ -52,15 +52,12 @@ unsigned char *aux_uni2str(unsigned unic)
buf[0] = (unsigned char) (0xc0 | (unic >> 6));
buf[1] = (unsigned char) (0x80 | (unic & 0x3f));
buf[2] = '\0';
- } else if (unic >= 0x110000) {
- buf[0] = (unsigned char) (unic - 0x110000);
- buf[1] = '\0';
} else if (unic < 0x10000) {
buf[0] = (unsigned char) (0xe0 | (unic >> 12));
buf[1] = (unsigned char) (0x80 | ((unic >> 6) & 0x3f));
buf[2] = (unsigned char) (0x80 | (unic & 0x3f));
buf[3] = '\0';
- } else {
+ } else if (unic < 0x110000) {
unic -= 0x10000;
int u = (int) (((unic & 0xf0000) >> 16) + 1);
buf[0] = (unsigned char) (0xf0 | (u >> 2));