From 5d31142615a7efaf833be860e412995b42bf9c3c Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Fri, 21 Oct 2022 21:37:10 +0200 Subject: 2022-10-21 21:13:00 --- source/luametatex/CMakeLists.txt | 4 +- source/luametatex/source/lua/lmtinterface.h | 2 - source/luametatex/source/lua/lmtnodelib.c | 4 +- source/luametatex/source/lua/lmttexlib.c | 69 +++++-------- source/luametatex/source/lua/lmttokenlib.c | 110 +++++++++++---------- source/luametatex/source/lua/lmttokenlib.h | 2 +- .../luametatex/source/luacore/lua54/src/ltable.c | 4 +- source/luametatex/source/luametatex.h | 6 +- source/luametatex/source/tex/texequivalents.c | 2 +- source/luametatex/source/tex/texexpand.c | 2 +- source/luametatex/source/tex/texfileio.c | 4 +- source/luametatex/source/tex/texlanguage.c | 4 +- source/luametatex/source/tex/texlinebreak.c | 4 +- source/luametatex/source/tex/texmaincontrol.c | 23 ++--- source/luametatex/source/tex/texmath.c | 32 ------ source/luametatex/source/tex/texmath.h | 4 - source/luametatex/source/tex/texprimitive.c | 14 +-- source/luametatex/source/tex/texprinting.c | 40 ++------ source/luametatex/source/tex/texscanning.c | 15 +-- source/luametatex/source/tex/texstringpool.c | 18 ++++ source/luametatex/source/tex/texstringpool.h | 64 ++++++------ source/luametatex/source/tex/textoken.c | 26 +++-- source/luametatex/source/tex/textoken.h | 8 +- source/luametatex/source/utilities/auxunistring.c | 5 +- 24 files changed, 196 insertions(+), 270 deletions(-) (limited to 'source/luametatex') diff --git a/source/luametatex/CMakeLists.txt b/source/luametatex/CMakeLists.txt index d972a0312..a5021595f 100644 --- a/source/luametatex/CMakeLists.txt +++ b/source/luametatex/CMakeLists.txt @@ -205,8 +205,8 @@ if (DEFINED LMT_OPTIMIZE) # set_property(TARGET luametatex PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) # # mingw64: 2865664, nocona: 2819584, lto: 2835968 (around 1% gain on manual) - - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) + # + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) # else() # No message needed, just accept the fact. diff --git a/source/luametatex/source/lua/lmtinterface.h b/source/luametatex/source/lua/lmtinterface.h index 3fd87db2f..e761252a8 100644 --- a/source/luametatex/source/lua/lmtinterface.h +++ b/source/luametatex/source/lua/lmtinterface.h @@ -865,9 +865,7 @@ make_lua_key(L, match);\ make_lua_key(L, math);\ make_lua_key(L, mathspec);\ make_lua_key(L, math_accent);\ -make_lua_key(L, math_char_given);\ make_lua_key(L, math_char_number);\ -make_lua_key(L, math_char_xgiven);\ make_lua_key(L, math_choice);\ make_lua_key(L, math_component);\ make_lua_key(L, math_fence);\ diff --git a/source/luametatex/source/lua/lmtnodelib.c b/source/luametatex/source/lua/lmtnodelib.c index 7d9d7bcbe..eb29611e2 100644 --- a/source/luametatex/source/lua/lmtnodelib.c +++ b/source/luametatex/source/lua/lmtnodelib.c @@ -2997,7 +2997,7 @@ static int nodelib_direct_getdata(lua_State *L) } case mark_node: if (lua_toboolean(L, 2)) { - lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0); + lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0, 0); } else { lmt_token_list_to_lua(L, mark_ptr(n)); } @@ -6555,7 +6555,7 @@ static int nodelib_common_getfield(lua_State *L, int direct, halfword n) lua_pushinteger(L, mark_index(n)); } else if (lua_key_eq(s, data) || lua_key_eq(s, mark)) { if (lua_toboolean(L, 3)) { - lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0); + lmt_token_list_to_luastring(L, mark_ptr(n), 0, 0, 0); } else { lmt_token_list_to_lua(L, mark_ptr(n)); } diff --git a/source/luametatex/source/lua/lmttexlib.c b/source/luametatex/source/lua/lmttexlib.c index d44127330..afafb068b 100644 --- a/source/luametatex/source/lua/lmttexlib.c +++ b/source/luametatex/source/lua/lmttexlib.c @@ -1715,10 +1715,9 @@ static int texlib_gettoks(lua_State *L) if (lua_toboolean(L, slot)) { lmt_token_register_to_lua(L, state ? toks_parameter(index) : toks_register(index)); } else { + strnumber value = tex_get_tex_toks_register(index, state); - char *s = tex_makecstring(value); - lua_pushstring(L, s); - lmt_memory_free(s); + lua_pushstring(L, tex_to_cstring(value)); tex_flush_str(value); } } else { @@ -1754,7 +1753,7 @@ static int texlib_getmark(lua_State *L) if (num >= 0 && num <= lmt_mark_state.mark_data.ptr) { halfword ptr = tex_get_some_mark(mrk, num); if (ptr) { - char *str = tex_tokenlist_to_tstring(ptr, 1, NULL, 0, 0, 0); + char *str = tex_tokenlist_to_tstring(ptr, 1, NULL, 0, 0, 0, 0); if (str) { lua_pushstring(L, str); } else { @@ -1783,7 +1782,6 @@ int lmt_get_box_id(lua_State *L, int i, int report) int cmd = eq_type(cs); switch (cmd) { case char_given_cmd: - // case math_char_given_cmd: case integer_cmd: index = eq_value(cs); break; @@ -2597,7 +2595,6 @@ static int texlib_newindex(lua_State *L) static int texlib_aux_convert(lua_State *L, int cur_code) { int i = -1; - char *str = NULL; switch (cur_code) { /* ignored (yet) */ case insert_progress_code: /* arg */ @@ -2632,19 +2629,22 @@ static int texlib_aux_convert(lua_State *L, int cur_code) if (cur_code < 32) { int texstr = tex_the_convert_string(cur_code, i); if (texstr) { - str = tex_makecstring(texstr); + lua_pushstring(L, tex_to_cstring(texstr)); tex_flush_str(texstr); + + // int allocated = 0; + // char *str = tex_makecstring(texstr, &allocated); + // lua_pushstring(L, str); + // if (allocated) { + // lmt_memory_free(str); + // } + // tex_flush_str(texstr); + return 1; } } break; } - /* end */ - if (str) { - lua_pushstring(L, str); - lmt_memory_free(str); - } else { - lua_pushnil(L); - } + lua_pushnil(L); return 1; } @@ -2687,10 +2687,9 @@ static int texlib_aux_scan_internal(lua_State *L, int cmd, int code, int values) default: { int texstr = tex_the_scanned_result(); - char *str = tex_makecstring(texstr); + char *str = tex_to_cstring(texstr); if (str) { lua_pushstring(L, str); - lmt_memory_free(str); } else { lua_pushnil(L); } @@ -3013,7 +3012,6 @@ static int texlib_get_internal(lua_State *L, int index, int all) case set_auxiliary_cmd: case set_page_property_cmd: case char_given_cmd: - // case math_char_given_cmd: case integer_cmd: case dimension_cmd: case gluespec_cmd: @@ -3407,26 +3405,22 @@ static int texlib_hashtokens(lua_State *L) int nx = 0; int all = lua_toboolean(L, 1); lua_createtable(L, hash_size, 0); + /* todo: check active characters as these have three bogus bytes in front */ if (all) { while (cs <= hash_size) { - /* because strings never get freed we can as well directly access |s|. */ strnumber s = cs_text(cs); if (s > 0) { halfword n = cs_next(cs); - char *ss = tex_makecstring(s); if (n) { int mt = 0; lua_createtable(L, 2, 0); - lua_pushstring(L, ss); - lmt_memory_free(ss); + lua_pushstring(L, tex_to_cstring(s)); ++nt; lua_rawseti(L, -2, ++mt); while (n) { s = cs_text(n); if (s) { - ss = tex_makecstring(s); - lua_pushstring(L, ss); - lmt_memory_free(ss); + lua_pushstring(L, tex_to_cstring(s)); lua_rawseti(L, -2, ++mt); ++nt; ++nx; @@ -3434,8 +3428,7 @@ static int texlib_hashtokens(lua_State *L) n = cs_next(n); } } else { - lua_pushstring(L, ss); - lmt_memory_free(ss); + lua_pushstring(L, tex_to_cstring(s)); ++nt; } } else { @@ -3449,16 +3442,12 @@ static int texlib_hashtokens(lua_State *L) strnumber s = cs_text(cs); if (s > 0) { halfword n = cs_next(cs); - char *ss = tex_makecstring(s); - lua_pushstring(L, ss); - lmt_memory_free(ss); + lua_pushstring(L, tex_to_cstring(s)); lua_rawseti(L, -2, ++nt); while (n) { s = cs_text(n); if (s) { - ss = tex_makecstring(s); - lua_pushstring(L, ss); - lmt_memory_free(ss); + lua_pushstring(L, tex_to_cstring(s)); lua_rawseti(L, -2, ++nt); ++nx; } @@ -3482,9 +3471,7 @@ static int texlib_primitives(lua_State *L) while (cs < prim_size) { strnumber s = get_prim_text(cs); if (s > 0 && (get_prim_origin(cs) != no_command)) { - char *ss = tex_makecstring(s); - lua_pushstring(L, ss); - lmt_memory_free(ss); + lua_pushstring(L, tex_to_cstring(s)); lua_rawseti(L, -2, ++nt); } cs++; @@ -3518,9 +3505,7 @@ static int texlib_extraprimitives(lua_State *L) while (cs < prim_size) { strnumber s = get_prim_text(cs); if (s > 0 && (get_prim_origin(cs) & mask)) { - char *ss = tex_makecstring(s); - lua_pushstring(L, ss); - lmt_memory_free(ss); + lua_pushstring(L, tex_to_cstring(s)); lua_rawseti(L, -2, ++nt); } cs++; @@ -3531,7 +3516,7 @@ static int texlib_extraprimitives(lua_State *L) static void texlib_aux_enableprimitive(const char *pre, size_t prel, const char *prm) { strnumber s = tex_maketexstring(prm); - halfword prm_val = tex_prim_lookup(s); + halfword prm_val = tex_prim_lookup(s); /* todo: no need for tex string */ tex_flush_str(s); if (prm_val != undefined_primitive && get_prim_origin(prm_val) != no_command) { char *newprm; @@ -3594,10 +3579,8 @@ static int texlib_enableprimitives(lua_State *L) for (int cs = 0; cs < prim_size; cs++) { strnumber s = get_prim_text(cs); if (s > 0) { - /* there is actually no need to copy */ - char *prm = tex_makecstring(s); + char *prm = tex_to_cstring(s); texlib_aux_enableprimitive(pre, lpre, prm); - lmt_memory_free(prm); } } } @@ -4561,8 +4544,6 @@ static int texlib_mathchardef(lua_State *L) d.index = lmt_optinteger(L, 8, 0); if (class_in_range(m.class_value) && family_in_range(m.family_value) && character_in_range(m.character_value)) { tex_define(flags, cs, mathspec_cmd, tex_new_math_dict_spec(d, m, umath_mathcode)); - // halfword code = math_packed_character(m.class_value, m.family_value, m.character_value); - // tex_define(flags, cs, (quarterword) math_char_xgiven_cmd, code); } else { tex_normal_error("lua", "mathchardef needs proper class, family and character codes"); } diff --git a/source/luametatex/source/lua/lmttokenlib.c b/source/luametatex/source/lua/lmttokenlib.c index ac69241ba..feac7cd46 100644 --- a/source/luametatex/source/lua/lmttokenlib.c +++ b/source/luametatex/source/lua/lmttokenlib.c @@ -156,8 +156,6 @@ void lmt_tokenlib_initialize(void) lmt_interface.command_names[math_shift_cs_cmd] = (command_item) { .id = math_shift_cs_cmd, .lua = lua_key_index(math_shift_cs), .name = lua_key(math_shift_cs), .kind = regular_command_item, .min = 0, .max = last_math_shift_cs_code, .base = 0, .fixedvalue = 0 }; /* a bit too tolerant */ lmt_interface.command_names[end_cs_name_cmd] = (command_item) { .id = end_cs_name_cmd, .lua = lua_key_index(end_cs_name), .name = lua_key(end_cs_name), .kind = regular_command_item, .min = 0, .max = 0, .base = 0, .fixedvalue = 0 }; lmt_interface.command_names[char_given_cmd] = (command_item) { .id = char_given_cmd, .lua = lua_key_index(char_given), .name = lua_key(char_given), .kind = character_command_item, .min = 0, .max = max_character_code, .base = 0, .fixedvalue = 0 }; - // lmt_interface.command_names[math_char_given_cmd] = (command_item) { .id = math_char_given_cmd, .lua = lua_key_index(math_char_given), .name = lua_key(math_char_given), .kind = character_command_item, .min = 0, .max = max_character_code, .base = 0, .fixedvalue = 0 }; - // lmt_interface.command_names[math_char_xgiven_cmd] = (command_item) { .id = math_char_xgiven_cmd, .lua = lua_key_index(math_char_xgiven), .name = lua_key(math_char_xgiven), .kind = character_command_item, .min = 0, .max = max_character_code, .base = 0, .fixedvalue = 0 }; lmt_interface.command_names[some_item_cmd] = (command_item) { .id = some_item_cmd, .lua = lua_key_index(some_item), .name = lua_key(some_item), .kind = regular_command_item, .min = 0, .max = last_some_item_code, .base = 0, .fixedvalue = 0 }; lmt_interface.command_names[internal_toks_cmd] = (command_item) { .id = internal_toks_cmd, .lua = lua_key_index(internal_toks), .name = lua_key(internal_toks), .kind = internal_command_item, .min = first_toks_code, .max = last_toks_code, .base = internal_toks_base, .fixedvalue = 0 }; lmt_interface.command_names[register_toks_cmd] = (command_item) { .id = register_toks_cmd, .lua = lua_key_index(register_toks), .name = lua_key(register_toks), .kind = register_command_item, .min = 0, .max = biggest_reg, .base = register_toks_base, .fixedvalue = 0 }; @@ -433,10 +431,10 @@ void lmt_token_list_to_lua(lua_State *L, halfword p) } } -void lmt_token_list_to_luastring(lua_State *L, halfword p, int nospace, int strip) +void lmt_token_list_to_luastring(lua_State *L, halfword p, int nospace, int strip, int wipe) { int l; - char *s = tex_tokenlist_to_tstring(p, 1, &l, 0, nospace, strip); /* nasty ... preambles or not, could have been endmatchtoken */ + char *s = tex_tokenlist_to_tstring(p, 1, &l, 0, nospace, strip, wipe); /* nasty ... preambles or not, could have been endmatchtoken */ if (l) { lua_pushlstring(L, s, (size_t) l); } else { @@ -511,16 +509,19 @@ halfword lmt_token_code_from_lua(lua_State *L, int slot) # define is_active_string(s) (strlen(s) > 3 && *s == 0xEF && *(s+1) == 0xBF && *(s+2) == 0xBF) -static unsigned char *tokenlib_aux_get_cs_text(int cs) +static unsigned char *tokenlib_aux_get_cs_text(int cs, int *allocated) { if (cs == null_cs) { - return (unsigned char *) lmt_memory_strdup("\\csname\\endcsname"); + return (unsigned char *) "\\csname\\endcsname"; } else if ((cs_text(cs) < 0) || (cs_text(cs) >= lmt_string_pool_state.string_pool_data.ptr)) { - return (unsigned char *) lmt_memory_strdup(""); - } else if (tex_is_active_cs(cs_text(cs))) { - return (unsigned char *) tex_makecstring(cs_text(cs)); + return (unsigned char *) ""; + // } else { + // return (unsigned char *) tex_makecstring(cs_text(cs)); + } else if (cs_text(cs) < cs_offset_value) { + *allocated = 1; + return (unsigned char *) aux_uni2str((unsigned) cs_text(cs)); } else { - return (unsigned char *) tex_makecstring(cs_text(cs)); + return (unsigned char *) (str_length(cs_text(cs)) > 0 ? (unsigned char *) str_string(cs_text(cs)) : (unsigned char *) ""); } } @@ -871,14 +872,17 @@ static int tokenlib_scan_csname(lua_State *L) } t = cur_cs ? cs_token_flag + cur_cs : token_val (cur_cmd, cur_chr); if (t >= cs_token_flag) { - unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag); + int allocated = 0; + unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag, &allocated); if (s) { if (tex_is_active_cs(cs_text(t - cs_token_flag))) { lua_pushstring(L, (char *) (s + 3)); } else { lua_pushstring(L, (char *) s); } - lmt_memory_free(s); + if (allocated) { + lmt_memory_free(s); + } } else { lua_pushnil(L); } @@ -1408,8 +1412,7 @@ static int tokenlib_scan_string(lua_State *L) { halfword defref = lmt_input_state.def_ref; halfword result = tex_scan_toks_expand(1, NULL, 0); - lmt_token_list_to_luastring(L, result, 0, 0); - tex_flush_token_list(result); + lmt_token_list_to_luastring(L, result, 0, 0, 1); lmt_input_state.def_ref = defref; break; } @@ -1421,8 +1424,7 @@ static int tokenlib_scan_string(lua_State *L) case tolerant_semi_protected_call_cmd: { halfword t = token_link(cur_chr); - lmt_token_list_to_luastring(L, t, 0, 0); - tex_flush_token_list(t); + lmt_token_list_to_luastring(L, t, 0, 0, 1); break; } case letter_cmd: @@ -1463,8 +1465,7 @@ static int tokenlib_scan_argument(lua_State *L) halfword defref = lmt_input_state.def_ref; int expand = lua_type(L, 1) == LUA_TBOOLEAN ? lua_toboolean(L, 1) : 1; halfword result = expand ? tex_scan_toks_expand(1, NULL, 0) : tex_scan_toks_normal(1, NULL); - lmt_token_list_to_luastring(L, result, 0, 0); - tex_flush_token_list(result); + lmt_token_list_to_luastring(L, result, 0, 0, 1); lmt_input_state.def_ref = defref; break; } @@ -1485,8 +1486,7 @@ static int tokenlib_scan_argument(lua_State *L) tex_back_input(cur_tok); result = tex_scan_toks_expand(1, NULL, 0); } - lmt_token_list_to_luastring(L, result, 0, 0); - tex_flush_token_list(result); + lmt_token_list_to_luastring(L, result, 0, 0, 1); lmt_input_state.def_ref = defref; break; } @@ -1685,8 +1685,7 @@ static int tokenlib_scan_delimited(lua_State *L) FINALYDONE: /* */ lmt_input_state.def_ref = defref; - lmt_token_list_to_luastring(L, result, 0, 0); - tex_flush_token_list(result); + lmt_token_list_to_luastring(L, result, 0, 0, 1); } else { tex_back_input(cur_tok); lua_pushnil(L); @@ -1781,8 +1780,7 @@ static int tokenlib_grab_until(lua_State *L) } tokenlib_aux_unsave_tex_scanner(texstate); lmt_input_state.def_ref = defref; - lmt_token_list_to_luastring(L, result, nospace, strip); - tex_flush_token_list(result); + lmt_token_list_to_luastring(L, result, nospace, strip, 1); } else { lua_pushnil(L); } @@ -2013,8 +2011,7 @@ static int tokenlib_scan_value(lua_State *L) halfword defref = lmt_input_state.def_ref; result = tex_scan_toks_expand(1, NULL, 0); lmt_input_state.def_ref = defref; - lmt_token_list_to_luastring(L, result, 0, 0); - tex_flush_token_list(result); + lmt_token_list_to_luastring(L, result, 0, 0, 1); } break; /* @@ -2052,8 +2049,7 @@ static int tokenlib_scan_value(lua_State *L) halfword defref = lmt_input_state.def_ref; result = tex_scan_toks_expand(1, NULL, 0); lmt_input_state.def_ref = defref; - lmt_token_list_to_luastring(L, result, 0, 0); - tex_flush_token_list(result); + lmt_token_list_to_luastring(L, result, 0, 0, 1); luaL_addchar(&b, '{'); luaL_addvalue(&b); luaL_addchar(&b, '}'); @@ -2069,7 +2065,8 @@ static int tokenlib_scan_value(lua_State *L) /*tex We need to add a space. */ halfword t = cur_cs ? cs_token_flag + cur_cs : token_val(cur_cmd, cur_chr); if (t >= cs_token_flag) { - unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag); + int allocated = 0; + unsigned char *s = tokenlib_aux_get_cs_text(t - cs_token_flag, &allocated); if (s) { if (tex_is_active_cs(cs_text(t - cs_token_flag))) { lua_pushstring(L, (char *) (s + 3)); @@ -2080,7 +2077,9 @@ static int tokenlib_scan_value(lua_State *L) luaL_addvalue(&b); luaL_addchar(&b, ' '); } - lmt_memory_free(s); + if (allocated) { + lmt_memory_free(s); + } } } } @@ -2377,7 +2376,7 @@ static int tokenlib_getprimitives(lua_State *L) while (cs < prim_size) { strnumber s = get_prim_text(cs); if (s > 0 && (get_prim_origin(cs) != no_command)) { - char *ss = tex_makecstring(s); + char *ss = tex_to_cstring(s); int cmd = prim_eq_type(cs); int chr = prim_equiv(cs); if (! raw) { @@ -2393,7 +2392,6 @@ static int tokenlib_getprimitives(lua_State *L) lua_pushinteger(L, prim_origin(cs)); lua_rawseti(L, -2, 4); lua_rawseti(L, -2, ++nt); - lmt_memory_free(ss); } cs++; } @@ -2508,14 +2506,17 @@ inline static int tokenlib_get_csname(lua_State *L) lua_token *n = tokenlib_aux_check_istoken(L, 1); halfword tok = token_info(n->token); if (tok >= cs_token_flag) { - unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag); + int allocated = 0; + unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag, &allocated); if (s) { if (tex_is_active_cs(cs_text(tok - cs_token_flag))) { lua_pushstring(L, (char *) (s + 3)); } else { lua_pushstring(L, (char *) s); } - lmt_memory_free(s); + if (allocated) { + lmt_memory_free(s); + } return 1; } } @@ -2544,10 +2545,13 @@ inline static int tokenlib_get_active(lua_State *L) halfword tok = token_info(n->token); int result = 0; if (tok >= cs_token_flag) { - unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag); + int allocated = 0; + unsigned char *s = tokenlib_aux_get_cs_text(tok - cs_token_flag, &allocated); if (s) { result = tex_is_active_cs(cs_text(tok - cs_token_flag)); - lmt_memory_free(s); + if (allocated) { + lmt_memory_free(s); + } } } lua_pushboolean(L, result); @@ -2753,7 +2757,8 @@ static int tokenlib_get_fields(lua_State *L) lua_createtable(L, 0, onlyflags ? 0 : 5); if (tok >= cs_token_flag) { int t = tok - cs_token_flag; - unsigned char* str = tokenlib_aux_get_cs_text(t); + int allocated = 0; + unsigned char* str = tokenlib_aux_get_cs_text(t, &allocated); if (str) { if (! onlyflags) { lua_push_key(csname); @@ -2767,7 +2772,9 @@ static int tokenlib_get_fields(lua_State *L) } lua_rawset(L, -3); } - lmt_memory_free(str); + if (allocated) { + lmt_memory_free(str); + } } cmd = eq_type(t); chr = eq_value(t); @@ -2836,9 +2843,12 @@ static int tokenlib_tostring(lua_State* L) unsigned char* csp = NULL; const char* cmn = NULL; if (tok >= cs_token_flag) { + int allocated = 0; tok -= cs_token_flag; - csn = tokenlib_aux_get_cs_text(tok); - csp = csn; + csn = tokenlib_aux_get_cs_text(tok, &allocated); + if (allocated) { + csp = csn; + } if (csn && tex_is_active_cs(cs_text(tok))) { csn += 3; } @@ -2974,7 +2984,7 @@ static int tokenlib_get_meaning(lua_State *L) lmt_token_register_to_lua(L, chr); } } else { - char *str = tex_tokenlist_to_tstring(chr, 1, NULL, 0, 0, 0); + char *str = tex_tokenlist_to_tstring(chr, 1, NULL, 0, 0, 0, 0); lua_pushstring(L, str ? str : ""); } return 1; @@ -3041,10 +3051,9 @@ static int tokenlib_get_macro(lua_State *L) char *str = NULL; if (lua_toboolean(L, 2)) { tokenlib_aux_expand_macros_in_tokenlist(chr); // todo: use return value instead of def_ref - str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0); - tex_flush_token_list(lmt_input_state.def_ref); + str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0, 1); } else { - str = tex_tokenlist_to_tstring(chr, 1, NULL, 1, 0, 0); + str = tex_tokenlist_to_tstring(chr, 1, NULL, 1, 0, 0, 0); } lua_pushstring(L, str ? str : ""); return 1; @@ -3091,8 +3100,7 @@ char *lmt_get_expansion(halfword head, int *len) halfword ref = get_reference_token(); set_token_link(ref, head); tokenlib_aux_expand_macros_in_tokenlist(ref); // todo: use return value instead of def_ref - str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, len, 0, 0, 0); - tex_flush_token_list(lmt_input_state.def_ref); + str = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, len, 0, 0, 0, 1); tex_flush_token_list(ref); return str; } @@ -3113,12 +3121,9 @@ static int tokenlib_get_expansion(lua_State* L) char *s; int l; tex_parse_str_to_tok(h, &t, ct, str, len, 2); /* ignore unknown */ - tokenlib_aux_expand_macros_in_tokenlist(h); // todo: use return value instead of def_ref - s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, &l, 0, 0, 0); - tex_flush_token_list(lmt_input_state.def_ref); + s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, &l, 0, 0, 0, 1); tex_flush_token_list(h); - if (l > 0) { lua_pushlstring(L, (const char *) s, (size_t) l); return 1; @@ -3472,9 +3477,8 @@ static int tokenlib_serialize(lua_State *L) halfword t = n->token; char *s; tokenlib_aux_expand_macros_in_tokenlist(t); // todo: use return value instead of def_ref - s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0); + s = tex_tokenlist_to_tstring(lmt_input_state.def_ref, 1, NULL, 0, 0, 0, 1); lua_pushstring(L, s ? s : ""); - tex_flush_token_list(lmt_input_state.def_ref); } else { lua_pushnil(L); } @@ -3656,7 +3660,7 @@ void lmt_token_call(int p) /*tex The \TEX\ pointer to the token list. */ { LoadS ls; int l = 0; - ls.s = tex_tokenlist_to_tstring(p, 1, &l, 0, 0, 0); + ls.s = tex_tokenlist_to_tstring(p, 1, &l, 0, 0, 0, 0); ls.size = (size_t) l; if (ls.size > 0) { lua_State *L = lmt_lua_state.lua_instance; diff --git a/source/luametatex/source/lua/lmttokenlib.h b/source/luametatex/source/lua/lmttokenlib.h index 5339a80fe..450c6173a 100644 --- a/source/luametatex/source/lua/lmttokenlib.h +++ b/source/luametatex/source/lua/lmttokenlib.h @@ -28,7 +28,7 @@ typedef enum command_item_types { } command_item_types; extern void lmt_token_list_to_lua (lua_State *L, halfword p); -extern void lmt_token_list_to_luastring (lua_State *L, halfword p, int nospace, int strip); +extern void lmt_token_list_to_luastring (lua_State *L, halfword p, int nospace, int strip, int wipe); extern halfword lmt_token_list_from_lua (lua_State *L, int slot); extern halfword lmt_token_code_from_lua (lua_State *L, int slot); diff --git a/source/luametatex/source/luacore/lua54/src/ltable.c b/source/luametatex/source/luacore/lua54/src/ltable.c index 1b1cd2415..d03e74865 100644 --- a/source/luametatex/source/luacore/lua54/src/ltable.c +++ b/source/luametatex/source/luacore/lua54/src/ltable.c @@ -107,7 +107,7 @@ static const TValue absentkey = {ABSTKEYCONSTANT}; */ static Node *hashint (const Table *t, lua_Integer i) { lua_Unsigned ui = l_castS2U(i); - if (ui <= (unsigned int)INT_MAX) + if (ui <= cast_uint(INT_MAX)) return hashmod(t, cast_int(ui)); else return hashmod(t, ui); @@ -488,7 +488,7 @@ static void setnodevector (lua_State *L, Table *t, unsigned int size) { luaG_runerror(L, "table overflow"); size = twoto(lsize); t->node = luaM_newvector(L, size, Node); - for (i = 0; i < (int)size; i++) { + for (i = 0; i < cast_int(size); i++) { Node *n = gnode(t, i); gnext(n) = 0; setnilkey(n); diff --git a/source/luametatex/source/luametatex.h b/source/luametatex/source/luametatex.h index 32fd78f6d..736d21173 100644 --- a/source/luametatex/source/luametatex.h +++ b/source/luametatex/source/luametatex.h @@ -87,9 +87,9 @@ # include "tex/textypes.h" # define luametatex_version 210 -# define luametatex_revision 01 -# define luametatex_version_string "2.10.01" -# define luametatex_development_id 20221014 +# define luametatex_revision 02 +# define luametatex_version_string "2.10.02" +# define luametatex_development_id 20221019 # define luametatex_name_camelcase "LuaMetaTeX" # define luametatex_name_lowercase "luametatex" diff --git a/source/luametatex/source/tex/texequivalents.c b/source/luametatex/source/tex/texequivalents.c index 291126fff..aa434d8fd 100644 --- a/source/luametatex/source/tex/texequivalents.c +++ b/source/luametatex/source/tex/texequivalents.c @@ -1437,7 +1437,7 @@ void tex_forced_word_define(int g, halfword p, singleword f, halfword w) void tex_save_for_after_group(halfword t) { - if (cur_level > level_one && tex_room_on_save_stack()) { + if (t && cur_level > level_one && tex_room_on_save_stack()) { save_type(lmt_save_state.save_stack_data.ptr) = insert_tokens_save_type; save_level(lmt_save_state.save_stack_data.ptr) = level_zero; save_value(lmt_save_state.save_stack_data.ptr) = t; diff --git a/source/luametatex/source/tex/texexpand.c b/source/luametatex/source/tex/texexpand.c index 25dcccdf3..a0416d118 100644 --- a/source/luametatex/source/tex/texexpand.c +++ b/source/luametatex/source/tex/texexpand.c @@ -684,7 +684,7 @@ inline static halfword tex_aux_get_cs_name(void) if (tex_aux_collect_cs_tokens(&p, &n)) { /*tex Look up the characters of list |r| in the hash table, and set |cur_cs|. */ int siz; - char *s = tex_tokenlist_to_tstring(h, 1, &siz, 0, 0, 0); + char *s = tex_tokenlist_to_tstring(h, 1, &siz, 0, 0, 0, 0); cur_cs = (siz > 0) ? tex_string_locate((char *) s, siz, 1) : null_cs; } else { tex_aux_complain_missing_csname(); diff --git a/source/luametatex/source/tex/texfileio.c b/source/luametatex/source/tex/texfileio.c index 4f712401b..c29e7cece 100644 --- a/source/luametatex/source/tex/texfileio.c +++ b/source/luametatex/source/tex/texfileio.c @@ -875,10 +875,8 @@ char *tex_read_file_name(int optionalequal, const char * name, const char* ext) } } DONE: - s = tex_tokenlist_to_tstring(result, 1, &l, 0, 0, 0); + s = tex_tokenlist_to_tstring(result, 1, &l, 0, 0, 0, 1); fn = s ? tex_aux_pack_file_name(s, l, name, ext) : NULL; - /*tex Shouldn't we also free |result| ? */ - tex_flush_token_list(result); return fn; } diff --git a/source/luametatex/source/tex/texlanguage.c b/source/luametatex/source/tex/texlanguage.c index 6f3460c22..22627ee58 100644 --- a/source/luametatex/source/tex/texlanguage.c +++ b/source/luametatex/source/tex/texlanguage.c @@ -364,7 +364,7 @@ void tex_clear_patterns(struct tex_language *lang) void tex_load_tex_patterns(halfword curlang, halfword head) { - char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0); + char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0, 0); if (s) { tex_load_patterns(tex_get_language(curlang), (unsigned char *) s); } @@ -539,7 +539,7 @@ void tex_clear_hyphenation(struct tex_language *lang) void tex_load_tex_hyphenation(halfword curlang, halfword head) { - char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0); + char *s = tex_tokenlist_to_tstring(head, 1, NULL, 0, 0, 0, 0); if (s) { tex_load_hyphenation(tex_get_language(curlang), (unsigned char *) s); } diff --git a/source/luametatex/source/tex/texlinebreak.c b/source/luametatex/source/tex/texlinebreak.c index 2172277ec..2e0c945a7 100644 --- a/source/luametatex/source/tex/texlinebreak.c +++ b/source/luametatex/source/tex/texlinebreak.c @@ -3145,7 +3145,7 @@ static void tex_aux_post_line_break(const line_break_properties *properties, hal n = node_next(n); } } - if (! n) { + if (! n && normalize_line_mode_par) { /*tex For the moment: */ tex_normal_warning("tex", "right parinit skip is gone"); } @@ -3163,7 +3163,7 @@ static void tex_aux_post_line_break(const line_break_properties *properties, hal n = node_prev(n); } } - if (! n) { + if (! n && normalize_line_mode_par) { /*tex For the moment: */ tex_normal_warning("tex", "left parfill skip is gone"); } diff --git a/source/luametatex/source/tex/texmaincontrol.c b/source/luametatex/source/tex/texmaincontrol.c index a44aa1167..5e0bb2f92 100644 --- a/source/luametatex/source/tex/texmaincontrol.c +++ b/source/luametatex/source/tex/texmaincontrol.c @@ -1372,11 +1372,13 @@ static void tex_aux_run_after_something(void) { } while (cur_cmd == spacer_cmd); if (cur_cmd == left_brace_cmd) { halfword source = tex_scan_toks_normal(1, NULL); - if (source) { - tex_save_for_after_group(token_link(source)); - token_link(source) = null; + if (source) { + if (token_link(source)) { + tex_save_for_after_group(token_link(source)); + token_link(source) = null; + } + tex_put_available_token(source); } - tex_flush_token_list(source); } else { tex_handle_error( normal_error_type, @@ -1394,10 +1396,11 @@ static void tex_aux_run_after_something(void) { if (cur_cmd == left_brace_cmd) { halfword source = tex_scan_toks_normal(1, NULL); if (source) { + /*tex Always, also when empty. */ lmt_main_control_state.after_tokens = token_link(source); token_link(source) = null; + tex_put_available_token(source); } - tex_flush_token_list(source); } else { tex_handle_error( normal_error_type, @@ -1422,7 +1425,7 @@ static void tex_aux_run_after_something(void) { } token_link(p) = token_link(source); token_link(source) = null; - tex_flush_token_list(source); + tex_put_available_token(source); } else { update_tex_end_of_group(source); } @@ -4118,7 +4121,6 @@ static void tex_aux_set_shorthand_def(int a, int force) { mathcodeval mval = tex_scan_mathchar(tex_mathcode); tex_define(a, p, mathspec_cmd, tex_new_math_spec(mval, tex_mathcode)); - // tex_define(a, p, math_char_given_cmd, math_old_packed_character(mval.class_value,mval.family_value,mval.character_value)); break; } case math_dchar_def_code: @@ -4126,14 +4128,12 @@ static void tex_aux_set_shorthand_def(int a, int force) mathdictval dval = tex_scan_mathdict(); mathcodeval mval = tex_scan_mathchar(umath_mathcode); tex_define(a, p, mathspec_cmd, tex_new_math_dict_spec(dval, mval, umath_mathcode)); - // tex_define(a, p, math_char_xgiven_cmd, math_packed_character(mval.class_value,mval.family_value,mval.character_value)); break; } case math_xchar_def_code: { mathcodeval mval = tex_scan_mathchar(umath_mathcode); tex_define(a, p, mathspec_cmd, tex_new_math_spec(mval, umath_mathcode)); - // tex_define(a, p, math_char_xgiven_cmd, math_packed_character(mval.class_value,mval.family_value,mval.character_value)); break; } case count_def_code: @@ -5854,14 +5854,13 @@ static void tex_aux_run_message(void) strnumber s = tex_aux_scan_string(); if (error_help_par) { strnumber helpinfo = tex_tokens_to_string(error_help_par); - char *h = tex_makecstring(helpinfo); + char *h = tex_to_cstring(helpinfo); tex_handle_error( normal_error_type, "%T", s, h ); - lmt_memory_free(h); tex_flush_str(helpinfo); } else if (lmt_error_state.long_help_seen) { tex_handle_error( @@ -6298,8 +6297,6 @@ inline static void tex_aux_big_switch(int mode, int cmd) register_runner(italic_correction_cmd, tex_aux_run_illegal_case, tex_aux_run_text_italic_correction, tex_run_math_italic_correction); register_runner(math_char_number_cmd, tex_aux_run_math_non_math, tex_run_text_math_char_number, tex_run_math_math_char_number); - // register_runner(math_char_given_cmd, tex_aux_run_math_non_math, tex_run_text_math_char_given, tex_run_math_math_char_given); - // register_runner(math_char_xgiven_cmd, tex_aux_run_math_non_math, tex_run_text_math_char_xgiven, tex_run_math_math_char_xgiven); register_runner(mathspec_cmd, tex_aux_run_math_non_math, tex_run_text_math_spec, tex_run_math_math_spec); register_runner(vadjust_cmd, tex_aux_run_illegal_case, tex_run_vadjust, tex_run_vadjust); diff --git a/source/luametatex/source/tex/texmath.c b/source/luametatex/source/tex/texmath.c index 52e200657..3f452b1ff 100644 --- a/source/luametatex/source/tex/texmath.c +++ b/source/luametatex/source/tex/texmath.c @@ -1713,12 +1713,6 @@ static int tex_aux_scan_math(halfword target, halfword style, int usetextfont, h mval = tex_get_math_spec(cur_chr); dval = tex_get_math_dict(cur_chr); break; - // case math_char_given_cmd: - // mval = tex_mathchar_from_integer(cur_chr, tex_mathcode); - // break; - // case math_char_xgiven_cmd: - // mval = tex_mathchar_from_integer(cur_chr, umath_mathcode); - // break; case delimiter_number_cmd: switch (cur_chr) { case math_delimiter_code: @@ -1962,12 +1956,6 @@ int tex_scan_math_cmd_val(mathcodeval *mval, mathdictval *dval) tex_get_x_token(); } while (cur_cmd == spacer_cmd); switch (cur_cmd) { - // case math_char_given_cmd: - // *mval = tex_mathchar_from_integer(cur_chr, tex_mathcode); - // break; - // case math_char_xgiven_cmd: - // *mval = tex_mathchar_from_integer(cur_chr, umath_mathcode); - // break; case mathspec_cmd: *mval = tex_get_math_spec(cur_chr); break; @@ -2050,26 +2038,6 @@ void tex_run_math_math_char_number(void) { } } -/*tex We build up an argument to |append_math_char|: */ - -// void tex_run_text_math_char_given(void) { -// tex_aux_append_math_char_in_text(tex_mathchar_from_integer(cur_chr, tex_mathcode)); -// } -// -// void tex_run_math_math_char_given(void) { -// tex_aux_append_math_char(tex_mathchar_from_integer(cur_chr, tex_mathcode)); -// } - -/*tex We build up an argument to |append_math_char| the \LUATEX\ way: */ - -// void tex_run_text_math_char_xgiven(void) { -// tex_aux_append_math_char_in_text(tex_mathchar_from_integer(cur_chr, umath_mathcode)); -// } -// -// void tex_run_math_math_char_xgiven(void) { -// tex_aux_append_math_char(tex_mathchar_from_integer(cur_chr, umath_mathcode)); -// } - void tex_run_math_delimiter_number(void) { switch (cur_chr) { case math_delimiter_code: diff --git a/source/luametatex/source/tex/texmath.h b/source/luametatex/source/tex/texmath.h index e024a6ba2..cc7aa7e7f 100644 --- a/source/luametatex/source/tex/texmath.h +++ b/source/luametatex/source/tex/texmath.h @@ -642,10 +642,6 @@ extern void tex_run_math_math_char_number (void); extern void tex_run_text_math_char_number (void); extern void tex_run_math_char_number (void); extern void tex_run_math_delimiter_number (void); -// void tex_run_math_math_char_given (void); -// void tex_run_text_math_char_given (void); -// void tex_run_math_math_char_xgiven (void); -// void tex_run_text_math_char_xgiven (void); extern void tex_run_math_equation_number (void); extern void tex_run_math_shift (void); extern void tex_run_math_italic_correction (void); diff --git a/source/luametatex/source/tex/texprimitive.c b/source/luametatex/source/tex/texprimitive.c index bbeea1bc0..42344af15 100644 --- a/source/luametatex/source/tex/texprimitive.c +++ b/source/luametatex/source/tex/texprimitive.c @@ -185,7 +185,7 @@ static int tex_aux_room_in_hash(void) inline static halfword tex_aux_compute_hash(const char *j, int l) { halfword h = (unsigned const char) j[0]; - for (int k = 1; k < l; k++) { + for (unsigned k = 1; k < l; k++) { h = (h + h + (unsigned const char) j[k]) % hash_prime; } return h; @@ -765,18 +765,6 @@ void tex_print_cmd_chr(singleword cmd, halfword chr) tex_print_str_esc("char"); tex_print_qhex(chr); break; - // case math_char_given_cmd: - // /*tex - // Okay, it's better for old macro packages that mess with meaning to report a more - // traditional value. A compromise. But, this might be dropped. - // */ - // tex_print_str_esc("mathchar"); - // tex_show_mathcode_value(tex_mathchar_from_integer(chr, tex_mathcode), tex_mathcode); - // break; - // case math_char_xgiven_cmd: - // tex_print_str_esc("Umathchar"); - // tex_show_mathcode_value(tex_mathchar_from_integer(chr, umath_mathcode), umath_mathcode); - // break; case lua_call_cmd: tex_aux_show_lua_call("luacall", chr); break; diff --git a/source/luametatex/source/tex/texprinting.c b/source/luametatex/source/tex/texprinting.c index 9a7c1d729..deabb4b72 100644 --- a/source/luametatex/source/tex/texprinting.c +++ b/source/luametatex/source/tex/texprinting.c @@ -217,9 +217,9 @@ void tex_print_char(int s) The first 256 entries above the 17th unicode plane are used for a special trick: when \TEX\ has to print items in that range, it will instead print the character that results from substracting - 0x110000 from that value. This allows byte-oriented output to things like |\specials|. - - This feature will disappear. + 0x110000 from that value. This allows byte-oriented output to things like |\specials|. We dropped + this feature because it was never used (we used it as part of experiments with \LUATEX). The old + code branches can be found in the repository. */ @@ -245,13 +245,6 @@ static void tex_aux_uprint(int s) tex_print_char(0xE0 + (s / 0x1000)); tex_print_char(0x80 + ((s % 0x1000) / 0x40)); tex_print_char(0x80 + ((s % 0x1000) % 0x40)); - } else if (s >= 0x110000) { - int c = s - 0x110000; - if (c >= 256) { - tex_formatted_warning("print", "bad raw byte to print (c=%d), skipped",c); - } else { - tex_print_char(c); - } } else { tex_print_char(0xF0 + (s / 0x40000)); tex_print_char(0x80 + ((s % 0x40000) / 0x1000)); @@ -260,24 +253,6 @@ static void tex_aux_uprint(int s) } } -static void tex_aux_lprint(lstring *ss) { - /*tex current character code position */ - unsigned char *j = ss->s; - unsigned char *l = j + ss->l; - while (j < l) { - /*tex We don't bother checking the last two bytes explicitly */ - /* 0x110000 in utf=8: 0xF4 0x90 0x80 0x80 */ - if ((j < l - 4) && (*j == 0xF4) && (*(j + 1) == 0x90)) { - int c = (*(j + 2) - 128) * 64 + (*(j + 3) - 128); - tex_print_char(c); - j = j + 4; - } else { - tex_print_char(*j); - ++j; - } - } -} - void tex_print_tex_str(int s) { if (s >= lmt_string_pool_state.string_pool_data.ptr) { @@ -291,7 +266,10 @@ void tex_print_tex_str(int s) } else if (lmt_print_state.selector == new_string_selector_code) { tex_append_string(str_string(s), (unsigned) str_length(s)); } else { - tex_aux_lprint(&str_lstring(s)); + unsigned char *j = str_string(s); + for (unsigned i = 0; i < str_length(s); i++) { + tex_print_char(j[i]); + } } } @@ -453,7 +431,7 @@ void tex_print_tex_str_esc(strnumber s) { /*tex Set variable |c| to the current escape character: */ int c = escape_char_par; - if (c >= 0 && c < 0x110000) { + if (c >= 0) { tex_print_tex_str(c); } if (s) { @@ -467,7 +445,7 @@ void tex_print_str_esc(const char *s) { /*tex Set variable |c| to the current escape character: */ int c = escape_char_par; - if (c >= 0 && c < 0x110000) { + if (c >= 0) { tex_print_tex_str(c); } if (s) { diff --git a/source/luametatex/source/tex/texscanning.c b/source/luametatex/source/tex/texscanning.c index 5dd34b3b6..fec04356b 100644 --- a/source/luametatex/source/tex/texscanning.c +++ b/source/luametatex/source/tex/texscanning.c @@ -1088,18 +1088,13 @@ void tex_scan_something_simple(halfword cmd, halfword chr) switch (cmd) { /* begin of tex_aux_short_scan_something_internal */ case char_given_cmd: - // case math_char_given_cmd: - // case math_char_xgiven_cmd: cur_val = chr; cur_val_level = int_val_level; break; - -case mathspec_cmd: - cur_val = (chr && node_subtype(chr) == tex_mathcode) ? math_spec_value(chr) : 0; - cur_val_level = int_val_level; - break; - - + case mathspec_cmd: + cur_val = (chr && node_subtype(chr) == tex_mathcode) ? math_spec_value(chr) : 0; + cur_val_level = int_val_level; + break; case iterator_value_cmd: cur_val = chr > 0x100000 ? - (chr - 0x100000) : chr; cur_val_level = int_val_level; @@ -1254,8 +1249,6 @@ static halfword tex_aux_scan_something_internal(halfword cmd, halfword chr, int switch (cmd) { /* begin of tex_aux_short_scan_something_internal */ case char_given_cmd: - // case math_char_given_cmd: - // case math_char_xgiven_cmd: cur_val = chr; cur_val_level = int_val_level; break; diff --git a/source/luametatex/source/tex/texstringpool.c b/source/luametatex/source/tex/texstringpool.c index 8367447da..fd93758b5 100644 --- a/source/luametatex/source/tex/texstringpool.c +++ b/source/luametatex/source/tex/texstringpool.c @@ -435,6 +435,7 @@ char *tex_makeclstring(int s, size_t *len) } } +/* char *tex_makecstring(int s) { if (s < cs_offset_value) { @@ -443,6 +444,23 @@ char *tex_makecstring(int s) return lmt_memory_strdup((str_length(s) > 0) ? (const char *) str_string(s) : ""); } } +*/ + +/*tex + I might eventually replace this because in qite some calls we know that we knwo that we have + a pointer in string space. We can kin dof predict in what cases we are below |cs_offset_value| + anyway. +*/ + +char *tex_makecstring(int s, int *allocated) +{ + *allocated = s < cs_offset_value; + if (*allocated) { + return (char *) aux_uni2str((unsigned) s); + } else { + return str_length(s) > 0 ? (char *) str_string(s) : ""; + } +} /*tex diff --git a/source/luametatex/source/tex/texstringpool.h b/source/luametatex/source/tex/texstringpool.h index b3924a0fe..7302e7fb4 100644 --- a/source/luametatex/source/tex/texstringpool.h +++ b/source/luametatex/source/tex/texstringpool.h @@ -21,7 +21,7 @@ typedef struct lstring { unsigned char *s; const char *c; }; - size_t l; /* could be int, but this way we padd */ + size_t l; /* could be int, but this way we padd */ } lstring; typedef struct string_pool_info { @@ -76,35 +76,37 @@ extern string_pool_info lmt_string_pool_state; /*tex Forget the last character in the pool. */ -inline void tex_flush_char(void) { --lmt_string_pool_state.string_temp_top; } - -extern strnumber tex_make_string (void); -extern strnumber tex_push_string (const unsigned char *s, int l); -extern char *tex_take_string (int *len); -extern int tex_str_eq_buf (strnumber s, int k, int n); -extern int tex_str_eq_str (strnumber s, strnumber t); -extern int tex_str_eq_cstr (strnumber s, const char *, size_t); -extern int tex_get_strings_started (void); -extern void tex_reset_cur_string (void); -/* strnumber tex_search_string (strnumber search); */ -/* int tex_used_strings (void); */ -extern strnumber tex_maketexstring (const char *s); -extern strnumber tex_maketexlstring (const char *s, size_t); -extern void tex_append_char (unsigned char c); -extern void tex_append_string (const unsigned char *s, unsigned l); -extern char *tex_makecstring (int s); -extern char *tex_makeclstring (int s, size_t *len); -extern void tex_dump_string_pool (dumpstream f); -extern void tex_undump_string_pool (dumpstream f); -extern void tex_initialize_string_pool (void); -extern void tex_initialize_string_mem (void); -extern void tex_flush_str (strnumber s); -extern strnumber tex_save_cur_string (void); -extern void tex_restore_cur_string (strnumber u); - -/* void tex_increment_pool_string (int n); */ -/* void tex_decrement_pool_string (int n); */ - -extern void tex_compact_string_pool (void); +inline void tex_flush_char(void) { --lmt_string_pool_state.string_temp_top; } + +extern strnumber tex_make_string (void); +extern strnumber tex_push_string (const unsigned char *s, int l); +extern char *tex_take_string (int *len); +extern int tex_str_eq_buf (strnumber s, int k, int n); +extern int tex_str_eq_str (strnumber s, strnumber t); +extern int tex_str_eq_cstr (strnumber s, const char *, size_t); +extern int tex_get_strings_started (void); +extern void tex_reset_cur_string (void); +/* strnumber tex_search_string (strnumber search); */ +/* int tex_used_strings (void); */ +extern strnumber tex_maketexstring (const char *s); +extern strnumber tex_maketexlstring (const char *s, size_t); +extern void tex_append_char (unsigned char c); +extern void tex_append_string (const unsigned char *s, unsigned l); +extern char *tex_makecstring (int s, int *allocated); +extern char *tex_makeclstring (int s, size_t *len); +extern void tex_dump_string_pool (dumpstream f); +extern void tex_undump_string_pool (dumpstream f); +extern void tex_initialize_string_pool (void); +extern void tex_initialize_string_mem (void); +extern void tex_flush_str (strnumber s); +extern strnumber tex_save_cur_string (void); +extern void tex_restore_cur_string (strnumber u); + +/* void tex_increment_pool_string (int n); */ +/* void tex_decrement_pool_string (int n); */ + +extern void tex_compact_string_pool (void); + +inline char *tex_to_cstring (int s) { return str_length(s) > 0 ? (char *) str_string(s) : ""; } # endif diff --git a/source/luametatex/source/tex/textoken.c b/source/luametatex/source/tex/textoken.c index 0bb0f3e45..93bf3913a 100644 --- a/source/luametatex/source/tex/textoken.c +++ b/source/luametatex/source/tex/textoken.c @@ -2820,7 +2820,7 @@ void tex_run_convert_tokens(halfword code) /* halfword result = tex_scan_toks_expand(0, NULL, code == lua_token_string_code); */ lmt_token_state.in_lua_escape = 1; escape_char_par = '\\'; - str.s = (unsigned char *) tex_tokenlist_to_tstring(result, 0, &length, 0, 0, 0); + str.s = (unsigned char *) tex_tokenlist_to_tstring(result, 0, &length, 0, 0, 0, 0); str.l = (unsigned) length; lmt_token_state.in_lua_escape = saved_in_lua_escape; escape_char_par = saved_escape_char; @@ -3034,8 +3034,6 @@ static void tex_aux_append_uchar_to_buffer(int s) lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0xE0 + (s / 0x1000)); lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x1000) / 0x40)); lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x1000) % 0x40)); - } else if (s >= 0x110000) { - lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (s - 0x11000); } else { lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0xF0 + (s / 0x40000)); lmt_token_state.buffer[lmt_token_state.bufloc++] = (char) (0x80 + ((s % 0x40000) / 0x1000)); @@ -3078,7 +3076,7 @@ static void tex_aux_append_esc_to_buffer(const char *s) /* make two versions: macro and not */ -char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamble, int nospace, int strip) +char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamble, int nospace, int strip, int wipe) { if (pp) { /*tex We need to go beyond the reference. */ @@ -3100,6 +3098,8 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb int min = 0; int max = lmt_token_memory_state.tokens_data.top; int skip = 0; + int tail = p; + int count = 0; if (skippreamble) { skip = get_token_parameters(pp); } @@ -3210,7 +3210,8 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb if (txt < 0 || txt >= lmt_string_pool_state.string_pool_data.ptr) { tex_aux_append_str_to_buffer(error_string_nonexistent(36)); } else { - char *sh = tex_makecstring(txt); + int allocated = 0; + char *sh = tex_makecstring(txt, &allocated); char *s = sh; if (tex_is_active_cs(txt)) { s = s + 3; @@ -3219,7 +3220,7 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb s++; } } else { - if (e >= 0 && e < 0x110000) { + if (e >= 0) { tex_aux_append_uchar_to_buffer(e); } while (*s) { @@ -3230,10 +3231,14 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb tex_aux_append_char_to_buffer(' '); } } - lmt_memory_free(sh); + if (allocated) { + lmt_memory_free(sh); + } } } } + tail = p; + ++count; p = token_link(p); } } @@ -3251,7 +3256,14 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb if (siz) { *siz = lmt_token_state.bufloc; } + if (wipe) { + tex_flush_token_list_head_tail(pp, tail, count); + } return lmt_token_state.buffer; + } else { + if (wipe) { + tex_put_available_token(pp); + } } } if (siz) { diff --git a/source/luametatex/source/tex/textoken.h b/source/luametatex/source/tex/textoken.h index 1996f351c..ad67dfcb5 100644 --- a/source/luametatex/source/tex/textoken.h +++ b/source/luametatex/source/tex/textoken.h @@ -366,8 +366,7 @@ extern void tex_run_combine_the_toks (void); extern void tex_run_convert_tokens (halfword code); extern strnumber tex_the_convert_string (halfword c, int i); extern strnumber tex_tokens_to_string (halfword p); -/* char *tex_tokenlist_to_cstring (int p, int inhibit_par, int *siz); */ -extern char *tex_tokenlist_to_tstring (int p, int inhibit_par, int *siz, int skip, int nospace, int strip); +extern char *tex_tokenlist_to_tstring (int p, int inhibit_par, int *siz, int skip, int nospace, int strip, int wipe); extern halfword tex_get_tex_dimen_register (int j, int internal); extern halfword tex_get_tex_skip_register (int j, int internal); @@ -391,9 +390,6 @@ extern halfword tex_copy_token_list (halfword h, halfword *t); extern halfword tex_parse_str_to_tok (halfword head, halfword *tail, halfword ct, const char *str, size_t lstr, int option); -inline int tex_valid_token(int t) -{ - return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top)); -} +inline int tex_valid_token (int t) { return ((t >= 0) && (t <= (int) lmt_token_memory_state.tokens_data.top)); } # endif diff --git a/source/luametatex/source/utilities/auxunistring.c b/source/luametatex/source/utilities/auxunistring.c index e95854a93..30dc518e6 100644 --- a/source/luametatex/source/utilities/auxunistring.c +++ b/source/luametatex/source/utilities/auxunistring.c @@ -52,15 +52,12 @@ unsigned char *aux_uni2str(unsigned unic) buf[0] = (unsigned char) (0xc0 | (unic >> 6)); buf[1] = (unsigned char) (0x80 | (unic & 0x3f)); buf[2] = '\0'; - } else if (unic >= 0x110000) { - buf[0] = (unsigned char) (unic - 0x110000); - buf[1] = '\0'; } else if (unic < 0x10000) { buf[0] = (unsigned char) (0xe0 | (unic >> 12)); buf[1] = (unsigned char) (0x80 | ((unic >> 6) & 0x3f)); buf[2] = (unsigned char) (0x80 | (unic & 0x3f)); buf[3] = '\0'; - } else { + } else if (unic < 0x110000) { unic -= 0x10000; int u = (int) (((unic & 0xf0000) >> 16) + 1); buf[0] = (unsigned char) (0xf0 | (u >> 2)); -- cgit v1.2.3