diff options
Diffstat (limited to 'source')
24 files changed, 337 insertions, 168 deletions
diff --git a/source/luametatex/source/lua/lmtfontlib.c b/source/luametatex/source/lua/lmtfontlib.c index 042772b32..f8ca2a37a 100644 --- a/source/luametatex/source/lua/lmtfontlib.c +++ b/source/luametatex/source/lua/lmtfontlib.c @@ -825,7 +825,7 @@ static int fontlib_id(lua_State *L) if (lua_type(L, 1) == LUA_TSTRING) { size_t l; const char *s = lua_tolstring(L, 1, &l); - int cs = tex_string_locate(s, l, 0); + int cs = tex_string_locate_only(s, l); int f = -1; if (cs == undefined_control_sequence || cs == undefined_cs_cmd || eq_type(cs) != set_font_cmd) { lua_pushliteral(L, "not a valid font csname"); @@ -903,7 +903,7 @@ static int fontlib_getmathspec(lua_State *L) if (lua_type(L, 1) == LUA_TSTRING) { size_t lname = 0; const char *name = lua_tolstring(L, 1, &lname); - halfword cs = tex_string_locate(name, lname, 0); + halfword cs = tex_string_locate_only(name, lname); if (eq_type(cs) == mathspec_cmd) { halfword ms = eq_value(cs); if (ms) { @@ -923,7 +923,7 @@ static int fontlib_getfontspec(lua_State *L) if (lua_type(L, 1) == LUA_TSTRING) { size_t lname = 0; const char *name = lua_tolstring(L, 1, &lname); - halfword cs = tex_string_locate(name, lname, 0); + halfword cs = tex_string_locate_only(name, lname); if (eq_type(cs) == fontspec_cmd) { halfword fs = eq_value(cs); if (fs) { diff --git a/source/luametatex/source/lua/lmtinterface.h b/source/luametatex/source/lua/lmtinterface.h index ea6ead4ab..1dc47cf15 100644 --- a/source/luametatex/source/lua/lmtinterface.h +++ b/source/luametatex/source/lua/lmtinterface.h @@ -553,6 +553,7 @@ make_lua_key(L, conditional);\ make_lua_key(L, conditionalmathskip);\ make_lua_key(L, connectoroverlapmin);\ make_lua_key(L, constant);\ +make_lua_key(L, constant_call);\ make_lua_key(L, container);\ make_lua_key(L, contributehead);\ make_lua_key(L, convert);\ @@ -568,8 +569,9 @@ make_lua_key(L, cs_name);\ make_lua_key(L, csname);\ make_lua_key(L, current);\ make_lua_key(L, data);\ -make_lua_key(L, deep_frozen_cs_dont_expand);\ -make_lua_key(L, deep_frozen_cs_end_template);\ +make_lua_key(L, deep_frozen_dont_expand);\ +make_lua_key(L, deep_frozen_end_template);\ +make_lua_key(L, deep_frozen_keep_constant);\ make_lua_key(L, def);\ make_lua_key(L, deferred);\ make_lua_key(L, define_char_code);\ diff --git a/source/luametatex/source/lua/lmttexlib.c b/source/luametatex/source/lua/lmttexlib.c index 603454bfe..08809e198 100644 --- a/source/luametatex/source/lua/lmttexlib.c +++ b/source/luametatex/source/lua/lmttexlib.c @@ -557,7 +557,7 @@ static int texlib_mprint(lua_State *L) if (lua_type(L, 1) == LUA_TSTRING) { size_t lname = 0; const char *name = lua_tolstring(L, 1, &lname); - int cs = tex_string_locate(name, lname, 0); + int cs = tex_string_locate_only(name, lname); int cmd = eq_type(cs); if (is_call_cmd(cmd)) { texlib_aux_store_token(cs_token_flag + cs, partial_line_mode, default_catcode_table_preset); @@ -1185,7 +1185,7 @@ inline static int texlib_aux_valid_register_index(lua_State *L, int slot, int cm { size_t len; const char *str = lua_tolstring(L, 1, &len); - int cs = tex_string_locate(str, len, 0); + int cs = tex_string_locate_only(str, len); if (eq_type(cs) == cmd) { index = eq_value(cs) - base; } else if (eq_type(cs) == constant_cmd) { @@ -1213,7 +1213,7 @@ static int texlib_get_register_index(lua_State *L) { size_t len; const char *str = lua_tolstring(L, 1, &len); - int cs = tex_string_locate(str, len, 0); + int cs = tex_string_locate_only(str, len); int index = -1; switch (eq_type(cs)) { case register_toks_cmd : index = eq_value(cs) - register_toks_base; break; @@ -1348,7 +1348,7 @@ static int texlib_aux_check_for_index( { size_t len; const char *str = lua_tolstring(L, slot, &len); - int cs = tex_string_locate(str, len, 0); + int cs = tex_string_locate_only(str, len); if (eq_type(cs) == internal_cmd) { *index = eq_value(cs) - internal_base; return 1; @@ -1903,7 +1903,7 @@ int lmt_get_box_id(lua_State *L, int i, int report) { size_t len = 0; const char *str = lua_tolstring(L, i, &len); - int cs = tex_string_locate(str, len, 0); + int cs = tex_string_locate_only(str, len); int cmd = eq_type(cs); switch (cmd) { case char_given_cmd: @@ -2573,7 +2573,7 @@ static int texlib_set_item(lua_State* L, int index, int prefixes) size_t sl; const char *st = lua_tolstring(L, slot++, &sl); if (sl > 0) { - int cs = tex_string_locate(st, sl, 0); + int cs = tex_string_locate_only(st, sl); if (cs != undefined_control_sequence && has_eq_flag_bits(cs, primitive_flag_bit)) { int cmd = eq_type(cs); switch (cmd) { @@ -3173,7 +3173,7 @@ static int texlib_get_internal(lua_State *L, int index, int all) int cmd = get_prim_eq_type(cs); int code = get_prim_equiv(cs); */ - int cs = tex_string_locate(s, l, 0); + int cs = tex_string_locate_only(s, l); if (cs != undefined_control_sequence && has_eq_flag_bits(cs, primitive_flag_bit)) { int cmd = eq_type(cs); int code = eq_value(cs); @@ -4280,7 +4280,7 @@ static int texlib_runlocal(lua_State *L) { size_t lname = 0; const char *name = lua_tolstring(L, 1, &lname); - int cs = tex_string_locate(name, lname, 0); + int cs = tex_string_locate_only(name, lname); int cmd = eq_type(cs); if (cmd < call_cmd) { // is_call_cmd // todo: use the better register helpers and range checkers @@ -4850,7 +4850,7 @@ static int texlib_getintegervalue(lua_State *L) /* todo, now has duplicate in to size_t len; const char *str = lua_tolstring(L, 1, &len); if (len > 0) { - int cs = tex_string_locate(str, len, 0); + int cs = tex_string_locate_only(str, len); switch (eq_type(cs)) { case integer_cmd: lua_pushinteger(L, eq_value(cs)); @@ -4858,6 +4858,7 @@ static int texlib_getintegervalue(lua_State *L) /* todo, now has duplicate in to case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: return texlib_aux_getvalue(L, int_val_level, cs); default: /* twice a lookup but fast enough for now */ @@ -4875,7 +4876,7 @@ static int texlib_getfloatvalue(lua_State *L) /* todo, now has duplicate in toke size_t len; const char *str = lua_tolstring(L, 1, &len); if (len > 0) { - int cs = tex_string_locate(str, len, 0); + int cs = tex_string_locate_only(str, len); switch (eq_type(cs)) { case posit_cmd: lua_pushnumber(L, tex_posit_to_double(eq_value(cs))); @@ -4883,6 +4884,7 @@ static int texlib_getfloatvalue(lua_State *L) /* todo, now has duplicate in toke case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: return texlib_aux_getvalue(L, posit_val_level, cs); default: /* twice a lookup but fast enough for now */ @@ -4900,7 +4902,7 @@ static int texlib_getdimensionvalue(lua_State *L) /* todo, now has duplicate in size_t len; const char *str = lua_tolstring(L, 1, &len); if (len > 0) { - int cs = tex_string_locate(str, len, 0); + int cs = tex_string_locate_only(str, len); switch (eq_type(cs)) { case dimension_cmd: lua_pushinteger(L, eq_value(cs)); @@ -4911,6 +4913,7 @@ static int texlib_getdimensionvalue(lua_State *L) /* todo, now has duplicate in case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: return texlib_aux_getvalue(L, dimen_val_level, cs); default: /* twice a lookup but fast enough for now */ diff --git a/source/luametatex/source/lua/lmttokenlib.c b/source/luametatex/source/lua/lmttokenlib.c index fe82e7e40..e4ec867f0 100644 --- a/source/luametatex/source/lua/lmttokenlib.c +++ b/source/luametatex/source/lua/lmttokenlib.c @@ -215,12 +215,14 @@ void lmt_tokenlib_initialize(void) lmt_interface.command_names[get_mark_cmd] = (command_item) { .id = get_mark_cmd, .lua = lua_key_index(get_mark), .name = lua_key(get_mark), .kind = regular_command_item, .min = 0, .max = last_get_mark_code, .base = 0, .fixedvalue = 0 }; lmt_interface.command_names[call_cmd] = (command_item) { .id = call_cmd, .lua = lua_key_index(call), .name = lua_key(call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; lmt_interface.command_names[protected_call_cmd] = (command_item) { .id = protected_call_cmd, .lua = lua_key_index(protected_call), .name = lua_key(protected_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; - lmt_interface.command_names[semi_protected_call_cmd] = (command_item) { .id = semi_protected_call_cmd, .lua = lua_key_index(semi_protected_call), .name = lua_key(protected_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; + lmt_interface.command_names[semi_protected_call_cmd] = (command_item) { .id = semi_protected_call_cmd, .lua = lua_key_index(semi_protected_call), .name = lua_key(semi_protected_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; + lmt_interface.command_names[constant_call_cmd] = (command_item) { .id = constant_call_cmd, .lua = lua_key_index(constant_call), .name = lua_key(constant_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; lmt_interface.command_names[tolerant_call_cmd] = (command_item) { .id = tolerant_call_cmd, .lua = lua_key_index(tolerant_call), .name = lua_key(tolerant_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; lmt_interface.command_names[tolerant_protected_call_cmd] = (command_item) { .id = tolerant_protected_call_cmd, .lua = lua_key_index(tolerant_protected_call), .name = lua_key(tolerant_protected_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; - lmt_interface.command_names[tolerant_semi_protected_call_cmd] = (command_item) { .id = tolerant_semi_protected_call_cmd, .lua = lua_key_index(tolerant_semi_protected_call), .name = lua_key(tolerant_protected_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; - lmt_interface.command_names[deep_frozen_end_template_cmd] = (command_item) { .id = deep_frozen_end_template_cmd, .lua = lua_key_index(deep_frozen_cs_end_template), .name = lua_key(deep_frozen_cs_end_template), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; - lmt_interface.command_names[deep_frozen_dont_expand_cmd] = (command_item) { .id = deep_frozen_dont_expand_cmd, .lua = lua_key_index(deep_frozen_cs_dont_expand), .name = lua_key(deep_frozen_cs_dont_expand), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; + lmt_interface.command_names[tolerant_semi_protected_call_cmd] = (command_item) { .id = tolerant_semi_protected_call_cmd, .lua = lua_key_index(tolerant_semi_protected_call), .name = lua_key(tolerant_semi_protected_call), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; + lmt_interface.command_names[deep_frozen_end_template_cmd] = (command_item) { .id = deep_frozen_end_template_cmd, .lua = lua_key_index(deep_frozen_end_template), .name = lua_key(deep_frozen_end_template), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; + lmt_interface.command_names[deep_frozen_dont_expand_cmd] = (command_item) { .id = deep_frozen_dont_expand_cmd, .lua = lua_key_index(deep_frozen_dont_expand), .name = lua_key(deep_frozen_dont_expand), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; + lmt_interface.command_names[deep_frozen_keep_constant_cmd] = (command_item) { .id = deep_frozen_keep_constant_cmd, .lua = lua_key_index(deep_frozen_keep_constant), .name = lua_key(deep_frozen_keep_constant), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; lmt_interface.command_names[internal_glue_reference_cmd] = (command_item) { .id = internal_glue_reference_cmd, .lua = lua_key_index(internal_glue_reference), .name = lua_key(internal_glue_reference), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; lmt_interface.command_names[register_glue_reference_cmd] = (command_item) { .id = register_glue_reference_cmd, .lua = lua_key_index(register_glue_reference), .name = lua_key(register_glue_reference), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; lmt_interface.command_names[internal_mu_glue_reference_cmd] = (command_item) { .id = internal_mu_glue_reference_cmd, .lua = lua_key_index(internal_mu_glue_reference), .name = lua_key(internal_mu_glue_reference), .kind = token_command_item, .min = ignore_entry, .max = ignore_entry, .base = ignore_entry, .fixedvalue = 0 }; @@ -1445,6 +1447,7 @@ static int tokenlib_scan_detokened(lua_State *L) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -1497,10 +1500,12 @@ static int tokenlib_scan_string(lua_State *L) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: { + /*tex It is pretty unlikely to end up here, because we expand. */ halfword t = token_link(cur_chr); lmt_token_list_to_luastring(L, t, 0, 0, 1); break; @@ -1550,24 +1555,41 @@ static int tokenlib_scan_argument(lua_State *L) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + // case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: - { - halfword result; - halfword defref = lmt_input_state.def_ref; - tex_back_input(right_brace_token + '}'); - if (lua_type(L, 1) == LUA_TBOOLEAN && ! lua_toboolean(L, 1)) { - tex_expand_current_token(); - result = tex_scan_toks_normal(1, NULL); - } else { - tex_back_input(cur_tok); - result = tex_scan_toks_expand(1, NULL, 0); - } - lmt_token_list_to_luastring(L, result, 0, 0, 1); - lmt_input_state.def_ref = defref; - break; - } + { + halfword result; + halfword defref = lmt_input_state.def_ref; + tex_back_input(right_brace_token + '}'); + if (lua_type(L, 1) == LUA_TBOOLEAN && ! lua_toboolean(L, 1)) { + tex_expand_current_token(); + result = tex_scan_toks_normal(1, NULL); + } else { + tex_back_input(cur_tok); + result = tex_scan_toks_expand(1, NULL, 0); + } + lmt_token_list_to_luastring(L, result, 0, 0, 1); + lmt_input_state.def_ref = defref; + break; + } + case constant_call_cmd: + { + halfword h = token_link(cur_chr); + if (h) { + luaL_Buffer b; + luaL_buffinit(L, &b); + while (h) { + tokenlib_aux_add_utf_char_to_buffer(&b, token_chr(token_info(h))); + h = token_link(h); + } + luaL_pushresult(&b); + } else { + lua_pushliteral(L, ""); + } + break; + } case letter_cmd: case other_char_cmd: { @@ -1694,6 +1716,7 @@ static int tokenlib_scan_delimited(lua_State *L) switch (cur_cmd) { case call_cmd: case tolerant_call_cmd: + case constant_call_cmd: tex_expand_current_token(); goto PICKUP; case protected_call_cmd: @@ -2112,6 +2135,7 @@ static int tokenlib_scan_value(lua_State *L) break; /* case call_cmd: + case constant_call_cmd: { halfword t = cur_cs ? cs_token_flag + cur_cs : token_val(cur_cmd, cur_chr); if (t >= cs_token_flag) { @@ -2154,6 +2178,7 @@ static int tokenlib_scan_value(lua_State *L) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -2307,7 +2332,7 @@ static int tokenlib_is_defined(lua_State *L) size_t l; const char *s = lua_tolstring(L, 1, &l); if (l > 0) { - int cs = tex_string_locate(s, l, 0); + int cs = tex_string_locate_only(s, l); b = (cs != undefined_control_sequence) && (eq_type(cs) != undefined_cs_cmd); } } @@ -2402,7 +2427,7 @@ static int tokenlib_get_cmdchrcs(lua_State* L) size_t l; const char *s = lua_tolstring(L, 1, &l); if (l > 0) { - int cs = tex_string_locate(s, l, 0); + int cs = tex_string_locate_only(s, l); int cmd = eq_type(cs); int chr = eq_value(cs); if (! lua_toboolean(L, 2)) { @@ -2457,7 +2482,7 @@ static int tokenlib_get_cstoken(lua_State* L) size_t l; const char *s = lua_tolstring(L, 1, &l); if (l > 0) { - lua_pushinteger(L, (lua_Integer) tex_string_locate(s, l, 0) + cs_token_flag); + lua_pushinteger(L, (lua_Integer) tex_string_locate_only(s, l) + cs_token_flag); return 1; } return 0; @@ -2774,12 +2799,7 @@ inline static int tokenlib_get_constant(lua_State *L) { lua_token *n = tokenlib_aux_check_istoken(L, 1); halfword tok = token_info(n->token); - int result = 0; - if (tok >= cs_token_flag && is_call_cmd(eq_type(tok - cs_token_flag))) { - halfword v = eq_value(tok - cs_token_flag); - result = v && get_token_reference(v) == max_token_reference; - } - lua_pushboolean(L, result); + lua_pushboolean(L, tok >= cs_token_flag && (eq_type(tok - cs_token_flag) == constant_call_cmd)); return 1; } @@ -2848,7 +2868,7 @@ static int tokenlib_get_fields(lua_State *L) if (l > 0) { halfword cs; lua_createtable(L, 0, onlyflags ? 0 : 5); - cs = tex_string_locate(str, l, 0); + cs = tex_string_locate_only(str, l); cmd = eq_type(cs); chr = eq_value(cs); flags = eq_flag(cs); @@ -3087,7 +3107,7 @@ static int tokenlib_get_meaning(lua_State *L) if (lua_type(L, 1) == LUA_TSTRING) { size_t lname = 0; const char *name = lua_tolstring(L, 1, &lname); - halfword cs = tex_string_locate(name, lname, 0); + halfword cs = tex_string_locate_only(name, lname); halfword cmd = eq_type(cs); if (is_call_cmd(cmd)) { int chr = eq_value(cs); @@ -3160,7 +3180,7 @@ static int tokenlib_get_macro(lua_State *L) if (lua_type(L, 1) == LUA_TSTRING) { size_t lname = 0; const char *name = lua_tolstring(L, 1, &lname); - halfword cs = tex_string_locate(name, lname, 0); + halfword cs = tex_string_locate_only(name, lname); halfword cmd = eq_type(cs); if (is_call_cmd(cmd)) { halfword chr = eq_value(cs); @@ -3198,7 +3218,7 @@ static int tokenlib_push_macro(lua_State *L) // todo: just store cmd and flag to size_t lname = 0; const char *name = lua_tolstring(L, 1, &lname); if (lname > 0) { - halfword cs = tex_string_locate(name, lname, 0); + halfword cs = tex_string_locate_only(name, lname); singleword cmd = eq_type(cs); halfword chr = eq_value(cs); quarterword global = lua_toboolean(L, 2) ? add_global_flag(0) : 0; /* how */ @@ -3355,9 +3375,6 @@ static int tokenlib_set_macro(lua_State *L) /* todo: protected */ h = get_reference_token(); /*tex Options: 1=create (will trigger an error), 2=ignore. */ tex_parse_str_to_tok(h, null, ct, str, lstr, lua_toboolean(L, slot++) ? 2 : 1); - if (is_constant(flags)) { - set_token_reference(h, max_token_reference); - } } else { h = lmt_token_state.empty; // tex_add_token_reference(h); @@ -3379,7 +3396,7 @@ halfword lmt_macro_to_tok(lua_State *L, int slot, halfword *tail) { size_t lname = 0; const char *name = lua_tolstring(L, slot, &lname); - int cs = tex_string_locate(name, lname, 0); + int cs = tex_string_locate_only(name, lname); int cmd = eq_type(cs); if (is_call_cmd(cmd)) { tok = cs_token_flag + cs; @@ -3542,7 +3559,7 @@ static int tokenlib_get_constant_value(lua_State *L, halfword cmd) size_t l; const char *s = lua_tolstring(L, 1, &l); if (l > 0) { - int cs = tex_string_locate(s, l, 0); + int cs = tex_string_locate_only(s, l); if (eq_type(cs) == cmd) { lua_pushinteger(L, eq_value(cs)); return 1; diff --git a/source/luametatex/source/luametatex.h b/source/luametatex/source/luametatex.h index 68820d24d..bb39d8517 100644 --- a/source/luametatex/source/luametatex.h +++ b/source/luametatex/source/luametatex.h @@ -92,7 +92,7 @@ # define luametatex_version 210 # define luametatex_revision 9 # define luametatex_version_string "2.10.09" -# define luametatex_development_id 20230531 +# define luametatex_development_id 20230604 # define luametatex_name_camelcase "LuaMetaTeX" # define luametatex_name_lowercase "luametatex" diff --git a/source/luametatex/source/tex/texalign.c b/source/luametatex/source/tex/texalign.c index 8de3adff9..3332d82f9 100644 --- a/source/luametatex/source/tex/texalign.c +++ b/source/luametatex/source/tex/texalign.c @@ -532,6 +532,7 @@ static void tex_aux_get_preamble_token(void) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -851,7 +852,7 @@ void tex_finish_no_alignment_group(void) static void tex_aux_align_peek(void) { RESTART: - lmt_input_state.align_state = 1000000; + lmt_input_state.align_state = busy_alignment_state; AGAIN: tex_get_x_or_protected(); switch (cur_cmd) { @@ -863,6 +864,7 @@ static void tex_aux_align_peek(void) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -907,7 +909,7 @@ void tex_run_alignment_initialize(void) { halfword saved_cs = cur_cs; tex_aux_push_alignment(); - lmt_input_state.align_state = -1000000; + lmt_input_state.align_state = initial_alignment_state; /*tex When |\halign| is used as a displayed formula, there should be no other pieces of mlists present. @@ -947,7 +949,7 @@ void tex_run_alignment_initialize(void) lmt_alignment_state.cur_loop = null; lmt_input_state.scanner_status = scanner_is_aligning; lmt_input_state.warning_index = saved_cs; - lmt_input_state.align_state = -1000000; + lmt_input_state.align_state = initial_alignment_state; /*tex At this point, |cur_cmd = left_brace|. */ while (1) { /*tex Append the current tabskip glue to the preamble list. */ @@ -973,7 +975,7 @@ void tex_run_alignment_initialize(void) tex_aux_get_preamble_token(); if ((cur_cmd == alignment_cmd && cur_chr == align_content_code) || cur_cmd == parameter_cmd) { break; - } else if ((cur_cmd == alignment_cmd || cur_cmd == alignment_tab_cmd) && (lmt_input_state.align_state == -1000000)) { + } else if ((cur_cmd == alignment_cmd || cur_cmd == alignment_tab_cmd) && (lmt_input_state.align_state == initial_alignment_state)) { if ((current == lmt_alignment_state.hold_token_head) && (! lmt_alignment_state.cur_loop) && (cur_cmd == alignment_tab_cmd)) { lmt_alignment_state.cur_loop = lmt_alignment_state.cur_align; } else { @@ -1002,7 +1004,7 @@ void tex_run_alignment_initialize(void) token_link(current) = null; while (1) { tex_aux_get_preamble_token(); - if ((cur_cmd == alignment_cmd || cur_cmd == alignment_tab_cmd) && (lmt_input_state.align_state == -1000000)) { + if ((cur_cmd == alignment_cmd || cur_cmd == alignment_tab_cmd) && (lmt_input_state.align_state == initial_alignment_state)) { break; } else if ((cur_cmd == alignment_cmd && cur_chr == align_content_code) || cur_cmd == parameter_cmd) { tex_handle_error( @@ -1157,7 +1159,7 @@ void tex_insert_alignment_template(void) align_record_cmd(lmt_alignment_state.cur_align) = cur_cmd; align_record_chr(lmt_alignment_state.cur_align) = cur_chr; tex_begin_token_list(tok, template_post_text); - lmt_input_state.align_state = 1000000; + lmt_input_state.align_state = busy_alignment_state; lmt_alignment_state.cell_source = alignment_cell_source_par; if (alignment_wrap_source_par) { lmt_alignment_state.wrap_source = alignment_wrap_source_par; @@ -1226,7 +1228,7 @@ static int tex_aux_finish_column(void) halfword q = node_next(lmt_alignment_state.cur_align); if (! q) { tex_confusion("end template, case 2"); - } else if (lmt_input_state.align_state < 500000) { + } else if (lmt_input_state.align_state < interwoven_alignment_threshold) { tex_alignment_interwoven_error(1); } else { /*tex A few state variables. */ @@ -1379,7 +1381,7 @@ static int tex_aux_finish_column(void) tex_aux_initialize_span(record); } } - lmt_input_state.align_state = 1000000; + lmt_input_state.align_state = busy_alignment_state; do { tex_get_x_or_protected(); } while (cur_cmd == spacer_cmd); diff --git a/source/luametatex/source/tex/texcommands.c b/source/luametatex/source/tex/texcommands.c index ae22f9794..71793cb73 100644 --- a/source/luametatex/source/tex/texcommands.c +++ b/source/luametatex/source/tex/texcommands.c @@ -1360,6 +1360,10 @@ void tex_initialize_commands(void) set_eq_type(deep_frozen_cs_dont_expand_code, deep_frozen_dont_expand_cmd); set_eq_flag(deep_frozen_cs_dont_expand_code, 0); + cs_text(deep_frozen_cs_keep_constant_code) = tex_maketexstring("keepconstant"); + set_eq_type(deep_frozen_cs_keep_constant_code, deep_frozen_keep_constant_cmd); + set_eq_flag(deep_frozen_cs_keep_constant_code, 0); + cs_text(deep_frozen_cs_protection_code) = tex_maketexstring("inaccessible"); cs_text(deep_frozen_cs_end_write_code) = tex_maketexstring("endwrite"); diff --git a/source/luametatex/source/tex/texcommands.h b/source/luametatex/source/tex/texcommands.h index 04f8fb606..16dc47520 100644 --- a/source/luametatex/source/tex/texcommands.h +++ b/source/luametatex/source/tex/texcommands.h @@ -279,6 +279,7 @@ typedef enum tex_command_code { call_cmd, /*tex regular control sequence */ protected_call_cmd, /*tex idem but doesn't expand in edef like situations */ semi_protected_call_cmd, + constant_call_cmd, tolerant_call_cmd, /*tex control sequence with tolerant arguments */ tolerant_protected_call_cmd, /*tex idem but doesn't expand in edef like situations */ tolerant_semi_protected_call_cmd, @@ -287,6 +288,7 @@ typedef enum tex_command_code { */ deep_frozen_end_template_cmd, /*tex end of an alignment template */ deep_frozen_dont_expand_cmd, /*tex the following token was marked by |\noexpand|) */ + deep_frozen_keep_constant_cmd, /*tex The next bunch is never seen directly as they are shortcuts to registers and special data strutures. They are the internal register (pseudo) commands and are also needed for diff --git a/source/luametatex/source/tex/texconditional.c b/source/luametatex/source/tex/texconditional.c index b4541f342..c4cf7f902 100644 --- a/source/luametatex/source/tex/texconditional.c +++ b/source/luametatex/source/tex/texconditional.c @@ -407,6 +407,7 @@ inline static halfword tex_aux_grab_toks(int expand, int expandlist, int *head) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -756,11 +757,15 @@ void tex_conditional_if(halfword code, int unless) p = cur_cmd; q = cur_chr; tex_get_next(); - if (cur_cmd != p) { + if ((p == constant_call_cmd && cur_cmd == call_cmd) || (p == call_cmd && cur_cmd == constant_call_cmd)) { + /*tex This is a somewhat special case. */ + goto SOMECALLCMD; + } else if (cur_cmd != p) { result = 0; } else if (cur_cmd < call_cmd) { result = cur_chr == q; } else { + SOMECALLCMD: /*tex Test if two macro texts match. Note also that |\ifx| decides that macros |\a| and |\b| are different in examples like this: @@ -769,6 +774,9 @@ void tex_conditional_if(halfword code, int unless) \def\a{\c} \def\c{} \def\b{\d} \def\d{} \stoptyping + + We acctually have commands beyond valid call commands but they are never + seen here. */ p = token_link(cur_chr); /*tex Omit reference counts. */ @@ -951,6 +959,7 @@ void tex_conditional_if(halfword code, int unless) EMPTY_CHECK_AGAIN: switch (cur_cmd) { case call_cmd: + case constant_call_cmd: result = ! token_link(cur_chr); break; case internal_toks_reference_cmd: @@ -1047,10 +1056,17 @@ void tex_conditional_if(halfword code, int unless) */ if (lmt_input_state.cur_input.loc) { halfword t = token_info(lmt_input_state.cur_input.loc); - lmt_input_state.cur_input.loc = token_link(lmt_input_state.cur_input.loc); if (t < cs_token_flag && token_cmd(t) == parameter_reference_cmd) { + lmt_input_state.cur_input.loc = token_link(lmt_input_state.cur_input.loc); result = lmt_input_state.parameter_stack[lmt_input_state.cur_input.parameter_start + token_chr(t) - 1] != null ? 1 : 2; - } + } else { + /*tex + We have a replacement text so we check and backtrack. This is somewhat + tricky because a parameter can be a condition but we assume sane usage. + */ + tex_get_token(); + result = cur_cmd == if_test_cmd ? 2 : 1; + } } goto CASE; } diff --git a/source/luametatex/source/tex/texdumpdata.h b/source/luametatex/source/tex/texdumpdata.h index 94a153374..cd6943d6c 100644 --- a/source/luametatex/source/tex/texdumpdata.h +++ b/source/luametatex/source/tex/texdumpdata.h @@ -55,7 +55,7 @@ */ -# define luametatex_format_fingerprint 692 +# define luametatex_format_fingerprint 693 /* These end up in the string pool. */ diff --git a/source/luametatex/source/tex/texequivalents.c b/source/luametatex/source/tex/texequivalents.c index 036a4a3d5..5c917afb2 100644 --- a/source/luametatex/source/tex/texequivalents.c +++ b/source/luametatex/source/tex/texequivalents.c @@ -851,6 +851,7 @@ static void tex_aux_eq_destroy(memoryword w) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -975,6 +976,7 @@ inline static int tex_aux_equal_eq(halfword p, singleword cmd, singleword flag, case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: diff --git a/source/luametatex/source/tex/texequivalents.h b/source/luametatex/source/tex/texequivalents.h index af3d98000..19c6959b7 100644 --- a/source/luametatex/source/tex/texequivalents.h +++ b/source/luametatex/source/tex/texequivalents.h @@ -240,13 +240,19 @@ n= 65496 cs=46426 indirect=14512 \stoptyping + We don't use the 85% prime (and we % the accumulated hash. Somehow this also performs better, + but of course I migbe be wrong. + */ -// # define hash_size 65536 -// # define hash_prime 65497 +//define hash_size 65536 +//define hash_prime 65497 # define hash_size 131072 /*tex 128K */ -# define hash_prime 131041 /*tex Plenty of room for the frozen. */ +# define hash_prime 131041 /*tex not the 85% prime */ + +//define hash_size 262144 /*tex 256K */ +//define hash_prime 262103 /*tex not the 85% prime */ # define null_cs 1 /*tex equivalent of |\csname\| |\endcsname| */ # define hash_base (null_cs + 1) /*tex beginning of region 2, for the hash table */ @@ -265,6 +271,7 @@ typedef enum deep_frozen_cs_codes { deep_frozen_cs_relax_code, /*tex permanent |\relax| */ deep_frozen_cs_end_write_code, /*tex permanent |\endwrite| */ deep_frozen_cs_dont_expand_code, /*tex permanent |\notexpanded:| */ + deep_frozen_cs_keep_constant_code, /*tex permanent |\notexpanded:| */ deep_frozen_cs_null_font_code, /*tex permanent |\nullfont| */ deep_frozen_cs_undefined_code, } deep_frozen_cs_codes; @@ -1282,7 +1289,9 @@ typedef enum flag_bit { inline static singleword tex_flags_to_cmd(int flags) { - if (is_tolerant(flags)) { + if (is_constant(flags)) { + return constant_call_cmd; + } else if (is_tolerant(flags)) { return is_protected (flags) ? tolerant_protected_call_cmd : (is_semiprotected(flags) ? tolerant_semi_protected_call_cmd : tolerant_call_cmd); } else { diff --git a/source/luametatex/source/tex/texexpand.c b/source/luametatex/source/tex/texexpand.c index d0de0c9f1..613fba0aa 100644 --- a/source/luametatex/source/tex/texexpand.c +++ b/source/luametatex/source/tex/texexpand.c @@ -393,10 +393,10 @@ void tex_expand_current_token(void) |scanner_status| temporarily. */ halfword t; - halfword save_scanner_status = lmt_input_state.scanner_status; - lmt_input_state.scanner_status = scanner_is_normal; +// halfword save_scanner_status = lmt_input_state.scanner_status; +// lmt_input_state.scanner_status = scanner_is_normal; t = tex_get_token(); - lmt_input_state.scanner_status = save_scanner_status; +// lmt_input_state.scanner_status = save_scanner_status; tex_back_input(t); /*tex Now |start| and |loc| point to the backed-up token |t|. */ if (t >= cs_token_flag) { @@ -637,16 +637,24 @@ static int tex_aux_collect_cs_tokens(halfword *p, int *n) /* break; */ case call_cmd: case tolerant_call_cmd: - if (get_token_reference(cur_chr) == max_token_reference) { // ! get_token_parameters(cur_chr)) { - /* we avoid the macro stack and expansion and we don't trace either */ - halfword h = token_link(cur_chr); - while (h) { - *p = tex_store_new_token(*p, token_info(h)); - *n += 1; - h = token_link(h); - } - } else { - tex_aux_macro_call(cur_cs, cur_cmd, cur_chr); + tex_aux_macro_call(cur_cs, cur_cmd, cur_chr); + break; + case constant_call_cmd: + { + halfword h = token_link(cur_chr); + if (h) { + if (token_link(h)) { + *p = tex_store_new_token(*p, token_val(deep_frozen_keep_constant_cmd, cur_chr)); + } else { + *p = tex_store_new_token(*p, token_info(h)); + } + *n += 1; + } + // while (h) { + // *p = tex_store_new_token(*p, token_info(h)); + // *n += 1; + // h = token_link(h); + // } } break; case end_cs_name_cmd: @@ -695,10 +703,18 @@ int tex_is_valid_csname(void) int m = f; halfword l = token_link(h); while (l) { - m = tex_aux_uni_to_buffer(lmt_fileio_state.io_buffer, m, token_chr(token_info(l))); + if (token_cmd(token_info(l)) == deep_frozen_keep_constant_cmd) { + halfword h = token_link(token_chr(token_info(l))); + while (h) { + m = tex_aux_uni_to_buffer(lmt_fileio_state.io_buffer, m, token_chr(token_info(h))); + h = token_link(h); + } + } else { + m = tex_aux_uni_to_buffer(lmt_fileio_state.io_buffer, m, token_chr(token_info(l))); + } l = token_link(l); } - cs = tex_id_locate(f, m - f, 0); /*tex Don't create a new cs! */ + cs = tex_id_locate_only(f, m - f); b = (cs != undefined_control_sequence) && (eq_type(cs) != undefined_cs_cmd); } } diff --git a/source/luametatex/source/tex/texinputstack.c b/source/luametatex/source/tex/texinputstack.c index 92a9c8a0c..4ef3307c3 100644 --- a/source/luametatex/source/tex/texinputstack.c +++ b/source/luametatex/source/tex/texinputstack.c @@ -789,7 +789,7 @@ void tex_end_token_list(void) case parameter_text: break; case template_pre_text: - if (lmt_input_state.align_state > 500000) { + if (lmt_input_state.align_state > interwoven_alignment_threshold) { lmt_input_state.align_state = 0; } else { tex_alignment_interwoven_error(7); @@ -840,7 +840,7 @@ void tex_cleanup_input_state(void) case parameter_text: break; case template_pre_text: - if (lmt_input_state.align_state > 500000) { + if (lmt_input_state.align_state > interwoven_alignment_threshold) { lmt_input_state.align_state = 0; } else { tex_alignment_interwoven_error(7); @@ -1083,7 +1083,7 @@ void tex_initialize_inputstack(void) lmt_token_state.luacstrings = 0; lmt_input_state.cur_input.cattable = default_catcode_table_preset; lmt_input_state.cur_input.partial = 0; - lmt_input_state.align_state = 1000000; + lmt_input_state.align_state = busy_alignment_state; } /*tex diff --git a/source/luametatex/source/tex/texmaincontrol.c b/source/luametatex/source/tex/texmaincontrol.c index 29b652ab7..2ee450375 100644 --- a/source/luametatex/source/tex/texmaincontrol.c +++ b/source/luametatex/source/tex/texmaincontrol.c @@ -2071,6 +2071,10 @@ static void tex_aux_run_glue(void) halfword v = tex_scan_glue(glue_val_level, 0); halfword g = tex_new_glue_node(v, user_skip_glue); /* glue_data(g) = glue_data_par; */ + if (cur_mode == mmode) { + /*tex This could be an option. */ + glue_options(g) |= glue_option_no_auto_break; + } tex_tail_append(g); tex_flush_node(v); break; @@ -4638,12 +4642,6 @@ static void tex_aux_set_def(int a, int force) if (force || tex_define_permitted(cur_cs, a)) { halfword p = cur_cs; halfword t = expand == 2 ? tex_scan_toks_expand(0, null, 1) : (expand ? tex_scan_macro_expand() : tex_scan_macro_normal()); - if (is_constant(a)) { - /* todo: check if already defined or just accept a leak */ - set_token_reference(t, max_token_reference); - // } else if (! token_link(t)) { - // t = lmt_token_state.empty; /* leaks */ - } tex_define(a, p, tex_flags_to_cmd(a), t); } } diff --git a/source/luametatex/source/tex/texmath.c b/source/luametatex/source/tex/texmath.c index e93714000..d849d4d52 100644 --- a/source/luametatex/source/tex/texmath.c +++ b/source/luametatex/source/tex/texmath.c @@ -4066,6 +4066,7 @@ void tex_run_math_fence(void) halfword leftclass = unset_noad_class; halfword rightclass = unset_noad_class; halfword source = 0; + halfword factor = scaling_factor; halfword attrlist = null; quarterword st = (quarterword) cur_chr; halfword style = cur_list.math_style; @@ -4090,7 +4091,7 @@ void tex_run_math_fence(void) } while (1) { /* todo: break down */ - switch (tex_scan_character("hdanlevpcrsutbHDANLEVPCRSUTB", 0, 1, 0)) { + switch (tex_scan_character("hdanlevpcrsutbfHDANLEVPCRSUTBF", 0, 1, 0)) { case 0: goto CHECK_PAIRING; case 'a': case 'A': @@ -4125,6 +4126,11 @@ void tex_run_math_fence(void) dp = tex_scan_dimen(0, 0, 0, 0, NULL); } break; + case 'f': case 'F': + if (tex_scan_mandate_keyword("factor", 1)) { + factor = tex_scan_int(0, NULL); + } + break; case 'h': case 'H': if (tex_scan_mandate_keyword("height", 1)) { ht = tex_scan_dimen(0, 0, 0, 0, NULL); @@ -4353,12 +4359,14 @@ void tex_run_math_fence(void) switch (st) { case left_fence_side: tex_aux_append_math_fence(fence, open_noad_subtype); + fence_nesting_factor(fence) = factor; break; case middle_fence_side: tex_aux_append_math_fence(fence, middle_noad_subtype); break; case right_fence_side: tex_aux_append_math_fence(fence, close_noad_subtype); + fence_nesting_factor(fence) = factor; break; case left_operator_side: { diff --git a/source/luametatex/source/tex/texmlist.c b/source/luametatex/source/tex/texmlist.c index 13085dca1..0ec6db249 100644 --- a/source/luametatex/source/tex/texmlist.c +++ b/source/luametatex/source/tex/texmlist.c @@ -7047,6 +7047,26 @@ static void tex_mlist_to_hlist_finalize_list(mliststate *state) /*tex Here we have a left, right, middle */ current_type = simple_noad; /*tex Same kind of fields. */ current_subtype = noad_analyzed(current); + if (fence_nesting_factor(current) && fence_nesting_factor(current) != scaling_factor) { + switch(current_subtype) { + case open_noad_subtype: + boundarylevel++; + boundaryfactor = fence_nesting_factor(current); + break; + case close_noad_subtype: + if (boundarylevel > 0) { + boundarylevel--; + if (boundarylevel == 0) { + boundaryfactor = scaling_factor; + } else { + boundaryfactor = fence_nesting_factor(current); + } + } else { + boundaryfactor = scaling_factor; + } + break; + } + } packedfence = current; break; case style_node: @@ -7089,7 +7109,9 @@ static void tex_mlist_to_hlist_finalize_list(mliststate *state) case 3: if (boundarylevel > 0) { boundarylevel--; - if (l == 2) { + if (boundarylevel == 0) { + boundaryfactor = scaling_factor; + } else if (l == 2) { boundaryfactor = boundary_reserved(current) ? boundary_reserved(current) : scaling_factor; } } else { diff --git a/source/luametatex/source/tex/texnodes.c b/source/luametatex/source/tex/texnodes.c index 48a1bd2db..52125cc7c 100644 --- a/source/luametatex/source/tex/texnodes.c +++ b/source/luametatex/source/tex/texnodes.c @@ -833,7 +833,7 @@ void lmt_nodelib_initialize(void) { inline static void lmt_properties_push(lua_State * L) { - lmt_node_memory_state.lua_properties_level++ ; + lmt_node_memory_state.lua_properties_level++; if (lmt_node_memory_state.lua_properties_level == 1) { lua_rawgeti(L, LUA_REGISTRYINDEX, lmt_node_memory_state.node_properties_id); } @@ -844,7 +844,7 @@ inline static void lmt_properties_pop(lua_State * L) if (lmt_node_memory_state.lua_properties_level == 1) { lua_pop(L, 1); } - lmt_node_memory_state.lua_properties_level-- ; + lmt_node_memory_state.lua_properties_level--; } /*tex Resetting boils down to nilling. */ @@ -1114,7 +1114,7 @@ halfword tex_copy_node_only(halfword p) # define copy_sub_node(target,source) do { \ if (source) { \ halfword copy_stub = tex_copy_node(source); \ - target = copy_stub ; \ + target = copy_stub; \ } else { \ target = null; \ } \ @@ -1181,7 +1181,7 @@ halfword tex_copy_node(halfword p) /* how about null */ } break; case insert_node: - copy_sub_list(insert_list(r), insert_list(p)) ; + copy_sub_list(insert_list(r), insert_list(p)); break; case mark_node: tex_add_token_reference(mark_ptr(p)); @@ -1190,53 +1190,53 @@ halfword tex_copy_node(halfword p) /* how about null */ copy_sub_list(adjust_list(r), adjust_list(p)); break; case choice_node: - copy_sub_list(choice_display_mlist(r), choice_display_mlist(p)) ; - copy_sub_list(choice_text_mlist(r), choice_text_mlist(p)) ; - copy_sub_list(choice_script_mlist(r), choice_script_mlist(p)) ; - copy_sub_list(choice_script_script_mlist(r), choice_script_script_mlist(p)) ; + copy_sub_list(choice_display_mlist(r), choice_display_mlist(p)); + copy_sub_list(choice_text_mlist(r), choice_text_mlist(p)); + copy_sub_list(choice_script_mlist(r), choice_script_mlist(p)); + copy_sub_list(choice_script_script_mlist(r), choice_script_script_mlist(p)); break; case simple_noad: case radical_noad: case fraction_noad: case accent_noad: - copy_sub_list(noad_nucleus(r), noad_nucleus(p)) ; - copy_sub_list(noad_subscr(r), noad_subscr(p)) ; - copy_sub_list(noad_supscr(r), noad_supscr(p)) ; - copy_sub_list(noad_subprescr(r), noad_subprescr(p)) ; - copy_sub_list(noad_supprescr(r), noad_supprescr(p)) ; - copy_sub_list(noad_prime(r), noad_prime(p)) ; - // copy_sub_list(noad_state(r), noad_state(p)) ; + copy_sub_list(noad_nucleus(r), noad_nucleus(p)); + copy_sub_list(noad_subscr(r), noad_subscr(p)); + copy_sub_list(noad_supscr(r), noad_supscr(p)); + copy_sub_list(noad_subprescr(r), noad_subprescr(p)); + copy_sub_list(noad_supprescr(r), noad_supprescr(p)); + copy_sub_list(noad_prime(r), noad_prime(p)); + // copy_sub_list(noad_state(r), noad_state(p)); switch (t) { case radical_noad: - copy_sub_node(radical_left_delimiter(r), radical_left_delimiter(p)) ; - copy_sub_node(radical_right_delimiter(r), radical_right_delimiter(p)) ; - copy_sub_node(radical_top_delimiter(r), radical_top_delimiter(p)) ; - copy_sub_node(radical_bottom_delimiter(r), radical_bottom_delimiter(p)) ; - copy_sub_list(radical_degree(r), radical_degree(p)) ; + copy_sub_node(radical_left_delimiter(r), radical_left_delimiter(p)); + copy_sub_node(radical_right_delimiter(r), radical_right_delimiter(p)); + copy_sub_node(radical_top_delimiter(r), radical_top_delimiter(p)); + copy_sub_node(radical_bottom_delimiter(r), radical_bottom_delimiter(p)); + copy_sub_list(radical_degree(r), radical_degree(p)); break; case fraction_noad: - // copy_sub_list(fraction_numerator(r), fraction_numerator(p)) ; - // copy_sub_list(fraction_denominator(r), fraction_denominator(p)) ; - copy_sub_node(fraction_left_delimiter(r), fraction_left_delimiter(p)) ; - copy_sub_node(fraction_right_delimiter(r), fraction_right_delimiter(p)) ; - copy_sub_node(fraction_middle_delimiter(r), fraction_middle_delimiter(p)) ; + // copy_sub_list(fraction_numerator(r), fraction_numerator(p)); + // copy_sub_list(fraction_denominator(r), fraction_denominator(p); + copy_sub_node(fraction_left_delimiter(r), fraction_left_delimiter(p)); + copy_sub_node(fraction_right_delimiter(r), fraction_right_delimiter(p)); + copy_sub_node(fraction_middle_delimiter(r), fraction_middle_delimiter(p)); break; case accent_noad: - copy_sub_list(accent_top_character(r), accent_top_character(p)) ; - copy_sub_list(accent_bottom_character(r), accent_bottom_character(p)) ; - copy_sub_list(accent_middle_character(r), accent_middle_character(p)) ; + copy_sub_list(accent_top_character(r), accent_top_character(p)); + copy_sub_list(accent_bottom_character(r), accent_bottom_character(p)); + copy_sub_list(accent_middle_character(r), accent_middle_character(p)); break; } break; case fence_noad: /* in principle also scripts */ - copy_sub_node(fence_delimiter_list(r), fence_delimiter_list(p)) ; - copy_sub_node(fence_delimiter_top(r), fence_delimiter_top(p)) ; - copy_sub_node(fence_delimiter_bottom(r), fence_delimiter_bottom(p)) ; + copy_sub_node(fence_delimiter_list(r), fence_delimiter_list(p)); + copy_sub_node(fence_delimiter_top(r), fence_delimiter_top(p)); + copy_sub_node(fence_delimiter_bottom(r), fence_delimiter_bottom(p)); break; case sub_box_node: case sub_mlist_node: - copy_sub_list(kernel_math_list(r), kernel_math_list(p)) ; + copy_sub_list(kernel_math_list(r), kernel_math_list(p)); break; case par_node: /* can also be copy_sub_node */ @@ -1960,7 +1960,7 @@ halfword tex_current_attribute_list(void) } return current_attribute_state; } else { - return null ; + return null; } } @@ -4417,7 +4417,7 @@ halfword tex_reversed_node_list(halfword list) halfword next = node_next(list); tex_couple_nodes(list, prev); if (node_type(list) == dir_node) { - node_subtype(list) = node_subtype(list) == cancel_dir_subtype ? normal_dir_subtype : cancel_dir_subtype ; + node_subtype(list) = node_subtype(list) == cancel_dir_subtype ? normal_dir_subtype : cancel_dir_subtype; } if (next) { prev = list; @@ -4569,7 +4569,7 @@ void tex_set_disc_field(halfword target, halfword location, halfword source) } node_prev(source) = null; /* don't expose this one! */ if (source) { - node_head(target) = source ; + node_head(target) = source; node_tail(target) = tex_tail_of_node_list(source); } else { node_head(target) = null; @@ -4627,7 +4627,7 @@ halfword tex_flatten_discretionaries(halfword head, int *count, int nest) } else { tex_try_couple_nodes(node_prev(current), h); } - disc_no_break_head(d) = null ; + disc_no_break_head(d) = null; } else if (current == head) { head = next; } else { diff --git a/source/luametatex/source/tex/texnodes.h b/source/luametatex/source/tex/texnodes.h index 492e220f2..f67667d2a 100644 --- a/source/luametatex/source/tex/texnodes.h +++ b/source/luametatex/source/tex/texnodes.h @@ -1987,6 +1987,7 @@ typedef enum math_accent_subtypes { # define fence_noad_size noad_size # define fence_delimiter_list noad_extra_1 // not really a list +# define fence_nesting_factor noad_extra_2 # define fence_delimiter_top noad_extra_3 # define fence_delimiter_bottom noad_extra_4 # define fence_top_overshoot noad_extra_5 diff --git a/source/luametatex/source/tex/texprimitive.c b/source/luametatex/source/tex/texprimitive.c index 54ef9b1b1..27239acf4 100644 --- a/source/luametatex/source/tex/texprimitive.c +++ b/source/luametatex/source/tex/texprimitive.c @@ -170,6 +170,7 @@ static int tex_aux_room_in_hash(void) 231--258.] https://en.wikipedia.org/wiki/Coalesced_hashing + https://programming.guide/coalesced-hashing.html Because we seldom use uppercase we get many misses, multiplying a chr j[k] by k actually gives a better spread. @@ -211,7 +212,7 @@ halfword tex_prim_lookup(strnumber s) /*tex We start searching here; note that |0 <= h < hash_prime|. */ halfword p = h + 1; while (1) { - /* When using |halfword text = prim_text(p)| no intelliugense warning for first test in: */ + /* When using |halfword text = prim_text(p)| no intellisense warning for first test in: */ if (prim_text(p) > 0 && str_length(prim_text(p)) == l && tex_str_eq_str(prim_text(p), s)) { return p; } else if (prim_next(p)) { @@ -506,15 +507,14 @@ static halfword tex_aux_insert_id(halfword p, const unsigned char *j, unsigned i halfword tex_id_locate(int j, int l, int create) { /*tex The index in |hash| array: */ - halfword h = tex_aux_compute_hash((char *) (lmt_fileio_state.io_buffer + j), (unsigned) l); + halfword p = tex_aux_compute_hash((char *) (lmt_fileio_state.io_buffer + j), (unsigned) l) + hash_base; /*tex We start searching here. Note that |0 <= h < hash_prime|: */ - halfword p = h + hash_base; - /*tex The next one in a list: */ while (1) { strnumber s = cs_text(p); if ((s > 0) && (str_length(s) == (unsigned) l) && tex_str_eq_buf(s, j, l)) { return p; } else { + /*tex The next one in a chain: */ halfword n = cs_next(p); if (n) { p = n; @@ -528,6 +528,20 @@ halfword tex_id_locate(int j, int l, int create) return undefined_control_sequence; } +halfword tex_id_locate_only(int j, int l) +{ + halfword p = tex_aux_compute_hash((char *) (lmt_fileio_state.io_buffer + j), (unsigned) l) + hash_base; + while (p) { + strnumber s = cs_text(p); + if ((s > 0) && (str_length(s) == (unsigned) l) && tex_str_eq_buf(s, j, l)) { + return p; + } else { + p = cs_next(p); + } + } + return undefined_control_sequence; +} + /*tex Here is a similar subroutine for finding a primitive in the hash. This one is based on a \CCODE\ @@ -558,10 +572,22 @@ halfword tex_string_locate(const char *s, size_t l, int create) return undefined_control_sequence; } +halfword tex_string_locate_only(const char *s, size_t l) +{ + halfword p = tex_aux_compute_hash(s, (unsigned) l) + hash_base; + while (p) { + if (cs_text(p) > 0 && tex_str_eq_cstr(cs_text(p), s, (int) l)) { + return p; + } else { + p = cs_next(p); + } + } + return undefined_control_sequence; +} + halfword tex_located_string(const char *s) { - size_t l = strlen(s); - return tex_string_locate(s, l, 0); + return tex_string_locate_only(s, strlen(s)); } /*tex @@ -704,6 +730,9 @@ void tex_print_cmd_flags(halfword cs, halfword cmd, int flags, int escaped) if (is_instance (flags)) { (escaped ? tex_print_str_esc : tex_print_str)("instance " ); } if (is_untraced (flags)) { (escaped ? tex_print_str_esc : tex_print_str)("untraced " ); } } + if (is_constant_cmd(cmd)) { + (escaped ? tex_print_str_esc : tex_print_str)("constant " ); + } if (is_tolerant_cmd(cmd)) { (escaped ? tex_print_str_esc : tex_print_str)("tolerant " ); } @@ -791,6 +820,7 @@ void tex_print_cmd_chr(singleword cmd, halfword chr) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -899,6 +929,10 @@ void tex_print_cmd_chr(singleword cmd, halfword chr) /*tex Kind of special. */ tex_print_str_esc("notexpanded"); break; + case deep_frozen_keep_constant_cmd: + /*tex Kind of special. */ + tex_print_str_esc("keepconstant"); + break; case internal_box_reference_cmd: tex_print_str_esc("hiddenlocalbox"); break; diff --git a/source/luametatex/source/tex/texprimitive.h b/source/luametatex/source/tex/texprimitive.h index 640a6b232..f4cf6dd97 100644 --- a/source/luametatex/source/tex/texprimitive.h +++ b/source/luametatex/source/tex/texprimitive.h @@ -36,14 +36,18 @@ extern hash_state_info lmt_hash_state; We use no defines as a |hash| macro will clash with lua hash. Most hash accessors are in a few places where it makes sense to be explicit anyway. + The only reason why we have a dedicated primitive hash is because we can selectively enable + them but at some point we might just always do that. There is never a runtiem lookup (asuming + a format). It also gives is access to some primitive metadata. + */ # define cs_next(a) lmt_hash_state.hash[(a)].half0 /*tex link for coalesced lists */ # define cs_text(a) lmt_hash_state.hash[(a)].half1 /*tex string number for control sequence name */ # define undefined_primitive 0 -# define prim_size 2100 /*tex (can be 1000) maximum number of primitives (quite a bit more than needed) */ -# define prim_prime 1777 /*tex (can be 853) about 85 percent of |primitive_size| */ +# define prim_size 2100 /*tex maximum number of primitives (quite a bit more than needed) */ +# define prim_prime 1777 /*tex about 85 percent of |primitive_size| */ typedef struct primitive_info { halfword subids; /*tex number of name entries */ @@ -87,9 +91,11 @@ extern void tex_dump_hashtable (dumpstream f); extern void tex_undump_hashtable (dumpstream f); /* halfword tex_string_lookup (const char *s, size_t l); */ extern halfword tex_string_locate (const char *s, size_t l, int create); +extern halfword tex_string_locate_only (const char *s, size_t l); extern halfword tex_located_string (const char *s); /* halfword tex_id_lookup (int j, int l); */ extern halfword tex_id_locate (int j, int l, int create); +extern halfword tex_id_locate_only (int j, int l); extern void tex_print_cmd_flags (halfword cs, halfword cmd, int flags, int escape); # endif diff --git a/source/luametatex/source/tex/texscanning.c b/source/luametatex/source/tex/texscanning.c index 9559a4040..83abd12a3 100644 --- a/source/luametatex/source/tex/texscanning.c +++ b/source/luametatex/source/tex/texscanning.c @@ -3703,8 +3703,18 @@ halfword tex_scan_toks_expand(int left_brace_found, halfword *tail, int expandco switch (cur_cmd) { case call_cmd: case tolerant_call_cmd: +// case constant_call_cmd: tex_expand_current_token(); goto PICKUP; + case constant_call_cmd: + { + halfword h = token_link(cur_chr); + while (h) { + p = tex_store_new_token(p, token_info(h)); + h = token_link(h); + } + goto PICKUP; + } case protected_call_cmd: case tolerant_protected_call_cmd: cur_tok = cs_token_flag + cur_cs; @@ -4020,8 +4030,18 @@ halfword tex_scan_macro_expand(void) switch (cur_cmd) { case call_cmd: case tolerant_call_cmd: + // case constant_call_cmd: tex_expand_current_token(); goto PICKUP; + case constant_call_cmd: + { + halfword h = token_link(cur_chr); + while (h) { + p = tex_store_new_token(p, token_info(h)); + h = token_link(h); + } + goto PICKUP; + } case protected_call_cmd: case semi_protected_call_cmd: case tolerant_protected_call_cmd: diff --git a/source/luametatex/source/tex/textoken.c b/source/luametatex/source/tex/textoken.c index 7580d72d3..7e5c10831 100644 --- a/source/luametatex/source/tex/textoken.c +++ b/source/luametatex/source/tex/textoken.c @@ -173,6 +173,7 @@ void tex_compact_tokens(void) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -463,6 +464,7 @@ void tex_print_meaning(halfword code) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -470,14 +472,10 @@ void tex_print_meaning(halfword code) tex_print_cs(cur_cs); return; } else { - int constant = (cur_chr && get_token_reference(cur_chr) == max_token_reference); switch (code) { case meaning_code: case meaning_full_code: case meaning_ful_code: - if (constant) { - tex_print_str("constant "); - } tex_print_str("macro"); if (code == meaning_ful_code) { return; @@ -485,9 +483,6 @@ void tex_print_meaning(halfword code) goto FOLLOWUP; } case meaning_asis_code: - if (constant) { - tex_print_str_esc("constant "); - } // tex_print_format("%e%C %S ", def_cmd, def_code, cur_cs); tex_print_cmd_chr(def_cmd, def_code); tex_print_char(' '); @@ -1382,18 +1377,18 @@ static int tex_aux_get_next_file(void) case skip_blanks_state + left_brace_cmd: case new_line_state + left_brace_cmd: lmt_input_state.cur_input.state = mid_line_state; - lmt_input_state.align_state++; + ++lmt_input_state.align_state; break; case mid_line_state + left_brace_cmd: - lmt_input_state.align_state++; + ++lmt_input_state.align_state; break; case skip_blanks_state + right_brace_cmd: case new_line_state + right_brace_cmd: lmt_input_state.cur_input.state = mid_line_state; - lmt_input_state.align_state--; + --lmt_input_state.align_state; break; case mid_line_state + right_brace_cmd: - lmt_input_state.align_state--; + --lmt_input_state.align_state; break; case mid_line_state + math_shift_cmd: case mid_line_state + alignment_tab_cmd: @@ -1557,7 +1552,7 @@ static int tex_aux_process_sup_mark(void) /*tex The single character case: */ { int c1 = lmt_fileio_state.io_buffer[lmt_input_state.cur_input.loc + 1]; - if (c1 < 0200) { + if (c1 < 0x80) { lmt_input_state.cur_input.loc = lmt_input_state.cur_input.loc + 2; // if (is_hex(c1) && (iloc <= ilimit)) { // int c2 = fileio_state.io_buffer[iloc]; @@ -1568,7 +1563,7 @@ static int tex_aux_process_sup_mark(void) // } // } // /*tex The somewhat odd cases, often special control characters: */ - cur_chr = (c1 < 0100 ? c1 + 0100 : c1 - 0100); + cur_chr = (c1 < 0x40 ? c1 + 0x40 : c1 - 0x40); return 1; } } @@ -1664,7 +1659,7 @@ static int tex_aux_check_expanded_code(int *kk, halfword *chr) } } else { int c1 = lmt_fileio_state.io_buffer[k + 1]; - if (c1 < 0200) { /* really ? */ + if (c1 < 0x80) { /* really ? */ d = 1; if (is_hex(c1) && (k + 2) <= lmt_input_state.cur_input.limit) { int c2 = lmt_fileio_state.io_buffer[k + 2]; @@ -1672,10 +1667,10 @@ static int tex_aux_check_expanded_code(int *kk, halfword *chr) d = 2; *chr = tex_aux_two_hex_to_cur_chr(c1, c2); } else { - *chr = (c1 < 0100 ? c1 + 0100 : c1 - 0100); + *chr = (c1 < 0x40 ? c1 + 0x40 : c1 - 0x40); } } else { - *chr = (c1 < 0100 ? c1 + 0100 : c1 - 0100); + *chr = (c1 < 0x40 ? c1 + 0x40 : c1 - 0x40); } } } @@ -2132,7 +2127,7 @@ void tex_get_next(void) /*tex Parameter needs to be expanded. */ continue; } - if ((lmt_input_state.align_state == 0) && (cur_cmd == alignment_tab_cmd || cur_cmd == alignment_cmd)) { + if ((! lmt_input_state.align_state) && (cur_cmd == alignment_tab_cmd || cur_cmd == alignment_cmd)) { /*tex If an alignment entry has just ended, take appropriate action. */ tex_insert_alignment_template(); continue; @@ -2166,12 +2161,12 @@ void tex_get_next_non_spacer(void) continue; case alignment_tab_cmd: case alignment_cmd: - if (lmt_input_state.align_state == 0) { - /*tex If an alignment entry has just ended, take appropriate action. */ + /*tex If an alignment entry has just ended, take appropriate action. */ + if (lmt_input_state.align_state) { + return; + } else { tex_insert_alignment_template(); continue; - } else { - return; } default: return; @@ -2392,8 +2387,6 @@ halfword tex_cur_str_toks(halfword *tail) */ -/*tex Change the string |str_pool[b..pool_ptr]| to a token list. */ - halfword tex_str_scan_toks(int ct, lstring ls) { /*tex index into string */ @@ -2417,7 +2410,7 @@ halfword tex_str_scan_toks(int ct, lstring ls) unsigned char *name = k ; while (k < l) { t = (halfword) aux_str2uni_len((const unsigned char *) k, &s); - c = tex_get_cat_code(ct,t); + c = tex_get_cat_code(ct, t); if (c == 11) { k += s ; lname += s ; @@ -2431,7 +2424,7 @@ halfword tex_str_scan_toks(int ct, lstring ls) } if (s > 0) { /*tex We have a potential |\cs|. */ - halfword cs = tex_string_locate((const char *) name, lname, 0); + halfword cs = tex_string_locate_only((const char *) name, lname); if (cs == undefined_control_sequence) { /*tex Let's play safe and backtrack. */ t += cc * (1<<21); @@ -2444,14 +2437,14 @@ halfword tex_str_scan_toks(int ct, lstring ls) Just a character with some meaning, so |\unknown| becomes effectively |\unknown| assuming that |\\| has some useful meaning of course. */ - t += cc * (1<<21); + t += cc * (1 << 21); k = name ; } } else { /*tex Whatever token, so for instance $x^2$ just works given a \TEX\ catcode regime. */ - t += cc * (1<<21); + t += cc * (1 << 21); } p = tex_store_new_token(p, t); if (! h) { @@ -2970,6 +2963,7 @@ void tex_run_convert_tokens(halfword code) case call_cmd: case protected_call_cmd: case semi_protected_call_cmd: + case constant_call_cmd: case tolerant_call_cmd: case tolerant_protected_call_cmd: case tolerant_semi_protected_call_cmd: @@ -3440,6 +3434,15 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb tex_aux_append_esc_to_buffer("par"); } break; + case deep_frozen_keep_constant_cmd: + if (! skip) { + halfword h = token_link(chr); + while (h) { + tex_aux_append_uchar_to_buffer(token_chr(token_info(h))); + h = token_link(h); + } + } + break; default: tex_aux_append_str_to_buffer(tex_aux_special_cmd_string(cmd, chr, error_string_bad(33))); break; @@ -3492,7 +3495,7 @@ char *tex_tokenlist_to_tstring(int pp, int inhibit_par, int *siz, int skippreamb p = token_link(p); } } - EXIT: + EXIT: if (strip && lmt_token_state.bufloc > 1) { if (lmt_token_state.buffer[lmt_token_state.bufloc-1] == strip) { lmt_token_state.bufloc -= 1; diff --git a/source/luametatex/source/tex/textypes.h b/source/luametatex/source/tex/textypes.h index 5a4dce7e8..41c8b119b 100644 --- a/source/luametatex/source/tex/textypes.h +++ b/source/luametatex/source/tex/textypes.h @@ -228,6 +228,10 @@ extern halfword tex_badness( # define math_default_penalty (infinite_penalty+1) +# define initial_alignment_state -1000000 +# define busy_alignment_state 1000000 +# define interwoven_alignment_threshold 500000 + /*tex For practical reasons all these registers were max'd to 64K but that really makes no sense for |