diff options
Diffstat (limited to 'source/luametatex/source/tex/textoken.c')
| -rw-r--r-- | source/luametatex/source/tex/textoken.c | 341 |
1 files changed, 222 insertions, 119 deletions
diff --git a/source/luametatex/source/tex/textoken.c b/source/luametatex/source/tex/textoken.c index 79e2e5e64..dfdc9f73d 100644 --- a/source/luametatex/source/tex/textoken.c +++ b/source/luametatex/source/tex/textoken.c @@ -266,7 +266,7 @@ halfword tex_get_available_token(halfword t) lmt_token_memory_state.available = token_link(p); } else if (lmt_token_memory_state.tokens_data.top < lmt_token_memory_state.tokens_data.allocated) { p = ++lmt_token_memory_state.tokens_data.top; - } else { + } else { tex_aux_bump_token_memory(); p = ++lmt_token_memory_state.tokens_data.top; } @@ -336,8 +336,8 @@ void tex_add_token_reference(halfword p) { if (get_token_reference(p) < max_token_reference) { add_token_reference(p); - // } else { - // tex_overflow_error("reference count", max_token_reference); + // } else { + // tex_overflow_error("reference count", max_token_reference); } } @@ -352,24 +352,13 @@ void tex_increment_token_reference(halfword p, int n) } } -// void tex_delete_token_reference(halfword p) -// { -// if (p) { -// if (get_token_reference(p)) { -// sub_token_reference(p); -// } else { -// tex_flush_token_list(p); -// } -// } -// } - void tex_delete_token_reference(halfword p) { if (p) { halfword r = get_token_reference(p); if (! r) { tex_flush_token_list(p); - } if(r < max_token_reference) { + } if (r < max_token_reference) { sub_token_reference(p); } } @@ -499,12 +488,7 @@ void tex_print_meaning(halfword code) tex_print_cs(cur_cs); tex_print_char(' '); if (cur_chr && token_link(cur_chr)) { - halfword body = get_token_preamble(cur_chr) ? tex_show_token_list(token_link(cur_chr), null, default_token_show_max, 1) : token_link(cur_chr); - tex_print_char('{'); - if (body) { - tex_show_token_list(body, null, default_token_show_max, 0); - } - tex_print_char('}'); + tex_show_token_list(token_link(cur_chr), get_token_preamble(cur_chr)); } return; } @@ -514,7 +498,7 @@ void tex_print_meaning(halfword code) tex_print_cmd_chr((singleword) cur_cmd, cur_chr); tex_print_char(':'); tex_print_nlp(); - tex_token_show(tex_get_some_mark(cur_chr, 0), default_token_show_max); + tex_token_show(tex_get_some_mark(cur_chr, 0)); return; case lua_value_cmd: case lua_call_cmd: @@ -547,7 +531,7 @@ void tex_print_meaning(halfword code) tex_print_char(':'); DETAILS: tex_print_nlp(); - tex_token_show(cur_chr, default_token_show_max); + tex_token_show(cur_chr); } /*tex @@ -581,6 +565,9 @@ void tex_print_meaning(halfword code) In \LUAMETATEX\ we have some more node types and token types so we also have additional tracing. Because there is some more granularity in for instance nodes (subtypes) more detail is reported. + It made sense to split the |tex_show_token_list| funciton in two, ine specialized for showing + the context. That saves some testing and passing arguments. + */ static const char *tex_aux_special_cmd_string(halfword cmd, halfword chr, const char *unknown) @@ -600,42 +587,120 @@ static const char *tex_aux_special_cmd_string(halfword cmd, halfword chr, const } } -halfword tex_show_token_list(halfword p, halfword q, int l, int asis) +void tex_show_token_list(halfword p, int asis) +{ + if (p) { + unsigned char n = 0; + int max = lmt_token_memory_state.tokens_data.top; + while (p) { + if (p < 0 || p > max) { + tex_print_str(error_string_clobbered(41)); + return; + } else if (token_info(p) >= cs_token_flag) { + tex_print_cs_checked(token_info(p) - cs_token_flag); + } else if (token_info(p) > 0) { + int cmd = token_cmd(token_info(p)); + int chr = token_chr(token_info(p)); + switch (cmd) { + case left_brace_cmd: + case right_brace_cmd: + case math_shift_cmd: + case alignment_tab_cmd: + case superscript_cmd: + case subscript_cmd: + case spacer_cmd: + case letter_cmd: + case other_char_cmd: + case active_char_cmd: + case ignore_cmd: + tex_print_tex_str(chr); + break; + case parameter_cmd: + if (! lmt_token_state.in_lua_escape && (lmt_expand_state.cs_name_level == 0)) { + tex_print_tex_str(chr); + } + tex_print_tex_str(chr); + break; + case parameter_reference_cmd: + tex_print_tex_str(match_visualizer); + if (chr <= 9) { + tex_print_char(chr + '0'); + } else if (chr <= max_match_count) { + tex_print_char(chr + '0' + gap_match_count); + } else { + tex_print_char('!'); + return; + } + break; + case match_cmd: + tex_print_char(match_visualizer); + if (is_valid_match_ref(chr)) { + ++n; + } + tex_print_char(chr ? chr : '0'); + if (n > max_match_count) { + return; + } else { + break; + } + case end_match_cmd: + if (asis) { + tex_print_char('{'); + } else if (chr == 0) { + tex_print_str("->"); + } + break; + case ignore_something_cmd: + break; + case set_font_cmd: + tex_print_format("[font->%s]", font_original(cur_val)); + break; + case end_paragraph_cmd: + /* tex_print_format("%e%s", "par "); */ + tex_print_str_esc("par "); + break; + default: + tex_print_str(tex_aux_special_cmd_string(cmd, chr, error_string_bad(43))); + break; + } + } else { + tex_print_str(error_string_bad(42)); + } + p = token_link(p); + } + if (asis) { + tex_print_char('}'); + } + } +} + +void tex_show_token_list_context(halfword p, halfword q) { if (p) { /*tex the highest parameter number, as an \ASCII\ digit */ unsigned char n = 0; - int min = 0; int max = lmt_token_memory_state.tokens_data.top; lmt_print_state.tally = 0; -// if (l <= 0) { - l = extreme_token_show_max; -// } - while (p && (lmt_print_state.tally < l)) { + while (p) { if (p == q) { /*tex Do magic computation. We only end up here in context showing. */ tex_set_trick_count(); } /*tex Display token |p|, and |return| if there are problems. */ - if (p < min || p > max) { + if (p < 0 || p > max) { tex_print_str(error_string_clobbered(41)); - return null; + return; } else if (token_info(p) >= cs_token_flag) { // if (! ((print_state.inhibit_par_tokens) && (token_info(p) == token_state.par_token))) { tex_print_cs_checked(token_info(p) - cs_token_flag); // } - } else if (token_info(p) < 0) { - tex_print_str(error_string_bad(42)); - } else if (token_info(p) == 0) { - tex_print_str(error_string_bad(44)); - } else { + } else if (token_info(p) > 0) { int cmd = token_cmd(token_info(p)); int chr = token_chr(token_info(p)); /* Display the token (|cmd|,|chr|). The procedure usually \quote {learns} the character code used for macro parameters by seeing one in a |match| command before it runs - into any |out_param| commands. - + into any |out_param| commands. This is probably not true any longer. */ switch (cmd) { case left_brace_cmd: @@ -665,7 +730,7 @@ halfword tex_show_token_list(halfword p, halfword q, int l, int asis) tex_print_char(chr + '0' + gap_match_count); } else { tex_print_char('!'); - return null; + return; } break; case match_cmd: @@ -676,16 +741,12 @@ halfword tex_show_token_list(halfword p, halfword q, int l, int asis) tex_print_char(chr ? chr : '0'); if (n > max_match_count) { /*tex Can this happen at all? */ - return null; + return; } else { break; } case end_match_cmd: - if (asis) { - return token_link(p); - } else if (chr == 0) { - tex_print_str("->"); - } + tex_print_str("->"); break; case ignore_something_cmd: break; @@ -693,12 +754,17 @@ halfword tex_show_token_list(halfword p, halfword q, int l, int asis) tex_print_format("[font->%s]", font_original(cur_val)); break; case end_paragraph_cmd: - tex_print_format("%e%s", "par "); + /* tex_print_format("%e%s", "par "); */ + tex_print_str_esc("par "); break; default: tex_print_str(tex_aux_special_cmd_string(cmd, chr, error_string_bad(43))); break; } + // } else if (token_info(p) == 0) { + // tex_print_str(error_string_bad(44)); + } else { + tex_print_str(error_string_bad(42)); } p = token_link(p); } @@ -706,7 +772,6 @@ halfword tex_show_token_list(halfword p, halfword q, int l, int asis) tex_print_str_esc("ETC."); } } - return p; } /* @@ -736,10 +801,10 @@ inline static halfword get_unichar_from_buffer(int *b) */ -void tex_token_show(halfword p, int max) +void tex_token_show(halfword p) { if (p && token_link(p)) { - tex_show_token_list(token_link(p), null, max, 0); + tex_show_token_list(token_link(p), 0); } } @@ -807,7 +872,6 @@ static next_line_retval tex_aux_next_line(void); halfword tex_scan_character(const char *s, int left_brace, int skip_space, int skip_relax) { halfword save_cur_cs = cur_cs; -// (void) skip_space; /* some day */ while (1) { tex_get_x_token(); switch (cur_cmd) { @@ -942,8 +1006,8 @@ int tex_scan_mandate_keyword(const char *s, int offset) goto BAD_NEWS; } break; - // case spacer_cmd: /* normally spaces are not pushed back */ - // case relax_cmd: /* normally not, should be option */ + // case spacer_cmd: /* normally spaces are not pushed back */ + // case relax_cmd: /* normally not, should be option */ // if (done) { // back_input(cur_tok); // goto BAD_NEWS; @@ -952,7 +1016,7 @@ int tex_scan_mandate_keyword(const char *s, int offset) // } // default: // goto BAD_NEWS; - case spacer_cmd: /* normally spaces are not pushed back */ + case spacer_cmd: /* normally spaces are not pushed back */ if (done) { goto BAD_NEWS; } else { @@ -1713,59 +1777,61 @@ static int tex_aux_scan_control_sequence(void) static void tex_aux_file_warning(void) { - halfword cond_ptr = lmt_save_state.save_stack_data.ptr; /*tex saved value of |save_ptr| or |cond_ptr| */ - int cur_if = cur_group; /*tex saved value of |cur_group| or |cur_if| */ - int cur_unless = 0; - int if_step = 0; - int if_unless = 0; - int if_limit = cur_level; /*tex saved value of |cur_level| or |if_limit| */ - int if_line = 0; /*tex saved value of |if_line| */ - lmt_save_state.save_stack_data.ptr = cur_boundary; - while (lmt_input_state.in_stack[lmt_input_state.in_stack_data.ptr].group != lmt_save_state.save_stack_data.ptr) { - --cur_level; - tex_print_nlp(); - tex_print_format("Warning: end of file when %G is incomplete", 1); - cur_group = save_level(lmt_save_state.save_stack_data.ptr); - lmt_save_state.save_stack_data.ptr = save_value(lmt_save_state.save_stack_data.ptr); - } - /*tex Restore old values. */ - lmt_save_state.save_stack_data.ptr = cond_ptr; - cur_level = (quarterword) if_limit; - cur_group = (quarterword) cur_if; - cond_ptr = lmt_condition_state.cond_ptr; - cur_if = lmt_condition_state.cur_if; - cur_unless = lmt_condition_state.cur_unless; - if_step = lmt_condition_state.if_step; - if_unless = lmt_condition_state.if_unless; - if_limit = lmt_condition_state.if_limit; - if_line = lmt_condition_state.if_line; - while (lmt_input_state.in_stack[lmt_input_state.in_stack_data.ptr].if_ptr != lmt_condition_state.cond_ptr) { - /* todo, more info */ - tex_print_nlp(); - tex_print_format("Warning: end of file when %C", if_test_cmd, lmt_condition_state.cur_if); - if (lmt_condition_state.if_limit == fi_code) { - tex_print_str_esc("else"); + { + // save_state_info saved_save_stack_data = lmt_save_state; + halfword saved_stack_ptr = lmt_save_state.save_stack_data.ptr; + quarterword saved_group = cur_group; + quarterword saved_level = cur_level; + lmt_save_state.save_stack_data.ptr = cur_boundary; + while (lmt_input_state.in_stack[lmt_input_state.in_stack_data.ptr].group != lmt_save_state.save_stack_data.ptr) { + --cur_level; + tex_print_nlp(); + tex_print_format("Warning: end of file when %G is incomplete", 1); + cur_group = save_level(lmt_save_state.save_stack_data.ptr); + lmt_save_state.save_stack_data.ptr = save_value(lmt_save_state.save_stack_data.ptr); } - if (lmt_condition_state.if_line) { - tex_print_format(" entered on line %i", lmt_condition_state.if_line); + // lmt_save_state = saved_save_stack_data; + lmt_save_state.save_stack_data.ptr = saved_stack_ptr; + cur_level = saved_level; + cur_group = saved_group; + } + { + condition_state_info saved_condition_state = lmt_condition_state; + // halfword cond_ptr = lmt_condition_state.cond_ptr; + // int cur_if = lmt_condition_state.cur_if; + // int cur_unless = lmt_condition_state.cur_unless; + // int if_step = lmt_condition_state.if_step; + // int if_unless = lmt_condition_state.if_unless; + // int if_limit = lmt_condition_state.if_limit; + // int if_line = lmt_condition_state.if_line; + while (lmt_input_state.in_stack[lmt_input_state.in_stack_data.ptr].if_ptr != lmt_condition_state.cond_ptr) { + /* todo, more info */ + tex_print_nlp(); + tex_print_format("Warning: end of file when %C", if_test_cmd, lmt_condition_state.cur_if); + if (lmt_condition_state.if_limit == fi_code) { + tex_print_str_esc("else"); + } + if (lmt_condition_state.if_line) { + tex_print_format(" entered on line %i", lmt_condition_state.if_line); + } + tex_print_str(" is incomplete"); + lmt_condition_state.cur_if = if_limit_subtype(lmt_condition_state.cond_ptr); + lmt_condition_state.cur_unless = if_limit_unless(lmt_condition_state.cond_ptr); + lmt_condition_state.if_step = if_limit_step(lmt_condition_state.cond_ptr); + lmt_condition_state.if_unless = if_limit_stepunless(lmt_condition_state.cond_ptr); + lmt_condition_state.if_limit = if_limit_type(lmt_condition_state.cond_ptr); + lmt_condition_state.if_line = if_limit_line(lmt_condition_state.cond_ptr); + lmt_condition_state.cond_ptr = node_next(lmt_condition_state.cond_ptr); } - tex_print_str(" is incomplete"); - lmt_condition_state.cur_if = if_limit_subtype(lmt_condition_state.cond_ptr); - lmt_condition_state.cur_unless = if_limit_unless(lmt_condition_state.cond_ptr); - lmt_condition_state.if_step = if_limit_step(lmt_condition_state.cond_ptr); - lmt_condition_state.if_unless = if_limit_stepunless(lmt_condition_state.cond_ptr); - lmt_condition_state.if_limit = if_limit_type(lmt_condition_state.cond_ptr); - lmt_condition_state.if_line = if_limit_line(lmt_condition_state.cond_ptr); - lmt_condition_state.cond_ptr = node_next(lmt_condition_state.cond_ptr); - } - /*tex restore old values */ - lmt_condition_state.cond_ptr = cond_ptr; - lmt_condition_state.cur_if = cur_if; - lmt_condition_state.cur_unless = cur_unless; - lmt_condition_state.if_step = if_step; - lmt_condition_state.if_unless = if_unless; - lmt_condition_state.if_limit = if_limit; - lmt_condition_state.if_line = if_line; + lmt_condition_state = saved_condition_state; + // lmt_condition_state.cond_ptr = cond_ptr; + // lmt_condition_state.cur_if = cur_if; + // lmt_condition_state.cur_unless = cur_unless; + // lmt_condition_state.if_step = if_step; + // lmt_condition_state.if_unless = if_unless; + // lmt_condition_state.if_limit = if_limit; + // lmt_condition_state.if_line = if_line; + } tex_print_nlp(); if (tracing_nesting_par > 1) { tex_show_context(); @@ -1818,12 +1884,12 @@ static next_line_retval tex_aux_next_line(void) switch (lmt_input_state.cur_input.name) { case io_lua_input_code: { - halfword n = null; + halfword result = null; int cattable = 0; int partial = 0; int finalline = 0; - int t = lmt_cstring_input(&n, &cattable, &partial, &finalline); - switch (t) { + int type = lmt_cstring_input(&result, &cattable, &partial, &finalline); + switch (type) { case eof_tex_input: lmt_token_state.force_eof = 1; break; @@ -1841,24 +1907,24 @@ static next_line_retval tex_aux_next_line(void) break; case token_tex_input: /*tex token */ - if (n >= cs_token_flag && eq_type(n - cs_token_flag) == input_cmd && eq_value(n - cs_token_flag) == end_of_input_code && lmt_input_state.cur_input.index > 0) { + if (result >= cs_token_flag && eq_type(result - cs_token_flag) == input_cmd && eq_value(result - cs_token_flag) == end_of_input_code && lmt_input_state.cur_input.index > 0) { tex_end_file_reading(); } - tex_back_input(n); + tex_back_input(result); return next_line_restart; case token_list_tex_input: /*tex token */ - tex_begin_backed_up_list(n); + tex_begin_backed_up_list(result); return next_line_restart; case node_tex_input: /*tex node */ - if (node_token_overflow(n)) { - tex_back_input(token_val(ignore_cmd, node_token_lsb(n))); - tex_reinsert_token(token_val(node_cmd, node_token_msb(n))); + if (node_token_overflow(result)) { + tex_back_input(token_val(ignore_cmd, node_token_lsb(result))); + tex_reinsert_token(token_val(node_cmd, node_token_msb(result))); return next_line_restart; } else { /*tex |0x10FFFF == 1114111| */ - tex_back_input(token_val(node_cmd, n)); + tex_back_input(token_val(node_cmd, result)); return next_line_restart; } default: @@ -1871,12 +1937,12 @@ static next_line_retval tex_aux_next_line(void) case io_token_eof_input_code: { /* can be simplified but room for extensions now */ - halfword n = null; + halfword result = null; int cattable = 0; int partial = 0; int finalline = 0; - int t = lmt_cstring_input(&n, &cattable, &partial, &finalline); - switch (t) { + int type = lmt_cstring_input(&result, &cattable, &partial, &finalline); + switch (type) { case eof_tex_input: lmt_token_state.force_eof = 1; if (lmt_input_state.cur_input.name == io_token_eof_input_code && every_eof_par) { @@ -2062,6 +2128,43 @@ void tex_get_next(void) } } +void tex_get_next_non_spacer(void) +{ + while (1) { + cur_cs = 0; + if (lmt_input_state.cur_input.state != token_list_state) { + /*tex Input from external file, |goto restart| if no input found. */ + if (! tex_aux_get_next_file()) { + continue; + } else { + /*tex Check align state later on! */ + } + } else if (! lmt_input_state.cur_input.loc) { + /*tex List exhausted, resume previous level. */ + tex_end_token_list(); + continue; + } else if (! tex_aux_get_next_tokenlist()) { + /*tex Parameter needs to be expanded. */ + continue; + } + switch (cur_cmd) { + case spacer_cmd: + continue; + case alignment_tab_cmd: + case alignment_cmd: + if (lmt_input_state.align_state == 0) { + /*tex If an alignment entry has just ended, take appropriate action. */ + tex_insert_alignment_template(); + continue; + } else { + return; + } + default: + return; + } + } +} + /*tex Since |get_next| is used so frequently in \TEX, it is convenient to define three related @@ -2834,7 +2937,7 @@ void tex_run_convert_tokens(halfword code) lmt_input_state.scanner_status = saved_scanner_status; t = tex_get_available_token(cur_tok); push_selector; - tex_show_token_list(t, null, extreme_token_show_max, 0); + tex_show_token_list(t, 0); tex_put_available_token(t); pop_selector; break; @@ -3054,7 +3157,7 @@ strnumber tex_tokens_to_string(halfword p) } else { int saved_selector = lmt_print_state.selector; lmt_print_state.selector = new_string_selector_code; - tex_token_show(p, extreme_token_show_max); + tex_token_show(p); lmt_print_state.selector = saved_selector; return tex_make_string(); } |