diff options
Diffstat (limited to 'tex/context')
21 files changed, 1240 insertions, 454 deletions
diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index 605229e98..d908f260a 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2019.06.05 15:39} +\newcontextversion{2019.06.11 19:20} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index b8d1ff737..ada3588ea 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2019.06.05 15:39} +\edef\contextversion{2019.06.11 19:20} %D For those who want to use this: diff --git a/tex/context/base/mkii/mult-pe.mkii b/tex/context/base/mkii/mult-pe.mkii index e1d0e9b5f..d2efcd7e9 100644 --- a/tex/context/base/mkii/mult-pe.mkii +++ b/tex/context/base/mkii/mult-pe.mkii @@ -570,6 +570,7 @@ \setinterfacevariable{temporary}{موقتی} \setinterfacevariable{test}{تست} \setinterfacevariable{text}{متن} +\setinterfacevariable{textnote}{textnote} \setinterfacevariable{three}{سه} \setinterfacevariable{thursday}{پنجشنبه} \setinterfacevariable{tight}{tight} diff --git a/tex/context/base/mkiv/buff-ver.lua b/tex/context/base/mkiv/buff-ver.lua index 969a28055..ad06dceec 100644 --- a/tex/context/base/mkiv/buff-ver.lua +++ b/tex/context/base/mkiv/buff-ver.lua @@ -127,7 +127,8 @@ local signal = "\000" visualizers.signal = signal visualizers.signalpattern = P(signal) -local functions = { __index = { +local functions = { + __index = { emptyline = f_emptyline, newline = f_newline, default = f_default, diff --git a/tex/context/base/mkiv/char-def.lua b/tex/context/base/mkiv/char-def.lua index b51250ce5..953d33f4a 100644 --- a/tex/context/base/mkiv/char-def.lua +++ b/tex/context/base/mkiv/char-def.lua @@ -22006,7 +22006,7 @@ characters.data={ category="mn", description="BENGALI SIGN CANDRABINDU", direction="nsm", - indic="o", + indic="m", indicorder="ap", linebreak="cm", unicodeslot=0x981, @@ -22474,7 +22474,7 @@ characters.data={ category="mc", description="BENGALI VOWEL SIGN II", direction="l", - indic="d", + indic="o", indicmark="r", indicorder="ap", linebreak="cm", @@ -22530,7 +22530,7 @@ characters.data={ description="BENGALI VOWEL SIGN E", direction="l", indic="d", - indicmark="r", + indicmark="l", indicorder="bh", linebreak="cm", unicodeslot=0x9C7, @@ -22541,7 +22541,7 @@ characters.data={ description="BENGALI VOWEL SIGN AI", direction="l", indic="d", - indicmark="r", + indicmark="l", indicorder="bh", linebreak="cm", unicodeslot=0x9C8, @@ -22551,7 +22551,8 @@ characters.data={ category="mc", description="BENGALI VOWEL SIGN O", direction="l", - indic="s", + indic="d", + indicmark="s", linebreak="cm", specials={ "char", 0x9C7, 0x9BE }, unicodeslot=0x9CB, @@ -22561,7 +22562,8 @@ characters.data={ category="mc", description="BENGALI VOWEL SIGN AU", direction="l", - indic="s", + indic="d", + indicmark="s", linebreak="cm", specials={ "char", 0x9C7, 0x9D7 }, unicodeslot=0x9CC, @@ -22761,6 +22763,7 @@ characters.data={ description="BENGALI LETTER RA WITH MIDDLE DIAGONAL", direction="l", indic="o", + indicclass="ra", indicorder="as", linebreak="al", shcode=0x9B0, @@ -23653,6 +23656,7 @@ characters.data={ description="GUJARATI SIGN ANUSVARA", direction="nsm", indic="o", + indicmark="t", linebreak="cm", unicodeslot=0xA82, }, @@ -24021,6 +24025,7 @@ characters.data={ description="GUJARATI LETTER YA", direction="l", indic="c", + indicorder="ap", linebreak="al", unicodeslot=0xAAF, }, @@ -24977,7 +24982,7 @@ characters.data={ description="ORIYA VOWEL SIGN AI", direction="l", indic="d", - indicmark="l", + indicmark="s", linebreak="cm", specials={ "char", 0xB47, 0xB56 }, unicodeslot=0xB48, @@ -26394,7 +26399,7 @@ characters.data={ description="TELUGU VOWEL SIGN AI", direction="nsm", indic="d", - indicmark="t", + indicmark="s", linebreak="cm", specials={ "char", 0xC46, 0xC56 }, unicodeslot=0xC48, @@ -27146,7 +27151,7 @@ characters.data={ description="KANNADA VOWEL SIGN II", direction="l", indic="d", - indicmark="r", + indicmark="s", linebreak="cm", specials={ "char", 0xCBF, 0xCD5 }, unicodeslot=0xCC0, @@ -27206,7 +27211,7 @@ characters.data={ description="KANNADA VOWEL SIGN EE", direction="l", indic="d", - indicmark="r", + indicmark="s", linebreak="cm", specials={ "char", 0xCC6, 0xCD5 }, unicodeslot=0xCC7, @@ -27216,7 +27221,7 @@ characters.data={ description="KANNADA VOWEL SIGN AI", direction="l", indic="d", - indicmark="r", + indicmark="s", linebreak="cm", specials={ "char", 0xCC6, 0xCD6 }, unicodeslot=0xCC8, @@ -27226,7 +27231,7 @@ characters.data={ description="KANNADA VOWEL SIGN O", direction="l", indic="d", - indicmark="r", + indicmark="s", linebreak="cm", specials={ "char", 0xCC6, 0xCC2 }, unicodeslot=0xCCA, @@ -27236,7 +27241,7 @@ characters.data={ description="KANNADA VOWEL SIGN OO", direction="l", indic="d", - indicmark="r", + indicmark="s", linebreak="cm", specials={ "char", 0xCCA, 0xCD5 }, unicodeslot=0xCCB, @@ -27256,7 +27261,7 @@ characters.data={ combining=0x9, description="KANNADA SIGN VIRAMA", direction="nsm", - indic="s", + indic="o", indicclass="halant", linebreak="cm", unicodeslot=0xCCD, @@ -27265,7 +27270,8 @@ characters.data={ category="mc", description="KANNADA LENGTH MARK", direction="l", - indic="o", + indic="d", + indicmark="r", indicorder="as", linebreak="cm", unicodeslot=0xCD5, @@ -27274,7 +27280,8 @@ characters.data={ category="mc", description="KANNADA AI LENGTH MARK", direction="l", - indic="o", + indic="d", + indicmark="b", indicorder="as", linebreak="cm", unicodeslot=0xCD6, @@ -28029,7 +28036,7 @@ characters.data={ combining=0x9, description="MALAYALAM SIGN VIRAMA", direction="nsm", - indic="s", + indic="o", indicclass="halant", linebreak="cm", synonyms={ "malayalam chandrakkala", "malayalam vowel half-u" }, @@ -120160,7 +120167,7 @@ characters.data={ combining=0xE6, description="COMBINING DEVANAGARI LETTER VI", direction="nsm", - indic="m", + indic="o", indicmark="t", linebreak="cm", unicodeslot=0xA8F0, @@ -257399,4 +257406,4 @@ characters.data={ synonyms={ "vs17" }, unicodeslot=0xE0100, }, -}
\ No newline at end of file +} diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index 4af95a2c0..a4c86506c 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2019.06.05 15:39} +\newcontextversion{2019.06.11 19:20} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index 5189d1246..d9fa164d7 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -45,7 +45,7 @@ %D {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2019.06.05 15:39} +\edef\contextversion{2019.06.11 19:20} \edef\contextkind {beta} %D Kind of special: diff --git a/tex/context/base/mkiv/font-osd.lua b/tex/context/base/mkiv/font-osd.lua index 8530fc264..203c1d79d 100644 --- a/tex/context/base/mkiv/font-osd.lua +++ b/tex/context/base/mkiv/font-osd.lua @@ -143,6 +143,7 @@ local s_half = states.half local s_pref = states.pref local s_blwf = states.blwf local s_pstf = states.pstf +local s_init = states.init local replace_all_nbsp = nil @@ -209,10 +210,10 @@ if not indicgroups and characters then local indicorders = { bp = { }, -- before_postscript ap = { }, -- after_postscript - bs = { }, -- before_half - as = { }, -- after_half - bh = { }, -- before_subscript - ah = { }, -- after_subscript + bs = { }, -- before_subscript + as = { }, -- after_subscript + bh = { }, -- before_half + ah = { }, -- after_half bm = { }, -- before_main am = { }, -- after_main } @@ -324,17 +325,11 @@ local zw_char = { -- both_joiners_true } local dflt_true = { - dflt = true + dflt = true, } -local two_defaults = { - dev2 = dflt_true, -} - -local one_defaults = { - dev2 = dflt_true, -- set later - deva = dflt_true, -- set later -} +local two_defaults = { } +local one_defaults = { } local false_flags = { false, false, false, false } @@ -367,7 +362,7 @@ local sequence_reorder_reph = { } local sequence_reorder_pre_base_reordering_consonants = { - features = { dv03 = two_defaults }, + features = { dv03 = one_defaults }, flags = false_flags, name = "dv03_reorder_pre_base_reordering_consonants", order = { "dv03" }, @@ -399,22 +394,17 @@ local sequence_remove_joiners = { -- as it might depends on the font. Not that it's a bottleneck. local basic_shaping_forms = { - -- init = true, -- new - -- abvs = true, -- new akhn = true, blwf = true, - -- calt = true, -- new cjct = true, half = true, - -- haln = true, -- new nukt = true, pref = true, - -- pres = true, -- new pstf = true, - -- psts = true, -- new rkrf = true, rphf = true, vatu = true, + locl = true, } local valid = { @@ -439,6 +429,7 @@ local valid = { psts = true, haln = true, calt = true, + locl = true, } local scripts = { } @@ -446,8 +437,6 @@ local scripts = { } local scripts_one = { "deva", "mlym", "beng", "gujr", "guru", "knda", "orya", "taml", "telu" } local scripts_two = { "dev2", "mlm2", "bng2", "gjr2", "gur2", "knd2", "ory2", "tml2", "tel2" } -local scripts_old = { } for i=1,#scripts_one do local v = scripts_one[i] scripts_old[v] = v end -- self - local nofscripts = #scripts_one for i=1,nofscripts do @@ -455,7 +444,7 @@ for i=1,nofscripts do local two = scripts_two[i] scripts[one] = true scripts[two] = true - two_defaults[one] = dflt_true + two_defaults[two] = dflt_true one_defaults[one] = dflt_true one_defaults[two] = dflt_true end @@ -476,28 +465,75 @@ local function initializedevanagi(tfmdata) local sequences = resources.sequences local sharedfeatures = tfmdata.shared.features -- - local lastmatch = 0 - for s=1,#sequences do -- classify chars + gsubfeatures["dv01"] = two_defaults -- reorder matras + gsubfeatures["dv02"] = two_defaults -- reorder reph + gsubfeatures["dv03"] = one_defaults -- reorder pre base reordering consonants + gsubfeatures["dv04"] = one_defaults -- remove joiners + -- + local reorder_pre_base_reordering_consonants = copy(sequence_reorder_pre_base_reordering_consonants) + local reorder_reph = copy(sequence_reorder_reph) + local reorder_matras = copy(sequence_reorder_matras) + local remove_joiners = copy(sequence_remove_joiners) + + local lastmatch = 0 + for s=1,#sequences do -- classify chars and make sure basic_shaping_forms come first local features = sequences[s].features if features then for k, v in next, features do + if k == "locl" then + local steps = sequences[s].steps + local nofsteps = sequences[s].nofsteps + for i=1,nofsteps do + local step = steps[i] + local coverage = step.coverage + if coverage then + for k, v in next, pre_mark do + local locl = coverage[k] + if locl then + if #locl > 0 then --contextchain; KE: is this right? + for j=1,#locl do + local ck = locl[j] + local f = ck[4] + local chainlookups = ck[6] + if chainlookups then + local chainlookup = chainlookups[f] + for j=1,#chainlookup do + local chainstep = chainlookup[j] + local steps = chainstep.steps + local nofsteps = chainstep.nofsteps + for i=1,nofsteps do + local step = steps[i] + local coverage = step.coverage + if coverage then + locl = coverage[k] + end + end + end + end + end + end + if locl then + reorder_matras.steps[1].coverage[locl] = true + end + end + end + end + end + end if basic_shaping_forms[k] then - lastmatch = s + lastmatch = lastmatch + 1 + if s ~= lastmatch then + table.insert(sequences, lastmatch, table.remove(sequences, s)) + end end end end end local insertindex = lastmatch + 1 -- - gsubfeatures["dv01"] = two_defaults -- reorder matras - gsubfeatures["dv02"] = two_defaults -- reorder reph - gsubfeatures["dv03"] = two_defaults -- reorder pre base reordering consonants - gsubfeatures["dv04"] = one_defaults -- remove joiners - -- - local reorder_pre_base_reordering_consonants = copy(sequence_reorder_pre_base_reordering_consonants) - local reorder_reph = copy(sequence_reorder_reph) - local reorder_matras = copy(sequence_reorder_matras) - local remove_joiners = copy(sequence_remove_joiners) + if tfmdata.properties.language then + dflt_true[tfmdata.properties.language] = true + end -- insert(sequences,insertindex,reorder_pre_base_reordering_consonants) insert(sequences,insertindex,reorder_reph) @@ -505,6 +541,8 @@ local function initializedevanagi(tfmdata) insert(sequences,insertindex,remove_joiners) -- local blwfcache = { } + local vatucache = { } + local pstfcache = { } local seqsubset = { } local rephstep = { coverage = { } -- will be adapted each work @@ -513,6 +551,8 @@ local function initializedevanagi(tfmdata) reph = false, vattu = false, blwfcache = blwfcache, + vatucache = vatucache, + pstfcache = pstfcache, seqsubset = seqsubset, reorderreph = rephstep, @@ -525,8 +565,6 @@ local function initializedevanagi(tfmdata) -- resources.devanagari = devanagari -- - local old = scripts_old[script] or false - -- for s=1,#sequences do local sequence = sequences[s] local steps = sequence.steps @@ -534,41 +572,148 @@ local function initializedevanagi(tfmdata) local features = sequence.features local has_rphf = features.rphf local has_blwf = features.blwf - if has_rphf and has_rphf[old] then + local has_vatu = features.vatu + local has_pstf = features.pstf + if has_rphf and has_rphf[script] then devanagari.reph = true - elseif has_blwf and has_blwf[old] then + elseif (has_blwf and has_blwf[script] ) or (has_vatu and has_vatu[script] ) then devanagari.vattu = true for i=1,nofsteps do local step = steps[i] local coverage = step.coverage if coverage then for k, v in next, coverage do - if not blwfcache[k] then - blwfcache[k] = v + for h, w in next, halant do + if v[h] then + if not blwfcache[k] then + blwfcache[k] = v + end + end + if has_vatu and has_vatu[script] and not vatucache[k] then + vatucache[k] = v + end + end + end + end + end + elseif has_pstf and has_pstf[script] then + for i=1,nofsteps do + local step = steps[i] + local coverage = step.coverage + if coverage then + for k, v in next, coverage do + if not pstfcache[k] then + pstfcache[k] = v + end + end + for k, v in next, ra do + local r = coverage[k] + if r then + local found = false + if #r > 0 then -- contextchain; KE: is this right? + for j=1,#r do + local ck = r[j] + local f = ck[4] + local chainlookups = ck[6] + if chainlookups and chainlookups[f] then --KE: why is check for chainlookups[f] necessacy??? + local chainlookup = chainlookups[f] + for j=1,#chainlookup do + local chainstep = chainlookup[j] + local steps = chainstep.steps + local nofsteps = chainstep.nofsteps + for i=1,nofsteps do + local step = steps[i] + local coverage = step.coverage + if coverage then + local h = coverage[k] + if h then + for k, v in next, h do + found = v and v.ligature + if found then + pre_base_reordering_consonants[found] = true + break + end + end + if found then + break + end + end + end + end + end + end + end + else + for k, v in next, r do + found = v and v.ligature + if found then + pre_base_reordering_consonants[found] = true + break + end + end + end + if found then + break + end end end end end end for kind, spec in next, features do - -- if spec.dev2 and valid[kind] then if valid[kind] and valid_two(spec)then for i=1,nofsteps do local step = steps[i] local coverage = step.coverage if coverage then - local reph = false + local reph, rephbase = false, false if kind == "rphf" then -- rphf acts on consonant + halant for k, v in next, ra do local r = coverage[k] if r then + rephbase = k local h = false - for k, v in next, halant do - local h = r[k] - if h then - reph = h.ligature or false - break + if #r > 0 then --contextchain; KE: is this right? + for j=1,#r do + local ck = r[j] + local f = ck[4] + local chainlookups = ck[6] + if chainlookups then + local chainlookup = chainlookups[f] + for j=1,#chainlookup do + local chainstep = chainlookup[j] + local steps = chainstep.steps + local nofsteps = chainstep.nofsteps + for i=1,nofsteps do + local step = steps[i] + local coverage = step.coverage + if coverage then + local r = coverage[k] + if r then + for k, v in next, halant do + local h = r[k] + if h then + reph = h.ligature or false + break + end + end + if h then + break + end + end + end + end + end + end + end + else + for k, v in next, halant do + local h = r[k] + if h then + reph = h.ligature or false + break + end end end if reph then @@ -577,7 +722,7 @@ local function initializedevanagi(tfmdata) end end end - seqsubset[#seqsubset+1] = { kind, coverage, reph } + seqsubset[#seqsubset+1] = { kind, coverage, reph, rephbase } end end end @@ -592,11 +737,46 @@ local function initializedevanagi(tfmdata) local h = coverage[k] if h then local found = false - for k, v in next, h do - found = v and v.ligature - if found then - pre_base_reordering_consonants[k] = found - break + if #h > 0 then -- contextchain; KE: is this right? + for j=1,#h do + local ck = h[j] + local f = ck[4] + local chainlookups = ck[6] + if chainlookups then + local chainlookup = chainlookups[f] + for j=1,#chainlookup do + local chainstep = chainlookup[j] + local steps = chainstep.steps + local nofsteps = chainstep.nofsteps + for i=1,nofsteps do + local step = steps[i] + local coverage = step.coverage + if coverage then + local h = coverage[k] + if h then + for k, v in next, h do + found = v and v.ligature + if found then + pre_base_reordering_consonants[found] = true + break + end + end + if found then + break + end + end + end + end + end + end + end + else + for k, v in next, h do + found = v and v.ligature + if found then + pre_base_reordering_consonants[found] = true + break + end end end if found then @@ -610,90 +790,17 @@ local function initializedevanagi(tfmdata) end end -- - -- The following presets need checking (by Kai). Most of these scripts share a common - -- handling (some need less but that doesn't hurt). The question is: what to enable. - -- - -- dv01_reorder_matras - -- dv02_reorder_reph - -- dv03_reorder_pre_base_reordering_consonants - -- dv04_remove_joiners - -- - if script == "deva" then - sharedfeatures["dv04"] = true - elseif script == "dev2" then - sharedfeatures["dv01"] = true - sharedfeatures["dv02"] = true - sharedfeatures["dv03"] = true - sharedfeatures["dv04"] = true - - elseif script == "knda" then - -- needs checking - sharedfeatures["dv04"] = true - elseif script == "knd2" then - -- needs checking - sharedfeatures["dv01"] = true - sharedfeatures["dv02"] = true - sharedfeatures["dv03"] = true - sharedfeatures["dv04"] = true - - elseif script == "beng" then - -- needs checking - sharedfeatures["dv04"] = true - elseif script == "bng2" then - -- needs checking - sharedfeatures["dv01"] = true - sharedfeatures["dv02"] = true - sharedfeatures["dv03"] = true - sharedfeatures["dv04"] = true - - elseif script == "gurj" then - -- needs checking - sharedfeatures["dv04"] = true - elseif script == "grj2" then - -- needs checking - sharedfeatures["dv01"] = true - sharedfeatures["dv02"] = true - sharedfeatures["dv03"] = true - sharedfeatures["dv04"] = true - - elseif script == "guru" then - -- needs checking - sharedfeatures["dv04"] = true - elseif script == "gur2" then - -- needs checking - sharedfeatures["dv01"] = true - sharedfeatures["dv02"] = true - sharedfeatures["dv03"] = true - sharedfeatures["dv04"] = true - - elseif script == "telu" then - -- needs checking - sharedfeatures["dv04"] = true - elseif script == "tel2" then - -- needs checking - sharedfeatures["dv01"] = true - sharedfeatures["dv02"] = true - sharedfeatures["dv03"] = true - sharedfeatures["dv04"] = true - - elseif script == "mlym" then - sharedfeatures["pstf"] = true - elseif script == "mlm2" then - sharedfeatures["pstf"] = true - sharedfeatures["pref"] = true - sharedfeatures["dv03"] = true - gsubfeatures ["dv03"] = two_defaults - insert(sequences,insertindex,sequence_reorder_pre_base_reordering_consonants) - - elseif script == "taml" then - -- needs checking - sharedfeatures["dv04"] = true - sharedfeatures["pstf"] = true - elseif script == "tml2" then - -- needs checking - - else - report("todo: enable the right features for script %a",script) + if two_defaults[script] then + sharedfeatures["dv01"] = true -- dv01_reorder_matras + sharedfeatures["dv02"] = true -- dv02_reorder_reph + sharedfeatures["dv03"] = true -- dv03_reorder_pre_base_reordering_consonants + sharedfeatures["dv04"] = true -- dv04_remove_joiners + elseif one_defaults[script] then + sharedfeatures["dv03"] = true -- dv03_reorder_pre_base_reordering_consonants + sharedfeatures["dv04"] = true -- dv04_remove_joiners + end + if script == "mlym" or script == "taml" then + devanagari.left_matra_before_base = true end end end @@ -735,10 +842,13 @@ local function initialize_one(font,attr) -- we need a proper hook into the datas reph = false, vattu = false, blwfcache = { }, + vatucache = { }, + pstfcache = { }, } datasets.devanagari = devanagaridata local resources = tfmdata.resources local devanagari = resources.devanagari + for s=1,#datasets do local dataset = datasets[s] if dataset and dataset[1] then -- value @@ -746,31 +856,97 @@ local function initialize_one(font,attr) -- we need a proper hook into the datas if kind == "rphf" then -- deva devanagaridata.reph = true - elseif kind == "blwf" then + elseif kind == "blwf" or kind == "vatu" then -- deva devanagaridata.vattu = true -- dev2 devanagaridata.blwfcache = devanagari.blwfcache + devanagaridata.vatucache = devanagari.vatucache + devanagaridata.pstfcache = devanagari.pstfcache end end end end - return devanagaridata.reph, devanagaridata.vattu, devanagaridata.blwfcache + return devanagaridata.reph, devanagaridata.vattu, devanagaridata.blwfcache, devanagaridata.vatucache, devanagaridata.pstfcache + +end + +local function contextchain(contexts, n) + local char = getchar(n) + for k=1,#contexts do + local ck = contexts[k] + local seq = ck[3] + local f = ck[4] + local l = ck[5] + if (l - f) == 1 and seq[f+1][char] then + local ok = true + local c = n + for i=l+1,#seq do + c = getnext(c) + if not c or not seq[i][ischar(c)] then + ok = false + break + end + end + if ok then + c = getprev(n) + for i=1,f-1 do + c = getprev(c) + if not c or not seq[f-i][ischar(c)] then + ok = false + end + end + end + if ok then + return true + end + end + end + return false +end +local function order_matras(c) + local cn = getnext(c) + local char = getchar(cn) + while dependent_vowel[char] do + local next = getnext(cn) + local cc = c + local cchar = getchar(cc) + while cc ~= cn do + if (above_mark[char] and (below_mark[cchar] or post_mark[cchar])) or (below_mark[char] and (post_mark[cchar])) then + local prev, next = getboth(cn) + if next then + setprev(next,prev) + end + -- todo: setlink + setnext(prev,next) + setnext(getprev(cc),cn) + setprev(cn,getprev(cc)) + setnext(cn,cc) + setprev(cc,cn) + break + end + cc = getnext(cc) + cchar = getchar(cc) + end + cn = next + char = getchar(cn) + end end local function reorder_one(head,start,stop,font,attr,nbspaces) - local reph, vattu, blwfcache = initialize_one(font,attr) -- todo: a hash[font] + local reph, vattu, blwfcache, vatucache, pstfcache = initialize_one(font,attr) -- todo: a hash[font] - local current = start - local n = getnext(start) - local base = nil - local firstcons = nil - local lastcons = nil - local basefound = false + local devanagari = fontdata[font].resources.devanagari + local current = start + local n = getnext(start) + local base = nil + local firstcons = nil + local lastcons = nil + local basefound = false if reph and ra[getchar(start)] and halant[getchar(n)] then -- if syllable starts with Ra + H and script has 'Reph' then exclude Reph @@ -853,8 +1029,11 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) if not base then base = current elseif blwfcache[char] then - -- consonant has below-base (or post-base) form + -- consonant has below-base form setprop(current,a_state,s_blwf) + elseif pstfcache[char] then + -- consonant has post-base form + setprop(current,a_state,s_pstf) else base = current end @@ -933,16 +1112,14 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) current = next end - if base ~= stop and getprop(base,a_state) then + if base ~= stop and getprop(base,a_state) then -- a_state can also be init local next = getnext(base) if halant[getchar(next)] and not (next ~= stop and getchar(getnext(next)) == c_zwj) then setprop(base,a_state,unsetvalue) end end - -- ToDo: split two- or three-part matras into their parts. Then, move the left 'matra' part to the beginning of the syllable. - -- Not necessary for Devanagari. However it is necessay for other scripts, such as Tamil (e.g. TAMIL VOWEL SIGN O - 0BCA) - + -- split two- or three-part matras into their parts. Then, move the left 'matra' part to the beginning of the syllable. -- classify consonants and 'matra' parts as pre-base, above-base (Reph), below-base or post-base, and group elements of the syllable (consonants and 'matras') according to this classification local current, allreordered, moved = start, false, { [base] = true } @@ -968,6 +1145,17 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) n = getnext(n) ch = getchar(n) end + + local tpm = twopart_mark[ch] + while tpm do + local extra = copy_node(n) + copyinjection(extra,n) + ch = tpm[1] + setchar(n,ch) + setchar(extra,tpm[2]) + head = insert_node_after(head,current,extra) + tpm = twopart_mark[ch] + end while c ~= stop and dependent_vowel[ch] do c = n n = getnext(n) @@ -986,15 +1174,51 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) end end end - local bp = getprev(firstcons) - local cn = getnext(current) + local bp = getprev(firstcons) + local cn = getnext(current) local last = getnext(c) while cn ~= last do -- move pre-base matras... if pre_mark[getchar(cn)] then - if bp then - setnext(bp,cn) + if devanagari.left_matra_before_base then + local prev, next = getboth(cn) + setlink(prev,next) + if cn == stop then + stop = getprev(cn) + end + if base == start then + if head == start then + head = cn + end + start = cn + end + setlink(getprev(base),cn) + setlink(cn,base) + -- setlink(getprev(base),cn,base) -- maybe + cn = next + else + if bp then + setnext(bp,cn) + end + local prev, next = getboth(cn) + if next then + setprev(next,prev) + end + setnext(prev,next) + if cn == stop then + stop = prev + end + setprev(cn,bp) + setlink(cn,firstcons) + if firstcons == start then + if head == start then + head = cn + end + start = cn + end + cn = next end + elseif current ~= base and dependent_vowel[getchar(cn)] then local prev, next = getboth(cn) if next then setprev(next,prev) @@ -1003,17 +1227,19 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) if cn == stop then stop = prev end - setprev(cn,bp) - setlink(cn,firstcons) - if firstcons == start then - if head == start then - head = cn - end - start = cn + setlink(b,cn,getnext(b)) + order_matras(cn) + cn = next + elseif current == base and dependent_vowel[getchar(cn)] then + local cnn = getnext(cn) + order_matras(cn) + cn = cnn + while cn ~= last and dependent_vowel[getchar(cn)] do + cn = getnext(cn) end - break + else + cn = getnext(cn) end - cn = getnext(cn) end allreordered = c == stop current = getnext(c) @@ -1075,6 +1301,13 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) if halant[getchar(next)] then cns = next end + if not vatucache[char] then + next = getnext(cns) + while dependent_vowel[getchar(next)] do + cns = next + next = getnext(cns) + end + end elseif char == c_nbsp then nbspaces = nbspaces + 1 cns = current @@ -1082,6 +1315,13 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) if halant[getchar(next)] then cns = next end + if not vatucache[char] then + next = getnext(cns) + while dependent_vowel[getchar(next)] do + cns = next + next = getnext(cns) + end + end end end current = getnext(current) @@ -1090,6 +1330,9 @@ local function reorder_one(head,start,stop,font,attr,nbspaces) if getchar(base) == c_nbsp then nbspaces = nbspaces - 1 + if base == stop then + stop = getprev(stop) + end head = remove_node(head,base) flush_node(base) end @@ -1114,7 +1357,7 @@ function handlers.devanagari_reorder_matras(head,start) -- no leak local char = ischar(current,startfont) local next = getnext(current) if char and getprop(current,a_syllabe) == startattr then - if halant[char] and not getprop(current,a_state) then + if halant[char] then -- a_state can also be init if next then local char = ischar(next,startfont) if char and zw_char[char] and getprop(next,a_syllabe) == startattr then @@ -1130,6 +1373,18 @@ function handlers.devanagari_reorder_matras(head,start) -- no leak -- setlink(current,start,next) -- maybe start = startnext break + -- elseif consonant[char] and ( not getprop(current,a_state) or getprop(current,a_state) == s_init) then + -- startnext = getnext(start) + -- head = remove_node(head,start) + -- if current == head then + -- setlink(start,current) + -- head = start + -- else + -- setlink(getprev(current),start) + -- setlink(start,current) + -- end + -- start = startnext + -- break end else break @@ -1148,6 +1403,8 @@ end -- In Devanagari reph has reordering position 'before postscript' and dev2 only -- follows step 2, 4, and 6. +local rephbase = { } + function handlers.devanagari_reorder_reph(head,start) local current = getnext(start) local startnext = nil @@ -1159,6 +1416,12 @@ function handlers.devanagari_reorder_reph(head,start) -- -- If reph should be positioned after post-base consonant forms, proceed to step 5. -- + local char = ischar(start,startfont) + local rephbase = rephbase[startfont][char] + if char and after_subscript[rephbase] then + goto step_5 + end + -- ::step_2:: -- -- If the reph repositioning class is not after post-base: target position is after @@ -1169,39 +1432,64 @@ function handlers.devanagari_reorder_reph(head,start) -- fixed in shaping engine, there was no case where reph position will be found on -- this step. -- - while current do - local char = ischar(current,startfont) - if char and getprop(current,a_syllabe) == startattr then - if halant[char] and not getprop(current,a_state) then - local next = getnext(current) - if next then - local nextchar = ischar(next,startfont) - if nextchar and zw_char[nextchar] and getprop(next,a_syllabe) == startattr then - current = next - next = getnext(current) + if char and not after_postscript[rephbase] then + while current do + local char = ischar(current,startfont) + if char and getprop(current,a_syllabe) == startattr then + if halant[char] then + local next = getnext(current) + if next then + local nextchar = ischar(next,startfont) + if nextchar and zw_char[nextchar] and getprop(next,a_syllabe) == startattr then + current = next + next = getnext(current) + end end + startnext = getnext(start) + head = remove_node(head,start) + setlink(start,next) + setlink(current,start) + -- setlink(current,start,next) -- maybe + start = startnext + startattr = getprop(start,a_syllabe) + break end - startnext = getnext(start) - head = remove_node(head,start) - setlink(start,next) - setlink(current,start) - -- setlink(current,start,next) -- maybe - start = startnext - startattr = getprop(start,a_syllabe) + current = getnext(current) + else break end - current = getnext(current) - else - break end end + -- ::step_3:: -- - -- If reph should be repositioned after the main consonant: from the first consonant + -- If reph should be repositioned after the main consonant: find the first consonant -- not ligated with main, or find the first consonant that is not a potential -- pre-base reordering Ra. -- - -- Kai: todo + if not startnext then + if char and after_main[rephbase] then + current = getnext(start) + while current do + local char = ischar(current,startfont) + if char and getprop(current,a_syllabe) == startattr then + if consonant[char] and not getprop(current,a_state) == s_pref then + startnext = getnext(start) + head = remove_node(head,start) + setlink(current,start) + setlink(start,getnext(current)) + -- setlink(current,start,getnext(current)) -- maybe + start = startnext + startattr = getprop(start,a_syllabe) + break + end + current = getnext(current) + else + break + end + end + end + end -- ::step_4:: -- @@ -1210,23 +1498,37 @@ function handlers.devanagari_reorder_reph(head,start) -- position should be before the first matra, syllable modifier sign or vedic sign. -- if not startnext then - current = getnext(start) - while current do - local char = ischar(current,startfont) - if char and getprop(current,a_syllabe) == startattr then - if getprop(current,a_state) == s_pstf then -- post-base - startnext = getnext(start) - head = remove_node(head,start) - setlink(getprev(current),start) - setlink(start,current) - -- setlink(getprev(current),start,current) -- maybe - start = startnext - startattr = getprop(start,a_syllabe) + if char and before_postscript[rephbase] then + current = getnext(start) + local c = nil + while current do + local char = ischar(current,startfont) + if char and getprop(current,a_syllabe) == startattr then + if getprop(current,a_state) == s_pstf then -- post-base + startnext = getnext(start) + head = remove_node(head,start) + setlink(getprev(current),start) + setlink(start,current) + -- setlink(getprev(current),start,current) -- maybe + start = startnext + startattr = getprop(start,a_syllabe) + break + elseif not c and ( vowel_modifier[char] or stress_tone_mark[char] ) then + c = current + end + current = getnext(current) + else + if c then + startnext = getnext(start) + head = remove_node(head,start) + setlink(getprev(c),start) + setlink(start,c) + -- setlink(getprev(c),start,c) -- maybe + start = startnext + startattr = getprop(start,a_syllabe) + end break end - current = getnext(current) - else - break end end end @@ -1245,7 +1547,10 @@ function handlers.devanagari_reorder_reph(head,start) while current do local char = ischar(current,startfont) if char and getprop(current,a_syllabe) == startattr then - if not c and mark_above_below_post[char] and not after_subscript[char] then + local state = getprop(current,a_state) + if before_subscript[rephbase] and (state == s_blwf or state == s_pstf) then + c = current + elseif after_subscript[rephbase] and (state == s_pstf) then c = current end current = getnext(current) @@ -1310,67 +1615,82 @@ end -- return head, start, done -- end +local reordered_pre_base_reordering_consonants = { } -- shared ? not reset ? + function handlers.devanagari_reorder_pre_base_reordering_consonants(head,start) - local current = start - local startnext = nil - local startprev = nil + if reordered_pre_base_reordering_consonants[start] then + return head, start, true + end + local current = start -- we could cache attributes here local startfont = getfont(start) local startattr = getprop(start,a_syllabe) - -- can be fast for loop + caching state while current do local char = ischar(current,startfont) + local next = getnext(current) if char and getprop(current,a_syllabe) == startattr then - local next = getnext(current) - if halant[char] and not getprop(current,a_state) then + if halant[char] then -- a_state can also be init if next then - local nextchar = ischar(next,startfont) - if nextchar and getprop(next,a_syllabe) == startattr then - if nextchar == c_zwnj or nextchar == c_zwj then - current = next - next = getnext(current) - end + local char = ischar(next,startfont) + if char and zw_char[char] and getprop(next,a_syllabe) == startattr then + current = next + next = getnext(current) end end - startnext = getnext(start) - removenode(start,start) + -- can be optimzied + local startnext = getnext(start) + head = remove_node(head,start) setlink(start,next) setlink(current,start) -- setlink(current,start,next) -- maybe + reordered_pre_base_reordering_consonants[start] = true start = startnext - break + return head, start, true + -- elseif consonant[char] and ( not getprop(current,a_state) or getprop(current,a_state) == s_init) then + -- startnext = getnext(start) + -- head = remove_node(head,start) + -- if current == head then + -- setlink(start,current) + -- head = start + -- else + -- setlink(getprev(current),start) + -- setlink(start,current) + -- end + -- start = startnext + -- break end - current = next else break end + current = next end - if not startnext then - current = getnext(start) - startattr = getprop(start,a_syllabe) - while current do - local char = ischar(current,startfont) - if char and getprop(current,a_syllabe) == startattr then - if not consonant[char] and getprop(current,a_state) then -- main - startnext = getnext(start) - removenode(start,start) - setlink(getprev(current),start) - setlink(start,current) - -- setlink(getprev(current),start,current) -- maybe - start = startnext - break - end - current = getnext(current) + + local startattr = getprop(start,a_syllabe) + local current = getprev(start) + while current and getprop(current,a_syllabe) == startattr do + local char = ischar(current) + if ( not dependent_vowel[char] and not getprop(current,a_state) or getprop(current,a_state) == s_init) then + startnext = getnext(start) + head = remove_node(head,start) + if current == head then + setlink(start,current) + head = start else - break + setlink(getprev(current),start) + setlink(start,current) end + reordered_pre_base_reordering_consonants[start] = true + start = startnext + break end + current = getprev(current) end + return head, start, true end function handlers.devanagari_remove_joiners(head,start,kind,lookupname,replacement) local stop = getnext(start) - local font = getfont(start) -- hm + local font = getfont(start) local last = start while stop do local char = ischar(stop,font) @@ -1411,15 +1731,15 @@ end -- of the actions local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pass over (determine stop in sweep) - local seqsubset, reorderreph = initialize_two(font,attr) - local reph = false -- was nil ... probably went unnoticed because never assigned local halfpos = nil local basepos = nil local subpos = nil local postpos = nil - local locl = { } + + reorderreph.coverage = { } + rephbase[font] = { } for i=1,#seqsubset do @@ -1429,17 +1749,17 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas local kind = subset[1] local lookupcache = subset[2] if kind == "rphf" then - reph = subset[3] + reorderreph.coverage[subset[3]] = true -- neat + rephbase[font][subset[3]] = subset[4] local current = start local last = getnext(stop) while current ~= last do if current ~= stop then - local c = locl[current] or getchar(current) + local c = getchar(current) local found = lookupcache[c] if found then local next = getnext(current) - local n = locl[next] or getchar(next) - if found[n] then --above-base: rphf Consonant + Halant + if found[getchar(next)] or contextchain(found, next) then --above-base: rphf Consonant + Halant local afternext = next ~= stop and getnext(next) if afternext and zw_char[getchar(afternext)] then -- ZWJ and ZWNJ prevent creation of reph current = afternext -- getnext(next) @@ -1459,15 +1779,16 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas local last = getnext(stop) while current ~= last do if current ~= stop then - local c = locl[current] or getchar(current) + local c = getchar(current) local found = lookupcache[c] if found then -- pre-base: pref Halant + Consonant local next = getnext(current) - local n = locl[next] or getchar(next) - if found[n] then - setprop(current,a_state,s_pref) - setprop(next,a_state,s_pref) - current = next + if found[getchar(next)] or contextchain(found, next) then + if (not getprop(current,a_state) and not getprop(next,a_state)) then --KE: a_state can also be init... + setprop(current,a_state,s_pref) + setprop(next,a_state,s_pref) + current = next + end end end end @@ -1478,15 +1799,14 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas local last = getnext(stop) while current ~= last do if current ~= stop then - local c = locl[current] or getchar(current) + local c = getchar(current) local found = lookupcache[c] if found then local next = getnext(current) - local n = locl[next] or getchar(next) - if found[n] then + if found[getchar(next)] or contextchain(found, next) then if next ~= stop and getchar(getnext(next)) == c_zwnj then -- zwnj prevent creation of half current = next - else + elseif (not getprop(current,a_state)) then --KE: a_state can also be init... setprop(current,a_state,s_half) if not halfpos then halfpos = current @@ -1498,21 +1818,22 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas end current = getnext(current) end - elseif kind == "blwf" then -- below-base: blwf / Halant + Consonant + elseif kind == "blwf" or kind == "vatu" then -- below-base: blwf / Halant + Consonant local current = start local last = getnext(stop) while current ~= last do if current ~= stop then - local c = locl[current] or getchar(current) + local c = getchar(current) local found = lookupcache[c] if found then local next = getnext(current) - local n = locl[next] or getchar(next) - if found[n] then - setprop(current,a_state,s_blwf) - setprop(next,a_state,s_blwf) - current = next - subpos = current + if found[getchar(next)] or contextchain(found, next) then + if (not getprop(current,a_state) and not getprop(next,a_state)) then --KE: a_state can also be init... + setprop(current,a_state,s_blwf) + setprop(next,a_state,s_blwf) + current = next + subpos = current + end end end end @@ -1523,16 +1844,17 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas local last = getnext(stop) while current ~= last do if current ~= stop then - local c = locl[current] or getchar(current) + local c = getchar(current) local found = lookupcache[c] if found then local next = getnext(current) - local n = locl[next] or getchar(next) - if found[n] then - setprop(current,a_state,s_pstf) - setprop(next,a_state,s_pstf) - current = next - postpos = current + if found[getchar(next)] or contextchain(found, next) then + if (not getprop(current,a_state) and not getprop(next,a_state)) then --KE: a_state can also be init... + setprop(current,a_state,s_pstf) + setprop(next,a_state,s_pstf) + current = next + postpos = current + end end end end @@ -1541,12 +1863,6 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas end end - -- this one changes per word ... - - reorderreph.coverage = { [reph] = true } -- neat - - -- end of weird - local current, base, firstcons = start, nil, nil if getprop(start,a_state) == s_rphf then @@ -1612,7 +1928,7 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas end -- check whether consonant has below-base or post-base form or is pre-base reordering Ra local a = getprop(current,a_state) - if not (a == s_pref or a == s_blwf or a == s_pstf) then + if not (a == s_blwf or a == s_pstf or (a ~= s_rphf and a ~= s_blwf and ra[getchar(current)])) then base = current end end @@ -1630,7 +1946,7 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas end return head, stop, nbspaces else - if getprop(base,a_state) then + if getprop(base,a_state) then -- a_state can also be init setprop(base,a_state,unsetvalue) end basepos = base @@ -1647,24 +1963,27 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas -- Matra characters are classified and reordered by which consonant in a conjunct they have affinity for - local moved = { } + local moved = { } local current = start - local last = getnext(stop) + local last = getnext(stop) while current ~= last do - local char, target, cn = locl[current] or getchar(current), nil, getnext(current) + local char = getchar(current) + local target = nil + local cn = getnext(current) -- not so efficient (needed for malayalam) local tpm = twopart_mark[char] - if tpm then + while tpm do local extra = copy_node(current) copyinjection(extra,current) char = tpm[1] setchar(current,char) setchar(extra,tpm[2]) head = insert_node_after(head,current,extra) + tpm = twopart_mark[char] end -- if not moved[current] and dependent_vowel[char] then - if pre_mark[char] then -- Before first half form in the syllable + if pre_mark[char] then -- or: if before_main or before_half moved[current] = true -- can be helper to remove one node local prev, next = getboth(current) @@ -1672,17 +1991,47 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas if current == stop then stop = getprev(current) end - if halfpos == start then + + local pos + if before_main[char] then + pos = basepos + -- basepos = current -- is this correct? + else + -- must be before_half + pos = halfpos + -- halfpos = current -- is this correct? + end + + local ppos = getprev(pos) -- necessary? + while ppos and getprop(ppos,a_syllabe) == getprop(pos,a_syllabe) do + if getprop(ppos,a_state) == s_pref then + pos = ppos + end + ppos = getprev(ppos) + end + + local ppos = getprev(pos) -- necessary? + while ppos and getprop(ppos,a_syllabe) == getprop(pos,a_syllabe) and halant[ischar(ppos)] do + ppos = getprev(ppos) + if ppos and getprop(ppos,a_syllabe) == getprop(pos,a_syllabe) and consonant[ischar(ppos)] then + pos = ppos + ppos = getprev(ppos) + else + break + end + end + + if pos == start then if head == start then head = current end start = current end - setlink(getprev(halfpos),current) - setlink(current,halfpos) - -- setlink(getprev(halfpos),current,halfpos) -- maybe - halfpos = current - elseif above_mark[char] then -- After main consonant + setlink(getprev(pos),current) + setlink(current,pos) + -- setlink(getprev(pos),current,pos) -- maybe + elseif above_mark[char] then + -- after main consonant target = basepos if subpos == basepos then subpos = current @@ -1691,13 +2040,25 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas postpos = current end basepos = current - elseif below_mark[char] then -- After subjoined consonants + elseif below_mark[char] then + -- after subjoined consonants target = subpos if postpos == subpos then postpos = current end subpos = current - elseif post_mark[char] then -- After post-form consonant + elseif post_mark[char] then + -- after post-form consonant + local n = getnext(postpos) -- nukta and vedic sign come first - is that right? and also halant+ra + while n do + local v = ischar(n,font) + if nukta[v] or stress_tone_mark[v] or vowel_modifier[v] then + postpos = n + else + break + end + n = getnext(n) + end target = postpos postpos = current end @@ -1718,9 +2079,44 @@ local function reorder_two(head,start,stop,font,attr,nbspaces) -- maybe do a pas current = cn end + -- reorder halant+Ra + + local current = getnext(start) + local last = getnext(stop) + while current ~= last do + local char = getchar(current) + local cn = getnext(current) + if halant[char] and ra[ischar(cn)] and getprop(cn,a_state) ~= s_rphf and getprop(cn,a_state) ~= s_blwf then + if after_main[ischar(cn)] then + local prev = getprev(current) + local next = getnext(cn) + local bpn = getnext(basepos) + while bpn and dependent_vowel[ischar(bpn)] do + basepos = bpn + bpn = getnext(bpn) + end + if basepos ~= prev then + setlink(prev,next) + setlink(cn, getnext(basepos)) + setlink(basepos, current) + if cn == stop then + stop = prev + end + cn = next + end + end + -- after_postscript + -- after_subscript + -- before_postscript + -- before_subscript + end + current = cn + end + -- Reorder marks to canonical order: Adjacent nukta and halant or nukta and vedic sign are always repositioned if necessary, so that the nukta is first. - local current, c = start, nil + local current = start + local c = nil while current ~= stop do local char = getchar(current) if halant[char] or stress_tone_mark[char] then @@ -1849,7 +2245,25 @@ local function analyze_next_chars_one(c,font,variant) -- skip one dependent vowe if not v then return c end - if dependent_vowel[v] then + local already_pre_mark -- = false + local already_above_mark -- = false + local already_below_mark -- = false + local already_post_mark -- = false + while dependent_vowel[v] do + local vowels = twopart_mark[v] or { v } + for k, v in next, vowels do + if pre_mark[v] and not already_pre_mark then + already_pre_mark = true + elseif above_mark[v] and not already_above_mark then + already_above_mark = true + elseif below_mark[v] and not already_below_mark then + already_below_mark = true + elseif post_mark[v] and not already_post_mark then + already_post_mark = true + else + return c + end + end c = getnext(c) n = getnext(c) if not n then @@ -2021,7 +2435,25 @@ local function analyze_next_chars_two(c,font) else -- c = ms_matra(c) -- same as one - if dependent_vowel[v] then + local already_pre_mark -- = false + local already_above_mark -- = false + local already_below_mark -- = false + local already_post_mark -- = false + while dependent_vowel[v] do + local vowels = twopart_mark[v] or { v } + for k, v in next, vowels do + if pre_mark[v] and not already_pre_mark then + already_pre_mark = true + elseif above_mark[v] and not already_above_mark then + already_above_mark = true + elseif below_mark[v] and not already_below_mark then + already_below_mark = true + elseif post_mark[v] and not already_post_mark then + already_post_mark = true + else + return c + end + end c = n n = getnext(c) if not n then @@ -2093,6 +2525,7 @@ local function method_one(head,font,attr) local start = true local done = false local nbspaces = 0 + local syllabe = 0 while current do local char = ischar(current,font) if char then @@ -2242,6 +2675,15 @@ local function method_one(head,font,attr) end end if syllablestart ~= syllableend then + if syllableend then + syllabe = syllabe + 1 + local c = syllablestart + local n = getnext(syllableend) + while c ~= n do + setprop(c,a_syllabe,syllabe) + c = getnext(c) + end + end head, current, nbspaces = reorder_one(head,syllablestart,syllableend,font,attr,nbspaces) current = getnext(current) end @@ -2283,6 +2725,21 @@ local function method_one(head,font,attr) head = replace_all_nbsp(head) end + current = head + local n = 0 + while current do + local char = ischar(current,font) + if char then + if n == 0 and not getprop(current,a_state) then + setprop(current,a_state,s_init) + end + n = n + 1 + else + n = 0 + end + current = getnext(current) + end + return head, done end @@ -2363,7 +2820,7 @@ local function method_two(head,font,attr) end if not syllableend and show_syntax_errors then local char = ischar(current,font) - if char and not getprop(current,a_state) then + if char and not getprop(current,a_state) then -- a_state can also be init local mark = mark_four[char] if mark then head, current = inject_syntax_error(head,current,char) @@ -2378,6 +2835,21 @@ local function method_two(head,font,attr) head = replace_all_nbsp(head) end + current = head + local n = 0 + while current do + local char = ischar(current,font) + if char then + if n == 0 and not getprop(current,a_state) then -- a_state can also be init + setprop(current,a_state,s_init) + end + n = n + 1 + else + n = 0 + end + current = getnext(current) + end + return head, done end diff --git a/tex/context/base/mkiv/lpdf-emb.lua b/tex/context/base/mkiv/lpdf-emb.lua index 8eab10c3b..5255eb038 100644 --- a/tex/context/base/mkiv/lpdf-emb.lua +++ b/tex/context/base/mkiv/lpdf-emb.lua @@ -91,7 +91,7 @@ end -- A couple of shared helpers. -local tounicodedictionary, widtharray, collectindices, subsetname, includecidset, tocidsetdictionary +local tounicodedictionary, widtharray, collectindices, subsetname, includecidset, forcecidset, tocidsetdictionary do @@ -230,14 +230,20 @@ end end includecidset = false + forcecidset = false -- for private testing only + + directives.register("backend.pdf.forcecidset",function(v) + forcecidset = v + end) tocidsetdictionary = function(indices,min,max) - if includecidset then + if includecidset or forcecidset then local b = { } local m = idiv(max+7,8) for i=0,max do b[i] = 0 end + b[0] = bor(b[0],lshift(1,7)) -- force notdef into the file for i=min,max do if indices[i] then local bi = idiv(i,8) diff --git a/tex/context/base/mkiv/mlib-pps.lua b/tex/context/base/mkiv/mlib-pps.lua index 5708577fe..73257968b 100644 --- a/tex/context/base/mkiv/mlib-pps.lua +++ b/tex/context/base/mkiv/mlib-pps.lua @@ -253,7 +253,7 @@ local function preset(t,k) return v end -local function startjob(plugmode,kind) +local function startjob(plugmode,kind,mpx) insert(stack,top) top = { textexts = { }, -- all boxes, optionally with a different color @@ -261,6 +261,7 @@ local function startjob(plugmode,kind) texlast = 0, texdata = setmetatableindex({},preset), -- references to textexts in order or usage plugmode = plugmode, -- some day we can then skip all pre/postscripts + extradata = mpx and metapost.getextradata(mpx), } if trace_runs then report_metapost("starting %s run at level %i in %s mode", @@ -580,8 +581,8 @@ end -- side effect of going single pass). function metapost.graphic_base_pass(specification) - local top = startjob(true,"base") local mpx = specification.mpx -- mandate + local top = startjob(true,"base",mpx) local data = specification.data or "" local inclusions = specification.inclusions or "" local initializations = specification.initializations or "" @@ -629,7 +630,7 @@ function metapost.process(specification,...) if type(specification) ~= "table" then oldschool(specification,...) else - startjob(specification.incontext or specification.useplugins,"process") + startjob(specification.incontext or specification.useplugins,"process",false) runmetapost(specification) stopjob() end @@ -910,7 +911,7 @@ local tx_reset, tx_process do end tx_process = function(object,prescript,before,after) - local data = top.texdata[metapost.properties.number] + local data = top.texdata[metapost.properties.number] -- the current figure number, messy local index = tonumber(prescript.tx_index) if index then if trace_textexts then @@ -932,19 +933,45 @@ local tx_reset, tx_process do top.texlast = mp_target -- local mp_text = top.texstrings[mp_index] + local mp_hash = prescript.tx_cache local box - if prescript.tx_cache == "no" then + if mp_hash == "no" then tex.runtoks("mptexttoks") box = textakebox("mptextbox") else - local hash = fmt(mp_text,mp_a or "-",mp_t or "-",mp_c or "-") - box = data.texhash[hash] + local cache = data.texhash + if mp_hash then + mp_hash = tonumber(mp_hash) + end + if mp_hash then + local extradata = top.extradata + if extradata then + cache = extradata.globalcache + if not cache then + cache = { } + extradata.globalcache = cache + end + if trace_runs then + if cache[mp_hash] then + report_textexts("reusing global entry %i",mp_hash) + else + report_textexts("storing global entry %i",mp_hash) + end + end + else + mp_hash = nil + end + end + if not mp_hash then + mp_hash = fmt(mp_text,mp_a or "-",mp_t or "-",mp_c or "-") + end + box = cache[mp_hash] if box then box = copy_list(box) else tex.runtoks("mptexttoks") box = textakebox("mptextbox") - data.texhash[hash] = box + cache[mp_hash] = box end end top.textexts[mp_target] = box diff --git a/tex/context/base/mkiv/mlib-run.lua b/tex/context/base/mkiv/mlib-run.lua index 3365e3d42..bd4818659 100644 --- a/tex/context/base/mkiv/mlib-run.lua +++ b/tex/context/base/mkiv/mlib-run.lua @@ -288,6 +288,11 @@ metapost.defaultmethod = "default" local mpxformats = { } local nofformats = 0 local mpxpreambles = { } +local mpxextradata = { } + +function metapost.getextradata(mpx) + return mpxextradata[mpx] +end function metapost.pushformat(specification,f,m) -- was: instance, name, method if type(specification) ~= "table" then @@ -338,6 +343,7 @@ function metapost.pushformat(specification,f,m) -- was: instance, name, method report_metapost("initializing instance %a using format %a and method %a",usedinstance,format,method) mpx = metapost.checkformat(format,method) mpxformats[usedinstance] = mpx + mpxextradata[mpx] = { } if mpp ~= "" then preamble = mpp end @@ -349,7 +355,6 @@ function metapost.pushformat(specification,f,m) -- was: instance, name, method return mpx end - -- luatex.wrapup(function() -- for k, mpx in next, mpxformats do -- mpx:finish() @@ -365,14 +370,16 @@ function metapost.reset(mpx) -- nothing elseif type(mpx) == "string" then if mpxformats[mpx] then - mpxformats[mpx]:finish() + mpxextradata[mpx] = nil mpxformats[mpx] = nil + mpxformats[mpx]:finish() end else for name, instance in next, mpxformats do if instance == mpx then + mpxextradata[mpx] = nil + mpxformats[mpx] = nil mpx:finish() - mpxformats[name] = nil break end end diff --git a/tex/context/base/mkiv/mult-fun.lua b/tex/context/base/mkiv/mult-fun.lua index 71d612156..57cf4778a 100644 --- a/tex/context/base/mkiv/mult-fun.lua +++ b/tex/context/base/mkiv/mult-fun.lua @@ -65,7 +65,7 @@ return { "withmask", "bitmapimage", "colordecimals", "ddecimal", "dddecimal", "ddddecimal", "colordecimalslist", "textext", "thetextext", "rawtextext", "textextoffset", "texbox", "thetexbox", "rawtexbox", "istextext", - "notcached", + "notcached", "keepcached", "verbatim", "thelabel", "label", "autoalign", diff --git a/tex/context/base/mkiv/spac-ver.mkiv b/tex/context/base/mkiv/spac-ver.mkiv index 27f9ffb70..c76555cba 100644 --- a/tex/context/base/mkiv/spac-ver.mkiv +++ b/tex/context/base/mkiv/spac-ver.mkiv @@ -2335,7 +2335,7 @@ \spac_vspacing_define_same_step\recurselevel\plustwo}% % whatever \global\c_spac_vspacing_special_done#1\relax} -\spac_vspacing_define_same_page{10} % 10 levels should be more than enough as a start +\spac_vspacing_define_same_page{12} % 12 levels should be more than enough as a start \def\spac_vspacing_same_page#1#2% level offset (starts at 0) {\ifnum#1>\c_spac_vspacing_special_done diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf Binary files differindex 1c524b8d1..70635c11f 100644 --- a/tex/context/base/mkiv/status-files.pdf +++ b/tex/context/base/mkiv/status-files.pdf diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf Binary files differindex f6553771d..f857f29b5 100644 --- a/tex/context/base/mkiv/status-lua.pdf +++ b/tex/context/base/mkiv/status-lua.pdf diff --git a/tex/context/base/mkiv/strc-flt.mkvi b/tex/context/base/mkiv/strc-flt.mkvi index 2af60b41d..8e8975a05 100644 --- a/tex/context/base/mkiv/strc-flt.mkvi +++ b/tex/context/base/mkiv/strc-flt.mkvi @@ -826,11 +826,47 @@ \strc_floats_finish_placement} \vbox} +%D \starttyping +%D \definefloat +%D [one] [figure] +%D [default=right, +%D rightmargindistance=-20cm, +%D criterium=129pt, +%D fallback=rightmargin] +%D +%D \definefloat +%D [two] [figure] +%D [default=right, +%D rightmargindistance=-20cm, +%D criterium=129pt, +%D fallback=three] +%D +%D \definefloat +%D [three] [figure] +%D [default=rightmargin, +%D rightmargindistance=0cm] +%D +%D \placefloat[one]{}{\blackrule[width=30pt]} \samplefile{tufte} +%D \placefloat[one]{}{\blackrule[width=60pt]} \samplefile{tufte} +%D \placefloat[one]{}{\blackrule[width=90pt]} \samplefile{tufte} +%D \placefloat[one]{}{\blackrule[width=130pt]} \samplefile{tufte} +%D \placefloat[two]{}{\blackrule[width=130pt]} \samplefile{tufte} +%D \stoptyping + \def\strc_floats_finish_placement {\doifsomething{\floatparameter\c!criterium} {\ifdim\wd\nextbox>\floatparameter\c!criterium\relax \edef\forcedfloatmethod{\floatparameter\c!fallback}% - \ifx\forcedfloatmethod\empty\let\forcedfloatmethod\v!here\fi + \ifx\forcedfloatmethod\empty \else + \doifelsecommandhandler\??float\forcedfloatmethod + {\let\currentfloat\forcedfloatmethod + \edef\floatlocation{\floatparameter\c!default}% + \let\forcedfloatmethod\floatlocation} + \donothing + \fi + \ifx\forcedfloatmethod\empty + \let\forcedfloatmethod\v!here + \fi \fi}% \strc_floats_check_extra_actions \strc_floats_analyze_variables_two diff --git a/tex/context/base/mkiv/util-jsn.lua b/tex/context/base/mkiv/util-jsn.lua index acbf16090..68c6a712e 100644 --- a/tex/context/base/mkiv/util-jsn.lua +++ b/tex/context/base/mkiv/util-jsn.lua @@ -20,155 +20,391 @@ if not modules then modules = { } end modules ['util-jsn'] = { local P, V, R, S, C, Cc, Cs, Ct, Cf, Cg = lpeg.P, lpeg.V, lpeg.R, lpeg.S, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cf, lpeg.Cg local lpegmatch = lpeg.match local format, gsub = string.format, string.gsub +local formatters = string.formatters local utfchar = utf.char -local concat = table.concat +local concat, sortedkeys = table.concat, table.sortedkeys local tonumber, tostring, rawset, type, next = tonumber, tostring, rawset, type, next local json = utilities.json or { } utilities.json = json --- \\ \/ \b \f \n \r \t \uHHHH - -local lbrace = P("{") -local rbrace = P("}") -local lparent = P("[") -local rparent = P("]") -local comma = P(",") -local colon = P(":") -local dquote = P('"') - -local whitespace = lpeg.patterns.whitespace -local optionalws = whitespace^0 - -local escapes = { - ["b"] = "\010", - ["f"] = "\014", - ["n"] = "\n", - ["r"] = "\r", - ["t"] = "\t", -} +do --- todo: also handle larger utf16 + -- \\ \/ \b \f \n \r \t \uHHHH -local escape_un = P("\\u")/"" * (C(R("09","AF","af")^-4) / function(s) - return utfchar(tonumber(s,16)) -end) + local lbrace = P("{") + local rbrace = P("}") + local lparent = P("[") + local rparent = P("]") + local comma = P(",") + local colon = P(":") + local dquote = P('"') -local escape_bs = P([[\]]) / "" * (P(1) / escapes) -- if not found then P(1) is returned i.e. the to be escaped char + local whitespace = lpeg.patterns.whitespace + local optionalws = whitespace^0 -local jstring = dquote * Cs((escape_un + escape_bs + (1-dquote))^0) * dquote -local jtrue = P("true") * Cc(true) -local jfalse = P("false") * Cc(false) -local jnull = P("null") * Cc(nil) -local jnumber = (1-whitespace-rparent-rbrace-comma)^1 / tonumber + local escapes = { + ["b"] = "\010", + ["f"] = "\014", + ["n"] = "\n", + ["r"] = "\r", + ["t"] = "\t", + } -local key = jstring + -- todo: also handle larger utf16 -local jsonconverter = { "value", - hash = lbrace * Cf(Ct("") * (V("pair") * (comma * V("pair"))^0 + optionalws),rawset) * rbrace, - pair = Cg(optionalws * key * optionalws * colon * V("value")), - array = Ct(lparent * (V("value") * (comma * V("value"))^0 + optionalws) * rparent), --- value = optionalws * (jstring + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber + #rparent) * optionalws, - value = optionalws * (jstring + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber) * optionalws, -} + local escape_un = P("\\u")/"" * (C(R("09","AF","af")^-4) / function(s) + return utfchar(tonumber(s,16)) + end) + + local escape_bs = P([[\]]) / "" * (P(1) / escapes) -- if not found then P(1) is returned i.e. the to be escaped char + + local jstring = dquote * Cs((escape_un + escape_bs + (1-dquote))^0) * dquote + local jtrue = P("true") * Cc(true) + local jfalse = P("false") * Cc(false) + local jnull = P("null") * Cc(nil) + local jnumber = (1-whitespace-rparent-rbrace-comma)^1 / tonumber + + local key = jstring + + local jsonconverter = { "value", + hash = lbrace * Cf(Ct("") * (V("pair") * (comma * V("pair"))^0 + optionalws),rawset) * rbrace, + pair = Cg(optionalws * key * optionalws * colon * V("value")), + array = Ct(lparent * (V("value") * (comma * V("value"))^0 + optionalws) * rparent), + -- value = optionalws * (jstring + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber + #rparent) * optionalws, + value = optionalws * (jstring + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber) * optionalws, + } + + -- local jsonconverter = { "value", + -- hash = lbrace * Cf(Ct("") * (V("pair") * (comma * V("pair"))^0 + optionalws),rawset) * rbrace, + -- pair = Cg(optionalws * V("string") * optionalws * colon * V("value")), + -- array = Ct(lparent * (V("value") * (comma * V("value"))^0 + optionalws) * rparent), + -- string = jstring, + -- value = optionalws * (V("string") + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber) * optionalws, + -- } + + -- lpeg.print(jsonconverter) -- size 181 --- local jsonconverter = { "value", --- hash = lbrace * Cf(Ct("") * (V("pair") * (comma * V("pair"))^0 + optionalws),rawset) * rbrace, --- pair = Cg(optionalws * V("string") * optionalws * colon * V("value")), --- array = Ct(lparent * (V("value") * (comma * V("value"))^0 + optionalws) * rparent), --- string = jstring, --- value = optionalws * (V("string") + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber) * optionalws, --- } + function json.tolua(str) + return lpegmatch(jsonconverter,str) + end --- lpeg.print(jsonconverter) -- size 181 + function json.load(filename) + local data = io.loaddata(filename) + if data then + return lpegmatch(jsonconverter,data) + end + end -function json.tolua(str) - return lpegmatch(jsonconverter,str) end -local escaper +do -local function tojson(value,t,n) -- we could optimize #t - local kind = type(value) - if kind == "table" then - local done = false - local size = #value - if size == 0 then - for k, v in next, value do - if done then - n = n + 1 ; t[n] = "," + -- It's pretty bad that JSON doesn't allow the trailing comma ... it's a + -- typical example of a spec that then forces all generators to check for + -- this. It's a way to make sure programmers keep jobs. + + local escaper + + local f_start_hash = formatters[ '%w{' ] + local f_start_array = formatters[ '%w[' ] + local f_start_hash_new = formatters[ "\n" .. '%w{' ] + local f_start_array_new = formatters[ "\n" .. '%w[' ] + local f_start_hash_key = formatters[ "\n" .. '%w"%s" : {' ] + local f_start_array_key = formatters[ "\n" .. '%w"%s" : [' ] + + local f_stop_hash = formatters[ "\n" .. '%w}' ] + local f_stop_array = formatters[ "\n" .. '%w]' ] + + local f_key_val_seq = formatters[ "\n" .. '%w"%s" : %s' ] + local f_key_val_str = formatters[ "\n" .. '%w"%s" : "%s"' ] + local f_key_val_num = f_key_val_seq + local f_key_val_yes = formatters[ "\n" .. '%w"%s" : true' ] + local f_key_val_nop = formatters[ "\n" .. '%w"%s" : false' ] + + local f_val_num = formatters[ "\n" .. '%w%s' ] + local f_val_str = formatters[ "\n" .. '%w"%s"' ] + local f_val_yes = formatters[ "\n" .. '%wtrue' ] + local f_val_nop = formatters[ "\n" .. '%wfalse' ] + local f_val_seq = f_val_num + + -- no empty tables because unknown if table or hash + + local t = { } + local n = 0 + + local function is_simple_table(tt) -- also used in util-tab so maybe public + local l = #tt + if l > 0 then + for i=1,l do + if type(tt[i]) == "table" then + return false + end + end + local nn = n + n = n + 1 t[n] = "[ " + for i=1,l do + if i > 1 then + n = n + 1 t[n] = ", " + end + local v = tt[i] + local tv = type(v) + if tv == "number" then + n = n + 1 t[n] = v + elseif tv == "string" then + n = n + 1 t[n] = '"' + n = n + 1 t[n] = lpegmatch(escaper,v) or v + n = n + 1 t[n] = '"' + elseif tv == "boolean" then + n = n + 1 t[n] = v and "true" or "false" else - n = n + 1 ; t[n] = "{" - done = true + n = n + 1 t[n] = tostring(v) end - n = n + 1 ; t[n] = format("%q:",k) - t, n = tojson(v,t,n) end - if done then - n = n + 1 ; t[n] = "}" + n = n + 1 t[n] = " ]" + local s = concat(t,"",nn+1,n) + n = nn + return s + end + return false + end + + local function tojsonpp(root,name,depth,level,size) + if root then + local indexed = size > 0 + n = n + 1 + if level == 0 then + if indexed then + t[n] = f_start_array(depth) + else + t[n] = f_start_hash(depth) + end + elseif name then + if tn == "string" then + name = lpegmatch(escaper,name) or name + elseif tn ~= "number" then + name = tostring(name) + end + if indexed then + t[n] = f_start_array_key(depth,name) + else + t[n] = f_start_hash_key(depth,name) + end else - n = n + 1 ; t[n] = "{}" + if indexed then + t[n] = f_start_array_new(depth) + else + t[n] = f_start_hash_new(depth) + end end - elseif size == 1 then - -- we can optimize for non tables - n = n + 1 ; t[n] = "[" - t, n = tojson(value[1],t,n) - n = n + 1 ; t[n] = "]" - else - for i=1,size do + depth = depth + 1 + if indexed then -- indexed + for i=1,size do + if i > 1 then + n = n + 1 t[n] = "," + end + local v = root[i] + local tv = type(v) + if tv == "number" then + n = n + 1 t[n] = f_val_num(depth,v) + elseif tv == "string" then + v = lpegmatch(escaper,v) or v + n = n + 1 t[n] = f_val_str(depth,v) + elseif tv == "table" then + if next(v) then + local st = is_simple_table(v) + if st then + n = n + 1 t[n] = f_val_seq(depth,st) + else + tojsonpp(v,k,depth,level+1,0) + end + end + elseif tv == "boolean" then + n = n + 1 + if v then + t[n] = f_val_yes(depth,v) + else + t[n] = f_val_nop(depth,v) + end + end + end + elseif next(root) then + local sk = sortedkeys(root) + for i=1,#sk do + if i > 1 then + n = n + 1 t[n] = "," + end + local k = sk[i] + local v = root[k] + local tv = type(v) + local tk = type(k) + if tv == "number" then + if tk == "number" then + n = n + 1 t[n] = f_key_val_num(depth,k,v) + elseif tk == "string" then + k = lpegmatch(escaper,k) or k + n = n + 1 t[n] = f_key_val_str(depth,k,v) + end + elseif tv == "string" then + if tk == "number" then + v = lpegmatch(escaper,v) or v + n = n + 1 t[n] = f_key_val_num(depth,k,v) + elseif tk == "string" then + k = lpegmatch(escaper,k) or k + v = lpegmatch(escaper,v) or v + n = n + 1 t[n] = f_key_val_str(depth,k,v) + end + elseif tv == "table" then + local l = #v + if l > 0 then + local st = is_simple_table(v) + if not st then + tojsonpp(v,k,depth,level+1,l) + elseif tk == "number" then + n = n + 1 t[n] = f_key_val_seq(depth,k,st) + elseif tk == "string" then + k = lpegmatch(escaper,k) or k + n = n + 1 t[n] = f_key_val_seq(depth,k,st) + end + elseif next(v) then + tojsonpp(v,k,depth,level+1,0) + end + elseif tv == "boolean" then + if tk == "number" then + n = n + 1 + if v then + t[n] = f_key_val_yes(depth,k) + else + t[n] = f_key_val_nop(depth,k) + end + elseif tk == "string" then + k = lpegmatch(escaper,k) or k + n = n + 1 + if v then + t[n] = f_key_val_yes(depth,k) + else + t[n] = f_key_val_nop(depth,k) + end + end + end + end + end + n = n + 1 + if indexed then + t[n] = f_stop_array(depth-1) + else + t[n] = f_stop_hash(depth-1) + end + end + end + + local function tojson(value,n) + local kind = type(value) + if kind == "table" then + local done = false + local size = #value + if size == 0 then + for k, v in next, value do + if done then + -- n = n + 1 ; t[n] = "," + n = n + 1 ; t[n] = ',"' + else + -- n = n + 1 ; t[n] = "{" + n = n + 1 ; t[n] = '{"' + done = true + end + n = n + 1 ; t[n] = lpegmatch(escaper,k) or k + n = n + 1 ; t[n] = '":' + t, n = tojson(v,n) + end if done then - n = n + 1 ; t[n] = "," + n = n + 1 ; t[n] = "}" else - n = n + 1 ; t[n] = "[" - done = true + n = n + 1 ; t[n] = "{}" end - t, n = tojson(value[i],t,n) + elseif size == 1 then + -- we can optimize for non tables + n = n + 1 ; t[n] = "[" + t, n = tojson(value[1],n) + n = n + 1 ; t[n] = "]" + else + for i=1,size do + if done then + n = n + 1 ; t[n] = "," + else + n = n + 1 ; t[n] = "[" + done = true + end + t, n = tojson(value[i],n) + end + n = n + 1 ; t[n] = "]" end - n = n + 1 ; t[n] = "]" + elseif kind == "string" then + n = n + 1 ; t[n] = '"' + n = n + 1 ; t[n] = lpegmatch(escaper,value) or value + n = n + 1 ; t[n] = '"' + elseif kind == "number" then + n = n + 1 ; t[n] = value + elseif kind == "boolean" then + n = n + 1 ; t[n] = tostring(value) end - elseif kind == "string" then - n = n + 1 ; t[n] = '"' - n = n + 1 ; t[n] = lpegmatch(escaper,value) or value - n = n + 1 ; t[n] = '"' - elseif kind == "number" then - n = n + 1 ; t[n] = value - elseif kind == "boolean" then - n = n + 1 ; t[n] = tostring(value) + return t, n end - return t, n -end -function json.tostring(value) - -- todo optimize for non table - local kind = type(value) - if kind == "table" then - if not escaper then - local escapes = { - ["\\"] = "\\u005C", - ["\""] = "\\u0022", - } - for i=0,0x20 do - escapes[utfchar(i)] = format("\\u%04X",i) - end - escaper = Cs( ( - (R('\0\x20') + S('\"\\')) / escapes - + P(1) - )^1 ) + -- escaping keys can become an option + + local function jsontostring(value,pretty) + -- todo optimize for non table + local kind = type(value) + if kind == "table" then + if not escaper then + local escapes = { + ["\\"] = "\\u005C", + ["\""] = "\\u0022", + } + for i=0,0x1F do + escapes[utfchar(i)] = format("\\u%04X",i) + end + escaper = Cs( ( + (R('\0\x20') + S('\"\\')) / escapes + + P(1) + )^1 ) + end + -- local to the closure (saves wrapping and local functions) + t = { } + n = 0 + if pretty then + tojsonpp(value,name,0,0,#value) +-- value = concat(t,"\n",1,n) + value = concat(t,"",1,n) + else + tojson(value,0) + value = concat(t,"",1,n) + end + t = nil + n = 0 + return value + elseif kind == "string" or kind == "number" then + return lpegmatch(escaper,value) or value + else + return tostring(value) end - return concat((tojson(value,{},0))) - elseif kind == "string" or kind == "number" then - return lpegmatch(escaper,value) or value - else - return tostring(value) end + + json.tostring = jsontostring + + function json.tojson(value) + return jsontostring(value,true) + end + end --- local tmp = [[ { "t" : "foobar", "a" : true, "b" : [ 123 , 456E-10, { "a" : true, "b" : [ 123 , 456 ] } ] } ]] +-- local tmp = [[ { "t\nt t" : "foo bar", "a" : true, "b" : [ 123 , 456E-10, { "a" : true, "b" : [ 123 , 456 ] } ] } ]] -- tmp = json.tolua(tmp) -- inspect(tmp) --- tmp = json.tostring(tmp) +-- tmp = json.tostring(tmp,true) -- inspect(tmp) -- tmp = json.tolua(tmp) -- inspect(tmp) @@ -176,13 +412,6 @@ end -- inspect(tmp) -- inspect(json.tostring(true)) -function json.load(filename) - local data = io.loaddata(filename) - if data then - return lpegmatch(jsonconverter,data) - end -end - -- local s = [[\foo"bar"]] -- local j = json.tostring { s = s } -- local l = json.tolua(j) diff --git a/tex/context/base/mkiv/util-pck.lua b/tex/context/base/mkiv/util-pck.lua index 83b85cd94..b90853fb6 100644 --- a/tex/context/base/mkiv/util-pck.lua +++ b/tex/context/base/mkiv/util-pck.lua @@ -55,7 +55,6 @@ local function pack(t,keys,skip,hash,index) local k = sk[i] if not skip or not skip[k] then local v = t[k] - -- if type(v) == "table" then pack(v,keys,skip,hash,index) if keys[k] then diff --git a/tex/context/interface/mkii/keys-pe.xml b/tex/context/interface/mkii/keys-pe.xml index 160f4f3fc..8532d9894 100644 --- a/tex/context/interface/mkii/keys-pe.xml +++ b/tex/context/interface/mkii/keys-pe.xml @@ -573,6 +573,7 @@ <cd:variable name='temporary' value='موقتی'/> <cd:variable name='test' value='تست'/> <cd:variable name='text' value='متن'/> + <cd:variable name='textnote' value='textnote'/> <cd:variable name='three' value='سه'/> <cd:variable name='thursday' value='پنجشنبه'/> <cd:variable name='tight' value='tight'/> diff --git a/tex/context/interface/mkiv/i-context.pdf b/tex/context/interface/mkiv/i-context.pdf Binary files differindex 746f0fb36..04e0e44ae 100644 --- a/tex/context/interface/mkiv/i-context.pdf +++ b/tex/context/interface/mkiv/i-context.pdf diff --git a/tex/context/interface/mkiv/i-readme.pdf b/tex/context/interface/mkiv/i-readme.pdf Binary files differindex 847353e60..4b14223aa 100644 --- a/tex/context/interface/mkiv/i-readme.pdf +++ b/tex/context/interface/mkiv/i-readme.pdf |