beta 2012.05.07 23:33

21 files changed, 722 insertions, 346 deletions

diff --git a/tex/context/base/attr-neg.lua b/tex/context/base/attr-neg.lua
index 766295a16..4d89cb49b 100644
--- a/tex/context/base/attr-neg.lua
+++ b/tex/context/base/attr-neg.lua
@@ -29,6 +29,8 @@ local negatives         = attributes.negatives
 
 local a_negative        = attributes.private("negative")
 
+local v_none            = interfaces.variables.none
+
 negatives.data          = allocate()
 negatives.attribute     = a_negative
 
@@ -41,7 +43,7 @@ local data       = negatives.data
 local registered = negatives.registered
 
 local function extender(negatives,key)
-    if key == "none" then
+    if key == "none" then -- v_none then
         local d = data[1]
         negatives.none = d
         return d
diff --git a/tex/context/base/attr-neg.mkiv b/tex/context/base/attr-neg.mkiv
index dc8f5e4f9..0fc5070f1 100644
--- a/tex/context/base/attr-neg.mkiv
+++ b/tex/context/base/attr-neg.mkiv
@@ -19,7 +19,7 @@
 
 % positive and negative are preregistered
 
-\def\dotriggernegative#1{\ctxcommand{triggernegative('#1'))}}
+\def\dotriggernegative#1{\ctxcommand{triggernegative('#1')}}
 
 \unexpanded\def\startnegative{\dotriggernegative\v!negative}
 \unexpanded\def\stopnegative {\dotriggernegative\v!positive}
diff --git a/tex/context/base/chem-str.lua b/tex/context/base/chem-str.lua
index ac7176bf1..6e2af50fb 100644
--- a/tex/context/base/chem-str.lua
+++ b/tex/context/base/chem-str.lua
@@ -406,8 +406,8 @@ function chemicals.start(settings)
         scale = tonumber(scale)
         if not scale or scale == 0 then
             scale = 750
-        elseif scale < 500 then
-            scale = 500
+        elseif scale < 10 then
+            scale = 10
         end
     end
     if width == variables.fit then
diff --git a/tex/context/base/colo-ext.mkiv b/tex/context/base/colo-ext.mkiv
index c95c919aa..af6c3830e 100644
--- a/tex/context/base/colo-ext.mkiv
+++ b/tex/context/base/colo-ext.mkiv
@@ -29,13 +29,22 @@
 %D
 %D will negate the colors in box zero.
 
+% \unexpanded\def\negatecolorbox#1%
+%   {\setbox#1\hbox
+%      {\startnegative % might change
+%       \startcolor[\s!white]\vrule\!!height\ht#1\!!depth\dp#1\!!width\wd#1\stopcolor
+%       \hskip-\wd#1%
+%       \box#1%
+%       \stopnegative}}
+
 \unexpanded\def\negatecolorbox#1%
   {\setbox#1\hbox
-     {\startnegativeproperty % might change
-      \startcolor[\s!white]\vrule\!!height\ht#1\!!depth\dp#1\!!width\wd#1\stopcolor
+     {\startnegative % might change
+%       \startcolor[\s!white]\vrule\!!height\ht#1\!!depth\dp#1\!!width\wd#1\stopcolor
+      \blackrule[\c!color=\s!white,\c!height=\ht#1,\c!depth=\dp#1,\c!width=\wd#1]%
       \hskip-\wd#1%
       \box#1%
-      \stopnegativeproperty}}
+      \stopnegative}}
 
 %D There are in principle two ways to handle overprint: bound to colors
 %D or independent. For the moment we only support independent overprint
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index f9ca34b97..103b9dcc2 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.05.06 21:53}
+\newcontextversion{2012.05.07 23:33}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 3e3df80b8..b90f0a90b 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.05.06 21:53}
+\newcontextversion{2012.05.07 23:33}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 55a7f5e2e..ac28b6b6b 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png
index 36eefb786..5edc6dfd8 100644
--- a/tex/context/base/context-version.png
+++ b/tex/context/base/context-version.png
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 8dfcf312d..3c582bdeb 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.05.06 21:53}
+\edef\contextversion{2012.05.07 23:33}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index a8be45840..502a61ef0 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -23,7 +23,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.05.06 21:53}
+\edef\contextversion{2012.05.07 23:33}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/font-pre.mkiv b/tex/context/base/font-pre.mkiv
index a03609f22..5ccf1a988 100644
--- a/tex/context/base/font-pre.mkiv
+++ b/tex/context/base/font-pre.mkiv
@@ -153,7 +153,7 @@
 %D We define some colors that are used in tracing (for instance \OPENTYPE\
 %D features). We cannot yet inherit because no colors are predefined.
 
-\definecolor[trace:0][s=.6]
+\definecolor[trace:0][s=.4]
 \definecolor[trace:1][r=.6]
 \definecolor[trace:2][g=.6]
 \definecolor[trace:3][b=.6]
diff --git a/tex/context/base/lpdf-col.lua b/tex/context/base/lpdf-col.lua
index b7d4b0eff..0c2a49ebe 100644
--- a/tex/context/base/lpdf-col.lua
+++ b/tex/context/base/lpdf-col.lua
@@ -40,6 +40,8 @@ local forcedmodel          = colors.forcedmodel
 
 local c_transparency = pdfconstant("Transparency")
 
+local report_color = logs.reporter("colors","backend")
+
 -- page groups (might move to lpdf-ini.lua)
 
 local colorspaceconstants = { -- v_none is ignored
@@ -221,7 +223,13 @@ local function registersomespotcolor(name,noffractions,names,p,colorspace,range,
             elseif n == "black" then
                 name = "Black"
             else
-                colorants[name]   = pdfreference(spotcolorhash[name] or spotcolorhash[n])
+                local sn = spotcolorhash[name] or spotcolorhash[n]
+                if sn then
+                    colorants[name] = pdfreference(sn)
+                else
+                    report_color("unknown colorant %s, using black instead",name or n)
+                    name = "Black"
+                end
             end
             cnames[#cnames+1] = pdfconstant(name)
             domain[#domain+1] = 0
diff --git a/tex/context/base/lpdf-mis.lua b/tex/context/base/lpdf-mis.lua
index f427fa3db..174d17427 100644
--- a/tex/context/base/lpdf-mis.lua
+++ b/tex/context/base/lpdf-mis.lua
@@ -57,10 +57,10 @@ local function initializenegative()
         Range        = a,
         Domain       = a,
     }
-    local negative = pdfdictionary { Type = g, TR = pdfreference(pdfflushstreamobject("1 exch sub",d)) }
+    local negative = pdfdictionary { Type = g, TR = pdfreference(pdfflushstreamobject("{ 1 exch sub }",d)) }
     local positive = pdfdictionary { Type = g, TR = pdfconstant("Identity") }
     lpdf.adddocumentextgstate("GSnegative", pdfreference(pdfflushobject(negative)))
-    lpdf.adddocumentextgstate("GSPositive", pdfreference(pdfflushobject(positive)))
+    lpdf.adddocumentextgstate("GSpositive", pdfreference(pdfflushobject(positive)))
     initializenegative = nil
 end
 
diff --git a/tex/context/base/m-units.mkiv b/tex/context/base/m-units.mkiv
index 19cb05557..b0db8d548 100644
--- a/tex/context/base/m-units.mkiv
+++ b/tex/context/base/m-units.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-%D Best use the buitl in unit handle!
+%D Best use the built in unit handle!
 
 %D Scientific units can be typeset in math mode pretty well,
 %D but occasionally one has to take care of spacing.
@@ -70,6 +70,8 @@
 
 \unprotect
 
+\writestatus{\m!units}{The units module is obsolete because functionality is built into the core.}
+
 \let\unit\undefined
 
 \definesynonyms
diff --git a/tex/context/base/mult-low.lua b/tex/context/base/mult-low.lua
index 794a1f6de..743f851c4 100644
--- a/tex/context/base/mult-low.lua
+++ b/tex/context/base/mult-low.lua
@@ -97,6 +97,9 @@ return {
         "starttext", "stoptext", "startdocument", "stopdocument", "documentvariable",
         "startmodule", "stopmodule", "usemodule",
         --
+        "startTEXpage", "stopTEXpage",
+    --  "startMPpage", "stopMPpage", -- already catched by nested lexer
+        --
         "enablemode", "disablemode", "preventmode", "pushmode", "popmode",
         --
         "typescriptone", "typescripttwo", "typescriptthree", "mathsizesuffix",
diff --git a/tex/context/base/page-box.mkvi b/tex/context/base/page-box.mkvi
index d9b2af881..42db40e5f 100644
--- a/tex/context/base/page-box.mkvi
+++ b/tex/context/base/page-box.mkvi
@@ -115,11 +115,11 @@
 
 \def\page_boxes_apply_orientate_indeed#odd#even#box%
   {\setbox#box\vbox
-     {\edef\somerotation{\number\ifdoublesided\ifodd\realpageno#odd\else#even\fi\else##odd\fi}%
+     {\edef\somerotation{\number\ifdoublesided\ifodd\realpageno#odd\else#even\fi\else#odd\fi}%
       \dorotatebox\somerotation\hbox{\box#box}}}
 
 \def\page_boxes_apply_mirror#box%
-  {\setbox#box\vbox{\domirrorbox\vbox{\box#box}}}
+  {\setbox#box\vbox{\mirror{\box#box}}}
 
 \def\page_boxes_apply_mirror_paper#box{\ifconditional\c_page_target_paper_mirror\page_boxes_apply_mirror{#box}\fi}
 \def\page_boxes_apply_mirror_print#box{\ifconditional\c_page_target_print_mirror\page_boxes_apply_mirror{#box}\fi}
diff --git a/tex/context/base/scrp-cjk.lua b/tex/context/base/scrp-cjk.lua
index d817c28cf..3463a8eb5 100644
--- a/tex/context/base/scrp-cjk.lua
+++ b/tex/context/base/scrp-cjk.lua
@@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['scrp-cjk'] = {
     license   = "see context related readme files"
 }
 
+local utfchar = utf.char
+
 local has_attribute      = node.has_attribute
 local insert_node_after  = node.insert_after
 local insert_node_before = node.insert_before
@@ -13,6 +15,7 @@ local remove_node        = nodes.remove
 
 local nodepool           = nodes.pool
 local new_glue           = nodepool.glue
+local new_kern           = nodepool.kern
 local new_penalty        = nodepool.penalty
 
 local nodecodes          = nodes.nodecodes
@@ -30,59 +33,123 @@ local fonthashes         = fonts.hashes
 local fontdata           = fonthashes.identifiers
 local quaddata           = fonthashes.quads
 
+local trace_details      = false  trackers.register("scripts.details", function(v) trace_details = v end)
+
+local report_details     = logs.reporter("scripts","detail")
+
 -- raggedleft is controlled by leftskip and we might end up with a situation where
 -- the intercharacter spacing interferes with this; the solution is to patch the
 -- nodelist but better is to use veryraggedleft
 
-local inter_char_stretch        = 0
-local inter_char_half_shrink    = 0
-local inter_char_hangul_penalty = 0
+local inter_char_shrink          = 0
+local inter_char_stretch         = 0
+local inter_char_half_shrink     = 0
+local inter_char_half_stretch    = 0
+local inter_char_quarter_shrink  = 0
+local inter_char_quarter_stretch = 0
+
+local full_char_width            = 0
+local half_char_width            = 0
+local quarter_char_width         = 0
+
+local inter_char_hangul_penalty  = 0
 
 local function set_parameters(font,data)
     -- beware: parameters can be nil in e.g. punk variants
     local quad = quaddata[font]
-    inter_char_half_shrink    = data.inter_char_half_shrink_factor * quad
-    inter_char_stretch        = data.inter_char_stretch_factor * quad
-    inter_char_hangul_penalty = data.inter_char_hangul_penalty
+    full_char_width            = quad
+    half_char_width            = quad/2
+    quarter_char_width         = quad/4
+    inter_char_shrink          = data.inter_char_shrink_factor          * quad
+    inter_char_stretch         = data.inter_char_stretch_factor         * quad
+    inter_char_half_shrink     = data.inter_char_half_shrink_factor     * quad
+    inter_char_half_stretch    = data.inter_char_half_stretch_factor    * quad
+    inter_char_quarter_shrink  = data.inter_char_quarter_shrink_factor  * quad
+    inter_char_quarter_stretch = data.inter_char_quarter_stretch_factor * quad
+    inter_char_hangul_penalty  = data.inter_char_hangul_penalty
 end
 
 -- a test version did compensate for crappy halfwidth but we can best do that
 -- at font definition time and/or just assume a correct font
 
+local function trace_detail(current,what)
+    local prev = current.prev
+    local c_id = current.id
+    local p_id = prev and prev.id
+    if c_id == glyph_code then
+        local c_ch = current.char
+        if p_id == glyph_code then
+            local p_ch = p_id and prev.char
+            report_details("[U+%05X %s %s] [%s] [U+%05X %s %s]",p_ch,utfchar(p_ch),hash[p_ch] or "unknown",what,c_ch,utfchar(c_ch),hash[c_ch] or "unknown")
+        else
+            report_details("[%s] [U+%05X %s %s]",what,c_ch,utfchar(c_ch),hash[c_ch] or "unknown")
+        end
+    else
+        if p_id == glyph_code then
+            local p_ch = p_id and prev.char
+            report_details("[U+%05X %s %s] [%s]",p_ch,utfchar(p_ch),hash[p_ch] or "unknown",what)
+        else
+            report_details("[%s]",what)
+        end
+    end
+end
+
 local function nobreak(head,current)
+    if trace_details then
+        trace_detail(current,"break")
+    end
     insert_node_before(head,current,new_penalty(10000))
 end
 
 local function stretch_break(head,current)
+    if trace_details then
+        trace_detail(current,"stretch break")
+    end
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
 end
 
 local function shrink_break(head,current)
+    if trace_details then
+        trace_detail(current,"shrink break")
+    end
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
 local function nobreak_stretch(head,current)
+    if trace_details then
+        trace_detail(current,"no break stretch")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
 end
 
 local function korean_break(head,current)
+    if trace_details then
+        trace_detail(current,"korean break")
+    end
     insert_node_before(head,current,new_penalty(inter_char_hangul_penalty))
 end
 
 local function nobreak_shrink(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak shrink")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
 local function nobreak_autoshrink(head,current)
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
+    if trace_details then
+        trace_detail(current,"nobreak autoshrink")
     end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
 local function nobreak_stretch_nobreak_shrink(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak stretch nobreak shrink")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
     insert_node_before(head,current,new_penalty(10000))
@@ -90,15 +157,19 @@ local function nobreak_stretch_nobreak_shrink(head,current)
 end
 
 local function nobreak_stretch_nobreak_autoshrink(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak stretch nobreak autoshrink")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
-    end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
 local function nobreak_shrink_nobreak_stretch(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak shrink nobreak stretch")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
     insert_node_before(head,current,new_penalty(10000))
@@ -106,29 +177,37 @@ local function nobreak_shrink_nobreak_stretch(head,current)
 end
 
 local function nobreak_autoshrink_nobreak_stretch(head,current)
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
+    if trace_details then
+        trace_detail(current,"nobreak autoshrink nobreak stretch")
     end
     insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
+    insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
 end
 
 local function nobreak_shrink_break_stretch(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak shrink break stretch")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
 end
 
 local function nobreak_autoshrink_break_stretch(head,current)
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
+    if trace_details then
+        trace_detail(current,"nobreak autoshrink break stretch")
     end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
 end
 
 local function nobreak_shrink_break_stretch_nobreak_shrink(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak shrink break stretch nobreak shrink")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
@@ -136,187 +215,191 @@ local function nobreak_shrink_break_stretch_nobreak_shrink(head,current)
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
 end
 
+local function japanese_between_full_close_open(head,current) -- todo: check width
+    if trace_details then
+        trace_detail(current,"japanese between full close open")
+    end
+    insert_node_before(head,current,new_kern(-half_char_width))
+    insert_node_before(head,current,new_glue(half_char_width,0,inter_char_half_shrink))
+    insert_node_before(head,current,new_kern(-half_char_width))
+end
+
+local function japanese_between_full_close_full_close(head,current) -- todo: check width
+    if trace_details then
+        trace_detail(current,"japanese between full close full close")
+    end
+    insert_node_before(head,current,new_kern(-half_char_width))
+ -- insert_node_before(head,current,new_glue(half_char_width,0,inter_char_half_shrink))
+end
+
+local function japanese_before_full_width_punct(head,current) -- todo: check width
+    if trace_details then
+        trace_detail(current,"japanese before full width punct")
+    end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(quarter_char_width,0,inter_char_quarter_shrink))
+    insert_node_before(head,current,new_kern(-quarter_char_width))
+end
+
+local function japanese_after_full_width_punct(head,current) -- todo: check width
+    if trace_details then
+        trace_detail(current,"japanese after full width punct")
+    end
+    insert_node_before(head,current,new_kern(-quarter_char_width))
+    insert_node_before(head,current,new_glue(quarter_char_width,0,inter_char_quarter_shrink))
+end
+
 local function nobreak_autoshrink_break_stretch_nobreak_autoshrink(head,current)
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
+    if trace_details then
+        trace_detail(current,"nobreak autoshrink break stretch nobreak autoshrink")
     end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
-    end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
 local function nobreak_autoshrink_break_stretch_nobreak_shrink(head,current)
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
+    if trace_details then
+        trace_detail(current,"nobreak autoshrink break stretch nobreak shrink")
     end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
 local function nobreak_shrink_break_stretch_nobreak_autoshrink(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak shrink break stretch nobreak autoshrink")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
-    if true then
-        insert_node_before(head,current,new_penalty(10000))
-        insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
-    end
+    insert_node_before(head,current,new_penalty(10000))
+    insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
 end
 
 local function nobreak_stretch_break_shrink(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak stretch break shrink")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
     insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
 local function nobreak_stretch_break_autoshrink(head,current)
+    if trace_details then
+        trace_detail(current,"nobreak stretch break autoshrink")
+    end
     insert_node_before(head,current,new_penalty(10000))
     insert_node_before(head,current,new_glue(0,inter_char_stretch,0))
-    if true then
-        insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
-    end
+    insert_node_before(head,current,new_glue(0,0,inter_char_half_shrink))
 end
 
--- hangul (korean)
+-- Korean: hangul
+
+local korean_0 = {
+}
+
+local korean_1 = {
+    jamo_initial     = korean_break,
+    korean           = korean_break,
+    chinese          = korean_break,
+    hiragana         = korean_break,
+    katakana         = korean_break,
+    half_width_open  = stretch_break,
+    half_width_close = nobreak,
+    full_width_open  = stretch_break,
+    full_width_close = nobreak,
+    full_width_punct = nobreak,
+--  hyphen           = nil,
+    non_starter      = korean_break,
+    other            = korean_break,
+}
+
+local korean_2 = {
+    jamo_initial     = stretch_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = stretch_break,
+    half_width_close = nobreak,
+    full_width_open  = stretch_break,
+    full_width_close = nobreak,
+    full_width_punct = nobreak,
+--  hyphen           = nil,
+    non_starter      = stretch_break,
+    other            = stretch_break,
+}
+
+local korean_3 = {
+    jamo_initial     = stretch_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = stretch_break,
+    half_width_close = nobreak,
+    full_width_open  = stretch_break,
+    full_width_close = nobreak,
+    full_width_punct = nobreak,
+--  hyphen           = nil,
+    non_starter      = nobreak,
+    other            = nobreak,
+}
+
+local korean_4 = {
+    jamo_initial     = nobreak,
+    korean           = nobreak,
+    chinese          = nobreak,
+    hiragana         = nobreak,
+    katakana         = nobreak,
+    half_width_open  = nobreak,
+    half_width_close = nobreak,
+    full_width_open  = nobreak,
+    full_width_close = nobreak,
+    full_width_punct = nobreak,
+    hyphen           = nobreak,
+    non_starter      = nobreak,
+    other            = nobreak,
+}
+
+local korean_5 = {
+    jamo_initial     = stretch_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = stretch_break,
+    half_width_close = nobreak_stretch,
+    full_width_open  = stretch_break,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+    hyphen           = nobreak_stretch,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
 
 local injectors = { -- [previous] [current]
-    jamo_final = {
-        jamo_initial     = korean_break,
-        korean           = korean_break,
-        chinese          = korean_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak,
-    --  hyphen           = nil,
-        non_starter      = korean_break,
-        other            = korean_break,
-    },
-    korean = {
-        jamo_initial     = korean_break,
-        korean           = korean_break,
-        chinese          = korean_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak,
-    --  hyphen           = nil,
-        non_starter      = korean_break,
-        other            = korean_break,
-    },
-    chinese = {
-        jamo_initial     = korean_break,
-        korean           = korean_break,
-        chinese          = korean_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak,
-    --  hyphen           = nil,
-        non_starter      = korean_break,
-        other            = korean_break,
-    },
-    hyphen = {
-        jamo_initial     = stretch_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak,
-    --  hyphen           = nil,
-        non_starter      = stretch_break,
-        other            = stretch_break,
-    },
-    start = {
-    --  jamo_initial     = nil,
-    --  korean           = nil,
-    --  chinese          = nil,
-    --  half_width_open  = nil,
-    --  half_width_close = nil,
-    --  full_width_open  = nil,
-    --  full_width_close = nil,
-    --  hyphen           = nil,
-    --  non_starter      = nil,
-    --  other            = nil,
-    },
-    other = {
-        jamo_initial     = stretch_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak,
-    --  hyphen           = nil,
-        non_starter      = stretch_break,
-        other            = stretch_break,
-    },
-    non_starter = {
-        jamo_initial     = stretch_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak,
-    --  hyphen           = nil,
-        non_starter      = nobreak,
-        other            = nobreak,
-    },
-    full_width_open = {
-        jamo_initial     = nobreak,
-        korean           = nobreak,
-        chinese          = nobreak,
-        half_width_open  = nobreak,
-        half_width_close = nobreak,
-        full_width_open  = nobreak,
-        full_width_close = nobreak,
-        hyphen           = nobreak,
-        non_starter      = nobreak,
-        other            = nobreak,
-    },
-    half_width_open = {
-        jamo_initial     = nobreak,
-        korean           = nobreak,
-        chinese          = nobreak,
-        half_width_open  = nobreak,
-        half_width_close = nobreak,
-        full_width_open  = nobreak,
-        full_width_close = nobreak,
-        hyphen           = nobreak,
-        non_starter      = nobreak,
-        other            = nobreak,
-    },
-    full_width_close = {
-        jamo_initial     = stretch_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak_stretch,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak_stretch,
-        hyphen           = nobreak_stretch,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
-    half_width_close = {
-        jamo_initial     = stretch_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = stretch_break,
-        half_width_close = nobreak_stretch,
-        full_width_open  = stretch_break,
-        full_width_close = nobreak_stretch,
-        hyphen           = nobreak_stretch,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
+    jamo_final       = korean_1,
+    korean           = korean_1,
+    chinese          = korean_1,
+    hiragana         = korean_1,
+    katakana         = korean_1,
+    hyphen           = korean_2,
+    start            = korean_0,
+    other            = korean_2,
+    non_starter      = korean_3,
+    full_width_open  = korean_4,
+    half_width_open  = korean_4,
+    full_width_close = korean_5,
+    full_width_punct = korean_5,
+    half_width_close = korean_5,
 }
 
 local function process(head,first,last)
@@ -377,150 +460,167 @@ end
 scripts.installmethod {
     name     = "hangul",
     process  = process,
-    datasets = {
+    datasets = { -- todo: metatables
         default = {
-            inter_char_stretch_factor     = 0.50, -- of quad
-            inter_char_half_shrink_factor = 0.50, -- of quad
-            inter_char_hangul_penalty     =   50,
+            inter_char_shrink_factor          = 0.50, -- of quad
+            inter_char_stretch_factor         = 0.50, -- of quad
+            inter_char_half_shrink_factor     = 0.50, -- of quad
+            inter_char_half_stretch_factor    = 0.50, -- of quad
+            inter_char_quarter_shrink_factor  = 0.50, -- of quad
+            inter_char_quarter_stretch_factor = 0.50, -- of quad
+            inter_char_hangul_penalty         =   50,
         },
     },
 }
 
--- hanzi (chinese)
+-- Chinese: hanzi
+
+local chinese_0 = {
+}
+
+local chinese_1 = {
+    jamo_initial     = korean_break,
+    korean           = korean_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+--  hyphen           = nil,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local chinese_2 = {
+    jamo_initial     = korean_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+    hyphen           = nobreak_stretch,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local chinese_3 = {
+    jamo_initial     = korean_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+--  hyphen           = nil,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local chinese_4 = {
+--  jamo_initial     = nil,
+--  korean           = nil,
+--  chinese          = nil,
+--  hiragana         = nil,
+--  katakana         = nil,
+    half_width_open  = nobreak_autoshrink,
+    half_width_close = nil,
+    full_width_open  = nobreak_shrink,
+    full_width_close = nobreak,
+    full_width_punct = nobreak,
+--  hyphen           = nil,
+    non_starter      = nobreak,
+--  other            = nil,
+}
+
+local chinese_5 = {
+    jamo_initial     = stretch_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+--  hyphen           = nil,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local chinese_6 = {
+    jamo_initial     = nobreak_stretch,
+    korean           = nobreak_stretch,
+    chinese          = nobreak_stretch,
+    hiragana         = nobreak_stretch,
+    katakana         = nobreak_stretch,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+    hyphen           = nobreak_stretch,
+    non_starter      = nobreak_stretch,
+    other            = nobreak_stretch,
+}
+
+local chinese_7 = {
+    jami_initial     = nobreak_shrink_break_stretch,
+    korean           = nobreak_shrink_break_stretch,
+    chinese          = stretch_break, -- nobreak_shrink_break_stretch,
+    hiragana         = stretch_break, -- nobreak_shrink_break_stretch,
+    katakana         = stretch_break, -- nobreak_shrink_break_stretch,
+    half_width_open  = nobreak_shrink_break_stretch_nobreak_autoshrink,
+    half_width_close = nobreak_shrink_nobreak_stretch,
+    full_width_open  = nobreak_shrink_break_stretch_nobreak_shrink,
+    full_width_close = nobreak_shrink_nobreak_stretch,
+    full_width_punct = nobreak_shrink_nobreak_stretch,
+    hyphen           = nobreak_shrink_break_stretch,
+    non_starter      = nobreak_shrink_break_stretch,
+    other            = nobreak_shrink_break_stretch,
+}
+
+local chinese_8 = {
+    jami_initial     = nobreak_shrink_break_stretch,
+    korean           = nobreak_autoshrink_break_stretch,
+    chinese          = stretch_break, -- nobreak_autoshrink_break_stretch,
+    hiragana         = stretch_break, -- nobreak_autoshrink_break_stretch,
+    katakana         = stretch_break, -- nobreak_autoshrink_break_stretch,
+    half_width_open  = nobreak_autoshrink_break_stretch_nobreak_autoshrink,
+    half_width_close = nobreak_autoshrink_nobreak_stretch,
+    full_width_open  = nobreak_autoshrink_break_stretch_nobreak_shrink,
+    full_width_close = nobreak_autoshrink_nobreak_stretch,
+    full_width_punct = nobreak_autoshrink_nobreak_stretch,
+    hyphen           = nobreak_autoshrink_break_stretch,
+    non_starter      = nobreak_autoshrink_break_stretch,
+    other            = nobreak_autoshrink_break_stretch,
+}
 
 local injectors = { -- [previous] [current]
-    jamo_final = {
-        jamo_initial     = korean_break,
-        korean           = korean_break,
-        chinese          = stretch_break,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_break_shrink,
-        full_width_close = nobreak_stretch,
-    --  hyphen           = nil,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
-    korean = {
-        jamo_initial     = korean_break,
-        korean           = korean_break,
-        chinese          = stretch_break,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_break_shrink,
-        full_width_close = nobreak_stretch,
-    --  hyphen           = nil,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
-    chinese = {
-        jamo_initial     = korean_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_break_shrink,
-        full_width_close = nobreak_stretch,
-        hyphen           = nobreak_stretch,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
-    hyphen = {
-        jamo_initial     = korean_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_break_shrink,
-        full_width_close = nobreak_stretch,
-    --  hyphen           = nil,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
-    start = {
-    --  jamo_initial     = nil,
-    --  korean           = nil,
-    --  chinese          = nil,
-        half_width_open  = nobreak_autoshrink,
-        half_width_close = nil,
-        full_width_open  = nobreak_shrink,
-        full_width_close = nobreak,
-    --  hyphen           = nil,
-        non_starter      = nobreak,
-    --  other            = nil,
-    },
-    other = {
-        jamo_initial     = stretch_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_break_shrink,
-        full_width_close = nobreak_stretch,
-    --  hyphen           = nil,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
-    non_starter = {
-        jamo_initial     = stretch_break,
-        korean           = stretch_break,
-        chinese          = stretch_break,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_break_shrink,
-        full_width_close = nobreak_stretch,
-    --  hyphen           = nil,
-        non_starter      = nobreak_stretch,
-        other            = stretch_break,
-    },
-    full_width_open = {
-        jamo_initial     = nobreak_stretch,
-        korean           = nobreak_stretch,
-        chinese          = nobreak_stretch,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_break_shrink,
-        full_width_close = nobreak_stretch,
-        hyphen           = nobreak_stretch,
-        non_starter      = nobreak_stretch,
-        other            = nobreak_stretch,
-    },
-    half_width_open = {
-        jamo_initial     = nobreak_stretch,
-        korean           = nobreak_stretch,
-        chinese          = nobreak_stretch,
-        half_width_open  = nobreak_stretch_break_autoshrink,
-        half_width_close = nobreak_stretch,
-        full_width_open  = nobreak_stretch_nobreak_shrink,
-        full_width_close = nobreak_stretch,
-        hyphen           = nobreak_stretch,
-        non_starter      = nobreak_stretch,
-        other            = nobreak_stretch,
-    },
-    full_width_close = {
-        jami_initial     = nobreak_shrink_break_stretch,
-        korean           = nobreak_shrink_break_stretch,
-        chinese          = stretch_break, -- nobreak_shrink_break_stretch,
-        half_width_open  = nobreak_shrink_break_stretch_nobreak_autoshrink,
-        half_width_close = nobreak_shrink_nobreak_stretch,
-        full_width_open  = nobreak_shrink_break_stretch_nobreak_shrink,
-        full_width_close = nobreak_shrink_nobreak_stretch,
-        hyphen           = nobreak_shrink_break_stretch,
-        non_starter      = nobreak_shrink_break_stretch,
-        other            = nobreak_shrink_break_stretch,
-    },
-    half_width_close = {
-        jami_initial     = nobreak_shrink_break_stretch,
-        korean           = nobreak_autoshrink_break_stretch,
-        chinese          = stretch_break, -- nobreak_autoshrink_break_stretch,
-        half_width_open  = nobreak_autoshrink_break_stretch_nobreak_autoshrink,
-        half_width_close = nobreak_autoshrink_nobreak_stretch,
-        full_width_open  = nobreak_autoshrink_break_stretch_nobreak_shrink,
-        full_width_close = nobreak_autoshrink_nobreak_stretch,
-        hyphen           = nobreak_autoshrink_break_stretch,
-        non_starter      = nobreak_autoshrink_break_stretch,
-        other            = nobreak_autoshrink_break_stretch,
-    },
+    jamo_final       = chinese_1,
+    korean           = chinese_1,
+    chinese          = chinese_2,
+    hiragana         = chinese_2,
+    katakana         = chinese_2,
+    hyphen           = chinese_3,
+    start            = chinese_4,
+    other            = chinese_5,
+    non_starter      = chinese_5,
+    full_width_open  = chinese_6,
+    half_width_open  = chinese_6,
+    full_width_close = chinese_7,
+    full_width_punct = chinese_7,
+    half_width_close = chinese_8,
 }
 
 local function process(head,first,last)
@@ -551,11 +651,11 @@ local function process(head,first,last)
                     if pid == glyph_code and nid == glyph_code then
                         local pa, na = has_attribute(p,a_prestat), has_attribute(n,a_prestat)
                         local pcjk, ncjk = pa and numbertocategory[pa], na and numbertocategory[na]
-                        if not pcjk                 or not ncjk
-                            or pcjk == "korean"     or ncjk == "korean"
-                            or pcjk == "other"      or ncjk == "other"
-                         -- or pcjk == "jamo_final" or ncjk == "jamo_initial" then
-                            or pcjk == "jamo_final" or ncjk == "jamo_initial" or pcjk == "half_width_close" or ncjk == "half_width_open" then
+                        if not pcjk                       or not ncjk
+                            or pcjk == "korean"           or ncjk == "korean"
+                            or pcjk == "other"            or ncjk == "other"
+                            or pcjk == "jamo_final"       or ncjk == "jamo_initial"
+                            or pcjk == "half_width_close" or ncjk == "half_width_open" then -- extra compared to korean
                             previous = "start"
                         else -- if head ~= first then
                             remove_node(head,first,true)
@@ -584,9 +684,236 @@ scripts.installmethod {
     process  = process,
     datasets = {
         default = {
-            inter_char_stretch_factor     = 0.50, -- of quad
-            inter_char_half_shrink_factor = 0.50, -- of quad
-            inter_char_hangul_penalty     =   50,
+            inter_char_shrink_factor          = 0.50, -- of quad
+            inter_char_stretch_factor         = 0.50, -- of quad
+            inter_char_half_shrink_factor     = 0.50, -- of quad
+            inter_char_half_stretch_factor    = 0.50, -- of quad
+            inter_char_quarter_shrink_factor  = 0.50, -- of quad
+            inter_char_quarter_stretch_factor = 0.50, -- of quad
+            inter_char_hangul_penalty         =   50,
+        },
+    },
+}
+
+-- Japanese: idiographic, hiragana, katakana, romanji / jis
+
+local japanese_0 = {
+}
+
+local japanese_1 = {
+    jamo_initial     = korean_break,
+    korean           = korean_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+--  hyphen           = nil,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local japanese_2 = {
+    jamo_initial     = korean_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = japanese_before_full_width_punct, -- nobreak_stretch,
+    hyphen           = nobreak_stretch,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local japanese_3 = {
+    jamo_initial     = korean_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+--  hyphen           = nil,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local japanese_4 = {
+--  jamo_initial     = nil,
+--  korean           = nil,
+--  chinese          = nil,
+--  hiragana         = nil,
+--  katakana         = nil,
+    half_width_open  = nobreak_autoshrink,
+    half_width_close = nil,
+    full_width_open  = nobreak_shrink,
+    full_width_close = nobreak,
+    full_width_punct = nobreak,
+--  hyphen           = nil,
+    non_starter      = nobreak,
+--  other            = nil,
+}
+
+local japanese_5 = {
+    jamo_initial     = stretch_break,
+    korean           = stretch_break,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+--  hyphen           = nil,
+    non_starter      = nobreak_stretch,
+    other            = stretch_break,
+}
+
+local japanese_6 = {
+    jamo_initial     = nobreak_stretch,
+    korean           = nobreak_stretch,
+    chinese          = nobreak_stretch,
+    hiragana         = nobreak_stretch,
+    katakana         = nobreak_stretch,
+    half_width_open  = nobreak_stretch_break_autoshrink,
+    half_width_close = nobreak_stretch,
+    full_width_open  = nobreak_stretch_break_shrink,
+    full_width_close = nobreak_stretch,
+    full_width_punct = nobreak_stretch,
+    hyphen           = nobreak_stretch,
+    non_starter      = nobreak_stretch,
+    other            = nobreak_stretch,
+}
+
+local japanese_7 = {
+    jami_initial     = nobreak_shrink_break_stretch,
+    korean           = nobreak_shrink_break_stretch,
+    chinese          = japanese_after_full_width_punct, -- stretch_break
+    hiragana         = japanese_after_full_width_punct, -- stretch_break
+    katakana         = japanese_after_full_width_punct, -- stretch_break
+    half_width_open  = nobreak_shrink_break_stretch_nobreak_autoshrink,
+    half_width_close = nobreak_shrink_nobreak_stretch,
+    full_width_open  = japanese_between_full_close_open, -- !!
+    full_width_close = japanese_between_full_close_full_close, -- nobreak_shrink_nobreak_stretch,
+    full_width_punct = nobreak_shrink_nobreak_stretch,
+    hyphen           = nobreak_shrink_break_stretch,
+    non_starter      = nobreak_shrink_break_stretch,
+    other            = nobreak_shrink_break_stretch,
+}
+
+local japanese_8 = {
+    jami_initial     = nobreak_shrink_break_stretch,
+    korean           = nobreak_autoshrink_break_stretch,
+    chinese          = stretch_break,
+    hiragana         = stretch_break,
+    katakana         = stretch_break,
+    half_width_open  = nobreak_autoshrink_break_stretch_nobreak_autoshrink,
+    half_width_close = nobreak_autoshrink_nobreak_stretch,
+    full_width_open  = nobreak_autoshrink_break_stretch_nobreak_shrink,
+    full_width_close = nobreak_autoshrink_nobreak_stretch,
+    full_width_punct = nobreak_autoshrink_nobreak_stretch,
+    hyphen           = nobreak_autoshrink_break_stretch,
+    non_starter      = nobreak_autoshrink_break_stretch,
+    other            = nobreak_autoshrink_break_stretch,
+}
+
+local injectors = { -- [previous] [current]
+    jamo_final       = japanese_1,
+    korean           = japanese_1,
+    chinese          = japanese_2,
+    hiragana         = japanese_2,
+    katakana         = japanese_2,
+    hyphen           = japanese_3,
+    start            = japanese_4,
+    other            = japanese_5,
+    non_starter      = japanese_5,
+    full_width_open  = japanese_6,
+    half_width_open  = japanese_6,
+    full_width_close = japanese_7,
+    full_width_punct = japanese_7,
+    half_width_close = japanese_8,
+}
+
+local function process(head,first,last)
+    if first ~= last then
+        local lastfont, previous, last = nil, "start", nil
+        while true do
+            local upcoming, id = first.next, first.id
+            if id == glyph_code then
+                local a = has_attribute(first,a_prestat)
+                local current = numbertocategory[a]
+                local action = injectors[previous]
+                if action then
+                    action = action[current]
+                    if action then
+                        local font = first.font
+                        if font ~= lastfont then
+                            lastfont = font
+                            set_parameters(font,numbertodataset[has_attribute(first,a_preproc)])
+                        end
+                        action(head,first)
+                    end
+                end
+                previous = current
+            else -- glue
+                local p, n = first.prev, upcoming
+                if p and n then
+                    local pid, nid = p.id, n.id
+                    if pid == glyph_code and nid == glyph_code then
+                        local pa, na = has_attribute(p,a_prestat), has_attribute(n,a_prestat)
+                        local pcjk, ncjk = pa and numbertocategory[pa], na and numbertocategory[na]
+                        if not pcjk                       or not ncjk
+                            or pcjk == "korean"           or ncjk == "korean"
+                            or pcjk == "other"            or ncjk == "other"
+                            or pcjk == "jamo_final"       or ncjk == "jamo_initial"
+                            or pcjk == "half_width_close" or ncjk == "half_width_open" then -- extra compared to korean
+                            previous = "start"
+                        else -- if head ~= first then
+                            remove_node(head,first,true)
+                            previous = pcjk
+                    --    else
+                    --        previous = pcjk
+                        end
+                    else
+                        previous = "start"
+                    end
+                else
+                    previous = "start"
+                end
+            end
+            if upcoming == last then -- was stop
+                break
+            else
+                first = upcoming
+            end
+        end
+    end
+end
+
+scripts.installmethod {
+    name     = "nihongo", -- what name to use?
+    process  = process,
+    datasets = {
+        default = {
+            inter_char_shrink_factor          = 0.50, -- of quad
+            inter_char_stretch_factor         = 0.50, -- of quad
+            inter_char_half_shrink_factor     = 0.50, -- of quad
+            inter_char_half_stretch_factor    = 0.50, -- of quad
+            inter_char_quarter_shrink_factor  = 0.25, -- of quad
+            inter_char_quarter_stretch_factor = 0.25, -- of quad
+            inter_char_hangul_penalty         =   50,
         },
     },
 }
+
diff --git a/tex/context/base/scrp-ini.lua b/tex/context/base/scrp-ini.lua
index 6e85a2c5e..e55dc0780 100644
--- a/tex/context/base/scrp-ini.lua
+++ b/tex/context/base/scrp-ini.lua
@@ -6,7 +6,8 @@ if not modules then modules = { } end modules ['scrp-ini'] = {
     license   = "see context related readme files"
 }
 
--- we need to rewrite this a bit ... rather old code
+-- We need to rewrite this a bit ... rather old code ... will be done when japanese
+-- is finished.
 
 local attributes, nodes, node = attributes, nodes, node
 
@@ -48,7 +49,7 @@ scripts.handlers      = handlers
 
 storage.register("scripts/hash", hash, "scripts.hash")
 
-if not next(hash) then
+if not next(hash) then -- this might move to char-def
 
     hash = { -- no local
         --
@@ -74,7 +75,6 @@ if not next(hash) then
         [0xFF08] = "full_width_open", -- （   left parenthesis
         [0xFF3B] = "full_width_open", -- ［   left square brackets
         [0xFF5B] = "full_width_open", -- ｛   left curve bracket
-        [0xFF62] = "full_width_open", --     left corner bracket
         --
         -- half width closing parenthesis
         [0x0029] = "half_width_close",
@@ -99,7 +99,9 @@ if not next(hash) then
         [0xFF09] = "full_width_close", -- ）   right parenthesis
         [0xFF3D] = "full_width_close", -- ］   right square brackets
         [0xFF5D] = "full_width_close", -- ｝   right curve brackets
-        [0xFF63] = "full_width_close", --     right corner bracket
+
+        [0xFF62] = "half_width_open", --     left corner bracket
+        [0xFF63] = "half_width_close", --     right corner bracket
         --
         -- vertical opening vertical
         --
@@ -131,13 +133,14 @@ if not next(hash) then
         -- full width closing punctuation
         [0x3001] = "full_width_close", -- 、
         [0x3002] = "full_width_close", -- 。
-        [0xFF01] = "full_width_close", -- ！
         [0xFF0C] = "full_width_close", -- ，
         [0xFF0E] = "full_width_close", -- ．
-        [0xFF1A] = "full_width_close", -- ：
-        [0xFF1B] = "full_width_close", -- ；
+        -- depends on font
+        [0xFF01] = "full_width_close", -- ！
         [0xFF1F] = "full_width_close", -- ？
         --
+        [0xFF1A] = "full_width_punct", -- ：
+        [0xFF1B] = "full_width_punct", -- ；
         -- non starter
         --
         [0x3005] = "non_starter", [0x3041] = "non_starter", [0x3043] = "non_starter", [0x3045] = "non_starter", [0x3047] = "non_starter",
@@ -157,13 +160,13 @@ if not next(hash) then
         [0x2014] = "hyphen", -- —   hyphen
     }
 
-    for i=0x03040,0x0309F do if not hash[i] then hash[i] = "chinese"      end end
-    for i=0x030A0,0x030FF do if not hash[i] then hash[i] = "chinese"      end end
-    for i=0x031F0,0x031FF do if not hash[i] then hash[i] = "chinese"      end end
+    for i=0x03040,0x030FF do if not hash[i] then hash[i] = "katakana"     end end -- had tag 'chinese'
+    for i=0x031F0,0x031FF do if not hash[i] then hash[i] = "katakana"     end end -- had tag 'chinese'
+    for i=0x032D0,0x032FE do if not hash[i] then hash[i] = "katakana"     end end -- had tag 'chinese'
     for i=0x03400,0x04DFF do if not hash[i] then hash[i] = "chinese"      end end
     for i=0x04E00,0x09FFF do if not hash[i] then hash[i] = "chinese"      end end
     for i=0x0F900,0x0FAFF do if not hash[i] then hash[i] = "chinese"      end end
-    for i=0x0FF00,0x0FFEF do if not hash[i] then hash[i] = "chinese"      end end
+    for i=0x0FF00,0x0FFEF do if not hash[i] then hash[i] = "katakana"     end end -- had tag 'chinese'
     for i=0x20000,0x2A6DF do if not hash[i] then hash[i] = "chinese"      end end
     for i=0x2F800,0x2FA1F do if not hash[i] then hash[i] = "chinese"      end end
     for i=0x0AC00,0x0D7A3 do if not hash[i] then hash[i] = "korean"       end end
@@ -171,11 +174,8 @@ if not next(hash) then
     for i=0x01160,0x011A7 do if not hash[i] then hash[i] = "jamo_medial"  end end
     for i=0x011A8,0x011FF do if not hash[i] then hash[i] = "jamo_final"   end end
 
---  for i=0x03041,0x030FF do if not hash[i] then hash[i] = "japanese"     end end
-
     for i=0x01200,0x0139F do hash[i] = "ethiopic_syllable" end
 
-
     hash[0x01361] = "ethiopic_word"
     hash[0x01362] = "ethiopic_sentence"
 
@@ -190,6 +190,18 @@ local numbertohandler = allocate()
 
 scripts.numbertodataset = numbertodataset
 
+local defaults = {
+    inter_char_shrink_factor          = 0,
+    inter_char_stretch_factor         = 0,
+    inter_char_half_shrink_factor     = 0,
+    inter_char_half_stretch_factor    = 0,
+    inter_char_quarter_shrink_factor  = 0,
+    inter_char_quarter_stretch_factor = 0,
+    inter_char_hangul_penalty         = 0,
+}
+
+scripts.defaults = defaults -- so we can add more
+
 function scripts.installmethod(handler)
     local name = handler.name
     handlers[name] = handler
@@ -197,6 +209,10 @@ function scripts.installmethod(handler)
     local datasets = handler.datasets
     if not datasets or not datasets.default then
         report_preprocessing("missing (default) dataset in script '%s'",name)
+        datasets.default = { } -- slower but an error anyway
+    end
+    for k, v in next, datasets do
+        table.setmetatableindex(v,defaults)
     end
     setmetatable(attributes, {
         __index = function(t,k)
@@ -268,14 +284,19 @@ function scripts.reset()
 end
 
 -- the following tables will become a proper installer (move to cjk/eth)
+--
+-- 0=gray 1=red 2=green 3=blue 4=yellow 5=magenta 6=cyan 7=x-yellow 8=x-magenta 9=x-cyan
 
 local scriptcolors = allocate {  -- todo: just named colors
     korean            = "trace:0",
     chinese           = "trace:0",
+    katakana          = "trace:0",
+    hiragana          = "trace:0",
     full_width_open   = "trace:1",
     full_width_close  = "trace:2",
     half_width_open   = "trace:3",
     half_width_close  = "trace:4",
+    full_width_punct  = "trace:5",
     hyphen            = "trace:5",
     non_starter       = "trace:6",
     jamo_initial      = "trace:7",
@@ -291,10 +312,13 @@ scripts.colors = scriptcolors
 local numbertocategory = allocate { -- rather bound to cjk ... will be generalized
     "korean",
     "chinese",
+    "katakana",
+    "hiragana",
     "full_width_open",
     "full_width_close",
     "half_width_open",
     "half_width_close",
+    "full_width_punct",
     "hyphen",
     "non_starter",
     "jamo_initial",
diff --git a/tex/context/base/scrp-ini.mkiv b/tex/context/base/scrp-ini.mkiv
index b9112a61f..8182368d7 100644
--- a/tex/context/base/scrp-ini.mkiv
+++ b/tex/context/base/scrp-ini.mkiv
@@ -58,6 +58,7 @@
 
 \definescript[hangul]  [\c!method=hangul]
 \definescript[hanzi]   [\c!method=hanzi]
+\definescript[nihongo] [\c!method=nihongo]
 
 \definescript[ethiopic][\c!method=ethiopic]
 
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 4ffaf6620..cb42c66a9 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 88e501874..05e2b6410 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf