beta 2013.10.08 21:49

12 files changed, 63 insertions, 36 deletions

diff --git a/context/data/scite/lexers/scite-context-lexer-lua.lua b/context/data/scite/lexers/scite-context-lexer-lua.lua
index b0297cf83..59cdd64f7 100644
--- a/context/data/scite/lexers/scite-context-lexer-lua.lua
+++ b/context/data/scite/lexers/scite-context-lexer-lua.lua
@@ -156,6 +156,9 @@ local number        = token("number", lexer.float + integer)
 
 local validword     = R("AZ","az","__") * R("AZ","az","__","09")^0
 
+local utf8character = P(1) * R("\128\191")^1 -- unchecked but fast
+local validword     = (R("AZ","az","__") + utf8character) * (R("AZ","az","__","09") + utf8character)^0
+
 local identifier    = token("default",validword)
 
 ----- operator      = token("special", P('..') + P('~=') + S('+-*/%^#=<>;:,.{}[]()')) -- maybe split off {}[]()
diff --git a/context/data/scite/lexers/scite-context-lexer-tex.lua b/context/data/scite/lexers/scite-context-lexer-tex.lua
index 34c88b16b..a509fadab 100644
--- a/context/data/scite/lexers/scite-context-lexer-tex.lua
+++ b/context/data/scite/lexers/scite-context-lexer-tex.lua
@@ -399,9 +399,11 @@ local inlinelua              = P("\\") * (
                                )
 
 local startlua               = P("\\start") * Cmt(luaenvironment,startdisplaylua)
+                             + P("<?lua") * Cmt(P(true),startdisplaylua)
                              + inlinelua * space^0 * ( Cmt(P("{"),startinlinelua) )
 
 local stoplua                = P("\\stop") * Cmt(luaenvironment,stopdisplaylua)
+                             + P("?>") * Cmt(P(true),stopdisplaylua)
                              + Cmt(P("{"),stopinlinelua_b)
                              + Cmt(P("}"),stopinlinelua_e)
 
diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index 7548650e6..a94aa6b5f 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -167,7 +167,7 @@ local blocks = allocate {
     ["cjkradicalssupplement"]                      = { first = 0x02E80, last = 0x02EFF, otf="hang", description = "CJK Radicals Supplement" },
     ["cjkstrokes"]                                 = { first = 0x031C0, last = 0x031EF, otf="hang", description = "CJK Strokes" },
     ["cjksymbolsandpunctuation"]                   = { first = 0x03000, last = 0x0303F, otf="hang", description = "CJK Symbols and Punctuation" },
-    ["cjkunifiedideographs"]                       = { first = 0x04E00, last = 0x09FFF, otf="hang", description = "CJK Unified Ideographs" },
+    ["cjkunifiedideographs"]                       = { first = 0x04E00, last = 0x09FFF, otf="hang", description = "CJK Unified Ideographs", catcode = "letter" },
     ["cjkunifiedideographsextensiona"]             = { first = 0x03400, last = 0x04DBF, otf="hang", description = "CJK Unified Ideographs Extension A" },
     ["cjkunifiedideographsextensionb"]             = { first = 0x20000, last = 0x2A6DF, otf="hang", description = "CJK Unified Ideographs Extension B" },
     ["combiningdiacriticalmarks"]                  = { first = 0x00300, last = 0x0036F,             description = "Combining Diacritical Marks" },
@@ -1065,13 +1065,20 @@ function characters.define(tobelettered, tobeactivated) -- catcodetables
                 end
                 local range = chr.range
                 if range then
-                    for i=1,range.first,range.last do
+                    for i=1,range.first,range.last do -- tricky as not all are letters
                         texsetcatcode(i,11)
                     end
                 end
             end
             texsetcatcode(0x200C,11) -- non-joiner
             texsetcatcode(0x200D,11) -- joiner
+            for k, v in next, blocks do
+                if v.catcode == "letter" then
+                    for i=v.first,v.last do
+                        texsetcatcode(i,11)
+                    end
+                end
+            end
         end
         tex.catcodetable = saved
     end
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index ee4a28692..8a6ea178b 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.10.07 23:31}
+\newcontextversion{2013.10.08 21:49}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index abfbcd64e..df2daae04 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index cb46ecc14..fb6dadb07 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.10.07 23:31}
+\edef\contextversion{2013.10.08 21:49}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/lang-ini.mkiv b/tex/context/base/lang-ini.mkiv
index a6ba21f39..17d00033b 100644
--- a/tex/context/base/lang-ini.mkiv
+++ b/tex/context/base/lang-ini.mkiv
@@ -374,15 +374,24 @@
    \fi
    \lang_basics_synchronize_min_max}
 
-\unexpanded\def\nohyphens % % % % % not clever, we still hyphenate but supress application
+% \unexpanded\def\nohyphens % % % % % not clever, we still hyphenate but supress application
+%   {\ifx\dohyphens\relax
+%      \unexpanded\edef\dohyphens
+%        {\hyphenpenalty  \the\hyphenpenalty
+%         \exhyphenpenalty\the\exhyphenpenalty
+%         \relax}%
+%    \fi
+%    \hyphenpenalty  \plustenthousand
+%    \exhyphenpenalty\plustenthousand}
+
+\unexpanded\def\nohyphens % nicer for url's
   {\ifx\dohyphens\relax
      \unexpanded\edef\dohyphens
-       {\hyphenpenalty  \the\hyphenpenalty
-        \exhyphenpenalty\the\exhyphenpenalty
-        \relax}%
+       {\hyphenminoffset\the\hyphenminoffset\relax
+        \lang_basics_synchronize_min_max}%
    \fi
-   \hyphenpenalty  \plustenthousand
-   \exhyphenpenalty\plustenthousand}
+   \hyphenminoffset\plusthousand
+   \lang_basics_synchronize_min_max}
 
 \let\dohyphens\relax
 
diff --git a/tex/context/base/luat-mac.lua b/tex/context/base/luat-mac.lua
index c8be06b63..282dc8ce3 100644
--- a/tex/context/base/luat-mac.lua
+++ b/tex/context/base/luat-mac.lua
@@ -76,21 +76,23 @@ local function pop()
     top = remove(stack)
 end
 
-local leftbrace   = P("{")   -- will be in patterns
-local rightbrace  = P("}")
-local escape      = P("\\")
+local leftbrace      = P("{")   -- will be in patterns
+local rightbrace     = P("}")
+local escape         = P("\\")
 
-local space       = patterns.space
-local spaces      = space^1
-local newline     = patterns.newline
-local nobrace     = 1 - leftbrace - rightbrace
+local space          = patterns.space
+local spaces         = space^1
+local newline        = patterns.newline
+local nobrace        = 1 - leftbrace - rightbrace
 
 local longleft       = leftbrace  -- P("(")
 local longright      = rightbrace -- P(")")
 local nolong         = 1 - longleft - longright
 
-local name           = R("AZ","az")^1
-local csname         = (R("AZ","az") + S("@?!_"))^1
+local utf8character  = P(1) * R("\128\191")^1 -- unchecked but fast
+
+local name           = (R("AZ","az") + utf8character)^1
+local csname         = (R("AZ","az") + S("@?!_") + utf8character)^1
 local longname       = (longleft/"") * (nolong^1) * (longright/"")
 local variable       = P("#") * Cs(name + longname)
 local escapedname    = escape * csname
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 89e383feb..b3d3fa61c 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 6ead455d8..c2a791633 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/strc-reg.lua b/tex/context/base/strc-reg.lua
index 7e7736e2b..b0d8a8a25 100644
--- a/tex/context/base/strc-reg.lua
+++ b/tex/context/base/strc-reg.lua
@@ -664,13 +664,10 @@ local function collapsepages(pages)
     return #pages
 end
 
--- todo: determine max
-
 function registers.flush(data,options,prefixspec,pagespec)
     local collapse_singles = options.compress == variables.yes
     local collapse_ranges  = options.compress == variables.all
     local result = data.result
-    local done = { } -- reused
     local maxlevel = 0
     --
     for i=1,#result do
@@ -688,15 +685,17 @@ function registers.flush(data,options,prefixspec,pagespec)
     end
     --
     context.startregisteroutput()
+local done = { }
     for i=1,#result do
      -- ranges need checking !
         local sublist = result[i]
+     -- local done = { false, false, false, false }
+for i=1,maxlevel do
+    done[i] = false
+end
         local data = sublist.data
         local d, n = 0, 0
         context.startregistersection(sublist.tag)
-        for i=1,maxlevel do
-            done[i] = false
-        end
         for d=1,#data do
             local entry = data[d]
             if entry.metadata.kind == "see" then
@@ -709,26 +708,28 @@ function registers.flush(data,options,prefixspec,pagespec)
                 end
             end
         end
-        local e = { } -- reused
+        -- ok, this is tricky: we use e[i] delayed so we need it to be local
+        -- but we don't want to allocate too many entries so there we go
         while d < #data do
             d = d + 1
             local entry = data[d]
+            local e = { false, false, false }
+for i=3,maxlevel do
+    e[i] = false
+end
             local metadata = entry.metadata
             local kind = metadata.kind
             local list = entry.list
             for i=1,maxlevel do
-                e[i] = false
-            end
-            for i=1,maxlevel do
                 if list[i] then
                     e[i] = list[i][1]
                 end
                 if e[i] ~= done[i] then
                     if e[i] and e[i] ~= "" then
                         done[i] = e[i]
-                        for j=i+1,maxlevel do
-                            done[j] = false
-                        end
+for j=i+1,maxlevel do
+    done[j] = false
+end
                         if n == i then
                             context.stopregisterentries()
                             context.startregisterentries(n)
@@ -745,6 +746,8 @@ function registers.flush(data,options,prefixspec,pagespec)
                         local internal  = entry.references.internal or 0
                         local seeparent = entry.references.seeparent or ""
                         local processor = entry.processors and entry.processors[1] or ""
+                        -- so, we need to keep e as is (local), or we need local title = e[i] ... which might be
+                        -- more of a problem
                         if metadata then
                             context.registerentry(processor,internal,seeparent,function() helpers.title(e[i],metadata) end)
                         else -- ?
@@ -752,9 +755,9 @@ function registers.flush(data,options,prefixspec,pagespec)
                         end
                     else
                         done[i] = false
-                        for j=i,maxlevel do
-                            done[j] = false
-                        end
+for j=i+1,maxlevel do
+    done[j] = false
+end
                     end
                 end
             end
@@ -885,6 +888,7 @@ function registers.flush(data,options,prefixspec,pagespec)
     data.metadata.sorted = false
 end
 
+
 function registers.analyze(class,options)
     context(registers.analyzed(class,options))
 end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 4720633d1..211093502 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 10/07/13 23:31:19
+-- merge date  : 10/08/13 21:49:42
 
 do -- begin closure to overcome local limits and interference