summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--scripts/context/lua/mtxrun.lua28
-rw-r--r--scripts/context/stubs/mswin/mtxrun.lua28
-rw-r--r--scripts/context/stubs/unix/mtxrun28
-rw-r--r--tex/context/base/cont-new.mkiv2
-rw-r--r--tex/context/base/context-version.pdfbin4110 -> 4105 bytes
-rw-r--r--tex/context/base/context.mkiv2
-rw-r--r--tex/context/base/l-lpeg.lua10
-rw-r--r--tex/context/base/l-unicode.lua12
-rw-r--r--tex/context/base/meta-ini.mkiv6
-rw-r--r--tex/context/base/node-fin.mkiv8
-rw-r--r--tex/context/base/node-shp.lua76
-rw-r--r--tex/context/base/page-imp.mkiv2
-rw-r--r--tex/context/base/status-files.pdfbin24725 -> 24755 bytes
-rw-r--r--tex/context/base/status-lua.log2
-rw-r--r--tex/generic/context/luatex/luatex-fonts-merged.lua12
15 files changed, 150 insertions, 66 deletions
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 2a647d043..a5894b8d4 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -415,7 +415,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true
--- original size: 28391, stripped down to: 15460
+-- original size: 28471, stripped down to: 15475
if not modules then modules={} end modules ['l-lpeg']={
version=1.001,
@@ -457,11 +457,11 @@ local dquote=P('"')
local space=P(" ")
local period=P(".")
local comma=P(",")
-local utfbom_32_be=P('\000\000\254\255')
-local utfbom_32_le=P('\255\254\000\000')
-local utfbom_16_be=P('\255\254')
-local utfbom_16_le=P('\254\255')
-local utfbom_8=P('\239\187\191')
+local utfbom_32_be=P('\000\000\254\255')
+local utfbom_32_le=P('\255\254\000\000')
+local utfbom_16_be=P('\254\255')
+local utfbom_16_le=P('\255\254')
+local utfbom_8=P('\239\187\191')
local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8
local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8")
local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0)
@@ -4056,7 +4056,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["l-unicode"] = package.loaded["l-unicode"] or true
--- original size: 26810, stripped down to: 11943
+-- original size: 26964, stripped down to: 12091
if not modules then modules={} end modules ['l-unicode']={
version=1.001,
@@ -4472,11 +4472,17 @@ local function big(c)
end
local _,l_remap=utf.remapper(little)
local _,b_remap=utf.remapper(big)
+function utf.utf8_to_utf16_be(str)
+ return char(254,255)..lpegmatch(b_remap,str)
+end
+function utf.utf8_to_utf16_le(str)
+ return char(255,254)..lpegmatch(l_remap,str)
+end
function utf.utf8_to_utf16(str,littleendian)
if littleendian then
- return char(255,254)..lpegmatch(l_remap,str)
+ return utf.utf8_to_utf16_le(str)
else
- return char(254,255)..lpegmatch(b_remap,str)
+ return utf.utf8_to_utf16_be(str)
end
end
local pattern=Cs (
@@ -16360,8 +16366,8 @@ end -- of closure
-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
-- skipped libraries : -
--- original bytes : 668227
--- stripped bytes : 234912
+-- original bytes : 668461
+-- stripped bytes : 234983
-- end library merge
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 2a647d043..a5894b8d4 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -415,7 +415,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true
--- original size: 28391, stripped down to: 15460
+-- original size: 28471, stripped down to: 15475
if not modules then modules={} end modules ['l-lpeg']={
version=1.001,
@@ -457,11 +457,11 @@ local dquote=P('"')
local space=P(" ")
local period=P(".")
local comma=P(",")
-local utfbom_32_be=P('\000\000\254\255')
-local utfbom_32_le=P('\255\254\000\000')
-local utfbom_16_be=P('\255\254')
-local utfbom_16_le=P('\254\255')
-local utfbom_8=P('\239\187\191')
+local utfbom_32_be=P('\000\000\254\255')
+local utfbom_32_le=P('\255\254\000\000')
+local utfbom_16_be=P('\254\255')
+local utfbom_16_le=P('\255\254')
+local utfbom_8=P('\239\187\191')
local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8
local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8")
local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0)
@@ -4056,7 +4056,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["l-unicode"] = package.loaded["l-unicode"] or true
--- original size: 26810, stripped down to: 11943
+-- original size: 26964, stripped down to: 12091
if not modules then modules={} end modules ['l-unicode']={
version=1.001,
@@ -4472,11 +4472,17 @@ local function big(c)
end
local _,l_remap=utf.remapper(little)
local _,b_remap=utf.remapper(big)
+function utf.utf8_to_utf16_be(str)
+ return char(254,255)..lpegmatch(b_remap,str)
+end
+function utf.utf8_to_utf16_le(str)
+ return char(255,254)..lpegmatch(l_remap,str)
+end
function utf.utf8_to_utf16(str,littleendian)
if littleendian then
- return char(255,254)..lpegmatch(l_remap,str)
+ return utf.utf8_to_utf16_le(str)
else
- return char(254,255)..lpegmatch(b_remap,str)
+ return utf.utf8_to_utf16_be(str)
end
end
local pattern=Cs (
@@ -16360,8 +16366,8 @@ end -- of closure
-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
-- skipped libraries : -
--- original bytes : 668227
--- stripped bytes : 234912
+-- original bytes : 668461
+-- stripped bytes : 234983
-- end library merge
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 2a647d043..a5894b8d4 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -415,7 +415,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true
--- original size: 28391, stripped down to: 15460
+-- original size: 28471, stripped down to: 15475
if not modules then modules={} end modules ['l-lpeg']={
version=1.001,
@@ -457,11 +457,11 @@ local dquote=P('"')
local space=P(" ")
local period=P(".")
local comma=P(",")
-local utfbom_32_be=P('\000\000\254\255')
-local utfbom_32_le=P('\255\254\000\000')
-local utfbom_16_be=P('\255\254')
-local utfbom_16_le=P('\254\255')
-local utfbom_8=P('\239\187\191')
+local utfbom_32_be=P('\000\000\254\255')
+local utfbom_32_le=P('\255\254\000\000')
+local utfbom_16_be=P('\254\255')
+local utfbom_16_le=P('\255\254')
+local utfbom_8=P('\239\187\191')
local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8
local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8")
local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0)
@@ -4056,7 +4056,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["l-unicode"] = package.loaded["l-unicode"] or true
--- original size: 26810, stripped down to: 11943
+-- original size: 26964, stripped down to: 12091
if not modules then modules={} end modules ['l-unicode']={
version=1.001,
@@ -4472,11 +4472,17 @@ local function big(c)
end
local _,l_remap=utf.remapper(little)
local _,b_remap=utf.remapper(big)
+function utf.utf8_to_utf16_be(str)
+ return char(254,255)..lpegmatch(b_remap,str)
+end
+function utf.utf8_to_utf16_le(str)
+ return char(255,254)..lpegmatch(l_remap,str)
+end
function utf.utf8_to_utf16(str,littleendian)
if littleendian then
- return char(255,254)..lpegmatch(l_remap,str)
+ return utf.utf8_to_utf16_le(str)
else
- return char(254,255)..lpegmatch(b_remap,str)
+ return utf.utf8_to_utf16_be(str)
end
end
local pattern=Cs (
@@ -16360,8 +16366,8 @@ end -- of closure
-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
-- skipped libraries : -
--- original bytes : 668227
--- stripped bytes : 234912
+-- original bytes : 668461
+-- stripped bytes : 234983
-- end library merge
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 2d8f76d93..a33c1c93f 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.
-\newcontextversion{2013.08.28 23:08}
+\newcontextversion{2013.08.29 20:45}
%D This file is loaded at runtime, thereby providing an excellent place for
%D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 465b5c002..b61f77a54 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
Binary files differ
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 4a3fd8877..e7b7d92e0 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
%D up and the dependencies are more consistent.
\edef\contextformat {\jobname}
-\edef\contextversion{2013.08.28 23:08}
+\edef\contextversion{2013.08.29 20:45}
\edef\contextkind {beta}
%D For those who want to use this:
diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua
index b33df96b7..78b503b8e 100644
--- a/tex/context/base/l-lpeg.lua
+++ b/tex/context/base/l-lpeg.lua
@@ -109,11 +109,11 @@ local space = P(" ")
local period = P(".")
local comma = P(",")
-local utfbom_32_be = P('\000\000\254\255')
-local utfbom_32_le = P('\255\254\000\000')
-local utfbom_16_be = P('\255\254')
-local utfbom_16_le = P('\254\255')
-local utfbom_8 = P('\239\187\191')
+local utfbom_32_be = P('\000\000\254\255') -- 00 00 FE FF
+local utfbom_32_le = P('\255\254\000\000') -- FF FE 00 00
+local utfbom_16_be = P('\254\255') -- FE FF
+local utfbom_16_le = P('\255\254') -- FF FE
+local utfbom_8 = P('\239\187\191') -- EF BB BF
local utfbom = utfbom_32_be + utfbom_32_le
+ utfbom_16_be + utfbom_16_le
+ utfbom_8
diff --git a/tex/context/base/l-unicode.lua b/tex/context/base/l-unicode.lua
index 813ffd54b..cf0c93d4f 100644
--- a/tex/context/base/l-unicode.lua
+++ b/tex/context/base/l-unicode.lua
@@ -777,11 +777,19 @@ end
local _, l_remap = utf.remapper(little)
local _, b_remap = utf.remapper(big)
+function utf.utf8_to_utf16_be(str)
+ return char(254,255) .. lpegmatch(b_remap,str)
+end
+
+function utf.utf8_to_utf16_le(str)
+ return char(255,254) .. lpegmatch(l_remap,str)
+end
+
function utf.utf8_to_utf16(str,littleendian)
if littleendian then
- return char(255,254) .. lpegmatch(l_remap,str)
+ return utf.utf8_to_utf16_le(str)
else
- return char(254,255) .. lpegmatch(b_remap,str)
+ return utf.utf8_to_utf16_be(str)
end
end
diff --git a/tex/context/base/meta-ini.mkiv b/tex/context/base/meta-ini.mkiv
index 16ecb4ca1..e3e694448 100644
--- a/tex/context/base/meta-ini.mkiv
+++ b/tex/context/base/meta-ini.mkiv
@@ -436,9 +436,9 @@
% watch out, this is a type1 font because mp can only handle 8 bit fonts
-\startMPinitializations % scale is not yet ok
- defaultfont:="rm-lmtt10";
-\stopMPinitializations
+% \startMPinitializations % scale is not yet ok
+% defaultfont:="rm-lmtt10";
+% \stopMPinitializations
%D A signal that we're in combined \CONTEXT||\METAFUN mode:
diff --git a/tex/context/base/node-fin.mkiv b/tex/context/base/node-fin.mkiv
index 09bac6c08..2eb033fc1 100644
--- a/tex/context/base/node-fin.mkiv
+++ b/tex/context/base/node-fin.mkiv
@@ -23,8 +23,12 @@
% we might have two variants at some point (efficiency)
-\def\finalizeobjectbox #1{\ctxcommand{finalizebox(\number#1)}}
-\def\finalizeshipoutbox#1{\ctxcommand{finalizebox(\number#1)}}
+\unexpanded\def\finalizeobjectbox #1{\ctxcommand{finalizebox(\number#1)}}
+\unexpanded\def\finalizeshipoutbox#1{\ctxcommand{finalizebox(\number#1)}}
+
+% Experimental (for Aditya):
+
+\unexpanded\def\cleanupbox#1{\ctxcommand{cleanupbox(\number#1)}}
% Tricky stuff: this might become obsolete.
diff --git a/tex/context/base/node-shp.lua b/tex/context/base/node-shp.lua
index 32024e5da..6ebfd767f 100644
--- a/tex/context/base/node-shp.lua
+++ b/tex/context/base/node-shp.lua
@@ -14,6 +14,7 @@ local concat, sortedpairs = table.concat, table.sortedpairs
local setmetatableindex = table.setmetatableindex
local nodecodes = nodes.nodecodes
+local whatsitcodes = nodes.whatsitcodes
local tasks = nodes.tasks
local handlers = nodes.handlers
@@ -23,6 +24,7 @@ local disc_code = nodecodes.disc
local mark_code = nodecodes.mark
local kern_code = nodecodes.kern
local glue_code = nodecodes.glue
+local whatsit_code = nodecodes.whatsit
local texgetbox = tex.getbox
@@ -30,21 +32,44 @@ local free_node = node.free
local remove_node = node.remove
local traverse_nodes = node.traverse
-local function cleanup(head) -- rough
+local removables = {
+ [whatsitcodes.open] = true,
+ [whatsitcodes.close] = true,
+ [whatsitcodes.write] = true,
+ [whatsitcodes.pdfdest] = true,
+ [whatsitcodes.pdfsavepos] = true,
+ [whatsitcodes.latelua] = true,
+}
+
+local function cleanup_redundant(head)
local start = head
while start do
local id = start.id
- if id == disc_code or (id == glue_code and not start.writable) or (id == kern_code and start.kern == 0) or id == mark_code then
- head, start, tmp = remove_node(head,start)
- free_node(tmp)
+ if id == disc_code then
+ head, start = remove_node(head,start,true)
+ -- elseif id == glue_code then
+ -- if start.writable then
+ -- start = start.next
+ -- elseif some_complex_check_on_glue_spec then
+ -- head, start = remove_node(head,start,true)
+ -- else
+ -- start = start.next
+ -- end
+ elseif id == kern_code then
+ if start.kern == 0 then
+ head, start = remove_node(head,start,true)
+ else
+ start = start.next
+ end
+ elseif id == mark_code then
+ head, start = remove_node(head,start,true)
elseif id == hlist_code or id == vlist_code then
local sl = start.list
if sl then
- start.list = cleanup(sl)
+ start.list = cleanup_redundant(sl)
start = start.next
else
- head, start, tmp = remove_node(head,start)
- free_node(tmp)
+ head, start = remove_node(head,start,true)
end
else
start = start.next
@@ -53,21 +78,50 @@ local function cleanup(head) -- rough
return head
end
-directives.register("backend.cleanup", function()
- tasks.enableaction("shipouts","nodes.handlers.cleanuppage")
-end)
+local function cleanup_flushed(head) -- rough
+ local start = head
+ while start do
+ local id = start.id
+ if id == whatsit_code and removables[start.subtype] then
+ head, start = remove_node(head,start,true)
+ elseif id == hlist_code or id == vlist_code then
+ local sl = start.list
+ if sl then
+ start.list = cleanup_flushed(sl)
+ start = start.next
+ else
+ head, start = remove_node(head,start,true)
+ end
+ else
+ start = start.next
+ end
+ end
+ return head
+end
function handlers.cleanuppage(head)
-- about 10% of the nodes make no sense for the backend
- return cleanup(head), true
+ return cleanup_redundant(head), true
end
+function handlers.cleanupbox(head)
+ return cleanup_flushed(head), true
+end
+
+directives.register("backend.cleanup", function()
+ tasks.enableaction("shipouts","nodes.handlers.cleanuppage")
+end)
+
local actions = tasks.actions("shipouts") -- no extra arguments
function handlers.finalize(head) -- problem, attr loaded before node, todo ...
return actions(head)
end
+function commands.cleanupbox(n)
+ cleanup_flushed(texgetbox(n))
+end
+
-- handlers.finalize = actions
-- interface
diff --git a/tex/context/base/page-imp.mkiv b/tex/context/base/page-imp.mkiv
index c22e9e646..cfa535ab2 100644
--- a/tex/context/base/page-imp.mkiv
+++ b/tex/context/base/page-imp.mkiv
@@ -293,7 +293,7 @@
\fi
\fi}
-\def\installpagearrangement #1 % will lchange, no space
+\def\installpagearrangement #1 % will change, no space
{\setgvalue{\??layoutarranger#1}}
\def\checkinstalledpagearrangement#1% can be empty: aaa,,bbb
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index a984f1c6b..88393860d 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
Binary files differ
diff --git a/tex/context/base/status-lua.log b/tex/context/base/status-lua.log
index ae1d43e6c..2afe02313 100644
--- a/tex/context/base/status-lua.log
+++ b/tex/context/base/status-lua.log
@@ -1,6 +1,6 @@
(cont-yes.mkiv
-ConTeXt ver: 2013.08.28 23:08 MKIV beta fmt: 2013.8.28 int: english/english
+ConTeXt ver: 2013.08.29 20:45 MKIV beta fmt: 2013.8.29 int: english/english
system > 'cont-new.mkiv' loaded
(cont-new.mkiv)
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index ff4758068..88b45182e 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
-- merged file : luatex-fonts-merged.lua
-- parent file : luatex-fonts.lua
--- merge date : 08/28/13 23:08:58
+-- merge date : 08/29/13 20:45:59
do -- begin closure to overcome local limits and interference
@@ -127,11 +127,11 @@ local dquote=P('"')
local space=P(" ")
local period=P(".")
local comma=P(",")
-local utfbom_32_be=P('\000\000\254\255')
-local utfbom_32_le=P('\255\254\000\000')
-local utfbom_16_be=P('\255\254')
-local utfbom_16_le=P('\254\255')
-local utfbom_8=P('\239\187\191')
+local utfbom_32_be=P('\000\000\254\255')
+local utfbom_32_le=P('\255\254\000\000')
+local utfbom_16_be=P('\254\255')
+local utfbom_16_le=P('\255\254')
+local utfbom_8=P('\239\187\191')
local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8
local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8")
local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0)