stable 2006.04.22 10:41

10 files changed, 375 insertions, 132 deletions

diff --git a/scripts/context/ruby/base/ctx.rb b/scripts/context/ruby/base/ctx.rb
index b1fda3c3a..6c8dd5a44 100644
--- a/scripts/context/ruby/base/ctx.rb
+++ b/scripts/context/ruby/base/ctx.rb
@@ -52,17 +52,43 @@ class CtxRunner
         end
 
         if not @ctxname then
-            report('provide ctx file')
+            report('no ctx file specified')
             return
         end
 
-        if not FileTest.file?(@ctxname) and defaultname and FileTest.file?(defaultname) then
-            @ctxname = defaultname
-        end
+        # name can be kpse:res-make.ctx
 
         if not FileTest.file?(@ctxname) then
-            report('provide ctx file')
-            return
+            fullname, done = '', false
+            if @ctxname =~ /^kpse:/ then
+                begin
+                    if fullname = Kpse.found(@ctxname.sub(/^kpse:/,'')) then
+                        @ctxname, done = fullname, true
+                    end
+                rescue
+                    # should not happen
+                end
+            else
+                ['..','../..'].each do |path|
+                    begin
+                        fullname = File.join(path,@ctxname)
+                        if FileTest.file?(fullname) then
+                            @ctxname, done = fullname, true
+                        end
+                    rescue
+                        # probably strange join
+                    end
+                    break if done
+                end
+            end
+            if ! done && defaultname && FileTest.file?(defaultname) then
+                report("using default ctxfile #{defaultname}")
+                @ctxname = defaultname
+            end
+            if not done then
+                report('no ctx file found')
+                return false
+            end
         end
 
         @xmldata = IO.read(@ctxname)
@@ -196,17 +222,27 @@ class CtxRunner
                         name = e.attributes.get_attribute(attribute).to_s
                         name = e.text.to_s if name.empty?
                         name.strip! if name
-                        if name and not name.empty? and FileTest.file?(name) then
-                            if f = File.open(name,'r') and i = REXML::Document.new(f) then
-                                report("including ctx file #{name}")
-                                REXML::XPath.each(i.root,"*") do |ii|
-                                    xmldata.root.insert_after(e,ii)
-                                    more = true
+                        done = false
+                        if name and not name.empty? then
+                            ['.',File.dirname(@ctxname),'..','../..'].each do |path|
+                                begin
+                                    fullname = if path == '.' then name else File.join(path,name) end
+                                    if FileTest.file?(fullname) then
+                                        if f = File.open(fullname,'r') and i = REXML::Document.new(f) then
+                                            report("including ctx file #{name}")
+                                            REXML::XPath.each(i.root,"*") do |ii|
+                                                xmldata.root.insert_after(e,ii)
+                                                more = true
+                                            end
+                                        end
+                                        done = true
+                                    end
+                                rescue
                                 end
                             end
-                        else
-                            report("no valid ctx inclusion file #{name}")
+                            break if done
                         end
+                        report("no valid ctx inclusion file #{name}") unless done
                     rescue Exception
                         # skip this file
                     ensure
diff --git a/scripts/context/ruby/base/kpse.rb b/scripts/context/ruby/base/kpse.rb
index d05600b8f..0e3292d00 100644
--- a/scripts/context/ruby/base/kpse.rb
+++ b/scripts/context/ruby/base/kpse.rb
@@ -18,6 +18,18 @@ require 'rbconfig'
 #
 # miktex has mem|fmt|base paths
 
+class String
+
+    def split_path
+        self.split(/\:\;/)
+    end
+
+    def join_path
+        self.join(FILE::PATH_SEPARATOR)
+    end
+
+end
+
 module Kpse
 
     @@located       = Hash.new
@@ -138,7 +150,7 @@ module Kpse
         # maybe we should check for writeability
         unless @@paths.key?('formatpaths') then
             begin
-                setpath('formatpaths',run("--show-path=fmt").gsub(/\\/,'/').split(File::PATH_SEPARATOR))
+                setpath('formatpaths',run("--show-path=fmt").gsub(/\\/,'/').split_path)
             rescue
                 setpath('formatpaths',[])
             end
@@ -195,7 +207,7 @@ module Kpse
                 end
                 # locate writable path
                 if ! formatpath.empty? then
-                    formatpath.split(File::PATH_SEPARATOR).each do |fp|
+                    formatpath.split_path.each do |fp|
                         fp.gsub!(/\\/,'/')
                         # remove funny patterns
                         fp.sub!(/^!!/,'')
diff --git a/scripts/context/ruby/base/kpsedirect.rb b/scripts/context/ruby/base/kpsedirect.rb
index d38e892b1..6fa8c8601 100644
--- a/scripts/context/ruby/base/kpsedirect.rb
+++ b/scripts/context/ruby/base/kpsedirect.rb
@@ -3,21 +3,19 @@ class KpseDirect
     attr_accessor :progname, :format, :engine
 
     def initialize
-        @progname = ''
-        @format = ''
-        @engine = ''
+        @progname, @format, @engine = '', '', ''
     end
 
     def expand_path(str)
-        `kpsewhich -expand-path=#{str}`.chomp
+        clean_name(`kpsewhich -expand-path=#{str}`.chomp)
     end
 
     def expand_var(str)
-        `kpsewhich -expand-var=#{str}`.chomp
+        clean_name(`kpsewhich -expand-var=#{str}`.chomp)
     end
 
     def find_file(str)
-        `kpsewhich #{_progname_} #{_format_} #{str}`.chomp
+        clean_name(`kpsewhich #{_progname_} #{_format_} #{str}`.chomp)
     end
 
     def _progname_
@@ -27,4 +25,10 @@ class KpseDirect
         if @format.empty?   then '' else "-format=\"#{@format}\"" end
     end
 
+    private
+
+    def clean_name(str)
+        str.gsub(/\\/,'/')
+    end
+
 end
diff --git a/scripts/context/ruby/base/kpsefast.rb b/scripts/context/ruby/base/kpsefast.rb
index 231c6221c..dbc9f55e8 100644
--- a/scripts/context/ruby/base/kpsefast.rb
+++ b/scripts/context/ruby/base/kpsefast.rb
@@ -50,7 +50,7 @@ module KpseUtil
         end
         filenames = Array.new
         if ENV['TEXMFCNF'] and not ENV['TEXMFCNF'].empty? then
-            ENV['TEXMFCNF'].split(File::PATH_SEPARATOR).each do |path|
+            ENV['TEXMFCNF'].split_path.each do |path|
                 filenames << File.join(path,@@texmfcnf)
             end
         elsif ENV['SELFAUTOPARENT'] == '.' then
@@ -79,6 +79,18 @@ module KpseUtil
 
 end
 
+class String
+
+    def split_path
+        self.split(/\:\;/)
+    end
+
+    def join_path
+        self.join(FILE::PATH_SEPARATOR)
+    end
+
+end
+
 class KpseFast
 
     # formats are an incredible inconsistent mess
@@ -281,7 +293,7 @@ class KpseFast
             end
             filenames = Array.new
             if @environment['TEXMFCNF'] and not @environment['TEXMFCNF'].empty? then
-                @environment['TEXMFCNF'].split(File::PATH_SEPARATOR).each do |path|
+                @environment['TEXMFCNF'].split_path.each do |path|
                     filenames << File.join(path,@@texmfcnf)
                 end
             elsif @environment['SELFAUTOPARENT'] == '.' then
@@ -600,7 +612,7 @@ class KpseFast
             puts ""
             puts @variables ["KPSE_TEST_PATTERN_#{i}"]
             puts ""
-            puts expand_path("KPSE_TEST_PATTERN_#{i}").split(File::PATH_SEPARATOR)
+            puts expand_path("KPSE_TEST_PATTERN_#{i}").split_path
             puts ""
         end
     end
@@ -612,11 +624,11 @@ class KpseFast
     # kpse stuff
 
     def expand_braces(str) # output variable and brace expansion of STRING.
-        _expanded_path_(original_variable(str).split(";")).join(File::PATH_SEPARATOR)
+        _expanded_path_(original_variable(str).split_path).join_path
     end
 
     def expand_path(str)   # output complete path expansion of STRING.
-        _expanded_path_(expanded_variable(str).split(";")).join(File::PATH_SEPARATOR)
+        _expanded_path_(expanded_variable(str).split_path).join_path
     end
 
     def expand_var(str)    # output variable expansion of STRING.
@@ -624,8 +636,7 @@ class KpseFast
     end
 
     def show_path(str)     # output search path for file type NAME
-        # expanded_path(var_of_format(str)).join(File::PATH_SEPARATOR)
-        expanded_path(str).join(File::PATH_SEPARATOR)
+        expanded_path(str).join_path
     end
 
     def var_value(str)     # output the value of variable $STRING.
diff --git a/scripts/context/ruby/base/kpseremote.rb b/scripts/context/ruby/base/kpseremote.rb
index f5a72b712..f1f3cb10f 100644
--- a/scripts/context/ruby/base/kpseremote.rb
+++ b/scripts/context/ruby/base/kpseremote.rb
@@ -80,25 +80,32 @@ class KpseRemote
         @kpse.expand_variables(@tree)
     end
     def expand_braces(str)
-        @kpse.expand_braces(@tree,str)
+        clean_name(@kpse.expand_braces(@tree,str))
     end
     def expand_path(str)
-        @kpse.expand_path(@tree,str)
+        clean_name(@kpse.expand_path(@tree,str))
     end
     def expand_var(str)
-        @kpse.expand_var(@tree,str)
+        clean_name(@kpse.expand_var(@tree,str))
     end
     def show_path(str)
-        @kpse.show_path(@tree,str)
+        clean_name(@kpse.show_path(@tree,str))
     end
     def var_value(str)
-        @kpse.var_value(@tree,str)
+        clean_name(@kpse.var_value(@tree,str))
     end
     def find_file(filename)
-        @kpse.find_file(@tree,filename)
+        clean_name(@kpse.find_file(@tree,filename))
     end
     def find_files(filename,first=false)
+        # dodo: each filename
         @kpse.find_files(@tree,filename,first)
     end
 
+    private
+
+    def clean_name(str)
+        str.gsub(/\\/,'/')
+    end
+
 end
diff --git a/scripts/context/ruby/base/switch.rb b/scripts/context/ruby/base/switch.rb
index 160b50aba..3113ad4d6 100644
--- a/scripts/context/ruby/base/switch.rb
+++ b/scripts/context/ruby/base/switch.rb
@@ -33,13 +33,26 @@ end
 
 class File
 
+    # def File.needsupdate(oldname,newname)
+        # begin
+            # if $mswindows then # we cannot use != due to some rounding error
+                # return File.stat(oldname).mtime > File.stat(newname).mtime
+            # else
+                # return File.stat(oldname).mtime != File.stat(newname).mtime
+            # end
+        # rescue
+            # return true
+        # end
+    # end
+
+    @@update_eps = 1
+
     def File.needsupdate(oldname,newname)
         begin
-            if $mswindows then
-                return File.stat(oldname).mtime > File.stat(newname).mtime
-            else
-                return File.stat(oldname).mtime != File.stat(newname).mtime
-            end
+            oldtime = File.stat(oldname).mtime.to_i
+            newtime = File.stat(newname).mtime.to_i
+            delta = newtime - oldtime
+            (delta  > @@update_eps) || (-delta > @@update_eps)
         rescue
             return true
         end
diff --git a/scripts/context/ruby/base/tex.rb b/scripts/context/ruby/base/tex.rb
index 0d1e01b95..5e9a9566f 100644
--- a/scripts/context/ruby/base/tex.rb
+++ b/scripts/context/ruby/base/tex.rb
@@ -781,32 +781,33 @@ class TEX
         if tmp = openedfile(File.suffixed(rawname,'run')) then
             tmp << "\\starttext\n"
             if forcexml then
-                if FileTest.file?(rawname) && (xml = File.open(rawname)) then
-                    xml.each do |line|
-                        case line
-                            when /<\?context\-directive\s+(\S+)\s+(\S+)\s+(\S+)\s*(.*?)\s*\?>/o then
-                                category, key, value, rest = $1, $2, $3, $4
-                                case category
-                                    when 'job' then
-                                        case key
-                                            when 'control' then
-                                                setvariable(value,if rest.empty? then true else rest end)
-                                            when 'mode', 'modes' then
-                                                tmp << "\\enablemode[#{value}]\n"
-                                            when 'stylefile', 'environment' then
-                                                tmp << "\\environment #{value}\n"
-                                            when 'module' then
-                                                tmp << "\\usemodule[#{value}]\n"
-                                            when 'interface' then
-                                                contextinterface = value
-                                        end
-                                end
-                            when /<[a-z]+/io then # beware of order, first pi test
-                                break
-                        end
-                    end
-                    xml.close
-                end
+                # if FileTest.file?(rawname) && (xml = File.open(rawname)) then
+                    # xml.each do |line|
+                        # case line
+                            # when /<\?context\-directive\s+(\S+)\s+(\S+)\s+(\S+)\s*(.*?)\s*\?>/o then
+                                # category, key, value, rest = $1, $2, $3, $4
+                                # case category
+                                    # when 'job' then
+                                        # case key
+                                            # when 'control' then
+                                                # setvariable(value,if rest.empty? then true else rest end)
+                                            # when 'mode', 'modes' then
+                                                # tmp << "\\enablemode[#{value}]\n"
+                                            # when 'stylefile', 'environment' then
+                                                # tmp << "\\environment #{value}\n"
+                                            # when 'module' then
+                                                # tmp << "\\usemodule[#{value}]\n"
+                                            # when 'interface' then
+                                                # contextinterface = value
+                                        # end
+                                # end
+                            # when /<[a-z]+/io then # beware of order, first pi test
+                                # break
+                        # end
+                    # end
+                    # xml.close
+                # end
+                tmp << checkxmlfile(rawname)
                 tmp << "\\processXMLfilegrouped{#{rawname}}\n"
             else
                 tmp << "\\processfile{#{rawname}}\n"
@@ -819,6 +820,40 @@ class TEX
         end
     end
 
+    def checkxmlfile(rawname)
+        tmp = ''
+        if FileTest.file?(rawname) && (xml = File.open(rawname)) then
+            xml.each do |line|
+                case line
+                    when /<\?context\-directive\s+(\S+)\s+(\S+)\s+(\S+)\s*(.*?)\s*\?>/o then
+                        category, key, value, rest = $1, $2, $3, $4
+                        case category
+                            when 'job' then
+                                case key
+                                    when 'control' then
+                                        setvariable(value,if rest.empty? then true else rest end)
+                                    when 'mode', 'modes' then
+                                        tmp << "\\enablemode[#{value}]\n"
+                                    when 'stylefile', 'environment' then
+                                        tmp << "\\environment #{value}\n"
+                                    when 'module' then
+                                        tmp << "\\usemodule[#{value}]\n"
+                                    when 'interface' then
+                                        contextinterface = value
+                                    when 'ctxfile' then
+                                        setvariable('ctxfile', value)
+                                        report("using source driven ctxfile #{value}")
+                                end
+                        end
+                    when /<[a-z]+/io then # beware of order, first pi test
+                        break
+                end
+            end
+            xml.close
+        end
+        return tmp
+    end
+
 end
 
 class TEX
@@ -1154,7 +1189,7 @@ checktestversion
                 begin
                     logger = Logger.new('TeXUtil')
                     if tu = TeXUtil::Converter.new(logger) and tu.loaded(fname) then
-                        tu.saved if tu.processed
+                        ok = tu.processed && tu.saved && tu.finalized
                     end
                 rescue
                     Kpse.runscript('texutil',fname,options)
@@ -1289,6 +1324,8 @@ checktestversion
             end
         end
 
+        jobsuffix = makestubfile(rawname,forcexml) if dummyfile || forcexml
+
         # preprocess files
 
         unless getvariable('noctx') then
@@ -1321,8 +1358,6 @@ checktestversion
 
         # end of preprocessing and merging
 
-        jobsuffix = makestubfile(rawname,forcexml) if dummyfile || forcexml
-
         if globalfile || FileTest.file?(rawname) then
 
             if not dummyfile and not globalfile then
@@ -1435,6 +1470,8 @@ checktestversion
                 PDFview.open(File.suffixed(if result.empty? then rawname else result end,'pdf'))
             end
 
+        else
+            report("nothing to process")
         end
 
     end
diff --git a/scripts/context/ruby/base/texutil.rb b/scripts/context/ruby/base/texutil.rb
index 583aa6bbf..86de0ee91 100644
--- a/scripts/context/ruby/base/texutil.rb
+++ b/scripts/context/ruby/base/texutil.rb
@@ -122,6 +122,16 @@ class TeXUtil
             end
         end
 
+        def finalizers
+            @plugins.each do |p|
+                begin
+                    eval("#{p}").finalizer(@logger)
+                rescue Exception
+                    @logger.report("fatal error in plugin finalizer #{p} (#{$!})")
+                end
+            end
+        end
+
     end
 
     class Sorter
@@ -342,6 +352,9 @@ end
                 end
             end
 
+            def MyFiles::finalizer(logger)
+            end
+
         end
 
     end
@@ -370,6 +383,9 @@ end
             def MyCommands::processor(logger)
             end
 
+            def MyCommands::finalizer(logger)
+            end
+
         end
 
     end
@@ -400,6 +416,14 @@ end
 
             def MyExtras::processor(logger)
                 @@programs.each do |p|
+                    # cmd = @@programs[p.to_i]
+                    # logger.report("running #{cmd}")
+                    # system(cmd)
+                end
+            end
+
+            def MyExtras::finalizer(logger)
+                @@programs.each do |p|
                     cmd = @@programs[p.to_i]
                     logger.report("running #{cmd}")
                     system(cmd)
@@ -486,6 +510,9 @@ end
                 end
             end
 
+            def MySynonyms::finalizer(logger)
+            end
+
         end
 
     end
@@ -742,6 +769,9 @@ end
                 end
             end
 
+            def MyRegisters::finalizer(logger)
+            end
+
         end
 
     end
@@ -779,6 +809,10 @@ end
                 @@plugins.processors if @@plugins
             end
 
+            def MyPlugins::finalizer(logger)
+                @@plugins.finalizers if @@plugins
+            end
+
         end
 
     end
@@ -832,6 +866,9 @@ end
                 logger.report("divisions : #{@@divisions.size}")  # logger.report(@@divisions.inspect)
             end
 
+            def MyKeys::finalizer(logger)
+            end
+
         end
 
     end
@@ -898,19 +935,27 @@ end
         def saved(filename=@filename)
             if @fatalerror then
                 report("fatal error, no tuo file saved")
+                return false
             else
-               begin
+                begin
                     if f = File.open(File.suffixed(filename,'tuo'),'w') then
                         @plugins.writers(f)
                         f.close
                     end
                 rescue
                     report("fatal error when saving file (#{$!})")
+                    return false
                 else
                     report("tuo file saved")
+                    return true
                 end
             end
+        end
+
+        def finalized
+            @plugins.finalizers
             @plugins.resets
+            return true # for the moment
         end
 
         def reset
diff --git a/scripts/context/ruby/texmfstart.rb b/scripts/context/ruby/texmfstart.rb
index 72c90ad8f..676b4351c 100644
--- a/scripts/context/ruby/texmfstart.rb
+++ b/scripts/context/ruby/texmfstart.rb
@@ -321,10 +321,10 @@ def expanded(arg) # no "other text files", too restricted
         method, original, resolved = $1, $2, ''
         if $program && ! $program.empty? then
             # pstrings = ["-progname=#{$program}"]
-pstrings = [$program]
+            pstrings = [$program]
         else
             # pstrings = ['','-progname=context']
-pstrings = ['','context']
+            pstrings = ['','context']
         end
         # auto suffix with texinputs as fall back
         if ENV["_CTX_K_V_#{original}_"] then
@@ -342,7 +342,7 @@ pstrings = ['','context']
                         # resolved = `#{command}`.chomp
                         $kpse.progname = pstr
                         $kpse.format = ''
-                        resolved = $kpse.find_file(original)
+                        resolved = $kpse.find_file(original).gsub(/\\/,'/')
                     rescue
                         resolved = ''
                     end
@@ -356,7 +356,7 @@ pstrings = ['','context']
                         # resolved = `#{command}`.chomp
                         $kpse.progname = pstr
                         $kpse.format = 'other text files'
-                        resolved = $kpse.find_file(original)
+                        resolved = $kpse.find_file(original).gsub(/\\/,'/')
                     rescue
                         resolved = ''
                     end
@@ -599,7 +599,7 @@ def find(filename,program)
                     report("using 'kpsewhich' to locate '#{filename}' in suffix space '#{suffix}' (1)")
                     # fullname = `kpsewhich -progname=#{program} -format=texmfscripts #{filename}.#{suffix}`.chomp
                     $kpse.format = 'texmfscripts'
-                    fullname = $kpse.find_file("#{filename}.#{suffix}")
+                    fullname = $kpse.find_file("#{filename}.#{suffix}").gsub(/\\/,'/')
                 rescue
                     report("kpsewhich cannot locate '#{filename}' in suffix space '#{suffix}' (1)")
                     fullname = nil
@@ -612,7 +612,7 @@ def find(filename,program)
                 report("using 'kpsewhich' to locate '#{filename}' in suffix space '#{suffix}' (2)")
                 # fullname = `kpsewhich -progname=#{program} -format="other text files" #{filename}.#{suffix}`.chomp
                 $kpse.format = 'other text files'
-                fullname = $kpse.find_file("#{filename}.#{suffix}")
+                fullname = $kpse.find_file("#{filename}.#{suffix}").gsub(/\\/,'/')
             rescue
                 report("kpsewhich cannot locate '#{filename}' in suffix space '#{suffix}' (2)")
                 fullname = nil
diff --git a/scripts/context/ruby/xmltools.rb b/scripts/context/ruby/xmltools.rb
index a57f74450..97b572e19 100644
--- a/scripts/context/ruby/xmltools.rb
+++ b/scripts/context/ruby/xmltools.rb
@@ -10,13 +10,16 @@
 # info      : j.hagen@xs4all.nl
 # www       : www.pragma-ade.com
 
+# todo : use kpse lib
+
 # This script will harbor some handy manipulations on tex
 # related files.
 
-banner = ['XMLTools', 'version 1.1.1', '2002/2005', 'PRAGMA ADE/POD']
+banner = ['XMLTools', 'version 1.2.0', '2002/2006', 'PRAGMA ADE/POD']
 
 unless defined? ownpath
     ownpath = $0.sub(/[\\\/][a-z0-9\-]*?\.rb/i,'')
+    # ownpath = File.dirname($0)
     $: << ownpath
 end
 
@@ -235,77 +238,150 @@ class Commands
 
     def analyze
 
-        file   = @commandline.argument('first')
-        result = @commandline.option('output')
+        file    = @commandline.argument('first')
+        result  = @commandline.option('output')
+        utf     = @commandline.option('utf')
+        process = @commandline.option('process')
 
         if FileTest.file?(file) then
             if data = IO.read(file) then
-                report("xml file #{file} loaded")
-                elements   = Hash.new
-                attributes = Hash.new
-                entities   = Hash.new
-                data.scan(/<([^>\s\/\!\?]+)([^>]*?)>/o) do
-                    element, attributelist = $1, $2
-                    if elements.key?(element) then
-                        elements[element] += 1
-                    else
-                        elements[element] = 1
-                    end
-                    attributelist.scan(/\s*([^\=]+)\=([\"\'])(.*?)(\2)/) do
-                        key, value = $1, $3
-                        attributes[element] = Hash.new unless attributes.key?(element)
-                        attributes[element][key] = Hash.new unless attributes[element].key?(key)
-                        if attributes[element][key].key?(value) then
-                            attributes[element][key][value] += 1
+                if data =~ /<?xml.*?version\=/ then
+                    report("xml file #{file} loaded")
+                    elements   = Hash.new
+                    attributes = Hash.new
+                    entities   = Hash.new
+                    chars      = Hash.new
+                    unicodes   = Hash.new
+                    names      = Hash.new
+                    data.scan(/<([^>\s\/\!\?]+)([^>]*?)>/o) do
+                        element, attributelist = $1, $2
+                        if elements.key?(element) then
+                            elements[element] += 1
                         else
-                            attributes[element][key][value] = 1
+                            elements[element] = 1
+                        end
+                        attributelist.scan(/\s*([^\=]+)\=([\"\'])(.*?)(\2)/) do
+                            key, value = $1, $3
+                            attributes[element] = Hash.new unless attributes.key?(element)
+                            attributes[element][key] = Hash.new unless attributes[element].key?(key)
+                            if attributes[element][key].key?(value) then
+                                attributes[element][key][value] += 1
+                            else
+                                attributes[element][key][value] = 1
+                            end
                         end
                     end
-                end
-                data.scan(/\&([^\;]+)\;/o) do
-                    entity = $1
-                    if entities.key?(entity) then
-                        entities[entity] += 1
-                    else
-                        entities[entity] = 1
+                    data.scan(/\&([^\;]+)\;/o) do
+                        entity = $1
+                        if entities.key?(entity) then
+                            entities[entity] += 1
+                        else
+                            entities[entity] = 1
+                        end
                     end
-                end
-                result = file.gsub(/\..*?$/, '') + '.xlg' if result.empty?
-                if f = File.open(result,'w') then
-                    report("saving report in #{result}")
-                    f.puts "<?xml version='1.0'?>\n"
-                    f.puts "<document>\n"
-                    if entities.length>0 then
-                        f.puts "  <entities>\n"
-                        entities.keys.asort.each do |entity|
-                            f.puts "    <entity name=#{entity.xstring} n=#{entities[entity].to_s.xstring}/>\n"
+                    if utf then
+                        data.scan(/(\w)/u) do
+                            chars[$1] = (chars[$1] || 0) + 1
+                        end
+                        if chars.size > 0 then
+                            begin
+                                # todo : use kpse lib
+                                filename, ownpath, foundpath = 'contextnames.txt', File.dirname($0), ''
+                                begin
+                                    foundpath = File.dirname(`kpsewhich -progname=context -format=\"other text files\" #{filename}`.chomp)
+                                rescue
+                                    foundpath = '.'
+                                else
+                                    foundpath = '.' if foundpath.empty?
+                                end
+                                [foundpath,ownpath,File.join(ownpath,'../../../context/data')].each do |path|
+                                    fullname = File.join(path,filename)
+                                    if FileTest.file?(fullname) then
+                                        report("loading '#{fullname}'")
+                                        # rough scan, we assume no valid lines after comments
+                                        IO.read(fullname).scan(/^([0-9A-F][0-9A-F][0-9A-F][0-9A-F])\s*\;\s*(.*?)\s*\;\s*(.*?)\s*\;\s*(.*?)\s*$/) do
+                                            names[$1.hex.to_i.to_s] = [$2,$3,$4]
+                                        end
+                                        break
+                                    end
+                                end
+                            rescue
+                            end
                         end
-                        f.puts "  </entities>\n"
                     end
-                    if elements.length>0 then
-                        f.puts "  <elements>\n"
-                        elements.keys.sort.each do |element|
-                            if attributes.key?(element) then
-                                f.puts "    <element name=#{element.xstring} n=#{elements[element].to_s.xstring}>\n"
+                    result = file.gsub(/\..*?$/, '') + '.xlg' if result.empty?
+                    if f = File.open(result,'w') then
+                        report("saving report in #{result}")
+                        f.puts "<?xml version='1.0'?>\n"
+                        f.puts "<document>\n"
+                        if entities.length>0 then
+                            total = 0
+                            entities.each do |k,v|
+                                total += v
+                            end
+                            f.puts "  <entities n=#{total.to_s.xstring}>\n"
+                            entities.keys.asort.each do |entity|
+                                f.puts "    <entity name=#{entity.xstring} n=#{entities[entity].to_s.xstring}/>\n"
+                            end
+                            f.puts "  </entities>\n"
+                        end
+                        if utf && (chars.size > 0) then
+                            total = 0
+                            chars.each do |k,v|
+                                total += v
+                            end
+                            f.puts "  <characters n=#{total.to_s.xstring}>\n"
+                            chars.each do |k,v|
+                                if k.length > 1 then
+                                    begin
+                                        u = k.unpack('U')
+                                        unicodes[u] = (unicodes[u] || 0) + v
+                                    rescue
+                                        report("invalid utf codes")
+                                    end
+                                end
+                            end
+                            unicodes.keys.sort.each do |u|
+                                ustr = u.to_s
+                                if names[ustr] then
+                                    f.puts "    <character number=#{ustr.xstring} pname=#{names[ustr][0].xstring} cname=#{names[ustr][1].xstring} uname=#{names[ustr][2].xstring} n=#{unicodes[u].to_s.xstring}/>\n"
+                                else
+                                    f.puts "    <character number=#{ustr.xstring} n=#{unicodes[u].to_s.xstring}/>\n"
+                                end
+                            end
+                            f.puts "  </characters>\n"
+                        end
+                        if elements.length>0 then
+                            f.puts "  <elements>\n"
+                            elements.keys.sort.each do |element|
                                 if attributes.key?(element) then
-                                    attributes[element].keys.asort.each do |attribute|
-                                        f.puts "      <attribute name=#{attribute.xstring}>\n"
-                                        attributes[element][attribute].keys.asort.each do |value|
-                                            f.puts "        <instance value=#{value.xstring} n=#{attributes[element][attribute][value].to_s.xstring}/>\n"
+                                    f.puts "    <element name=#{element.xstring} n=#{elements[element].to_s.xstring}>\n"
+                                    if attributes.key?(element) then
+                                        attributes[element].keys.asort.each do |attribute|
+                                            f.puts "      <attribute name=#{attribute.xstring}>\n"
+                                            attributes[element][attribute].keys.asort.each do |value|
+                                                f.puts "        <instance value=#{value.xstring} n=#{attributes[element][attribute][value].to_s.xstring}/>\n"
+                                            end
+                                            f.puts "      </attribute>\n"
                                         end
-                                        f.puts "      </attribute>\n"
                                     end
+                                    f.puts "    </element>\n"
+                                else
+                                    f.puts "    <element name=#{element.xstring} n=#{elements[element].to_s.xstring}/>\n"
                                 end
-                                f.puts "    </element>\n"
-                            else
-                                f.puts "    <element name=#{element.xstring} n=#{elements[element].to_s.xstring}/>\n"
                             end
+                            f.puts "  </elements>\n"
+                        end
+                        f.puts "</document>\n"
+                        f.close
+                        if process then
+                            system("texmfstart texexec --purgeall --pdf --use=xml-analyze #{result}")
                         end
-                        f.puts "  </elements>\n"
+                    else
+                        report("unable to open file '#{result}'")
                     end
-                    f.puts "</document>\n"
                 else
-                    report("unable to open file '#{result}'")
+                    report("invalid xml file '#{file}'")
                 end
             else
                 report("unable to load file '#{file}'")
@@ -322,7 +398,7 @@ commandline = CommandLine.new
 
 commandline.registeraction('dir',     'generate directory listing')
 commandline.registeraction('mmlpages','generate graphic from mathml')
-commandline.registeraction('analyze', 'report entities and elements')
+commandline.registeraction('analyze', 'report entities and elements [--utf --process]')
 
 # commandline.registeraction('dir',     'filename --pattern= --output= [--recurse --stripname --longname --url --root]')
 # commandline.registeraction('mmlpages','filename [--eps --jpg --png --style= --mode=]')
@@ -344,6 +420,8 @@ commandline.registervalue('root')
 commandline.registerflag('eps')
 commandline.registerflag('png')
 commandline.registerflag('jpg')
+commandline.registerflag('utf')
+commandline.registerflag('process')
 commandline.registervalue('style')
 commandline.registervalue('modes')