diff options
author | Marius <mariausol@gmail.com> | 2013-05-19 20:40:34 +0300 |
---|---|---|
committer | Marius <mariausol@gmail.com> | 2013-05-19 20:40:34 +0300 |
commit | 13ec4b540e0d46c97fd7b089e0b7413da81e0a9f (patch) | |
tree | bebfa563a17c06b3bd3bf8f6f4ba6d025e00d107 /tex/context/base/bibl-bib.lua | |
parent | 69ad13650cda027526271179e95b5294694143a1 (diff) | |
download | context-13ec4b540e0d46c97fd7b089e0b7413da81e0a9f.tar.gz |
beta 2013.05.19 19:27
Diffstat (limited to 'tex/context/base/bibl-bib.lua')
-rw-r--r-- | tex/context/base/bibl-bib.lua | 1532 |
1 files changed, 766 insertions, 766 deletions
diff --git a/tex/context/base/bibl-bib.lua b/tex/context/base/bibl-bib.lua index c86a0c0c2..ab38a0b28 100644 --- a/tex/context/base/bibl-bib.lua +++ b/tex/context/base/bibl-bib.lua @@ -1,766 +1,766 @@ -if not modules then modules = { } end modules ['bibl-bib'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -<p>This is a prelude to integrated bibliography support. This file just loads -bibtex files and converts them to xml so that the we access the content -in a convenient way. Actually handling the data takes place elsewhere.</p> ---ldx]]-- - -local lower, format, gsub, concat = string.lower, string.format, string.gsub, table.concat -local next = next -local utfchar = utf.char -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local textoutf = characters and characters.tex.toutf -local variables = interfaces and interfaces.variables -local settings_to_hash = utilities.parsers.settings_to_hash -local finalizers = xml.finalizers.tex -local xmlfilter, xmltext, getid = xml.filter, xml.text, lxml.getid -local formatters = string.formatters - -local P, R, S, C, Cc, Cs, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct - -local trace_bibxml = false trackers.register("publications.bibxml", function(v) trace_bibtex = v end) - -local report_xml = logs.reporter("publications","xml") - -bibtex = bibtex or { } -local bibtex = bibtex - -bibtex.statistics = bibtex.statistics or { } -local bibtexstats = bibtex.statistics - -bibtexstats.nofbytes = 0 -bibtexstats.nofdefinitions = 0 -bibtexstats.nofshortcuts = 0 - -local defaultshortcuts = { - jan = "1", - feb = "2", - mar = "3", - apr = "4", - may = "5", - jun = "6", - jul = "7", - aug = "8", - sep = "9", - oct = "10", - nov = "11", - dec = "12", -} - -local shortcuts = { } -local data = { } -local entries - --- Currently we expand shortcuts and for large ones (like the acknowledgements --- in tugboat.bib this is not that efficient. However, eventually strings get --- hashed again. - -local function do_shortcut(tag,key,value) - bibtexstats.nofshortcuts = bibtexstats.nofshortcuts + 1 - if lower(tag) == "@string" then - shortcuts[key] = value - end -end - -local function do_definition(tag,key,tab) -- maybe check entries here (saves memory) - if not entries or entries[key] then - bibtexstats.nofdefinitions = bibtexstats.nofdefinitions + 1 - local t = { } - for i=1,#tab,2 do - t[tab[i]] = tab[i+1] - end - local p = data[tag] - if not p then - data[tag] = { [key] = t } - else - p[key] = t - end - end -end - -local function resolve(s) - return shortcuts[s] or defaultshortcuts[s] or s -- can be number -end - -local percent = P("%") -local start = P("@") -local comma = P(",") -local hash = P("#") -local escape = P("\\") -local single = P("'") -local double = P('"') -local left = P('{') -local right = P('}') -local both = left + right -local lineending = S("\n\r") -local space = S(" \t\n\r\f") -local spacing = space^0 -local equal = P("=") -local collapsed = (space^1)/ " " - -local function add(a,b) if b then return a..b else return a end end - -local keyword = C((R("az","AZ","09") + S("@_:-"))^1) -- C((1-space)^1) -local s_quoted = ((escape*single) + collapsed + (1-single))^0 -local d_quoted = ((escape*double) + collapsed + (1-double))^0 -local balanced = lpegpatterns.balanced - -local s_value = (single/"") * s_quoted * (single/"") -local d_value = (double/"") * d_quoted * (double/"") -local b_value = (left /"") * balanced * (right /"") -local r_value = keyword/resolve - -local somevalue = s_value + d_value + b_value + r_value -local value = Cs((somevalue * ((spacing * hash * spacing)/"" * somevalue)^0)) - -local assignment = spacing * keyword * spacing * equal * spacing * value * spacing -local shortcut = keyword * spacing * left * spacing * (assignment * comma^0)^0 * spacing * right -local definition = keyword * spacing * left * spacing * keyword * comma * Ct((assignment * comma^0)^0) * spacing * right -local comment = keyword * spacing * left * (1-right)^0 * spacing * right -local forget = percent^1 * (1-lineending)^0 - --- todo \% - -local grammar = (space + forget + shortcut/do_shortcut + definition/do_definition + comment + 1)^0 - -function bibtex.convert(session,content) - statistics.starttiming(bibtex) - data, shortcuts, entries = session.data, session.shortcuts, session.entries - bibtexstats.nofbytes = bibtexstats.nofbytes + #content - session.nofbytes = session.nofbytes + #content - lpegmatch(grammar,content or "") - statistics.stoptiming(bibtex) -end - -function bibtex.load(session,filename) - local filename = resolvers.findfile(filename,"bib") - if filename ~= "" then - local data = io.loaddata(filename) or "" - if data == "" then - report_xml("empty file %a, no conversion to xml",filename) - elseif trace_bibxml then - report_xml("converting file %a to xml",filename) - end - bibtex.convert(session,data) - end -end - -function bibtex.new() - return { - data = { }, - shortcuts = { }, - xml = xml.convert("<?xml version='1.0' standalone='yes'?>\n<bibtex></bibtex>"), - nofbytes = 0, - entries = nil, - loaded = false, - } -end - -local p_escaped = lpegpatterns.xml.escaped - -local ihatethis = { - f = "\\f", - n = "\\n", - r = "\\r", - s = "\\s", - t = "\\t", - v = "\\v", - z = "\\z", -} - -local command = P("\\")/"" * Cc("\\bibtexcommand{") * (R("az","AZ")^1) * Cc("}") -local any = P(1) -local done = P(-1) -local one_l = P("{") / "" -local one_r = P("}") / "" -local two_l = P("{{") / "" -local two_r = P("}}") / "" - -local filter = Cs( - two_l * (command + any - two_r - done)^0 * two_r * done + - one_l * (command + any - one_r - done)^0 * one_r * done + - (command + any )^0 -) - -function bibtex.toxml(session,options) - if session.loaded then - return - else - session.loaded = true - end - -- we can always speed this up if needed - -- format slows down things a bit but who cares - statistics.starttiming(bibtex) - local result, r = { }, 0 - local options = settings_to_hash(options) - local convert = options.convert -- todo: interface - local strip = options.strip -- todo: interface - local entries = session.entries - r = r + 1 ; result[r] = "<?xml version='1.0' standalone='yes'?>" - r = r + 1 ; result[r] = "<bibtex>" - for id, categories in next, session.data do - id = lower(gsub(id,"^@","")) - for name, entry in next, categories do - if not entries or entries[name] then - r = r + 1 ; result[r] = formatters["<entry tag='%s' category='%s'>"](lower(name),id) - for key, value in next, entry do - value = gsub(value,"\\(.)",ihatethis) -- this really needs checking - value = lpegmatch(p_escaped,value) - if value ~= "" then - if convert then - value = textoutf(value,true) - end - if strip then - -- as there is no proper namespace in bibtex we need this - -- kind of hackery ... bibtex databases are quite unportable - value = lpegmatch(filter,value) or value - end - r = r + 1 ; result[r] = formatters[" <field name='%s'>%s</field>"](key,value) - end - end - r = r + 1 ; result[r] = "</entry>" - end - end - end - r = r + 1 ; result[r] = "</bibtex>" - result = concat(result,"\n") - -- alternatively we could use lxml.convert - session.xml = xml.convert(result, { - resolve_entities = true, - resolve_predefined_entities = true, -- in case we have escaped entities - -- unify_predefined_entities = true, -- & -> & - utfize_entities = true, - } ) - session.data = nil - session.shortcuts = nil - statistics.stoptiming(bibtex) -end - -statistics.register("bibtex load time", function() - local nofbytes = bibtexstats.nofbytes - if nofbytes > 0 then - return format("%s seconds (%s bytes, %s definitions, %s shortcuts)", - statistics.elapsedtime(bibtex),nofbytes,bibtexstats.nofdefinitions,bibtexstats.nofshortcuts) - else - return nil - end -end) - ---~ str = [[ ---~ @COMMENT { CRAP } ---~ @STRING{ hans = "h a n s" } ---~ @STRING{ taco = "t a c o" } ---~ @SOMETHING{ key1, abc = "t a c o" , def = "h a n s" } ---~ @SOMETHING{ key2, abc = hans # taco } ---~ @SOMETHING{ key3, abc = "hans" # taco } ---~ @SOMETHING{ key4, abc = hans # "taco" } ---~ @SOMETHING{ key5, abc = hans # taco # "hans" # "taco"} ---~ @SOMETHING{ key6, abc = {oeps {oeps} oeps} } ---~ ]] - ---~ local session = bibtex.new() ---~ bibtex.convert(session,str) ---~ bibtex.toxml(session) ---~ print(session.nofbytes,statistics.elapsedtime(bibtex)) - ---~ local session = bibtex.new() ---~ bibtex.load(session,"IEEEabrv.bib") ---~ bibtex.load(session,"IEEEfull.bib") ---~ bibtex.load(session,"IEEEexample.bib") ---~ bibtex.toxml(session) ---~ print(session.nofbytes,statistics.elapsedtime(bibtex)) - ---~ local session = bibtex.new() ---~ bibtex.load(session,"gut.bib") ---~ bibtex.load(session,"komoedie.bib") ---~ bibtex.load(session,"texbook1.bib") ---~ bibtex.load(session,"texbook2.bib") ---~ bibtex.load(session,"texbook3.bib") ---~ bibtex.load(session,"texgraph.bib") ---~ bibtex.load(session,"texjourn.bib") ---~ bibtex.load(session,"texnique.bib") ---~ bibtex.load(session,"tugboat.bib") ---~ bibtex.toxml(session) ---~ print(session.nofbytes,statistics.elapsedtime(bibtex)) - ---~ print(table.serialize(session.data)) ---~ print(table.serialize(session.shortcuts)) ---~ print(xml.serialize(session.xml)) - -if not characters then dofile(resolvers.findfile("char-def.lua")) end - -local chardata = characters.data -local concat = table.concat - -local lpeg = lpeg - -local P, Ct, lpegmatch, lpegpatterns = lpeg.P, lpeg.Ct, lpeg.match, lpeg.patterns - -local space, comma = P(" "), P(",") - -local andsplitter = lpeg.tsplitat(space^1 * "and" * space^1) -local commasplitter = lpeg.tsplitat(space^0 * comma * space^0) -local spacesplitter = lpeg.tsplitat(space^1) -local firstcharacter = lpegpatterns.utf8byte - -local function is_upper(str) - local first = lpegmatch(firstcharacter,str) - local okay = chardata[first] - return okay and okay.category == "lu" -end - -local function splitauthors(str) - local authors = lpegmatch(andsplitter,str) - for i=1,#authors do - local firstnames, vons, surnames, initials, juniors, words - local author = authors[i] - local split = lpegmatch(commasplitter,author) - local n = #split - if n == 1 then - --~ First von Last - words = lpegmatch(spacesplitter,author) - firstnames, vons, surnames = { }, { }, { } - local i, n = 1, #words - while i <= n do - local w = words[i] - if is_upper(w) then - firstnames[#firstnames+1], i = w, i + 1 - else - break - end - end - while i <= n do - local w = words[i] - if is_upper(w) then - break - else - vons[#vons+1], i = w, i + 1 - end - end - while i <= n do - surnames[#surnames+1], i = words[i], i + 1 - end - elseif n == 2 then - --~ von Last, First - words = lpegmatch(spacesplitter,split[2]) - surnames = lpegmatch(spacesplitter,split[1]) - firstnames, vons = { }, { } - local i, n = 1, #words - while i <= n do - local w = words[i] - if is_upper(w) then - firstnames[#firstnames+1], i = w, i + 1 - else - break - end - end - while i <= n do - vons[#vons+1], i = words[i], i + 1 - end - else - --~ von Last, Jr ,First - firstnames = lpegmatch(spacesplitter,split[1]) - juniors = lpegmatch(spacesplitter,split[2]) - surnames = lpegmatch(spacesplitter,split[3]) - if n > 3 then - -- error - end - end - if #surnames == 0 then - surnames[1] = firstnames[#firstnames] - firstnames[#firstnames] = nil - end - if firstnames then - initials = { } - for i=1,#firstnames do - initials[i] = utfchar(lpegmatch(firstcharacter,firstnames[i])) - end - end - authors[i] = { - original = author, - firstnames = firstnames, - vons = vons, - surnames = surnames, - initials = initials, - juniors = juniors, - } - end - authors.original = str - return authors -end - -local function the_initials(initials,symbol) - local t, symbol = { }, symbol or "." - for i=1,#initials do - t[i] = initials[i] .. symbol - end - return t -end - --- authors - -bibtex.authors = bibtex.authors or { } - -local authors = bibtex.authors - -local defaultsettings = { - firstnamesep = " ", - vonsep = " ", - surnamesep = " ", - juniorsep = " ", - surnamejuniorsep = ", ", - juniorjuniorsep = ", ", - surnamefirstnamesep = ", ", - surnameinitialsep = ", ", - namesep = ", ", - lastnamesep = " and ", - finalnamesep = " and ", -} - -function authors.normal(author,settings) - local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors - local result, settings = { }, settings or defaultsettings - if firstnames and #firstnames > 0 then - result[#result+1] = concat(firstnames," ") - result[#result+1] = settings.firstnamesep or defaultsettings.firstnamesep - end - if vons and #vons > 0 then - result[#result+1] = concat(vons," ") - result[#result+1] = settings.vonsep or defaultsettings.vonsep - end - if surnames then - result[#result+1] = concat(surnames," ") - end - if juniors and #juniors > 0 then - result[#result+1] = concat(juniors," ") - result[#result+1] = settings.surnamesep or defaultsettings.surnamesep - end - return concat(result) -end - -function authors.normalshort(author,settings) - local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors - local result, settings = { }, settings or defaultsettings - if firstnames and #firstnames > 0 then - result[#result+1] = concat(firstnames," ") - result[#result+1] = settings.firstnamesep or defaultsettings.firstnamesep - end - if vons and #vons > 0 then - result[#result+1] = concat(vons," ") - result[#result+1] = settings.vonsep or defaultsettings.vonsep - end - if surnames then - result[#result+1] = concat(surnames," ") - end - if juniors and #juniors > 0 then - result[#result+1] = concat(juniors," ") - result[#result+1] = settings.surnamejuniorsep or defaultsettings.surnamejuniorsep - end - return concat(result) -end - -function authors.inverted(author,settings) - local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors - local result, settings = { }, settings or defaultsettings - if vons and #vons > 0 then - result[#result+1] = concat(vons," ") - result[#result+1] = settings.vonsep or defaultsettings.vonsep - end - if surnames then - result[#result+1] = concat(surnames," ") - end - if juniors and #juniors > 0 then - result[#result+1] = settings.juniorjuniorsep or defaultsettings.juniorjuniorsep - result[#result+1] = concat(juniors," ") - end - if firstnames and #firstnames > 0 then - result[#result+1] = settings.surnamefirstnamesep or defaultsettings.surnamefirstnamesep - result[#result+1] = concat(firstnames," ") - end - return concat(result) -end - -function authors.invertedshort(author,settings) - local vons, surnames, initials, juniors = author.vons, author.surnames, author.initials, author.juniors - local result, settings = { }, settings or defaultsettings - if vons and #vons > 0 then - result[#result+1] = concat(vons," ") - result[#result+1] = settings.vonsep or defaultsettings.vonsep - end - if surnames then - result[#result+1] = concat(surnames," ") - end - if juniors and #juniors > 0 then - result[#result+1] = settings.juniorjuniorsep or defaultsettings.juniorjuniorsep - result[#result+1] = concat(juniors," ") - end - if initials and #initials > 0 then - result[#result+1] = settings.surnameinitialsep or defaultsettings.surnameinitialsep - result[#result+1] = concat(the_initials(initials)," ") - end - return concat(result) -end - -local lastconcatsize = 1 - -local function bibtexconcat(t,settings) - local namesep = settings.namesep or defaultsettings.namesep or ", " - local lastnamesep = settings.lastnamesep or defaultsettings.lastnamesep or namesep - local finalnamesep = settings.finalnamesep or defaultsettings.finalnamesep or lastnamesep - local lastconcatsize = #t - if lastconcatsize > 2 then - local s = { } - for i=1,lastconcatsize-2 do - s[i] = t[i] .. namesep - end - s[lastconcatsize-1], s[lastconcatsize] = t[lastconcatsize-1] .. finalnamesep, t[lastconcatsize] - return concat(s) - elseif lastconcatsize > 1 then - return concat(t,lastnamesep) - elseif lastconcatsize > 0 then - return t[1] - else - return "" - end -end - -function authors.concat(author,combiner,what,settings) - if type(combiner) == "string" then - combiner = authors[combiner or "normal"] or authors.normal - end - local split = splitauthors(author) - local setting = settings[what] - local etallimit, etaldisplay, etaltext = 1000, 1000, "" - if setting then - etallimit = settings.etallimit or 1000 - etaldisplay = settings.etaldisplay or etallimit - etalltext = settings.etaltext or "" - end - local max = #split - if max > etallimit and etaldisplay < max then - max = etaldisplay - end - for i=1,max do - split[i] = combiner(split[i],settings) - end - local result = bibtexconcat(split,settings) - if max < #split then - return result - else - return result .. etaltext - end -end - -function authors.short(author,year) - local result = { } - if author then - local authors = splitauthors(author) - for a=1,#authors do - local aa = authors[a] - local initials = aa.initials - for i=1,#initials do - result[#result+1] = initials[i] - end - local surnames = aa.surnames - for s=1,#surnames do - result[#result+1] = utfchar(lpegmatch(firstcharacter,surnames[s])) - end - end - end - if year then - result[#result+1] = year - end - return concat(result) -end - --- We can consider creating a hashtable key -> entry but I wonder if --- pays off. - -local function collectauthoryears(id,list) - list = settings_to_hash(list) - id = getid(id) - local found = { } - for e in xml.collected(id,"/bibtex/entry") do - if list[e.at.tag] then - local year = xmlfilter(e,"xml:///field[@name='year']/text()") - local author = xmlfilter(e,"xml:///field[@name='author']/text()") - if author and year then - local a = found[author] - if not a then - a = { } - found[author] = a - end - local y = a[year] - if not y then - y = { } - a[year] = y - end - y[#y+1] = e - end - end - end - -- found = { author = { year_1 = { e1, e2, e3 } } } - local done = { } - for author, years in next, found do - local yrs = { } - for year, entries in next, years do - if subyears then - -- -- add letters to all entries of an author and if so shouldn't - -- -- we tag all years of an author as soon as we do this? - -- if #entries > 1 then - -- for i=1,#years do - -- local entry = years[i] - -- -- years[i] = year .. string.char(i + string.byte("0") - 1) - -- end - -- end - else - yrs[#yrs+1] = year - end - end - done[author] = yrs - end - return done -end - -local method, settings = "normal", { } - -function authors.setsettings(s) - settings = s or settings -end - -if commands then - - local sessions = { } - - function commands.definebibtexsession(name) - sessions[name] = bibtex.new() - end - - function commands.preparebibtexsession(name,xmlname,options) - bibtex.toxml(sessions[name],options) - lxml.register(xmlname,sessions[name].xml) - end - - function commands.registerbibtexfile(name,filename) - bibtex.load(sessions[name],filename) - end - - function commands.registerbibtexentry(name,entry) - local session = sessions[name] - local entries = session.entries - if not entries then - session.entries = { [entry] = true } -- here we can keep more info - else - entries[entry] = true - end - end - - -- commands.bibtexconcat = bibtexconcat - - -- finalizers can be rather dumb as we have just text and no embedded xml - - function finalizers.bibtexconcat(collected,method,what) - if collected then - local author = collected[1].dt[1] or "" - if author ~= "" then - context(authors.concat(author,method,what,settings)) - end - end - end - - function finalizers.bibtexshort(collected) - if collected then - local c = collected[1] - local year = xmlfilter(c,"xml://field[@name='year']/text()") - local author = xmlfilter(c,"xml://field[@name='author']/text()") - context(authors.short(author,year)) - end - end - - -- experiment: - - --~ -- alternative approach: keep data at the tex end - - --~ local function xbibtexconcat(t,sep,finalsep,lastsep) - --~ local n = #t - --~ if n > 0 then - --~ context(t[1]) - --~ if n > 1 then - --~ if n > 2 then - --~ for i=2,n-1 do - --~ context.bibtexpublicationsparameter("sep") - --~ context(t[i]) - --~ end - --~ context.bibtexpublicationsparameter("finalsep") - --~ else - --~ context.bibtexpublicationsparameter("lastsep") - --~ end - --~ context(t[n]) - --~ end - --~ end - --~ end - - -- todo : sort - - -- todo: choose between bibtex or commands namespace - - function bibtex.authorref(id,list) - local result = collectauthoryears(id,list,method,what) - for author, years in next, result do - context(authors.concat(author,method,what,settings)) - end - end - - function bibtex.authoryearref(id,list) - local result = collectauthoryears(id,list,method,what) - for author, years in next, result do - context("%s (%s)",authors.concat(author,method,what,settings),concat(years,", ")) - end - end - - function bibtex.authoryearsref(id,list) - local result = collectauthoryears(id,list,method,what) - for author, years in next, result do - context("(%s, %s)",authors.concat(author,method,what,settings),concat(years,", ")) - end - end - - function bibtex.singularorplural(singular,plural) - if lastconcatsize and lastconcatsize > 1 then - context(plural) - else - context(singular) - end - end - -end - - ---~ local function test(sample) ---~ local authors = splitauthors(sample) ---~ print(table.serialize(authors)) ---~ for i=1,#authors do ---~ local author = authors[i] ---~ print(normalauthor (author,settings)) ---~ print(normalshortauthor (author,settings)) ---~ print(invertedauthor (author,settings)) ---~ print(invertedshortauthor(author,settings)) ---~ end ---~ print(concatauthors(sample,settings,normalauthor)) ---~ print(concatauthors(sample,settings,normalshortauthor)) ---~ print(concatauthors(sample,settings,invertedauthor)) ---~ print(concatauthors(sample,settings,invertedshortauthor)) ---~ end - ---~ local sample_a = "Hagen, Hans and Hoekwater, Taco Whoever T. Ex. and Henkel Hut, Hartmut Harald von der" ---~ local sample_b = "Hans Hagen and Taco Whoever T. Ex. Hoekwater and Hartmut Harald von der Henkel Hut" - ---~ test(sample_a) ---~ test(sample_b) +if not modules then modules = { } end modules ['bibl-bib'] = {
+ version = 1.001,
+ comment = "this module is the basis for the lxml-* ones",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files"
+}
+
+--[[ldx--
+<p>This is a prelude to integrated bibliography support. This file just loads
+bibtex files and converts them to xml so that the we access the content
+in a convenient way. Actually handling the data takes place elsewhere.</p>
+--ldx]]--
+
+local lower, format, gsub, concat = string.lower, string.format, string.gsub, table.concat
+local next = next
+local utfchar = utf.char
+local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
+local textoutf = characters and characters.tex.toutf
+local variables = interfaces and interfaces.variables
+local settings_to_hash = utilities.parsers.settings_to_hash
+local finalizers = xml.finalizers.tex
+local xmlfilter, xmltext, getid = xml.filter, xml.text, lxml.getid
+local formatters = string.formatters
+
+local P, R, S, C, Cc, Cs, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct
+
+local trace_bibxml = false trackers.register("publications.bibxml", function(v) trace_bibtex = v end)
+
+local report_xml = logs.reporter("publications","xml")
+
+bibtex = bibtex or { }
+local bibtex = bibtex
+
+bibtex.statistics = bibtex.statistics or { }
+local bibtexstats = bibtex.statistics
+
+bibtexstats.nofbytes = 0
+bibtexstats.nofdefinitions = 0
+bibtexstats.nofshortcuts = 0
+
+local defaultshortcuts = {
+ jan = "1",
+ feb = "2",
+ mar = "3",
+ apr = "4",
+ may = "5",
+ jun = "6",
+ jul = "7",
+ aug = "8",
+ sep = "9",
+ oct = "10",
+ nov = "11",
+ dec = "12",
+}
+
+local shortcuts = { }
+local data = { }
+local entries
+
+-- Currently we expand shortcuts and for large ones (like the acknowledgements
+-- in tugboat.bib this is not that efficient. However, eventually strings get
+-- hashed again.
+
+local function do_shortcut(tag,key,value)
+ bibtexstats.nofshortcuts = bibtexstats.nofshortcuts + 1
+ if lower(tag) == "@string" then
+ shortcuts[key] = value
+ end
+end
+
+local function do_definition(tag,key,tab) -- maybe check entries here (saves memory)
+ if not entries or entries[key] then
+ bibtexstats.nofdefinitions = bibtexstats.nofdefinitions + 1
+ local t = { }
+ for i=1,#tab,2 do
+ t[tab[i]] = tab[i+1]
+ end
+ local p = data[tag]
+ if not p then
+ data[tag] = { [key] = t }
+ else
+ p[key] = t
+ end
+ end
+end
+
+local function resolve(s)
+ return shortcuts[s] or defaultshortcuts[s] or s -- can be number
+end
+
+local percent = P("%")
+local start = P("@")
+local comma = P(",")
+local hash = P("#")
+local escape = P("\\")
+local single = P("'")
+local double = P('"')
+local left = P('{')
+local right = P('}')
+local both = left + right
+local lineending = S("\n\r")
+local space = S(" \t\n\r\f")
+local spacing = space^0
+local equal = P("=")
+local collapsed = (space^1)/ " "
+
+local function add(a,b) if b then return a..b else return a end end
+
+local keyword = C((R("az","AZ","09") + S("@_:-"))^1) -- C((1-space)^1)
+local s_quoted = ((escape*single) + collapsed + (1-single))^0
+local d_quoted = ((escape*double) + collapsed + (1-double))^0
+local balanced = lpegpatterns.balanced
+
+local s_value = (single/"") * s_quoted * (single/"")
+local d_value = (double/"") * d_quoted * (double/"")
+local b_value = (left /"") * balanced * (right /"")
+local r_value = keyword/resolve
+
+local somevalue = s_value + d_value + b_value + r_value
+local value = Cs((somevalue * ((spacing * hash * spacing)/"" * somevalue)^0))
+
+local assignment = spacing * keyword * spacing * equal * spacing * value * spacing
+local shortcut = keyword * spacing * left * spacing * (assignment * comma^0)^0 * spacing * right
+local definition = keyword * spacing * left * spacing * keyword * comma * Ct((assignment * comma^0)^0) * spacing * right
+local comment = keyword * spacing * left * (1-right)^0 * spacing * right
+local forget = percent^1 * (1-lineending)^0
+
+-- todo \%
+
+local grammar = (space + forget + shortcut/do_shortcut + definition/do_definition + comment + 1)^0
+
+function bibtex.convert(session,content)
+ statistics.starttiming(bibtex)
+ data, shortcuts, entries = session.data, session.shortcuts, session.entries
+ bibtexstats.nofbytes = bibtexstats.nofbytes + #content
+ session.nofbytes = session.nofbytes + #content
+ lpegmatch(grammar,content or "")
+ statistics.stoptiming(bibtex)
+end
+
+function bibtex.load(session,filename)
+ local filename = resolvers.findfile(filename,"bib")
+ if filename ~= "" then
+ local data = io.loaddata(filename) or ""
+ if data == "" then
+ report_xml("empty file %a, no conversion to xml",filename)
+ elseif trace_bibxml then
+ report_xml("converting file %a to xml",filename)
+ end
+ bibtex.convert(session,data)
+ end
+end
+
+function bibtex.new()
+ return {
+ data = { },
+ shortcuts = { },
+ xml = xml.convert("<?xml version='1.0' standalone='yes'?>\n<bibtex></bibtex>"),
+ nofbytes = 0,
+ entries = nil,
+ loaded = false,
+ }
+end
+
+local p_escaped = lpegpatterns.xml.escaped
+
+local ihatethis = {
+ f = "\\f",
+ n = "\\n",
+ r = "\\r",
+ s = "\\s",
+ t = "\\t",
+ v = "\\v",
+ z = "\\z",
+}
+
+local command = P("\\")/"" * Cc("\\bibtexcommand{") * (R("az","AZ")^1) * Cc("}")
+local any = P(1)
+local done = P(-1)
+local one_l = P("{") / ""
+local one_r = P("}") / ""
+local two_l = P("{{") / ""
+local two_r = P("}}") / ""
+
+local filter = Cs(
+ two_l * (command + any - two_r - done)^0 * two_r * done +
+ one_l * (command + any - one_r - done)^0 * one_r * done +
+ (command + any )^0
+)
+
+function bibtex.toxml(session,options)
+ if session.loaded then
+ return
+ else
+ session.loaded = true
+ end
+ -- we can always speed this up if needed
+ -- format slows down things a bit but who cares
+ statistics.starttiming(bibtex)
+ local result, r = { }, 0
+ local options = settings_to_hash(options)
+ local convert = options.convert -- todo: interface
+ local strip = options.strip -- todo: interface
+ local entries = session.entries
+ r = r + 1 ; result[r] = "<?xml version='1.0' standalone='yes'?>"
+ r = r + 1 ; result[r] = "<bibtex>"
+ for id, categories in next, session.data do
+ id = lower(gsub(id,"^@",""))
+ for name, entry in next, categories do
+ if not entries or entries[name] then
+ r = r + 1 ; result[r] = formatters["<entry tag='%s' category='%s'>"](lower(name),id)
+ for key, value in next, entry do
+ value = gsub(value,"\\(.)",ihatethis) -- this really needs checking
+ value = lpegmatch(p_escaped,value)
+ if value ~= "" then
+ if convert then
+ value = textoutf(value,true)
+ end
+ if strip then
+ -- as there is no proper namespace in bibtex we need this
+ -- kind of hackery ... bibtex databases are quite unportable
+ value = lpegmatch(filter,value) or value
+ end
+ r = r + 1 ; result[r] = formatters[" <field name='%s'>%s</field>"](key,value)
+ end
+ end
+ r = r + 1 ; result[r] = "</entry>"
+ end
+ end
+ end
+ r = r + 1 ; result[r] = "</bibtex>"
+ result = concat(result,"\n")
+ -- alternatively we could use lxml.convert
+ session.xml = xml.convert(result, {
+ resolve_entities = true,
+ resolve_predefined_entities = true, -- in case we have escaped entities
+ -- unify_predefined_entities = true, -- & -> &
+ utfize_entities = true,
+ } )
+ session.data = nil
+ session.shortcuts = nil
+ statistics.stoptiming(bibtex)
+end
+
+statistics.register("bibtex load time", function()
+ local nofbytes = bibtexstats.nofbytes
+ if nofbytes > 0 then
+ return format("%s seconds (%s bytes, %s definitions, %s shortcuts)",
+ statistics.elapsedtime(bibtex),nofbytes,bibtexstats.nofdefinitions,bibtexstats.nofshortcuts)
+ else
+ return nil
+ end
+end)
+
+--~ str = [[
+--~ @COMMENT { CRAP }
+--~ @STRING{ hans = "h a n s" }
+--~ @STRING{ taco = "t a c o" }
+--~ @SOMETHING{ key1, abc = "t a c o" , def = "h a n s" }
+--~ @SOMETHING{ key2, abc = hans # taco }
+--~ @SOMETHING{ key3, abc = "hans" # taco }
+--~ @SOMETHING{ key4, abc = hans # "taco" }
+--~ @SOMETHING{ key5, abc = hans # taco # "hans" # "taco"}
+--~ @SOMETHING{ key6, abc = {oeps {oeps} oeps} }
+--~ ]]
+
+--~ local session = bibtex.new()
+--~ bibtex.convert(session,str)
+--~ bibtex.toxml(session)
+--~ print(session.nofbytes,statistics.elapsedtime(bibtex))
+
+--~ local session = bibtex.new()
+--~ bibtex.load(session,"IEEEabrv.bib")
+--~ bibtex.load(session,"IEEEfull.bib")
+--~ bibtex.load(session,"IEEEexample.bib")
+--~ bibtex.toxml(session)
+--~ print(session.nofbytes,statistics.elapsedtime(bibtex))
+
+--~ local session = bibtex.new()
+--~ bibtex.load(session,"gut.bib")
+--~ bibtex.load(session,"komoedie.bib")
+--~ bibtex.load(session,"texbook1.bib")
+--~ bibtex.load(session,"texbook2.bib")
+--~ bibtex.load(session,"texbook3.bib")
+--~ bibtex.load(session,"texgraph.bib")
+--~ bibtex.load(session,"texjourn.bib")
+--~ bibtex.load(session,"texnique.bib")
+--~ bibtex.load(session,"tugboat.bib")
+--~ bibtex.toxml(session)
+--~ print(session.nofbytes,statistics.elapsedtime(bibtex))
+
+--~ print(table.serialize(session.data))
+--~ print(table.serialize(session.shortcuts))
+--~ print(xml.serialize(session.xml))
+
+if not characters then dofile(resolvers.findfile("char-def.lua")) end
+
+local chardata = characters.data
+local concat = table.concat
+
+local lpeg = lpeg
+
+local P, Ct, lpegmatch, lpegpatterns = lpeg.P, lpeg.Ct, lpeg.match, lpeg.patterns
+
+local space, comma = P(" "), P(",")
+
+local andsplitter = lpeg.tsplitat(space^1 * "and" * space^1)
+local commasplitter = lpeg.tsplitat(space^0 * comma * space^0)
+local spacesplitter = lpeg.tsplitat(space^1)
+local firstcharacter = lpegpatterns.utf8byte
+
+local function is_upper(str)
+ local first = lpegmatch(firstcharacter,str)
+ local okay = chardata[first]
+ return okay and okay.category == "lu"
+end
+
+local function splitauthors(str)
+ local authors = lpegmatch(andsplitter,str)
+ for i=1,#authors do
+ local firstnames, vons, surnames, initials, juniors, words
+ local author = authors[i]
+ local split = lpegmatch(commasplitter,author)
+ local n = #split
+ if n == 1 then
+ --~ First von Last
+ words = lpegmatch(spacesplitter,author)
+ firstnames, vons, surnames = { }, { }, { }
+ local i, n = 1, #words
+ while i <= n do
+ local w = words[i]
+ if is_upper(w) then
+ firstnames[#firstnames+1], i = w, i + 1
+ else
+ break
+ end
+ end
+ while i <= n do
+ local w = words[i]
+ if is_upper(w) then
+ break
+ else
+ vons[#vons+1], i = w, i + 1
+ end
+ end
+ while i <= n do
+ surnames[#surnames+1], i = words[i], i + 1
+ end
+ elseif n == 2 then
+ --~ von Last, First
+ words = lpegmatch(spacesplitter,split[2])
+ surnames = lpegmatch(spacesplitter,split[1])
+ firstnames, vons = { }, { }
+ local i, n = 1, #words
+ while i <= n do
+ local w = words[i]
+ if is_upper(w) then
+ firstnames[#firstnames+1], i = w, i + 1
+ else
+ break
+ end
+ end
+ while i <= n do
+ vons[#vons+1], i = words[i], i + 1
+ end
+ else
+ --~ von Last, Jr ,First
+ firstnames = lpegmatch(spacesplitter,split[1])
+ juniors = lpegmatch(spacesplitter,split[2])
+ surnames = lpegmatch(spacesplitter,split[3])
+ if n > 3 then
+ -- error
+ end
+ end
+ if #surnames == 0 then
+ surnames[1] = firstnames[#firstnames]
+ firstnames[#firstnames] = nil
+ end
+ if firstnames then
+ initials = { }
+ for i=1,#firstnames do
+ initials[i] = utfchar(lpegmatch(firstcharacter,firstnames[i]))
+ end
+ end
+ authors[i] = {
+ original = author,
+ firstnames = firstnames,
+ vons = vons,
+ surnames = surnames,
+ initials = initials,
+ juniors = juniors,
+ }
+ end
+ authors.original = str
+ return authors
+end
+
+local function the_initials(initials,symbol)
+ local t, symbol = { }, symbol or "."
+ for i=1,#initials do
+ t[i] = initials[i] .. symbol
+ end
+ return t
+end
+
+-- authors
+
+bibtex.authors = bibtex.authors or { }
+
+local authors = bibtex.authors
+
+local defaultsettings = {
+ firstnamesep = " ",
+ vonsep = " ",
+ surnamesep = " ",
+ juniorsep = " ",
+ surnamejuniorsep = ", ",
+ juniorjuniorsep = ", ",
+ surnamefirstnamesep = ", ",
+ surnameinitialsep = ", ",
+ namesep = ", ",
+ lastnamesep = " and ",
+ finalnamesep = " and ",
+}
+
+function authors.normal(author,settings)
+ local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if firstnames and #firstnames > 0 then
+ result[#result+1] = concat(firstnames," ")
+ result[#result+1] = settings.firstnamesep or defaultsettings.firstnamesep
+ end
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = concat(juniors," ")
+ result[#result+1] = settings.surnamesep or defaultsettings.surnamesep
+ end
+ return concat(result)
+end
+
+function authors.normalshort(author,settings)
+ local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if firstnames and #firstnames > 0 then
+ result[#result+1] = concat(firstnames," ")
+ result[#result+1] = settings.firstnamesep or defaultsettings.firstnamesep
+ end
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = concat(juniors," ")
+ result[#result+1] = settings.surnamejuniorsep or defaultsettings.surnamejuniorsep
+ end
+ return concat(result)
+end
+
+function authors.inverted(author,settings)
+ local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = settings.juniorjuniorsep or defaultsettings.juniorjuniorsep
+ result[#result+1] = concat(juniors," ")
+ end
+ if firstnames and #firstnames > 0 then
+ result[#result+1] = settings.surnamefirstnamesep or defaultsettings.surnamefirstnamesep
+ result[#result+1] = concat(firstnames," ")
+ end
+ return concat(result)
+end
+
+function authors.invertedshort(author,settings)
+ local vons, surnames, initials, juniors = author.vons, author.surnames, author.initials, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = settings.juniorjuniorsep or defaultsettings.juniorjuniorsep
+ result[#result+1] = concat(juniors," ")
+ end
+ if initials and #initials > 0 then
+ result[#result+1] = settings.surnameinitialsep or defaultsettings.surnameinitialsep
+ result[#result+1] = concat(the_initials(initials)," ")
+ end
+ return concat(result)
+end
+
+local lastconcatsize = 1
+
+local function bibtexconcat(t,settings)
+ local namesep = settings.namesep or defaultsettings.namesep or ", "
+ local lastnamesep = settings.lastnamesep or defaultsettings.lastnamesep or namesep
+ local finalnamesep = settings.finalnamesep or defaultsettings.finalnamesep or lastnamesep
+ local lastconcatsize = #t
+ if lastconcatsize > 2 then
+ local s = { }
+ for i=1,lastconcatsize-2 do
+ s[i] = t[i] .. namesep
+ end
+ s[lastconcatsize-1], s[lastconcatsize] = t[lastconcatsize-1] .. finalnamesep, t[lastconcatsize]
+ return concat(s)
+ elseif lastconcatsize > 1 then
+ return concat(t,lastnamesep)
+ elseif lastconcatsize > 0 then
+ return t[1]
+ else
+ return ""
+ end
+end
+
+function authors.concat(author,combiner,what,settings)
+ if type(combiner) == "string" then
+ combiner = authors[combiner or "normal"] or authors.normal
+ end
+ local split = splitauthors(author)
+ local setting = settings[what]
+ local etallimit, etaldisplay, etaltext = 1000, 1000, ""
+ if setting then
+ etallimit = settings.etallimit or 1000
+ etaldisplay = settings.etaldisplay or etallimit
+ etalltext = settings.etaltext or ""
+ end
+ local max = #split
+ if max > etallimit and etaldisplay < max then
+ max = etaldisplay
+ end
+ for i=1,max do
+ split[i] = combiner(split[i],settings)
+ end
+ local result = bibtexconcat(split,settings)
+ if max < #split then
+ return result
+ else
+ return result .. etaltext
+ end
+end
+
+function authors.short(author,year)
+ local result = { }
+ if author then
+ local authors = splitauthors(author)
+ for a=1,#authors do
+ local aa = authors[a]
+ local initials = aa.initials
+ for i=1,#initials do
+ result[#result+1] = initials[i]
+ end
+ local surnames = aa.surnames
+ for s=1,#surnames do
+ result[#result+1] = utfchar(lpegmatch(firstcharacter,surnames[s]))
+ end
+ end
+ end
+ if year then
+ result[#result+1] = year
+ end
+ return concat(result)
+end
+
+-- We can consider creating a hashtable key -> entry but I wonder if
+-- pays off.
+
+local function collectauthoryears(id,list)
+ list = settings_to_hash(list)
+ id = getid(id)
+ local found = { }
+ for e in xml.collected(id,"/bibtex/entry") do
+ if list[e.at.tag] then
+ local year = xmlfilter(e,"xml:///field[@name='year']/text()")
+ local author = xmlfilter(e,"xml:///field[@name='author']/text()")
+ if author and year then
+ local a = found[author]
+ if not a then
+ a = { }
+ found[author] = a
+ end
+ local y = a[year]
+ if not y then
+ y = { }
+ a[year] = y
+ end
+ y[#y+1] = e
+ end
+ end
+ end
+ -- found = { author = { year_1 = { e1, e2, e3 } } }
+ local done = { }
+ for author, years in next, found do
+ local yrs = { }
+ for year, entries in next, years do
+ if subyears then
+ -- -- add letters to all entries of an author and if so shouldn't
+ -- -- we tag all years of an author as soon as we do this?
+ -- if #entries > 1 then
+ -- for i=1,#years do
+ -- local entry = years[i]
+ -- -- years[i] = year .. string.char(i + string.byte("0") - 1)
+ -- end
+ -- end
+ else
+ yrs[#yrs+1] = year
+ end
+ end
+ done[author] = yrs
+ end
+ return done
+end
+
+local method, settings = "normal", { }
+
+function authors.setsettings(s)
+ settings = s or settings
+end
+
+if commands then
+
+ local sessions = { }
+
+ function commands.definebibtexsession(name)
+ sessions[name] = bibtex.new()
+ end
+
+ function commands.preparebibtexsession(name,xmlname,options)
+ bibtex.toxml(sessions[name],options)
+ lxml.register(xmlname,sessions[name].xml)
+ end
+
+ function commands.registerbibtexfile(name,filename)
+ bibtex.load(sessions[name],filename)
+ end
+
+ function commands.registerbibtexentry(name,entry)
+ local session = sessions[name]
+ local entries = session.entries
+ if not entries then
+ session.entries = { [entry] = true } -- here we can keep more info
+ else
+ entries[entry] = true
+ end
+ end
+
+ -- commands.bibtexconcat = bibtexconcat
+
+ -- finalizers can be rather dumb as we have just text and no embedded xml
+
+ function finalizers.bibtexconcat(collected,method,what)
+ if collected then
+ local author = collected[1].dt[1] or ""
+ if author ~= "" then
+ context(authors.concat(author,method,what,settings))
+ end
+ end
+ end
+
+ function finalizers.bibtexshort(collected)
+ if collected then
+ local c = collected[1]
+ local year = xmlfilter(c,"xml://field[@name='year']/text()")
+ local author = xmlfilter(c,"xml://field[@name='author']/text()")
+ context(authors.short(author,year))
+ end
+ end
+
+ -- experiment:
+
+ --~ -- alternative approach: keep data at the tex end
+
+ --~ local function xbibtexconcat(t,sep,finalsep,lastsep)
+ --~ local n = #t
+ --~ if n > 0 then
+ --~ context(t[1])
+ --~ if n > 1 then
+ --~ if n > 2 then
+ --~ for i=2,n-1 do
+ --~ context.bibtexpublicationsparameter("sep")
+ --~ context(t[i])
+ --~ end
+ --~ context.bibtexpublicationsparameter("finalsep")
+ --~ else
+ --~ context.bibtexpublicationsparameter("lastsep")
+ --~ end
+ --~ context(t[n])
+ --~ end
+ --~ end
+ --~ end
+
+ -- todo : sort
+
+ -- todo: choose between bibtex or commands namespace
+
+ function bibtex.authorref(id,list)
+ local result = collectauthoryears(id,list,method,what)
+ for author, years in next, result do
+ context(authors.concat(author,method,what,settings))
+ end
+ end
+
+ function bibtex.authoryearref(id,list)
+ local result = collectauthoryears(id,list,method,what)
+ for author, years in next, result do
+ context("%s (%s)",authors.concat(author,method,what,settings),concat(years,", "))
+ end
+ end
+
+ function bibtex.authoryearsref(id,list)
+ local result = collectauthoryears(id,list,method,what)
+ for author, years in next, result do
+ context("(%s, %s)",authors.concat(author,method,what,settings),concat(years,", "))
+ end
+ end
+
+ function bibtex.singularorplural(singular,plural)
+ if lastconcatsize and lastconcatsize > 1 then
+ context(plural)
+ else
+ context(singular)
+ end
+ end
+
+end
+
+
+--~ local function test(sample)
+--~ local authors = splitauthors(sample)
+--~ print(table.serialize(authors))
+--~ for i=1,#authors do
+--~ local author = authors[i]
+--~ print(normalauthor (author,settings))
+--~ print(normalshortauthor (author,settings))
+--~ print(invertedauthor (author,settings))
+--~ print(invertedshortauthor(author,settings))
+--~ end
+--~ print(concatauthors(sample,settings,normalauthor))
+--~ print(concatauthors(sample,settings,normalshortauthor))
+--~ print(concatauthors(sample,settings,invertedauthor))
+--~ print(concatauthors(sample,settings,invertedshortauthor))
+--~ end
+
+--~ local sample_a = "Hagen, Hans and Hoekwater, Taco Whoever T. Ex. and Henkel Hut, Hartmut Harald von der"
+--~ local sample_b = "Hans Hagen and Taco Whoever T. Ex. Hoekwater and Hartmut Harald von der Henkel Hut"
+
+--~ test(sample_a)
+--~ test(sample_b)
|