From d9825168b0fb7b07e315879584487f9d4d9e9494 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 11 Feb 2014 06:27:04 +0100 Subject: [*] move mkcharacters, mktests, mkglyphlist, mkstatus to separate subdirectory scripts --- scripts/mkcharacters | 156 +++++++++++++++++++++++++++++++ scripts/mkglyphlist | 173 ++++++++++++++++++++++++++++++++++ scripts/mkstatus | 149 +++++++++++++++++++++++++++++ scripts/mktests | 260 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 738 insertions(+) create mode 100755 scripts/mkcharacters create mode 100755 scripts/mkglyphlist create mode 100755 scripts/mkstatus create mode 100755 scripts/mktests (limited to 'scripts') diff --git a/scripts/mkcharacters b/scripts/mkcharacters new file mode 100755 index 0000000..5d4a2f4 --- /dev/null +++ b/scripts/mkcharacters @@ -0,0 +1,156 @@ +#!/usr/bin/env texlua +----------------------------------------------------------------------- +-- FILE: mkcharacters.lua +-- USAGE: ./mkcharacters.lua +-- DESCRIPTION: import parts of char-def.lua +-- REQUIREMENTS: lua, ConTeXt, the lualibs package +-- AUTHOR: Philipp Gesang (Phg), +-- VERSION: 2.4 +-- CREATED: 2013-05-17 12:41:39+0200 +----------------------------------------------------------------------- +-- we create a stripped-down version of char-def.lua +----------------------------------------------------------------------- + +----------------------------------------------------------------------- +-- config +----------------------------------------------------------------------- +local charfile = "./luaotfload-characters.lua" +local chardef = "/home/phg/base/char-def.lua" + +--- for every code point char-def.lua provides a set of fields. they +--- are: +--- +--- * adobename +--- * category +--- * cjkwd +--- * comment +--- * contextname +--- * description +--- * direction +--- * lccode +--- * linebreak +--- * mathclass +--- * mathextensible +--- * mathfiller +--- * mathname +--- * mathspec +--- * mathstretch +--- * mathsymbol +--- * mirror +--- * shcode +--- * specials +--- * textclass +--- * uccode +--- * unicodeslot +--- * variants + +local import = { + "direction", "mirror", --> πολυγλωσσία/uax9 + "category", --> https://gist.github.com/phi-gamma/5812290 + "textclass", --> https://gist.github.com/phi-gamma/6488187 +} + +----------------------------------------------------------------------- +-- includes +----------------------------------------------------------------------- + +kpse.set_program_name"luatex" + +for _, lib in next, { "lualibs-lua.lua", + "lualibs-lpeg.lua", + "lualibs-table.lua", } do + local found = assert(kpse.find_file(lib, "lua"), + "Could not locate " .. lib .. ".\n" + .. "Please install the lualibs package.") + require(found) +end + +if not (chardef and lfs.isfile(chardef)) then + --- we could grab the file from contextgarden but as Context is part + --- of TL it’s not worth bothering + chardef = assert(kpse.find_file("char-def.lua", "lua"), + "Could not find ConTeXt.") +end + +----------------------------------------------------------------------- +-- functionality +----------------------------------------------------------------------- + +local get_characters = function ( ) + local data + local inchan = io.open(chardef, "r") + if not inchan then + io.write("Could not open file for reading: "..chardef.."\n.") + goto fail + end + data = inchan:read "*all" + inchan:close() + data = loadstring(data) + if data then + data() --> characters.data + data = nil + collectgarbage "collect" + if characters.data and next(characters.data) then + return characters.data + end + io.write "Character table empty.\n" + goto fail + end + io.write(chardef .. " is not a valid Lua file.\n") + ::fail:: + io.write "Emergency exit.\n" + os.exit(1) +end + +local extract_fields_indeed +extract_fields_indeed = function (data, acc, lastidx) + local idx, char = next(data, lastidx) + if idx then + local imported = { } + for i=1, #import do + local field = import[i] + imported[field] = char[field] + end + acc[idx] = imported + return extract_fields_indeed(data, acc, idx) + end + return acc +end + +local extract_fields = function (data) + return extract_fields_indeed(data, {}, nil) +end + +local writedata = function (data) + local outchan = io.open(charfile, "w") + if not outchan then + io.write("Could not open "..charfile.." for writing.\n") + return false + end + outchan:write(data) + outchan:close() + return true +end + +do + local chardata = get_characters() + local stripped = extract_fields(chardata) + local serialized = table.serialize(stripped, true, { + compact = true, + noquotes = true, + hexify = true, --- for consistency with char-def + }) + if writedata(serialized) then + goto done + end + goto fail +end + +::done:: + os.exit(0) + +::fail:: + io.write "Emergency exit.\n" + os.exit(1) + +--- vim:ft=lua:ts=2:et:sw=2 diff --git a/scripts/mkglyphlist b/scripts/mkglyphlist new file mode 100755 index 0000000..f7a1cb9 --- /dev/null +++ b/scripts/mkglyphlist @@ -0,0 +1,173 @@ +#!/usr/bin/env texlua +----------------------------------------------------------------------- +-- FILE: mkglyphlist.lua +-- USAGE: ./mkglyphlist.lua +-- DESCRIPTION: part of the luaotfload package +-- REQUIREMENTS: lua, lpeg, luasocket, the lualibs package +-- AUTHOR: Philipp Gesang (Phg), +-- VERSION: 2.4 +-- CREATED: 04/23/2013 12:42:17 PM CEST +----------------------------------------------------------------------- +-- interesting thread on the Context list: +-- http://www.ntg.nl/pipermail/ntg-context/2008/029057.html +----------------------------------------------------------------------- + + +----------------------------------------------------------------------- +-- config +----------------------------------------------------------------------- +local glyphfile = "./glyphlist.txt" +local font_age = "./luaotfload-glyphlist.lua" +local glyph_source = "http://partners.adobe.com/public/developer/en/opentype/glyphlist.txt" + +----------------------------------------------------------------------- +-- fallbacks +----------------------------------------------------------------------- +--- Hans adds a small list of mappings that are not in the original +--- glyph list but seem to be normalizations of some sort. I trust his +--- experience, so I’ll just include them here. Background: +--- http://www.ntg.nl/pipermail/ntg-context/2013/073089.html + +local fallbacks = { + ["SF10000"]=9484, ["SF20000"]=9492, ["SF30000"]=9488, + ["SF40000"]=9496, ["SF50000"]=9532, ["SF60000"]=9516, + ["SF70000"]=9524, ["SF80000"]=9500, ["SF90000"]=9508, + ["afii208"]=8213, +} + +----------------------------------------------------------------------- +-- includes +----------------------------------------------------------------------- +require"lpeg" +require"socket" + +kpse.set_program_name"luatex" +for _, lib in next, { "lualibs-lua.lua", + "lualibs-lpeg.lua", + "lualibs-table.lua", } do + local found = assert(kpse.find_file(lib, "lua"), + "Could not locate " .. lib) + require(found) +end + +local C, Cf, Cg, Ct, P, R = + lpeg.C, lpeg.Cf, lpeg.Cg, lpeg.Ct, lpeg.P, lpeg.R + +local http = socket.http + +----------------------------------------------------------------------- +-- functionality +----------------------------------------------------------------------- + +local dec_of_hex = function (hex) return tonumber(hex, 16) end + +local separator = P";" +local gartenzaun = P"#" +local eol = P"\n\r" + P"\r\n" + P"\r" + P"\n" +local space = P" " +local alphanum = R("az", "AZ", "09") +local hexdigit = R("af", "AF", "09") +local eof_tag = gartenzaun * P"--end" +local header_line = gartenzaun * (1-eol)^0 * eol +local codepoint = hexdigit^1 +local glyphname = alphanum^1 + +local definition = Cg(C(glyphname) * separator * (C(codepoint)/ dec_of_hex)) + --- With combined glyphs we take only the first + --- value as char-def and font-age do, and skip + --- the rest. + * (space * codepoint)^0 + * eol +local definitions = Cf(Ct"" * definition^1, rawset) + +local p_glyphs = header_line^0 * definitions * eof_tag + +local get_glyphs = function (data) + local res = lpeg.match(p_glyphs, data) + if not res then + print("error: could not parse glyph list") + os.exit(-1) + end + for name, glyph in next, fallbacks do + res[name] = res[name] or glyph + end + return res +end + +local file_header = [==[ +if not modules then modules = { } end modules ["font-age"] = { + version = 2.400, + comment = "part of the luaotfload package", + author = "luaotfload team / mkglyphlist", + copyright = "derived from %s", + original = "Adobe Glyph List, version 2.0, September 20, 2002", + dataonly = true, +} + +if context then + logs.report("fatal error","this module is not for context") + os.exit(-1) +end + +--[[doc-- +Everything below has been autogenerated. Run mkglyphlist to rebuild +luaotfload-glyphlist.lua. +--doc]]-- + +]==] + +local writedata = function (data) + data = table.serialize(data, true) + data = string.format(file_header, glyph_source) .. data + local fh = io.open(font_age, "wb") + if not fh then + print(string.format("error: %s not writable", font_age)) + os.exit(-1) + end + print(string.format("saving %d bytes to %s", #data, font_age)) + fh:write(data) + fh:close() +end + + +local get_raw get_raw = function (retry) + local fh = io.open(glyphfile, "rb") + if fh then + local data = fh:read"*all" + fh:close() + if data then return data end + elseif not retry then --- attempt download + print"info: retrieving glyph list from" + print(glyph_source) + local glyphdata = http.request(glyph_source) + if glyphdata then + local fh = io.open(glyphfile, "wb") + if not fh then + print"error: glyph file not writable" + os.exit(-1) + end + fh:write(glyphdata) + fh:close() + return get_raw(true) + end + print"error: download failed" + os.exit(-1) + end + print("error: could not obtain glyph data from "..glyphfile) + os.exit(-1) +end + +local main = function () + if arg[1] then glyphfile = arg[1] end + if arg[2] then font_age = arg[2] end + + local data = get_raw() + local parsed = get_glyphs(data) + writedata(parsed) + return 0 +end + + +return main() + +--- vim:ft=lua:ts=2:et:sw=2 diff --git a/scripts/mkstatus b/scripts/mkstatus new file mode 100755 index 0000000..6e6e375 --- /dev/null +++ b/scripts/mkstatus @@ -0,0 +1,149 @@ +#!/usr/bin/env texlua +----------------------------------------------------------------------- +-- FILE: mkstatus.lua +-- USAGE: ./mkstatus.lua +-- DESCRIPTION: writes the repository state +-- REQUIREMENTS: luatex, the lualibs package +-- AUTHOR: Philipp Gesang (Phg), +-- VERSION: 1.0 +-- CREATED: 2013-07-07 14:01:12+0200 +----------------------------------------------------------------------- +-- +-- This script generates a list of hashes that serves as the input +-- for the file integrity check (option --diagnose). md5 is all we can +-- assume in Luatex, so it’s really only a superficial test. + +kpse.set_program_name "luatex" + +local md5 = require "md5" + +require "lualibs" + +local stringformat = string.format +local md5sumhexa = md5.sumhexa +local ioloaddata = io.loaddata +local iosavedata = io.savedata +local iopopen = io.popen + +----------------------------------------------------------------------- +-- settings +----------------------------------------------------------------------- + +local filelist = "luaotfload-status.lua" --- result + +local names = { + --- only the runtime files and scripts + "luaotfload-auxiliary.lua", + "luaotfload-basics-gen.lua", + "luaotfload-basics-nod.lua", + "luaotfload-characters.lua", + "luaotfload-colors.lua", + "luaotfload-database.lua", + "luaotfload-diagnostics.lua", + "luaotfload-features.lua", + "luaotfload-fonts-cbk.lua", + "luaotfload-fonts-def.lua", + "luaotfload-fonts-enc.lua", + "luaotfload-fonts-ext.lua", + "luaotfload-fonts-lua.lua", + "luaotfload-fonts-tfm.lua", + "luaotfload-glyphlist.lua", + "luaotfload-letterspace.lua", + "luaotfload-loaders.lua", + "luaotfload-log.lua", + "luaotfload-main.lua", + "luaotfload-fontloader.lua", + "luaotfload-override.lua", + "luaotfload-parsers.lua", + "luaotfload-tool.lua", + "mkcharacters", + "mkglyphlist", + "mkstatus", +} + +----------------------------------------------------------------------- +-- helpers +----------------------------------------------------------------------- + +local die = function (...) + io.stderr:write "[fatal error]: " + io.stderr:write (stringformat (...)) + io.stderr:write "\naborting.\n" + os.exit (1) +end + +local gitcmd = "git log -1 \z + --format=\"return {\z + %n revision = [[%H]],\z + %n timestamp = [[%cd]],\z + %n committer = [[%cn <%ce>]],\z + %n}\" \z + --date=iso" + +local git_info = function () + --io.write (gitcmd) + --io.write "\n" + local chan = iopopen (gitcmd) + if not chan then + die ("this script needs to be run inside \z + the luaotfload git repository") + end + + local data = chan:read "*all" + chan:close () + if data and type (data) == "string" and data ~= "" then + data = load (data) + if not data then + die "cannot parse git information" + end + return data () + end + die "cannot read from pipe" +end + +----------------------------------------------------------------------- +-- functionality +----------------------------------------------------------------------- + +local hash_file = function (fname) + if not lfs.isfile (fname) then + die ("cannot find %s.", fname) + end + local raw = ioloaddata (fname) + if not raw then + die ("cannot read from %s.", fname) + end + return md5sumhexa (raw) +end + +local hash_all +hash_all = function (list, acc) + if list == nil then + return hash_all (table.fastcopy (names), { }) + end + + local fname = list[#list] + list[#list] = nil + if fname then + local sum = hash_file (fname) + acc[#acc+1] = { fname, sum } + return hash_all (list, acc) + end + return acc +end + +local main = function () + local hashes = hash_all () + local notes = git_info () + local serialized = table.serialize ({ notes = notes, + hashes = hashes }, true) + local success = io.savedata (filelist, serialized) + if success == false then + die ("could not write to %s.", filelist) + end + return 0 +end + +return main () + +--- vim:ft=lua:ts=2:et:sw=2 diff --git a/scripts/mktests b/scripts/mktests new file mode 100755 index 0000000..0bf3f64 --- /dev/null +++ b/scripts/mktests @@ -0,0 +1,260 @@ +#!/usr/bin/env texlua +----------------------------------------------------------------------- +-- FILE: mktests +-- USAGE: ./mktests +-- DESCRIPTION: test the behavior of Luaotfload +-- REQUIREMENTS: Luatex > 0.76, Luaotfload +-- AUTHOR: Philipp Gesang (Phg), +-- VERSION: 2.4 +-- MODIFIED: 2013-08-26 09:31:22+0200 +----------------------------------------------------------------------- +-- +--===================================================================-- +-- NOTE +-- this is a stub, to be completed long-term +-- suggestions welcome +--===================================================================-- + + +local tests = { } + +config = { luaotfload = { + names_dir = "names", + cache_dir = "fonts", + index_file = "luaotfload-names.lua", + resolver = "normal", + update_live = true, --- suppress db updates +}} + +kpse.set_program_name "luatex" + +require "lualibs" +require "luaotfload-basics-gen.lua" +require "luaotfload-log.lua" +require "luaotfload-parsers" +require "luaotfload-database" + +local names = fonts.names + +----------------------------------------------------------------------- +--- helper functions +----------------------------------------------------------------------- + +local pprint_resolve = function (input, output, result) + texio.write_nl (string.format ("[%s] “%s” -> “%s”", + result == true and "passed" or "failed", + input, + output)) +end + +local pprint_result = function (name, failed, total) + if failed == 0 then + texio.write_nl (string.format ("[%s] all %d passed", name, total)) + else + texio.write_nl (string.format ("[%s] %d of %d failed", + name, + failed, + total)) + end +end + +local pprint_spec = function (spec) + return string.format ("%s/%s*%.2fpt", + spec.specification, + spec.style or "regular", + spec.optsize or 0) +end + +----------------------------------------------------------------------- +--- tool tests +----------------------------------------------------------------------- + + + +----------------------------------------------------------------------- +--- font tests +----------------------------------------------------------------------- + +--- test sets + +local infer_regular_style = { + --- inferring which one is the correct style for “regular”; can be + --- obscured by synonyms like “book” etc. + { "Iwona", "Iwona-Regular.otf" }, -- trivial case + { "DejaVu Serif", "DejaVuSerif.ttf" }, + { "DejaVu Sans", "DejaVuSans.ttf" }, + { "Adobe Garamond Pro", "agaramondpro_regular.otf" }, + { "Garamond Premier Pro", "GaramondPremrPro.otf" }, + { "CMU Serif", "cmunrm.otf" }, + { "CMU Sans Serif", "cmunss.otf" }, + { "Minion Pro", "MinionPro-Regular.otf" }, + --- Below test will succeed only if we match for the + --- splainname (= sanitized tfmdata.fullname) field + --- explicitly. + { "Minion Pro Italic", "MinionPro-It.otf" }, +} + +local choose_optical_size = { + { { name = "Latin Modern Roman", optsize = 1 }, "lmroman5-regular.otf" }, + { { name = "Latin Modern Roman", optsize = 10 }, "lmroman10-regular.otf" }, + { { name = "Latin Modern Roman", optsize = 12 }, "lmroman12-regular.otf" }, + { { name = "Latin Modern Roman", optsize = 42 }, "lmroman17-regular.otf" }, + { { name = "EB Garamond", optsize = 1 }, "EBGaramond08-Regular.otf" }, + { { name = "EB Garamond", optsize = 8 }, "EBGaramond08-Regular.otf" }, + { { name = "EB Garamond", optsize = 12 }, "EBGaramond12-Regular.otf" }, + { { name = "EB Garamond", optsize = 42 }, "EBGaramond12-Regular.otf" }, + { { name = "Garamond Premier Pro", optsize = 1 }, "GaramondPremrPro-Capt.otf" }, + { { name = "Garamond Premier Pro", optsize = 10 }, "GaramondPremrPro.otf" }, + { { name = "Garamond Premier Pro", optsize = 15 }, "GaramondPremrPro-Subh.otf" }, + { { name = "Garamond Premier Pro", optsize = 42 }, "GaramondPremrPro-Disp.otf" }, +} + +local choose_style = { + { { name = "DejaVu Sans", style = "regular" }, "DejaVuSans.ttf" }, + { { name = "DejaVu Sans", style = "italic" }, "DejaVuSans-Oblique.ttf" }, + { { name = "DejaVu Sans", style = "bold" }, "DejaVuSans-Bold.ttf" }, + { { name = "DejaVu Sans", style = "bolditalic" }, "DejaVuSans-BoldOblique.ttf" }, + { { name = "Linux Libertine O", style = "regular" }, "LinLibertine_R.otf" }, + { { name = "Linux Libertine O", style = "italic" }, "LinLibertine_RI.otf" }, + { { name = "Linux Libertine O", style = "bold" }, "LinLibertine_RB.otf" }, + { { name = "Linux Libertine O", style = "bolditalic" }, "LinLibertine_RBI.otf" }, + { { name = "Liberation Serif", style = "regular" }, "LiberationSerif-Regular.ttf" }, + { { name = "Liberation Serif", style = "italic" }, "LiberationSerif-Italic.ttf" }, + { { name = "Liberation Serif", style = "bold" }, "LiberationSerif-Bold.ttf" }, + { { name = "Liberation Serif", style = "bolditalic" }, "LiberationSerif-BoldItalic.ttf" }, + { { name = "CMU Sans Serif", style = "regular" }, "cmunss.otf" }, -- no “regular” but “medium” + { { name = "CMU Sans Serif", style = "italic" }, "cmunsi.otf" }, -- no “italic” but “oblique” + { { name = "CMU Sans Serif", style = "bold" }, "cmunsx.otf" }, + { { name = "CMU Sans Serif", style = "bolditalic" }, "cmunso.otf" }, + --[[-- + Minion Pro Italic is exceptionally weird regarding identifiers in + that the postscript fontname and both info.fontname and + info.fullname are given as “minionproit”. Now its english fullname + (field 18) is “minionproital”. Only the value “fullname” in the root of + the tfmdata structure (not the one returned by fontloader.info()!) + accurately yields “Minion Pro Italic”. + + To complete the picture, the file naming isn’t very consistent either: + we find the suffixes “Regular” and “Bold”, but “It” and “BoldIt”. What + the hell were the designers smoking? + + Also, the full Minion Pro set comes with different optical sizes which + for monetary reasons cannot considered here. + --]]-- + { { name = "Minion Pro", style = "regular" }, "MinionPro-Regular.otf" }, + { { name = "Minion Pro", style = "italic" }, "MinionPro-It.otf" }, + { { name = "Minion Pro", style = "bold" }, "MinionPro-Bold.otf" }, + { { name = "Minion Pro", style = "bolditalic" }, "MinionPro-BoldIt.otf" }, +} + +--- this needs a database built with --formats=+pfa,pfb,afm + +local resolve_t1_font = { + { { name = "URW Gothic L", style = "regular" }, "a010013l.pfb" }, --> “book” +-- { { name = "URW Gothic L", style = "italic" }, "a010033l.pfb" }, --> “book oblique” +-- { { name = "URW Gothic L", style = "bold" }, "a010015l.pfb" }, --> “demi” +-- { { name = "URW Gothic L", style = "bolditalic" }, "a010035l.pfb" }, --> “demi oblique” + { { name = "Century Schoolbook L", style = "regular" }, "c059013l.pfb" }, + { { name = "Century Schoolbook L", style = "italic" }, "c059033l.pfb" }, + { { name = "Century Schoolbook L", style = "bold" }, "c059016l.pfb" }, + { { name = "Century Schoolbook L", style = "bolditalic" }, "c059036l.pfb" }, + { { name = "Nimbus Roman No9 L", style = "regular" }, "n021003l.pfb" }, + { { name = "Nimbus Roman No9 L", style = "italic" }, "n021023l.pfb" }, + { { name = "Nimbus Roman No9 L", style = "bold" }, "n021004l.pfb" }, --- medium, actually + { { name = "Nimbus Roman No9 L", style = "bolditalic" }, "n021024l.pfb" }, +} + +local translate_style = { + regular = "r", + italic = "i", + bold = "b", + bolditalic = "bi", +} + +local font_name_tests = { + infer_regular_style, + choose_optical_size, + choose_style, + resolve_t1_font, +} + +local default_spec = { + name = false, + lookup = "name", + specification = false, + optsize = 0, +} + +local resolve_font_name = function () + local failed, total = 0, 0 + local resolve_name = names.resolve_name + for nset = 1, #font_name_tests do + local set = font_name_tests[nset] + + for ntest = 1, #set do + local test = set[ntest] + local input, output = test[1], test[2] + + if type (input) == "string" then + local input_spec = table.copy (default_spec) + input_spec.name = input + input_spec.specification = input_spec.lookup .. ":" .. input + local result = resolve_name (input_spec) == output + total = total + 1 + if not result then + failed = failed + 1 + end + pprint_resolve (input, output, result) + + else + local input_spec, output = test[1], test[2] + input_spec.specification = (input_spec.lookup + or default_spec.lookup) + .. ":" .. input_spec.name + input_spec.optsize = input_spec.optsize or default_spec.optsize + input_spec.style = translate_style [input_spec.style] + local result = resolve_name (input_spec) == output + total = total + 1 + if not result then + failed = failed + 1 + end + pprint_resolve (pprint_spec (input_spec), output, result) + end + + end + end + return failed, total +end + +tests ["resolve_font_name"] = resolve_font_name + +----------------------------------------------------------------------- +--- runner +----------------------------------------------------------------------- + +local main = function () + local failed, total = 0, 0 + for name, test in next, tests do + texio.write_nl ("[" .. name .. "]") + local newfailed, newtotal = test () + total = total + 1 + pprint_result (name, newfailed, newtotal) + failed = failed + newfailed + total = total + newtotal + end + + if failed == 0 then + texio.write_nl (string.format ("[report] all %d tests passed.", total)) + else + texio.write_nl (string.format ("[report] %d of %d tests failed (%d %%).", + failed, + total, + failed / total * 100)) + end + texio.write_nl "" + os.exit (0) +end + +return main () + +--- vim:ft=lua:ts=2:et:sw=2 -- cgit v1.2.3