diff options
author | Karl Berry <karl@freefriends.org> | 2011-08-02 00:09:00 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2011-08-02 00:09:00 +0000 |
commit | 6189f60645741ffcb1b6e730325243882d7e5526 (patch) | |
tree | 62dfcc1190bd97244ece6865ae07c33d55d13e7f /Master/texmf-dist/scripts | |
parent | 8cb09a6490220c7e2df1022ea66743b9afa234be (diff) |
new luatex package interpreter (9jul11)
git-svn-id: svn://tug.org/texlive/trunk@23308 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts')
-rwxr-xr-x | Master/texmf-dist/scripts/interpreter/i-doc.lua | 253 | ||||
-rwxr-xr-x | Master/texmf-dist/scripts/interpreter/interpreter.lua | 379 |
2 files changed, 632 insertions, 0 deletions
diff --git a/Master/texmf-dist/scripts/interpreter/i-doc.lua b/Master/texmf-dist/scripts/interpreter/i-doc.lua new file mode 100755 index 00000000000..b8b3c106a46 --- /dev/null +++ b/Master/texmf-dist/scripts/interpreter/i-doc.lua @@ -0,0 +1,253 @@ +--[[ +Here's a description of "i-doc.lua", the file containing the interpretation +used for Interpreter's documentation. Remember that none of the TeX macros +used here is defined by Interpreter; instead, they are my own and should be +adapted if necessary. Also several options taken here are far from optimal but +are convenient examples. + +Shorthands for often used functions. +--]] +local gsub, match = string.gsub, string.match +local add_pattern = interpreter.add_pattern +local nomagic = interpreter.nomagic + +--[[ +Class 1 and 2 will be used for verbatim (thus protecting) and ``normal'' +patterns go into class 3 or higher. +--]] +interpreter.default_class = 3 + +--[[ +The reader might have observed that "interpreter-doc.txt" begins with a table +of contents. This table is useful for the source file only, and isn't typeset +by TeX, because the following pattern suppresses it: the entire paragraph +containing "TABLE OF CONTENTS" on a line of its own is deleted. Protecting the +paragraph is useless, but it makes things a little bit faster because the +paragraph won't be pointlessly searched for other patterns. +--]] +local function contents (buffer) + for n in ipairs(buffer) do + buffer[n] = "" + end + interpreter.protect() +end +add_pattern{ + pattern = "^%s*TABLE OF CONTENTS%s*$", + call = contents, + class = 1 +} + +--[[ +Sections headers are typeset as + + ====================================== section_tag + === Section title ==================== + ====================================== + +The first and third line are decorations and they are removed. The +"section_tag" is meant for the source only again (linking the section to the table +of contents). I could have used it to create PDF destinations, but that seemed +unnecessary in such a small file. The associated pattern is: at least four +equals signs. +--]] +add_pattern{ + pattern = "^====+.*", + replace = "" +} + +--[[ +The middle line is spotted with the tree equals sign at the beginning of the +line (the previous pattern being longer, the decoration lines have been +already removed and they won't be taken for section titles). The signs are +removed and replaced with "\section{" and "}". +--]] +local function section (buffer, num) + local l = buffer[num] + l = gsub(l, "^===%s*", "\\section{") + l = gsub(l, "%s*=+%s*", "}") + buffer[num] = l +end +add_pattern{ + pattern = "^===", + call = section +} + +--[[ +The following pattern simply turns "Interpreter" into "\ital{Interpreter}". The +meaning of the "\ital" command is obvious, I suppose. Note the offset: +starting at the backslash, this leads to the _n_ in Interpreter, thus avoiding +matching the pattern again. The Lua notation with double square brackets is +used for strings with no escape character (hence "\ital" and not "\\ital" as +would be necessary with a simple string). +--]] +add_pattern{ + pattern = "Interpreter", + replace = [[\ital{Interpreter}]], + offset = 7 +} + +--[[ +Turning "TeX" into TeX. This illustrates the use of a function as "replace"; +the point is that "\TeX" should be suffixed with a space if initially followed +by anything but a space or end of line (so as not to form a control sequence +with the following letters), and it should be suffixed with a control space if +initially followed by a space or end of line (so as to avoid gobbling the +space). So the function checks the second capture. Note that simply replacing +"TeX" with "\TeX{}" would be much simpler, but less instructive! +--]] +local function maketex (tex, next) + if next == " " or next == "" then + return [[\TeX\ ]] + else + return [[\TeX ]] .. next + end +end +add_pattern{ + pattern = "(TeX)(.?)", + replace = maketex, + offset = 2 +} + +--[[ +The following turns "<text>" into <text> and "_text_" into _text_. Setting a +class just so the patterns inherit the "nomagic" feature is of course an +overkill, but that's an example. +--]] +interpreter.set_class(4, {nomagic = true}) +add_pattern{ + pattern = "<...>", + replace = [[\arg{%1}]], + class = 4 +} +add_pattern{ + pattern = "_..._", + replace = [[\ital{%1}]], + class = 4 } + +--[[ +I use double quotes as protectors; they are replaced with a "\verb" command at +the very end of the processing (with class 0). +--]] +interpreter.protector('"') +add_pattern{ + pattern = nomagic'"..."', + replace = [[\verb`%1`]], + class = 0 +} + +--[[ +The description of functions (in red in the PDF file) are handled with the +"\describe" macro, which takes the function as its first argument and +additional information as its second one (typeset in italics in the PDF file). +In the source, it is simply marked as + + > function (arguments) [Additional information] + +with "[Additional information]" sometimes missing (i.e. there is no empty +pairs of square brackets). Descriptions of entries in pattern tables follows +the same syntax, except the line begins with ">>". So the pattern first spots +lines beginning with ">[>]" followed by at least one space, adds an empty pair of +brackets at the end if there isn't any, and turn the whole into "\describe". +The number of ">" symbols sets "\describe"'s third argument, which specifies +the level of the bookmark. +--]] +local function describe (buffer, num) + local l = buffer[num] + if not match (l, "%[.-%]%s*$") then + l = l .. " []" + end + local le = match(l, ">>") and 4 or 3 + buffer[num] = gsub(l, ">+%s+(.-)%s+%[(.-)%]", + [[\describe{%1}{%2}{]] .. le .. "}") +end +add_pattern{ + pattern = "^>+%s+", + call = describe +} + +--[[ +Here's how multiline verbatim is handled; in the source it is simply marked by +indenting the line with ten spaces; thus code is easily spotted when reading +the source without useless and annoying "<code>"/"</code>" or anything similar +to mark it. To be properly processed by TeX, the code should be surrounded by +"\verbatim" and "\verbatim/" (my way of signalling blocks). Those must be on +their own lines, so we insert a line at the beginning and at the end of the +paragraph: for the closing "\verbatim/", we can simply replace the last line +of the paragraph, which is the boundary line, unless we're at the end of the +file. But for the opening "\verbatim" a line must be added at the beginning of +the paragraph; thus line numbers in the original source file and in its +processed version don't match anymore, and this might be annoying when TeX +reports erros. Besides, blank verbatim lines aren't handled correctly and +create a new verbatim block instead. So this way of marking verbatim material +is good for small documents, but explicit marking is cleaner and more +powerful (albeit not so good-looking in the source file). + +Note that the verbatim pattern belongs to class 2 and the entire paragraph is +protected, so Interpreter leaves it alone afterward (remember the default +class is 3). Of course, the first ten space characters are removed. +--]] +local function verbatim (buffer) + for n, l in ipairs(buffer) do + buffer[n] = gsub(l, "%s%s%s%s%s%s%s%s%s%s","") + end + table.insert(buffer, 1, [[\verbatim]]) + if gsub(buffer[#buffer], + interpreter.paragraph, "") == "" then + buffer[#buffer] = [[\verbatim/]] + else + table.insert(buffer, [[\verbatim/]]) + end + interpreter.protect() +end +add_pattern{ + pattern = "^%s%s%s%s%s%s%s%s%s%s", + call = verbatim, + class = 2 +} + +--[[ +And now comes the fun part. I wanted "i-doc.lua" to be self-describing. The +source of what you're reading right now isn't "interpreter-doc.txt", but +"i-doc.lua" itself input in the latter file with + + \intepreterfile{doc}{i-doc.lua} + +How should code and comment be organized in "i-doc.lua"? Well, there is little +choice, since the file is a normal Lua file: comment lines should be prefixed +with "--" or surrounded with +\tcode{--[{}[} and \tcode{--]{}]}. % Sorry for the braces, I can't nest Lua comments! +I chose the latter option, which is simpler. But normal code should also be +typeset as verbatim material; I could have begun all lines with ten spaces, +but that would have seemed strange. Instead, \tcode{--]{}]} is turned into +"\source" and "\source/" is added at the end of the paragraph ("\source" is +just "\verbatim" with a different layout). Which means all paragraphs have the +same structure: comments between +\tcode{--[{}[} and \tcode{--]{}]} +and code immediately following (\tcode{--[{}[} is simply removed). The pattern +is in class 1 and the paragraph is protected, so that lines indented with ten +spaces or more aren't touched by the previous verbatim pattern (in class 2). +--]] +local function autoverbatim (buffer, line) + buffer[line] = [[\source]] + for n = line + 1, #buffer do + interpreter.protect(n) + end + if gsub(buffer[#buffer], + interpreter.paragraph, "") == "" then + buffer[#buffer] = [[\source/]] + else + table.insert(buffer, [[\source/]]) + end +end +add_pattern{ + pattern = nomagic"%^--]]", + call = autoverbatim, + class = 1 +} +local function test () + return "" +end +add_pattern{ + pattern = nomagic"%^--[[", + replace = test, +} diff --git a/Master/texmf-dist/scripts/interpreter/interpreter.lua b/Master/texmf-dist/scripts/interpreter/interpreter.lua new file mode 100755 index 00000000000..3c2d3391eec --- /dev/null +++ b/Master/texmf-dist/scripts/interpreter/interpreter.lua @@ -0,0 +1,379 @@ +-- This is the main Lua file for the Interpreter package. +-- Further information in interpreter-doc.pdf or interpreter-doc.txt. +-- Paul Isambert - zappathustra AT free DOT fr - July 2011 + +local find, gsub, match, sub = string.find, string.gsub, string.match, string.sub +local insert, sort = table.insert, table.sort + +-- Utility function sorting patterns by length (alphabetically if they are of +-- equal length). +local function string_order (a, b) + local a, b = a.pattern, b.pattern + return #a == #b and a < b or #a > #b +end + +interpreter = { +-- *** interpreter.active *** +-- Following paragraphs (as defined by interpreter.paragraph) are interpreted +-- iff this is not set to false. + active = true, +-- *** interpreter.default_class *** +-- Sets the default class for patterns which are added without specifying the +-- class. Default 1. + default_class = 1 + } +local _classes = {} + +-- *** interpreter.add_pattern (table) *** +-- Creates pattern <table>, which can contain the following entries: +-- pattern [string] = The pattern to match. Magic characters are obeyed! +-- replace [string] = The replacement for <pattern>. Can be a string, a +-- table or a function. A simple string.gsub() is +-- applied. +-- call [function] = The function applied to <pattern>; <replace> is applied +-- iff there is no <call>. +-- offset [number] = If <pattern> is used at index n, then the search on the +-- same line for the same pattern starts again at index n +-- + offset. Applied only when no <call> (in this case, +-- search starts again at the beginning of the line). By +-- default, offset = 0. This is needed to avoid infinite +-- loops with replacements which contain the pattern; +-- e.g. replacing "TeX" with "\TeX" will produce an +-- infinite loop, unless offset = 2. +-- nomagic [boolean] = Sets whether <replace> should be transformed with interpreter.nomagic. +-- class [number] = The pattern's <class> (classes of patterns are applied in +-- order, e.g. all patterns in class 1 are applied, then +-- all patterns in class 2, etc; class 0, however, is +-- always applied last). If <class> is not given, the +-- default_class number is used. Classes must be numbered +-- consecutively. +function interpreter.add_pattern (tb) + local class = tb.class or interpreter.default_class + interpreter.set_class(class, {}) + setmetatable(tb, _classes[class].meta) + if tb.nomagic then + tb.pattern = interpreter.nomagic(tb.pattern) + end + insert(_classes[class], tb) + sort(_classes[class], string_order) + return tb +end + +-- *** interpreter.set_class (number, table) *** +-- Sets default values (of the table normally specified in add_pattern) for +-- patterns of class <number>; patterns added to this class can still specify +-- different values, which will override defaults. In other words, this is a +-- metatable for patterns (which are tables) of that class. +function interpreter.set_class(num, tb) + _classes[num] = _classes[num] or { meta = { __index = function (_, k) return _classes[num].meta[k] end } } + for a, b in pairs(tb) do + _classes[num].meta[a] = b + end + return _classes[num] +end + +-- Class 0 must exist since it is always used at the end of the paragraph. +interpreter.set_class(0, {}) + +-- *** interpreter.nomagic (string) *** +-- Turns a normal string into a string with magic characters escaped, so it +-- can be used as a pattern. +local magic_characters = { + ["^"] = "%^", + ["$"] = "%$", + ["("] = "%(", + [")"] = "%)", + ["%"] = "%%", + ["."] = "%.", + ["["] = "%[", + ["]"] = "%]", + ["*"] = "%*", + ["+"] = "%+", + ["-"] = "%-", + ["?"] = "%?", +} +--function interpreter.nomagic (str) +-- return gsub(str, ".", magic_characters) +--end +function interpreter.nomagic (str) + local i, s = 1, "" + while i <= #str do + local c, c2, c3 = sub(str, i, i), sub(str, i + 1, i + 1), sub(str, i + 2, i + 2) + i = i + 1 + if c == "%" and magic_characters[c2] then + s = s .. c2 + i = i + 1 + elseif c == "." and c2 == "." and c3 == "." then + s = s .. "(.-)" + i = i + 2 + elseif magic_characters[c] then + s = s .. "%" .. c + else + s = s .. c + end + end + return s +end +-- *** interpreter.protect ([spec]) *** +-- Protects a set of lines in a paragraph; a protected line won't be +-- interpreted. If <spec> is a number, this protects line <spec> in the current +-- paragraph; if <spec> is true, this protects the entire current paragraph. Of +-- course, patterns that were applied to the line(s) or paragraph before +-- protection happened aren't undone. +local _protected +function interpreter.protect (num) + if type(num) == "number" then + if type(_protected) ~= "boolean" then + _protected = _protected or {} + _protected[num] = true + end + else + _protected = true + end +end + +-- Utility function making a replacement in a string but only from a certain +-- position and only once. We can't let gsub unrestricted, because some +-- part(s) of the string might be protected. +local function xsub (str, num, patt, rep) + return sub(str, 1, num-1) .. gsub(sub(str, num), patt, rep, 1) +end + +-- *** interpreter.protector (left [, right]) *** +-- Sets <left> and <right> (set to <left> if missing) as protectors, i.e. +-- enclosed material won't be processed even if the line is processed +-- otherwise. For instance: after interpreter.protector ("|"), the word +-- "little" in +-- +-- Hello, |little| world! +-- +-- will be left untouched; Interpreter is terribly smart (thanks to lpeg), so +-- in "|a| b |c|", "b" isn't protected, as intended, because the "|" on its +-- left doesn't match the one on its right but with the one before "a". An +-- example with <right> specified: interpreter.protector("[", "]") and +-- then: +-- +-- Hello, [little] world! +-- +-- achieves the same as above. Protectors AREN'T removed when the line is +-- finally passed to TeX; and there can be several protectors. Compare with +-- interpreter.escape. +local P, Cf, Cg, Cp, Ct, V = lpeg.P, lpeg.Cf, lpeg.Cg, lpeg.Cp, lpeg.Ct, lpeg.V +local _grammar +local function _protector (str, index) + local protections = Cf(Ct("") * Cg{ _grammar + 1 * V(1) }^1, rawset) + protections = protections:match(str) + if protections then + for a, b in pairs(protections) do + if index > a and index < b then + return nil, b + end + end + end + return index +end +function interpreter.protector (left, right) + right = right or left + local gram = P(Cp() * P(left) * (1 - P(right))^0 * Cp() * P(right)) + if _grammar then + _grammar = _grammar + gram + else + _grammar = gram + end +end + +-- *** interpreter.escape *** +-- A string used as an escape character: if a pattern matches, it is processed +-- iff the character immediately to its left isn't <escape>. The escape +-- character IS removed once the lines have been processed, so TeX never sees +-- it; also, only one escape character is allowed, and itself can't be escaped +-- (i.e. it doesn't mean anything to try to escape it). E.g.: +-- +-- interpreter.escape = "|" +-- ... this won't be |*processed* +-- +-- Assuming you have a pattern with stars, here it won't be applied. Instead +-- "this won't be *processed*" will be passed to TeX (note that the escape +-- character has disappeared). + +local function get_index (str, patt, index) + index = find(str, patt, index) + if index then + if sub(str, index-1, index-1) == interpreter.escape then + return get_index(str, patt, index + 1) + elseif _grammar then + local right + index, right = _protector(str, index, patt) + return index or get_index(str, patt, right + 1) + else + return index + end + end +end + +-- The main function that applies all the patterns in a <class> on a given +-- paragraph. +local function process_buffer (buffer, class) + for _, tb in ipairs(class) do + local num, idx = 1, 1 + while num <= #buffer do + local line = type(buffer[num]) == "string" and buffer[num] + if line then + local index = get_index(line, tb.pattern, idx) + if index then + if tb.call then + local l, o = tb.call(buffer, num, index, tb) + if o then + num, idx = l, o + elseif l then + idx = l + end + elseif tb.replace then + buffer[num] = xsub(line, index, tb.pattern, tb.replace) + idx = index + (tb.offset or 0) + end + -- If there was a function applied, protection might have been + -- enforced. Acts accordingly. + if _protected then + if type(_protected) == "table" then + for n, _ in pairs(_protected) do + if type(buffer[n]) == "string" then + buffer[n] = {buffer[n]} + end + end + if _protected[num] then + num = num + 1 + end + else + break + end + end + else + num = num + 1 + idx = 1 + end + else + num = num + 1 + idx = 1 + end + end + end +end + +-- The functions that iterates the previous one for each class. Note that +-- class 0 is processed last. +local function read_buffer (buffer) + for _, class in ipairs(_classes) do + if type(_protected) ~= "boolean" then + process_buffer(buffer, class) + end + end + if type(_protected) ~= "boolean" then + process_buffer(buffer, _classes[0]) + end + _protected = nil + -- Changes protected lines (tables) back to strings. + for n, _ in ipairs(buffer) do + if type(buffer[n]) == "table" then + buffer[n] = buffer[n][1] + end + end + return buffer +end + +local lines = { } + +-- *** interpreter.paragraph *** +-- The pattern that defines a line acting as a paragraph boundary, +-- prompting Interpreter to process the lines gathered up to now. Default is a +-- line composed of spaces at most. +interpreter.paragraph = "%s*" + +-- *** interpreter.direct (pattern) *** +-- Sets the pattern defining a line as direct Lua code: if a line begins with +-- <pattern> (which itself shouldn't contain the beginning-of-string character "^") +-- the code that follows is processed as Lua code, and the line is turned to +-- an empty string; note that this empty string will be seen as a paragraph +-- boundary if the line happened in the middle of a paragraph and +-- interpreter.paragraph has set paragraph boundary to empty string. Default +-- is "%%I " (two "%" followed by one "I" followed by at least one space +-- character). +interpreter.direct = "%%%%I%s+" + +-- At last, the function to be registered in open_read_file, defining the +-- function that reads a file. +function interpreter.read_file (fname) + -- *** interpreter.unregister () *** + -- The function used to remove read_file from the "open_read_file" callback. + -- Uses callback.register by default, or luatexbase.remove_from_callback if + -- detected. + if not interpreter.unregister then + if luatexbase and luatexbase.remove_from_callback then + function interpreter.unregister () + luatexbase.remove_from_callback("open_read_file", "interpreter") + end + else + function interpreter.unregister () + callback.register("open_read_file", nil) + end + end + end + interpreter.unregister() + + local f, line = io.open(fname), true + + local read_line = function () + + -- If no lines is left in store, we gather a new paragraph. + if #lines == 0 then + local line = f:read () + while line do + -- Checks for direct code. + if interpreter.direct and match(line, "^" .. interpreter.direct) then + loadstring(gsub(line, "^" .. interpreter.direct, ""))() + line = "" + end + table.insert(lines, line) + -- Checks for paragraph boundary, which breaks line gathering. + if gsub(line, interpreter.paragraph, "") == "" then + break + end + line = f:read () + end + + -- Pass the paragraph (a table of strings) to the bug functions above, + -- if interpreter is active; then remove escape characters, if any. + if interpreter.active then + lines = read_buffer (lines) + if interpreter.escape then + for num, line in ipairs(lines) do + lines[num] = gsub(line, interpreter.escape, "") + end + end + end + end + + return lines and table.remove (lines, 1) + end + return {reader = read_line} +end + +-- *** interpreter.reset () *** +-- Resets to default values; mostly to be used when calling a new +-- interpretation with \interpretfile. +function interpreter.reset () + interpreter.active = true + interpreter.default_class = 1 + _classes = {} + interpreter.set_class(0, {}) + _grammar = nil + interpreter.escape = nil + interpreter.paragraph = "%s*" + interpreter.direct = "%%%%I%s+" +end + +-- *** interpreter.register (function) *** +-- The function used to register the read_file function in the +-- "open_read_file" callback. If none is given, use callback.register, or +-- luatexbase.add_to_callback if detected (with "interpreter" as the name). +-- The function is defined in \interpretfile (see interpreter.tex). |