diff options
author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /macros/luatex/generic/interpreter |
Initial commit
Diffstat (limited to 'macros/luatex/generic/interpreter')
-rw-r--r-- | macros/luatex/generic/interpreter/README | 29 | ||||
-rw-r--r-- | macros/luatex/generic/interpreter/i-doc.lua | 254 | ||||
-rw-r--r-- | macros/luatex/generic/interpreter/interpreter-doc.pdf | bin | 0 -> 110003 bytes | |||
-rw-r--r-- | macros/luatex/generic/interpreter/interpreter-doc.tex | 115 | ||||
-rw-r--r-- | macros/luatex/generic/interpreter/interpreter-doc.txt | 732 | ||||
-rw-r--r-- | macros/luatex/generic/interpreter/interpreter.lua | 450 | ||||
-rw-r--r-- | macros/luatex/generic/interpreter/interpreter.sty | 17 | ||||
-rw-r--r-- | macros/luatex/generic/interpreter/interpreter.tex | 37 |
8 files changed, 1634 insertions, 0 deletions
diff --git a/macros/luatex/generic/interpreter/README b/macros/luatex/generic/interpreter/README new file mode 100644 index 0000000000..f3e1c56c52 --- /dev/null +++ b/macros/luatex/generic/interpreter/README @@ -0,0 +1,29 @@ +This is the README file for the Interpreter package. +Author: Paul Isambert. +E-mail: zappathustra AT free DOT fr +Comments and suggestions are welcome. +Date: June 2012. +Version: 1.2. + +Interpreter preprocesses input files on the fly (no external program) and +manipulates input lines, e.g. to turn some markup into proper TeX syntax. +Interpreter doesn't work with ConTeXt. + +LuaTeX is required, and the Lua side of the Gates package (i.e. gates.lua), +version at least 0.2. + +Relevant information can be found in interpreter-doc.pdf +or interpreter-doc.txt (source of the doc readable in a text editor). + +The files in this distribution are: + +interpreter.lua - main code +interpreter.tex - \input in plain TeX +interpreter.sty - wrapper for LaTeX +interpreter-doc.tex - master file for the doc +interpreter-doc.txt - text of the doc +interpreter-doc.pdf - typeset doc +i-doc.lua - interpretation for the doc (because, of course, the doc + is typeset with Interpreter) + +Licensing of this package is covered by LPPL. diff --git a/macros/luatex/generic/interpreter/i-doc.lua b/macros/luatex/generic/interpreter/i-doc.lua new file mode 100644 index 0000000000..12bc214eda --- /dev/null +++ b/macros/luatex/generic/interpreter/i-doc.lua @@ -0,0 +1,254 @@ +--[[ +Here's a description of "i-doc.lua", the file containing the interpretation +used for Interpreter's documentation. Remember that none of the TeX macros +used here is defined by Interpreter; instead, they are my own and should be +adapted if necessary. Also several options taken here are far from optimal but +are convenient examples. + +Shorthands for often used functions. +--]] +local gsub, match = string.gsub, string.match +local add_pattern = interpreter.add_pattern +local nomagic = interpreter.nomagic + +--[[ +Class 1 and 2 will be used for verbatim (thus protecting) and ``normal'' +patterns go into class 3 or higher. +--]] +interpreter.default_class = 3 + +--[[ +The reader might have observed that "interpreter-doc.txt" begins with a table +of contents. This table is useful for the source file only, and isn't typeset +by TeX, because the following pattern suppresses it: the entire paragraph +containing "TABLE OF CONTENTS" on a line of its own is deleted. Protecting the +paragraph is useless, but it makes things a little bit faster because the +paragraph won't be pointlessly searched for other patterns. +--]] +local function contents (buffer) + for n in ipairs(buffer) do + buffer[n] = "" + end + interpreter.protect() +end +add_pattern{ + pattern = "^%s*TABLE OF CONTENTS%s*$", + call = contents, + class = 1 +} + +--[[ +Sections headers are typeset as + + ====================================== section_tag + === Section title ==================== + ====================================== + +The first and third line are decorations and they are removed. The +"section_tag" is meant for the source only again (linking the section to the table +of contents). I could have used it to create PDF destinations, but that seemed +unnecessary in such a small file. The associated pattern is: at least four +equals signs. +--]] +add_pattern{ + pattern = "^====+.*", + replace = "" +} + +--[[ +The middle line is spotted with the tree equals sign at the beginning of the +line (the previous pattern being longer, the decoration lines have been +already removed and they won't be taken for section titles). The signs are +removed and replaced with "\section{" and "}". +--]] +local function section (buffer, num) + local l = buffer[num] + l = gsub(l, "^===%s*", "\\section{") + l = gsub(l, "%s*=+%s*", "}") + buffer[num] = l +end +add_pattern{ + pattern = "^===", + call = section +} + +--[[ +The following pattern simply turns "Interpreter" into "\ital{Interpreter}". The +meaning of the "\ital" command is obvious, I suppose. Note the offset: +starting at the backslash, this leads to the _n_ in Interpreter, thus avoiding +matching the pattern again. The Lua notation with double square brackets is +used for strings with no escape character (hence "\ital" and not "\\ital" as +would be necessary with a simple string). +--]] +add_pattern{ + pattern = "Interpreter", + replace = [[\ital{Interpreter}]], + offset = 7 +} + +--[[ +Turning "TeX" into TeX. This illustrates the use of a function as "replace"; +the point is that "\TeX" should be suffixed with a space if initially followed +by anything but a space or end of line (so as not to form a control sequence +with the following letters), and it should be suffixed with a control space if +initially followed by a space or end of line (so as to avoid gobbling the +space). So the function checks the second capture. Note that simply replacing +"TeX" with "\TeX{}" would be much simpler, but less instructive! +--]] +local function maketex (tex, next) + if next == " " or next == "" then + return [[\TeX\ ]] + else + return [[\TeX ]] .. next + end +end +add_pattern{ + pattern = "(TeX)(.?)", + replace = maketex, + offset = 2 +} + +--[[ +The following turns "<text>" into <text> and "_text_" into _text_. Setting a +class just so the patterns inherit the "nomagic" feature is of course an +overkill, but that's an example. +--]] +interpreter.set_class(4, {nomagic = true}) +add_pattern{ + pattern = "<...>", + replace = [[\arg{%1}]], + class = 4 +} +add_pattern{ + pattern = "_..._", + replace = [[\ital{%1}]], + class = 4 +} + +--[[ +I use double quotes as protectors; they are replaced with a "\verb" command at +the very end of the processing (with class 0). +--]] +interpreter.protector('"') +add_pattern{ + pattern = nomagic'"..."', + replace = [[\verb`%1`]], + class = 0 +} + +--[[ +The description of functions (in red in the PDF file) are handled with the +"\describe" macro, which takes the function as its first argument and +additional information as its second one (typeset in italics in the PDF file). +In the source, it is simply marked as + + > function (arguments) [Additional information] + +with "[Additional information]" sometimes missing (i.e. there is no empty +pairs of square brackets). Descriptions of entries in pattern tables follows +the same syntax, except the line begins with ">>". So the pattern first spots +lines beginning with ">[>]" followed by at least one space, adds an empty pair of +brackets at the end if there isn't any, and turn the whole into "\describe". +The number of ">" symbols sets "\describe"'s third argument, which specifies +the level of the bookmark. +--]] +local function describe (buffer, num) + local l = buffer[num] + if not match (l, "%[.-%]%s*$") then + l = l .. " []" + end + local le = match(l, ">>") and 4 or 3 + buffer[num] = gsub(l, ">+%s+(.-)%s+%[(.-)%]", + [[\describe{%1}{%2}{]] .. le .. "}") +end +add_pattern{ + pattern = "^>+%s+", + call = describe +} + +--[[ +Here's how multiline verbatim is handled; in the source it is simply marked by +indenting the line with ten spaces; thus code is easily spotted when reading +the source without useless and annoying "<code>"/"</code>" or anything similar +to mark it. To be properly processed by TeX, the code should be surrounded by +"\verbatim" and "\verbatim/" (my way of signalling blocks). Those must be on +their own lines, so we insert a line at the beginning and at the end of the +paragraph: for the closing "\verbatim/", we can simply replace the last line +of the paragraph, which is the boundary line, unless we're at the end of the +file. But for the opening "\verbatim" a line must be added at the beginning of +the paragraph; thus line numbers in the original source file and in its +processed version don't match anymore, and this might be annoying when TeX +reports erros. Besides, blank verbatim lines aren't handled correctly and +create a new verbatim block instead. So this way of marking verbatim material +is good for small documents, but explicit marking is cleaner and more +powerful (albeit not so good-looking in the source file). + +Note that the verbatim pattern belongs to class 2 and the entire paragraph is +protected, so Interpreter leaves it alone afterward (remember the default +class is 3). Of course, the first ten space characters are removed. +--]] +local function verbatim (buffer) + for n, l in ipairs(buffer) do + buffer[n] = gsub(l, "%s%s%s%s%s%s%s%s%s%s","", 1) + end + table.insert(buffer, 1, [[\verbatim]]) + if gsub(buffer[#buffer], + interpreter.paragraph, "") == "" then + buffer[#buffer] = [[\verbatim/]] + else + table.insert(buffer, [[\verbatim/]]) + end + interpreter.protect() +end +add_pattern{ + pattern = "^%s%s%s%s%s%s%s%s%s%s", + call = verbatim, + class = 2 +} + +--[[ +And now comes the fun part. I wanted "i-doc.lua" to be self-describing. The +source of what you're reading right now isn't "interpreter-doc.txt", but +"i-doc.lua" itself input in the latter file with + + \intepreterfile{doc}{i-doc.lua} + +How should code and comment be organized in "i-doc.lua"? Well, there is little +choice, since the file is a normal Lua file: comment lines should be prefixed +with "--" or surrounded with +\tcode{--[{}[} and \tcode{--]{}]}. % Sorry for the braces, I can't nest Lua comments! +I chose the latter option, which is simpler. But normal code should also be +typeset as verbatim material; I could have begun all lines with ten spaces, +but that would have seemed strange. Instead, \tcode{--]{}]} is turned into +"\source" and "\source/" is added at the end of the paragraph ("\source" is +just "\verbatim" with a different layout). Which means all paragraphs have the +same structure: comments between +\tcode{--[{}[} and \tcode{--]{}]} +and code immediately following (\tcode{--[{}[} is simply removed). The pattern +is in class 1 and the paragraph is protected, so that lines indented with ten +spaces or more aren't touched by the previous verbatim pattern (in class 2). +--]] +local function autoverbatim (buffer, line) + buffer[line] = [[\source]] + for n = line + 1, #buffer do + interpreter.protect(n) + end + if gsub(buffer[#buffer], + interpreter.paragraph, "") == "" then + buffer[#buffer] = [[\source/]] + else + table.insert(buffer, [[\source/]]) + end +end +add_pattern{ + pattern = nomagic"%^--]]", + call = autoverbatim, + class = 1 +} +local function remove_comment () + return "" +end +add_pattern{ + pattern = nomagic"%^--[[", + replace = remove_comment +} diff --git a/macros/luatex/generic/interpreter/interpreter-doc.pdf b/macros/luatex/generic/interpreter/interpreter-doc.pdf Binary files differnew file mode 100644 index 0000000000..2dcc32ea48 --- /dev/null +++ b/macros/luatex/generic/interpreter/interpreter-doc.pdf diff --git a/macros/luatex/generic/interpreter/interpreter-doc.tex b/macros/luatex/generic/interpreter/interpreter-doc.tex new file mode 100644 index 0000000000..30a1da52ba --- /dev/null +++ b/macros/luatex/generic/interpreter/interpreter-doc.tex @@ -0,0 +1,115 @@ +% This is the master file producing interpreter-doc.pdf. The version of the +% documentation readable in a text editor is interpreter-doc.txt (input below). +% +% Paul Isambert - zappathustra AT free DOT fr - June 2012 + +\input pitex + +\overfullrule=0pt + +\OutputRoutine remove {headers}{shipout} + +\setparameter page : + hsize = 25pc + left = 60pt + width = "\dimexpr 25pc + 120pt\relax" + lines = 35 + height = 20cm + +\setparameter section : + font = \bf + link = true + number = none + numbercommand = \llap + beforeskip = 1 + +\setparameter navigator : + title = "Interpreter documentation" + author = "Paul Isambert" + mode = outlines + +\setfont\mainfont: + name = "Chaparral Pro" + bold = Semibold + big = 18pt + +\setfont\codefont: + name = "Lucida Console" + slant = 15 + bold italic = none + size = 8pt + features = "-tlig, -trep, space = mono" + +\parfillskip=0pt plus 1fill +\def\describe#1#2#3{% + \unless\ifdim\lastskip=\baselineskip + \vskip\baselineskip + \fi + \needspace{2\baselineskip}% + \noindent\color{.8 0 0}{% + {\outline{#3}{\directlua{% + local t = string.gsub("\luaescapestring{#1}", "[ (].*", "") + tex.print(t)}}% + \codefont#1}% + \reverse\iffemptystring{#2} + {\kern1em \hfil\penalty0\hbox{\ital{(#2)}}}}% + \par + \removenextindent} + +\newverbatim\source{} + {\vskip\baselineskip + \hfuzz=1em + \codefont\parindent=0pt + \pdfcolorstack0 push {.8 0 0 rg} + \printverbatim + \pdfcolorstack0 pop + \vskip\baselineskip} + +% The verbatim facilities in PiTeX aren't gated yet, so I must rely on +% this horrible hack! +\directlua{ +function do_verbatim (name, exec) + if exec then + tex.print(pitex.verbatims[name]) + else + for n, l in ipairs(pitex.verbatims[name]) do + if n == \string#pitex.verbatims[name] then + tex.print("\noexpand\\penalty\noexpand\\widowpenalty") + end + tex.print(pitex.verbatims[name].regime, l) + if n == 1 and \string#pitex.verbatims[name] > 2 then + tex.print("\noexpand\\penalty\noexpand\\widowpenalty") + end + end + end +end +} + +\def\arg#1{{\codefont\char"2039 #1\char"203A}} +\pdfdef\ital#1{#1} +\pdfdef\verb`#1`{#1} + +% Not optimal, but hey, with all the "intepreter.core.classes" stuff... +\hyphenation{li-nes cla-sses} + +\input interpreter + +% Title +\vbox to 3\baselineskip{ +\hbox to \hsize{\big Interpreter\hfil\normalsize Paul Isambert} +\hbox to \hsize{v.1.2, June 2012 \hfil \tcode{zappathustra AT free DOT fr}} +\vfil +} + + + +% Bulk of the doc. +\interpretfile{doc}{interpreter-doc.txt} +\vskip0pt plus 1filll +\noindent +\bgroup\it +Typeset with Lua\TeX\ 0.71 in Chaparral Pro and Lucida Console +... nonetheless this documentation looks dull, I don't know why. +\egroup + +\bye diff --git a/macros/luatex/generic/interpreter/interpreter-doc.txt b/macros/luatex/generic/interpreter/interpreter-doc.txt new file mode 100644 index 0000000000..6257aa6733 --- /dev/null +++ b/macros/luatex/generic/interpreter/interpreter-doc.txt @@ -0,0 +1,732 @@ + TABLE OF CONTENTS + (Make a search on the tag in the right column to jump + to the associated section. + Vim users can simply type * on the tag; + Emacs users do that with C-s C-w (I think); + Other editors: I don't know!) +=========================================================== + Introduction intro_tag + Input files input_tag + Paragraphs paragraphs_tag + Declaring patterns patterns_tag + Classes classes_tag + Protecting input protect_tag + Technical stuff technical_tag + An example: i-doc.lua example_tag + The Gates in Interpreter gates_tag + The interpreter table interpreter_tag + The interpreter.core.tools table interpreter_tools_tag + The interpreter.core.reader table interpreter_reader_tag +=========================================================== + + + +================================== intro_tag +=== Introduction ================= +================================== + +Interpreter preprocesses input files before their contents is fed to TeX. It +is meant to write document with whatever markup one wishes to define while +using normal TeX macros in the background. As a simple example, suppose you +have a macro "\bold" to put text in boldface; then Interpreter lets you map +"*text*", or "<strong>text</strong>", or simply "!text", or anything else, to +"\bold{text}". Interpreter doesn't perform any trickery with active +characters; instead, it manipulates the strings representing the lines of +a file and search for patterns. + +There are two main advantages: first, TeX documents can be typeset with +a completely non-TeX syntax; second, if one uses some lightweight markup +language, the source file is much easier to read and might even be more useful +than the typeset PDF file, e.g. for some technical documentation you want to +read directly in your text editor while writing code (powerful editors +generally have their own documentation in such a format, for a good reason). +A third advantage, not explored in this documentation, is that while feeding +modified lines to TeX you can also translate the original lines into, say, +HTML, and write them to an external file, thus creating both PDF and HTML +output at once. + +Interpreter has been rewritten with the Gates package (actually, only +the Lua side) in version 1.1. That hasn't changed anything to its default +behavior, but now it can also be customized quite deeply, since its +code is a collection of small chunks with names that can be externally +controlled and/or augmented. See the Gates documentation for further +information. The last sections of this documentation describe the gates +in Interpreter. + + + +================================== input_tag +=== Input files ================== +================================== + +Once Interpreter is loaded with + + \input interpreter + +in plain TeX or + + \usepackage{interpreter} + +in LaTeX, files to be processed are input as follows: + + \interpretfile{<language>}{<file>} + +There should exist a file "i-<language>.lua" containing the language used in +<file>. For instance, the source of this documentation is +"interpreter-doc.txt", input in the master file "interpreter-doc.tex" with + + \interpretfile{doc}{interpreter-doc.txt} + +and the interpretation to be used is defined in "i-doc.lua". The contents of +such an interpretation file is the object of the rest of this documentation. + + + +================================== paragraphs_tag +=== Paragraphs =================== +================================== + +Interpreter doesn't process lines one by one. Instead, it gathers an entire +paragraph and then processes the lines. It is important because you can +manipulate an entire paragraph when a given pattern is detected, and modify +several lines according to what happens in only one. A paragraph in +Interpreter has nothing to do with what TeX considers a paragraph; instead, it +is defined by the following string. + +> interpreter.paragraph [Default: blank line with spaces ignored] + A string to be interpreted as a paragraph boundary when Interpreter collects + lines before processing them. The string actually represents a pattern, so + magic characters are obeyed. The default is "%s*", i.e. a blank line is + considered a paragraph boundary, spaces notwithstanding. Of course, the end + of the file itself is a paragraph boundary. + + + +================================== patterns_tag +=== Declaring patterns =========== +================================== + +Once the lines of a paragraph have been collected, Interpreter searches them +trying to match declared patterns, but it doesn't do so indiscriminately: +patterns are searched in a given order, as explained below. + +Patterns are searched for in each line only, i.e. no match can occur across +lines. However, since you can manipulate entire paragraphs based on a match in +one line, the limitation easily vanishes. + +> interpreter.add_pattern(<table>) + This is the basic function used to defined patterns. The <table> may + contain the following entries, along other entries Interpreter won't use + but which can be useful to you, especially with "call" below. The function + returns a table. + +>> class [Default: "intepreter.default_class"] + The class of the pattern. See the section on classes. + +>> pattern + The pattern to match. Lua's magic characters are in force and should be + escaped with "%" if necessary, unless "nomagic" is "true" (or the pattern + itself is the result of "interpreter.nomagic"). + +>> nomagic [Default: "false"] + A boolean deciding whether the pattern should be transformed with + "interpreter.nomagic". + +>> replace + The replacement for the pattern, applied only if there is no "call" entry. + This may be a string, a table or a function. Interpreter simply executes + something similar to "string.gsub()", hence the replacement follows this + function's ordinary syntax. More precisely, if "replace" is a string, the + pattern is replaced with it; in this string, "%n" may be used to denote the + _n_th capture in the pattern. If "replace" is a table, the first capture or + the entire match (if there is no capture) is used as the key, and the + associated value is used as the replacement. If "replace" is a function, it + is called with the captures passed as arguments, or the entire match if + there is no capture. For instance, the following pattern will replace all + "*text*" with "\bold{text}": + + interpreter.add_pattern{ + pattern = "%*(.-)%*", + replace = [[\bold{%1}]] + } + +>> offset [Default: 0] + The number of positions Interpreter should shift to the right after a match + has occurred. Normally, Interpreter starts searching for another occurrence + of the current pattern at the same position where it found the last one. + However, loops might easily occur: the replacement for a pattern may very + well contain another match for the same pattern, so Interpreter will get + stuck. Suppose for instance you want to replace "TeX" with "\TeX". The + first match will do that, but then Interpreter will start searching again + at the backslash, producing "\\TeX", then "\\\TeX", etc. In this case, if + you set "offset" to 2 in the pattern, then search will start again at the + "e" and no new match will occur. + +>> call + This entry shall contain a function to be called if there is a match (if + this entry exists, "replace" isn't applied). It is meant to perform complex + tasks that aren't amenable to simple string replacement. The function will + be executed as follows: + + function (paragraph, line, index, pattern) + + "paragraph" is a table representing the current paragraph; lines are stored + at successive indices. The last line of this paragraph is always the + paragraph boundary (see "interpreter.paragraph"), unless the paragraph + stopped at the end of the file. The second argument, "line", is a number + representing the index in "paragraph" containing the line where the pattern + was found; "index" is the position in this line where the match occurred. + Finally, "pattern" is the entire table declared with + "interpreter.add_pattern" and containing all the entries discussed here. + + The function may return zero, one, or two numbers. If it returns none, the + search for the next occurrence of the pattern will start again on the same + line (rather, on the line with the same position in the paragraph), at + "index". If it returns one number, the search will resume at the same line + but at position _n_, with _n_ the returned number. Finally, if two numbers + are returned, the search will resume at line _m_ at position _n_, _m_ and + _n_ being the returned values. Specifying which line should be examined + when the search resumes might be necessary if the function adds new lines + in the paragraph _before_ the current line, since Interpreter only keeps + count of line numbers. + + The entire paragraph can thus be modified if necessary. For instance, + suppose you want to declare comments in your source file with only + "!Comment" in the first line, i.e. TeX should ignore a paragraph such as: + + !Comment + This should be ignored + by TeX + + Then the following pattern will do (where the function requires only the + first argument): + + local function comment (paragraph) + for n, l in ipairs(paragraph) do + paragraph[n] = "%" .. l + end + end + interpreter.add_pattern{ + pattern = "^!Comment", + call = comment + } + +> interpreter.nomagic (string) + A function which reverses the usual Lua magic for patterns: ordinary magic + characters are normal characters here, unless they are prefixed with "%", in + which case they are magic again. For instance, a pattern like ".+" is + normally interpreted as ``one or more characters''. If passed to this + function, a pattern is returned meaning ``a dot followed by a plus sign''. + On the contrary, "%.%+" normally has the second interpretation, while with + "interpreter.nomagic" it has the first one. The function makes another + transformation: "..." is used to denote a capture "(.-)". Thus + "interpreter.nomagic('*...*')" returns a pattern matching any number of + characters surrounded by stars and capturing those characters; this would be + expressed in ordinary Lua magic as "%*(.-)%*". + + + +================================== classes_tag +=== Classes ====================== +================================== + +As already alluded to, the search for patterns isn't done at random. Instead, +patterns are organized in classes, which are applied one after the other. More +precisely, the process is as follows: Interpreter searches the entire +paragraph for the first pattern in class~1, then for the second pattern in the +same class, then for the third, etc., then when there is no pattern left in +class~1 it does the same with class~2, up to class~_n_, where _n_ is the +highest class number such that there exists a class _n - 1_ (in other words, +classes should be numbered consecutively). Finally, the same goes for the +patterns in class~0 (which always exists, even if it contains no pattern). + +Inside a class, patterns are ordered by length from long to short, or +alphabetically if two patterns have the same length. This means that if you +use e.g. "/text/" for italics and "//text//" for bold, you don't need to put +the second pattern in a class before the first to avoid "//text//" being +interpreted as two empty arguments in italics surrounding a text in roman. +Since the way the bold-pattern will be declared, e.g. "//(.-)//", is probably +longer than for the italic-pattern, e.g. "/(.-)/", it will always match first. + +That said, the sorting isn't very clever and simply relies on the number of +symbols, no matter what they mean; in the patterns above, the parentheses +denote a capture but they still count in the pattern's length as understood by +Interpreter. Alternatively, while ".*" denotes ``zero or more character'' and +"%+" means ``a plus sign'' ("+" being magic, you have to escape it to refer to +it), in Interpreter's eye the two patterns have the same length: two. Finally, +one should be aware that patterns declared with a "nomagic" entry set to +"true" are sorted after they've been transformed (so that their real length +might not be obvious). So classes are needed when patterns need a proper +ordering no matter their lengths. For instance, some patterns should always be +declared first, as they protect input from Interpreter (see next section), +while others might need to be declared last, as they rely on what previous +patterns might have done. Besides, classes are metatables for the patterns +they contain. + +> interpreter.default_class [Default: 1] + All patterns belong to a class, even though you may omit the "class" entry + when declaring one. In this case, the pattern is assigned to the class + denoted by this number. + +> interpreter.set_class(number, table) + Defines class "number" as "table". Classes don't need to be defined + beforehand for patterns to be added to them (rather, Interpreter defines + them implicitly when needed). However, classes are also metatables for the + patterns, so that if there lacks an entry in a pattern's table, the class's + entry is used if it exists. The function returns a table. + + + +================================== protect_tag +=== Protecting input ============= +================================== + +Sometimes you want Interpreter to refrain from interpreting; that is most +useful for verbatim code, for instance. There are various ways to do that. + +> interpreter.active [Default: true] + A boolean switching Interpreter on and off. Beware, the switching applies + only starting at the next paragraph. + +> interpreter.protect([line]) + A function protecting all or part of the current paragraph. If "line" is + given, it should be a number _n_, and line _n_ in the current paragraph will + be protected; without "line", the entire paragraph is protected. Protecting + means that the patterns not yet searched for will be ignored. For instance, + if you want material to be read verbatim when surrounded with "<code>" and + "</code>", you can declare a pattern as follows: + + local function verbatim (buffer) + buffer[1] = "\\verbatim" + buffer[#buffer - 1] = "\\endverbatim" + intepreter.protect() + end + interpreter.add_pattern{ + pattern = "^%s*<code>%*s$", + call = verbatim, + class = 1 + } + + This code is extremely simplified : it assumes that "<code>" and "</code>" + starts and ends the paragraph and that "</code>" isn't the last line of the + file (otherwise it'd also be the last line in the paragraph, whereas here + the last one is the paragraph boundary). An important point is that the + pattern belongs to the first class, so it is called before all other + patterns (provided there is no shorter pattern in class~1) and prevents them + from doing anything, since the entire paragraph is protected. (Typesetting + the material as verbatim material obviously depends on the "\verbatim" + macro, not on Interpreter.) + +> interpreter.escape + A character which prevents patterns from being replaced if immediately + preceded by it. As an example, if "interpreter.escape = '_'", and "*text*" + denotes italic, then "*text*" will produce _text_ while "_*text*" will + produce *text*. Once a paragraph has been processed, Interpreter removes all + escape characters. Only one character can be an escape character. + +> interpreter.protector(left[, right]) ["right" defaults to "left"] + Defines two characters to protect what they surround. In other words, + Interpreter replaces patterns only if the match isn't found between "left" + and "right". Unlike the escape character, you can define as many protectors + as you wish; and unlike the escape character again, Interpreter _doesn't_ + remove them once the paragraph has been processed, so you must take care of + them. For instance: + + intepreter.protector('"') + interpreter.add_pattern{ + pattern = '"(.-)"', + replace = '\\verb`%1`', + class = 0 + } + + Anything between double quotes will be left untouched; then, when the + paragraph has been processed for all other classes, a pattern in class~0 + calls the "\verb" command to take care of the argument. Note that the + protectors should enclose what they protect without coinciding with it; this + is not the case here, which is why the pattern is applied. + +> interpreter.direct [Default: two percent signs then "I" and at least one space] + A string, actually a pattern, signalling that the line which it begins + should be processed as Lua code. The default is "%%%%I%s+", i.e. "%%I" + followed by at least one space. The pattern shouldn't declare itself as + attached to the beginning of the line (as in "^%%%%I%s+") because they will + be matched at the beginning of the line only anyway. The line is processed + with the "loadstring" function, and then turned into an empty line. For + instance: + + %%I interpreter.active = false + This won't be interpreted... + %%I interpreter.active = true + + As this example shows, lines flagged with "interpreter.direct" don't obey + "interpreter.active" and are always processed as described above. + + + +================================== technical_tag +=== Technical stuff ============== +================================== + +You don't have to bother with this section if you don't mind how Interpreter +does its job; actually you won't learn much anyway. + +> interpreter.reset() + A function which resets everything to default and deletes classes. It is used + when calling "\interpretefile" so that new interpretetions start from zero. + +> interpreter.register(function) + A function called to put Interpreter's main function into the + "post_linebreak_filter" callback; you can redefine it at will. If it is + undefined, "callback.register()" is used, unless "luatexbase.add_to_callback()" + is detected. (The detection takes place at the first call to + "\interpretfile", so there is no need to load Interpreter after + "luatexbase".) + +> interpreter.unregister(function) + A function called to remove Interpreter's main function from the + "post_linebreak_filter" callback. It works similarly to the previous one. + + + +================================== example_tag +=== An example: i-doc.lua ======== +================================== + +\interpretfile{doc}{i-doc.lua} + + + +================================== gates_tag +=== The Gates in Interpreter ===== +================================== + +Interpreter is written with the Gates package (only the Lua side, +actually). It means that it can be hacked down to the core. Here I'll +simply list the gates involved; you should read the Gates +documentation to learn how to use them. + +There are three gates families: "interpreter", associated with the main +"interpreter" table, contains the user interface; "interpreter_tools", +associated with "interpreter.core.tools" table, contains internal +functions; finally "interpreter_reader", associated with the +"interpreter.core.reader" table, contains the main functions used to +read the file. + +Whenever I mention a conditional or a loop, I mean the local conditionals +and loops, relative to the l-gate where the gate appears. Also, the +syntax indicates the arguments a gate uses, not all the arguments that +are passed to it (which are simply what the previous gate has returned). + +As an example of customizing Interpreter with Gates, you could very +well add a bit of code which does something to all lines. Inserting a +small gate, say "everyline", after "check_direct" in "aggregate_lines" +below would do the trick, e.g.: + + + function interpreter.core.reader.everyline (file, line) + line = dosomething(line) + return file, line + end + interpreter.core.reader.add( + "everyline", "aggregate_lines", "after check_direct") + interpreter.core.reader.conditional( + "everyline", "aggregate_lines", function (f, l) return l end) + +(Note that it is important to check that the line really exist, +because one might have hit the end of the file; hence the conditional, +as with others gates in "aggregate_lines"). + + + + +================================== interpreter_tag +=== The "interpreter" table ======== +================================== + +All the user functions in "interpreter" are simple m-gates, so they can +be treated as ordinary functions, except "interpreter.add_pattern", +which is an l-gate containing, built as: + + add_pattern + . ensure_class + . apply_nomagic + . insert-pattern + . . do_insert + . . sort_class + +> ensure_class (<pattern>) [m-gate] + Creates the class of <pattern> if necessary, and set it as the metatable + for <pattern>. Classes themselves are kept in the "interpreter.core.classes" + table. The gate return <pattern> and the class number. + +> apply_nomagic (<pattern>, <class>) [m-gate] + Transforms the "pattern" entry in <pattern> with "intepreter.nomagic"; + tied to a conditional that returns <pattern>'s "nomagic" entry (so + the gate is executed only if "nomagic" is true); autoreturns both + arguments. + +> insert_pattern (<pattern>, <class>) [l-gate] + An autoreturning l-gate containing the following two gates. + +> do_insert (<pattern>, <class>) [m-gate] + Adds <pattern> to <class> (i.e. "interpreter.core.classes[<class>]"). + +> sort_class (<pattern>, <class>) [m-gate] + Sorts <class> with function "interpreter.core.tools.sort". This gate + can be skipped to apply the patterns in the order in which they were + declared. + + + +============================================= interpreter_tools_tag +=== The interpreter.core.tools table ======== +============================================= + +All the functions in the "interpreter.core.tools" all are simple m-gates. + +> sort (<patt1>, <patt2>) [m-gate] + Returns true if the pattern in "patt1" is longer than the one in + "patt2", or if they have the same length and the first ranks before + the second with respect to alphabetical order. The gate is used in + the "interpreter.sort_class" m-gate. + +> xsub (<string>, <index>, <pattern>, <replacement>) [m-gate] + Returns <string> with <pattern> replaced with <replacement>, but only + once, and only after <index>. + +> protector (<string>, <index>) [m-gate] + Checks whether <index> in <string> isn't between characters declared + with "interpreter.protector". If that is the case, the function returns + "nil" and the index of the second protector. Otherwise, it returns + <index>. + +> get_index (<string>, <pattern>, <index>) [m-gate] + Checks whether <pattern> occurs in <string>, starting at <index>. If + it does, but if <index>-1 is "interpreter.escape", calls itself with + <index>+1. Otherwise, calls "interpreter.core.tools.protector" to + check whether <index> is in a protected part of the string. If so, + calls itself with <right>+1 instead of <index>, where <right> is the + second return value of of "interpreter.core.tools.protector", i.e. + it searches again after the right protector. If <index> is found, + end of story, returns <index>, otherwise returns nothing. + +The "interpreter.core.tools" table also contains +"magic_characters", a table with an entry for each magic character in Lua except +`"."' and `"%"'; the values to those entries are the same characters +prefixed with `"%"'. The table is used by "interpreter.nomagic" to spot +and replace magic characters, with the dot and the percent sign dealt +with independantly. + + + +============================================= interpreter_reader_tag +=== The interpreter.core.reader table ======= +============================================= + +Interpreter works by hooking in the "open_read_file" callback; the +function registered there is the "interpreter.core.reader.input" +l-gate, built as follows: + + input + . unregister + . . set_unregister + . . use_unregister + . open_file + . set_reader + +> unregister (<filename>) [l-gate] + Contains the following two m-gates; <filename> is received from + "input", which itself receives it from the callback, i.e. that's the + file that's being input (the second argument to "\interpretfile"). + It is also automatically returned. + +> set_unregister () [m-gate] + Sets the function to remove "input" from the callback, namely + "interpreter.unregister"; the gate is called only if gate + "interpreter.unregister" doesn't already exits. If "luatexbase" is + detected, the functions there are used; otherwise, "callback.register" + is used with "nil" as the second argument. + +> use_unregister () [m-gate] + Calls "interpreter.unregister()". (You don't want the next input file + to be processed with Interpreter by default, that's why you remove + the callback function; not that the current one is nonetheless + processed with the current file, of course.) + +> open_file (<filename>) [m-gate] + Returns "io.open(<filename>)". + +> set_reader (<file>) [m-gate] + Returns a table with a "reader" entry containing a function whose + definition is + + function () + return interpreter.core.reader.read_file(f) + end + + That's the convention for the "open_read_file" callback: it should + return such a table, and the function will be called each time a + line is required from the input file. + +So most of the work is done by "interpreter.core.reader.read_file", +which is why it is so heavy; it receives a file handle: + + read_file + . make_paragraph + . . aggregate_lines + . . . read_line + . . . check_direct + . . . insert_line + . . apply_classes + . . . pass_class + . . . . pass_pattern + . . . . . process_lines + . . . . . . switch + . . . . . . call + . . . . . . replace + . . . . . . protect + . . . unprotect + . . . . undo_protected + . . . . unprotect_lines + . . . remove_escape + . return_line + +> make_paragraph (<file>) [l-gate] + The big l-gate that contains everything that follows, barring + "return_line". It is called if and only if the "interpreter.core.lines" + table is empty; that table is where lines of a paragraph are stored, + and it is emptied by "return_line". + +> aggregate_lines (<file>, <line>) [l-gate] + The main l-gate that reads line from <file> and stores them in + "interpreter.core.lines". It loops until <line> is "nil" or equivalent + to "interpreter.paragraph". (Of course, <line> is "nil" on the first + iteration, but the "loopuntil" conditional is evaluated after that + first iteration, during which the last subgate "insert_line" will + probably returns a line.) + +> read_line (<file>) [m-gate] + Reads the next line from <file>, and returns <file> and that line. + +> check_direct (<file>, <line>) [m-gate] + If <line> begins with "interpreter.direct", removes it and use + "loadstring" on the resulting string. Returns <file> and <line>, the + latter set to an empty string is the previous operation applied. The + gate depends on a conditional: <line> should be non-"nil" (of + course), and "interpreter.direct" should be defined. + +> insert_line (<file>, <line>) [m-gate] + Adds <line> to "interpreter.core.lines". Automatically returns the + two arguments (and if <line> isn't "nil" or equivalent to + "interpreter.paragraph", it will be executed again). + +> apply_classes () [l-gate] + The l-gate that applies transformations to the lines, once the paragraph + has been gathered, with the gates that follow. For each class, it + will apply each pattern on each line. It depends on a conditional: + "interpreter.core.lines" shouldn't be empty, and "interpreter.active" + should be true. + +> pass_class () [l-gate] + This gate iterates on all classes in "interpreter.core.classes" and + then on class 0. On each iteration it checks beforehand whether the + paragraph is protected, i.e. "interpreter.core.reader.protected" isn't + a boolean (see "unprotecte" below). On each iteration, the class + number and the class itself are returned. (This behavior is + implemented with a Gates iterator.) + +> pass_pattern (<ignored>, <class>) [l-gate] + Same as "pass_class", except it iterates on the patterns in "class": + it is executed as long as "interpreter.core.reader.protected" and + returns the patter number and the pattern itself. (The <ignored> + argument isn't used; it is for the "pass_class" iterator; the same + holds for the following gates.) + +> process_lines (<ignored>, <pattern>) [l-gate] + Again, this calls an iterator. It browses each line in "interpreter.core.lines" + and returns the line's number (provided it is valid, i.e. not a table, + see "protect" below), <pattern> and the current index in that line. + To keep track of the current line and index, two internal numbers are + used: "interpreter.core.reader.current_line" and + "interpreter.core.reader.current_index". + +> switch (<line>, <pattern>, <index>) [m-gate] + If <pattern> has a "call" entry, it sets the "call" gate below to + "ajar"; otherwise, if <pattern> has a "replace" entry, it sets the + "replace" gate to "ajar". + +> call (<line>, <pattern>, <index>) [m-gate] + This gate is closed by default and set to "ajar" by "switch" above. + If, starting at <index>, the <pattern>'s "pattern" entry can be + found in <line> with "interpreter.core.tools.get_index" (which makes + sure that protectors are obeyed and returns <newindex>, where the + pattern is found if it occurs), the <pattern>'s "call" entry is + applied as + + <pattern>.call(interpreter.core.lines, + <line>, <newindex>, <pattern>) + + This may returned zero, one or two values. If nothing is returned, + "interpreter.core.reader.current_index" is set to 0, which makes + the "process_lines" iterator consider the next line. If one value is + returned, it is the new current index and "process_lines" will not + update the line number. If two values are returned, the first is the + new current line number and the second the new current index. + +> replace (<line>, <pattern>, <index>) [m-gate] + This gate is closed by default and set to "ajar" by "switch" above. + This tries to find the <pattern>'s "pattern" like "call" above, and if + it is found, it applies "interpreter.core.tools.xsub" as: + + xsub (interpreter.core.lines[<line>], + <newindex>, <pattern>.pattern, <pattern>.replace) + + where <newindex> is defined as in "call". The return value of "xsub" + is assigned to "interpreter.core.lines[<line>]", and the current + index is set to <index> plus the <pattern>'s "offset" if any. If the + pattern wasn't found, the current index is set to 0, which makes + "process_lines" turn to the next line as explained in "call". + +> protect () [m-gate] + The "interpreter.protect()" function can either protect the whole + paragraph (when no argument is passed) or a single line (when a number + is passed). In the first case, "interpreter.core.tools.protected" + takes the value "true", which is checked in various gates above. In + the second case, "interpreter.core.tools.protected" is a table with + each index indicating a line to be protected. This gate implements + the protection in that case: it iterates on all entries in the table + with "pairs" and protects the line with the same index in + "interpreter.core.lines" by transforming it into a table (with a + single entry, the string representing the original line); the type + of the line is checked in the "process_lines" iterator above. The + gate's iterator doesn't take arguments, but the function itself is + defined as taking a number (the line). + +> unprotect () [l-gate] + Now all the patterns in all the classes have been applied to the + entire paragraph, and protection must be removed. This l-gate + contains the following two gates. + +> undo_protected () [m-gate] + Simply sets "interpreter.core.tools.protected" to "nil" so it is + ready for the next paragraph. + +> unprotect_lines () [m-gate] + Restores all the lines in "interpreter.core.lines" as simple strings. + The gate uses an iterator that simply runs "ipairs" on "interpreter.core.lines", + so the function's definition actually takes the line's number and + the line itself as arguments. + +> remove_escape () [m-gate] + If "interpreter.escape" is defined, removes all its occurrences in + each line of the paragraph. + +This is the end of the big "make_paragraph" l-gate. It won't be called +again until the paragraph has been fully passed to TeX, i.e. when +"interpreter.core.lines" is empty. + +> return_line () [m-gate] + Pops the first line from "interpreter.core.lines" and returns it. + Since this is the very last subgate of "read_file", the line is + passed to TeX. + diff --git a/macros/luatex/generic/interpreter/interpreter.lua b/macros/luatex/generic/interpreter/interpreter.lua new file mode 100644 index 0000000000..05de6eb791 --- /dev/null +++ b/macros/luatex/generic/interpreter/interpreter.lua @@ -0,0 +1,450 @@ +-- This is the main Lua file for the Interpreter package. +-- Further information in interpreter-doc.pdf or interpreter-doc.txt. +-- Paul Isambert - zappathustra AT free DOT fr - June 2012 +-- +-- Beware, this is written with Gates. Please read the Gates doc if +-- you want to understand something. + +local find, gsub, match, sub = string.find, string.gsub, string.match, string.sub +local insert, sort, remove = table.insert, table.sort, table.remove +local io_open = io.open +local ipairs, pairs, type = ipairs, pairs, type +require("gates.lua") +if not gates.iterator then + tex.error("Interpreter error: Your version of Gates should be at least v.0.2. I quit. Expect chaos") + return +end + +interpreter = gates.new("interpreter") + +-- *** interpreter.active *** +-- Following paragraphs (as defined by interpreter.paragraph) are interpreted +-- iff this is not set to false. +interpreter.active = true +-- *** interpreter.default_class *** +-- Sets the default class for patterns which are added without specifying the +-- class. Default 1. +interpreter.default_class = 1 + +interpreter.core = { + classes = {}, -- The classes of patterns. + lines = {}, -- The lines of the paragraph. + reader = gates.new("interpreter_reader"), -- The main processing functions. + tools = gates.new("interpreter_tools")} -- Auxiliary functions. + +-- Utility function sorting patterns by length (alphabetically if they are of +-- equal length). +function interpreter.core.tools.sort (a, b) + local a, b = a.pattern, b.pattern + return #a == #b and a < b or #a > #b +end + +-- *** interpreter.add_pattern (table) *** +-- Creates pattern <table>, which can contain the following entries: +-- pattern [string] = The pattern to match. Magic characters are obeyed! +-- replace [string] = The replacement for <pattern>. Can be a string, a +-- table or a function. A simple string.gsub() is +-- applied. +-- call [function] = The function applied to <pattern>; <replace> is applied +-- iff there is no <call>. +-- offset [number] = If <pattern> is used at index n, then the search on the +-- same line for the same pattern starts again at index n +-- + offset. Applied only when no <call> (in this case, +-- search starts again at the beginning of the line). By +-- default, offset = 0. This is needed to avoid infinite +-- loops with replacements which contain the pattern; +-- e.g. replacing "TeX" with "\TeX" will produce an +-- infinite loop, unless offset = 2. +-- nomagic [boolean] = Sets whether <replace> should be transformed with interpreter.nomagic. +-- class [number] = The pattern's <class> (classes of patterns are applied in +-- order, e.g. all patterns in class 1 are applied, then +-- all patterns in class 2, etc; class 0, however, is +-- always applied last). If <class> is not given, the +-- default_class number is used. Classes must be numbered +-- consecutively. +interpreter.list{"add_pattern", + {"ensure_class", + function (tb) + local class = tb.class or interpreter.default_class + interpreter.set_class(class, {}) + setmetatable(tb, interpreter.core.classes[class].meta) + return tb, class + end}, + {"apply_nomagic", conditional = function (tb) return tb.nomagic end, + autoreturn = true, + function (tb, class) + tb.pattern = interpreter.nomagic(tb.pattern) + end}, + {"insert_pattern", autoreturn = true, + {"do_insert", autoreturn = true, + function (tb, class) + insert(interpreter.core.classes[class], tb) + end}, + {"sort_class", autoreturn = true, + function (tb, class) + sort(interpreter.core.classes[class], interpreter.core.tools.sort) + end}}} + +-- *** interpreter.set_class (number, table) *** +-- Sets default values (of the table normally specified in add_pattern) for +-- patterns of class <number>; patterns added to this class can still specify +-- different values, which will override defaults. In other words, this is a +-- metatable for patterns (which are tables) of that class. +function interpreter.set_class (num, tb) + interpreter.core.classes[num] = interpreter.core.classes[num] or + { meta = { __index = function (_, k) return interpreter.core.classes[num].meta[k] end } } + for a, b in pairs(tb) do + interpreter.core.classes[num].meta[a] = b + end + return interpreter.core.classes[num] +end + +-- Class 0 must exist since it is always used at the end of the paragraph. +interpreter.set_class(0, {}) + +-- *** interpreter.nomagic (string) *** +-- Turns a normal string into a string with magic characters escaped, so it +-- can be used as a pattern. +interpreter.core.tools.magic_characters = { + ["^"] = "%^", + ["$"] = "%$", + ["("] = "%(", + [")"] = "%)", + ["%"] = "%%", + ["."] = "%.", + ["["] = "%[", + ["]"] = "%]", + ["*"] = "%*", + ["+"] = "%+", + ["-"] = "%-", + ["?"] = "%?", +} +function interpreter.nomagic (str) + local i, s = 1, "" + local magic_characters = interpreter.core.tools.magic_characters + while i <= #str do + local c, c2, c3 = sub(str, i, i), sub(str, i + 1, i + 1), sub(str, i + 2, i + 2) + i = i + 1 + if c == "%" and magic_characters[c2] then + s = s .. c2 + i = i + 1 + elseif c == "." and c2 == "." and c3 == "." then + s = s .. "(.-)" + i = i + 2 + elseif magic_characters[c] then + s = s .. "%" .. c + else + s = s .. c + end + end + return s +end + +-- *** interpreter.protect ([spec]) *** +-- Protects a set of lines in a paragraph; a protected line won't be +-- interpreted. If <spec> is a number, this protects line <spec> in the current +-- paragraph; if <spec> is true, this protects the entire current paragraph. Of +-- course, patterns that were applied to the line(s) or paragraph before +-- protection happened aren't undone. +function interpreter.protect (num) + if type(num) == "number" then + if type(interpreter.core.reader.protected) ~= "boolean" then + interpreter.core.reader.protected = interpreter.core.reader.protected or {} + interpreter.core.reader.protected[num] = true + end + else + interpreter.core.reader.protected = true + end +end + +-- Utility function making a replacement in a string but only from a certain +-- position and only once. We can't let gsub unrestricted, because some +-- part(s) of the string might be protected. +function interpreter.core.tools.xsub (str, num, patt, rep) + return sub(str, 1, num-1) .. gsub(sub(str, num), patt, rep, 1) +end + +-- *** interpreter.protector (left [, right]) *** +-- Sets <left> and <right> (set to <left> if missing) as protectors, i.e. +-- enclosed material won't be processed even if the line is processed +-- otherwise. For instance: after interpreter.protector ("|"), the word +-- "little" in +-- +-- Hello, |little| world! +-- +-- will be left untouched; Interpreter is terribly smart (thanks to lpeg), so +-- in "|a| b |c|", "b" isn't protected, as intended, because the "|" on its +-- left doesn't match the one on its right but with the one before "a". An +-- example with <right> specified: interpreter.protector("[", "]") and +-- then: +-- +-- Hello, [little] world! +-- +-- achieves the same as above. Protectors AREN'T removed when the line is +-- finally passed to TeX; and there can be several protectors. Compare with +-- interpreter.escape. +local P, Cf, Cg, Cp, Ct, V = lpeg.P, lpeg.Cf, lpeg.Cg, lpeg.Cp, lpeg.Ct, lpeg.V +local _grammar +function interpreter.core.tools.protector (str, index) + local protections = Cf(Ct("") * Cg{ _grammar + 1 * V(1) }^1, rawset) + protections = protections:match(str) + if protections then + for a, b in pairs(protections) do + if index > a and index < b then + return nil, b + end + end + end + return index +end +function interpreter.protector (left, right) + right = right or left + local gram = P(Cp() * P(left) * (1 - P(right))^0 * Cp() * P(right)) + if _grammar then + _grammar = _grammar + gram + else + _grammar = gram + end +end + +-- *** interpreter.escape *** +-- A string used as an escape character: if a pattern matches, it is processed +-- iff the character immediately to its left isn't <escape>. The escape +-- character IS removed once the lines have been processed, so TeX never sees +-- it; also, only one escape character is allowed, and itself can't be escaped +-- (i.e. it doesn't mean anything to try to escape it). E.g.: +-- +-- interpreter.escape = "|" +-- ... this won't be |*processed* +-- +-- Assuming you have a pattern with stars, here it won't be applied. Instead +-- "this won't be *processed*" will be passed to TeX (note that the escape +-- character has disappeared). + +function interpreter.core.tools.get_index (str, patt, index) + index = find(str, patt, index) + if index then + if sub(str, index-1, index-1) == interpreter.escape then + return interpreter.core.tools.get_index(str, patt, index + 1) + elseif _grammar then + local right + index, right = interpreter.core.tools.protector(str, index, patt) + return index or interpreter.core.tools.get_index(str, patt, right + 1) + else + return index + end + end +end + +-- *** interpreter.paragraph *** +-- The pattern that defines a line acting as a paragraph boundary, +-- prompting Interpreter to process the lines gathered up to now. Default is a +-- line composed of spaces at most. +interpreter.paragraph = "%s*" + +-- *** interpreter.direct (pattern) *** +-- Sets the pattern defining a line as direct Lua code: if a line begins with +-- <pattern> (which itself shouldn't contain the beginning-of-string character "^") +-- the code that follows is processed as Lua code, and the line is turned to +-- an empty string; note that this empty string will be seen as a paragraph +-- boundary if the line happened in the middle of a paragraph and +-- interpreter.paragraph has set paragraph boundary to empty string. Default +-- is "%%I " (two "%" followed by one "I" followed by at least one space +-- character). +interpreter.direct = "%%%%I%s+" + +-- At last, the function to be registered in open_read_file, defining the +-- function that reads a file. + +interpreter.core.reader.current_line = 0 +interpreter.core.reader.current_line = 0 + +interpreter.core.reader.list{"read_file", + {"make_paragraph", conditional = function () return #interpreter.core.lines == 0 end, + {"aggregate_lines", loopuntil = function (_, line) return not line or gsub(line, "^" .. interpreter.paragraph .. "$", "") == "" end, + {"read_line", + function (f) + return f, f:read() + end}, + {"check_direct", conditional = function (_, line) return line and interpreter.direct end, + function (f, line) + if match(line, "^" .. interpreter.direct) then + loadstring(gsub(line, "^" .. interpreter.direct, ""))() + line = "" + end + return f, line + end}, + {"insert_line", conditional = function (_, line) return line end, + autoreturn = true, + function (f, line) + insert(interpreter.core.lines, line) + end}}, + {"apply_classes", conditional = function () return #interpreter.core.lines > 0 and interpreter.active end, + {"pass_class", iterator = function () + local done_zero + local function f (t, i) + if type(interpreter.core.reader.protected) ~= "boolean" then + if not done_zero then + i = i+1 + local v = t[i] + if v then + return i, v + else + done_zero = true + return 0, t[0] + end + end + end + end + return f, interpreter.core.classes, 0 + end, + {"pass_pattern", iterator = function (_, class) + local function f (t, i) + if type(interpreter.core.reader.protected) ~= "boolean" then + i = i+1 + local v = t[i] + if v then + return i, v + end + end + end + return f, class, 0 + end, + {"process_lines", iterator = function (_, pattern) + interpreter.core.reader.current_line = 0 + interpreter.core.reader.current_index = 0 + return function () + if type(interpreter.core.reader.protected) ~= "boolean" then + local l = interpreter.core.reader.current_line + local i = interpreter.core.reader.current_index + if i == 0 then + l, i = l + 1, 1 + end + line = interpreter.core.lines[l] + -- When protected, a line is a table. + while type(line) == "table" do + l, i = l+1, 1 + line = interpreter.core.lines[l] + end + if line then + interpreter.core.reader.current_line = l + interpreter.core.reader.current_index = i + return l, pattern, i + end + end + end + end, + {"switch", autoreturn = true, + function (_, pattern) + if pattern.call then + interpreter.core.reader.ajar("call", "process_lines") + elseif pattern.replace then + interpreter.core.reader.ajar("replace", "process_lines") + end + end}, + {"call", status = "close", + function(i, pattern, ind) + local line = interpreter.core.lines[i] + local index = interpreter.core.tools.get_index(line, pattern.pattern, ind) + if index then + local L, O = pattern.call(interpreter.core.lines, i, index, pattern) + if O then + interpreter.core.reader.current_line = L + interpreter.core.reader.current_index = O + elseif L then + interpreter.core.reader.current_index = L + end + else + interpreter.core.reader.current_index = 0 + end + end}, + {"replace", status = "close", + function(i, pattern, ind) + local line = interpreter.core.lines[i] + local index = interpreter.core.tools.get_index(line, pattern.pattern, ind) + if index then + interpreter.core.lines[i] = interpreter.core.tools.xsub(line, index, pattern.pattern, pattern.replace) + interpreter.core.reader.current_index = index + (pattern.offset or 0) + else + interpreter.core.reader.current_index = 0 + end + end}, + {"protect", iterator = function () + if type(interpreter.core.reader.protected) == "table" then + return pairs(interpreter.core.reader.protected) + end + end, + function (n) + if type(interpreter.core.lines[n]) == "string" then + interpreter.core.lines[n] = {interpreter.core.lines[n]} + end + end}}}}, + {"unprotect", + {"undo_protected", + function () + interpreter.core.reader.protected = nil + end}, + {"unprotect_lines", iterator = function () return ipairs(interpreter.core.lines) end, + function (i, l) + if type(l) == "table" then + interpreter.core.lines[i] = l[1] + end + end}}, + {"remove_escape", conditional = function () return interpreter.escape end, + function () + for num, line in ipairs(interpreter.core.lines) do + interpreter.core.lines[num] = gsub(line, interpreter.escape, "") + end + end}}}, + {"return_line", + function () + return remove(interpreter.core.lines, 1) + end}} + +interpreter.core.reader.list{"input", + -- *** interpreter.unregister () *** + -- The function used to remove read_file from the "open_read_file" callback. + -- Uses callback.register by default, or luatexbase.remove_from_callback if + -- detected. + {"unregister", autoreturn = true, + {"set_unregister", conditional = function () return interpreter.type"unregister" == 0 end, + function () + if luatexbase and luatexbase.remove_from_callback then + function interpreter.unregister () + luatexbase.remove_from_callback("open_read_file", "interpreter") + end + else + function interpreter.unregister () + callback.register("open_read_file", nil) + end + end + end}, + {"use_unregister", + function () -- You can't use the `unregister' gate directly, because it isn't created yet. + interpreter.unregister() + end}}, + {"open_file", + function (fname) + return io_open(fname) + end}, + {"set_reader", + function (f) + return {reader = function () return interpreter.core.reader.read_file(f) end} + end}} + +function interpreter.reset () + interpreter.active = true + interpreter.default_class = 1 + interpreter.core.classes = {} + interpreter.set_class(0, {}) + _grammar = nil + interpreter.escape = nil + interpreter.paragraph = "%s*" + interpreter.direct = "%%%%I%s+" +end + +-- *** interpreter.register (function) *** +-- The function used to register the read_file function in the +-- "open_read_file" callback. If none is given, use callback.register, or +-- luatexbase.add_to_callback if detected (with "interpreter" as the name). +-- The function is defined in \interpretfile (see interpreter.tex). diff --git a/macros/luatex/generic/interpreter/interpreter.sty b/macros/luatex/generic/interpreter/interpreter.sty new file mode 100644 index 0000000000..ba16a86256 --- /dev/null +++ b/macros/luatex/generic/interpreter/interpreter.sty @@ -0,0 +1,17 @@ +% This is interpreter.sty, a style file to load +% the Interpreter package in LaTeX. Useful information +% can be found in interpreter-doc.pdf or interpreter-doc.txt. +% +% Author: Paul Isambert. +% E-mail: zappathustra AT free DOT fr +% Comments and suggestions are welcome. +% Date: June 2012. + +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{interpreter}[2012/06/10 v.1.2 Preprocessing input files on the fly.] +% Needed to prevent LaTeX check input, which would call open_read_file and +% thus remove Interpreter's main function (which is removed as soon as it is +% used). +\expandafter\let\expandafter\interpreterinput\csname @@input\endcsname +\input interpreter.tex +\endinput diff --git a/macros/luatex/generic/interpreter/interpreter.tex b/macros/luatex/generic/interpreter/interpreter.tex new file mode 100644 index 0000000000..3568fc90e9 --- /dev/null +++ b/macros/luatex/generic/interpreter/interpreter.tex @@ -0,0 +1,37 @@ +% This is the main TeX file for the Interpreter package. +% Further information in interpreter-doc.pdf or interpreter-doc.txt. +% +% Paul Isambert - zappathustra AT free DOT fr - June 2012 +% +\csname Oh no, Interpreter won't be loaded twice!\endcsname +\expandafter\let\csname Oh no, Interpreter won't be loaded twice!\endcsname\endinput +\directlua{require("interpreter.lua")} +\def\interpretergobble#1{} +\unless\ifdefined\interpreterinput + \let\interpreterinput\input +\fi +\def\interpretfile#1#2{% + \directlua{% + local f = kpse.find_file("i-#1.lua") + if f then + interpreter.reset() + dofile(f) + if interpreter.type"register" == 0 then + if luatexbase and luatexbase.add_to_callback then + function interpreter.register (f) + luatexbase.add_to_callback("open_read_file", f, "interpreter") + end + else + function interpreter.register (f) + callback.register("open_read_file", f) + end + end + end + interpreter.register(interpreter.core.reader.input) + tex.print([[\noexpand\interpreterinput]]) + else + tex.error("I can't find file `i-#1.lua'. I won't input file `#2'") + tex.print([[\noexpand\interpretergobble]]) + end + }{#2}% + } |