summaryrefslogtreecommitdiff
path: root/macros/luatex/generic/interpreter
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /macros/luatex/generic/interpreter
Initial commit
Diffstat (limited to 'macros/luatex/generic/interpreter')
-rw-r--r--macros/luatex/generic/interpreter/README29
-rw-r--r--macros/luatex/generic/interpreter/i-doc.lua254
-rw-r--r--macros/luatex/generic/interpreter/interpreter-doc.pdfbin0 -> 110003 bytes
-rw-r--r--macros/luatex/generic/interpreter/interpreter-doc.tex115
-rw-r--r--macros/luatex/generic/interpreter/interpreter-doc.txt732
-rw-r--r--macros/luatex/generic/interpreter/interpreter.lua450
-rw-r--r--macros/luatex/generic/interpreter/interpreter.sty17
-rw-r--r--macros/luatex/generic/interpreter/interpreter.tex37
8 files changed, 1634 insertions, 0 deletions
diff --git a/macros/luatex/generic/interpreter/README b/macros/luatex/generic/interpreter/README
new file mode 100644
index 0000000000..f3e1c56c52
--- /dev/null
+++ b/macros/luatex/generic/interpreter/README
@@ -0,0 +1,29 @@
+This is the README file for the Interpreter package.
+Author: Paul Isambert.
+E-mail: zappathustra AT free DOT fr
+Comments and suggestions are welcome.
+Date: June 2012.
+Version: 1.2.
+
+Interpreter preprocesses input files on the fly (no external program) and
+manipulates input lines, e.g. to turn some markup into proper TeX syntax.
+Interpreter doesn't work with ConTeXt.
+
+LuaTeX is required, and the Lua side of the Gates package (i.e. gates.lua),
+version at least 0.2.
+
+Relevant information can be found in interpreter-doc.pdf
+or interpreter-doc.txt (source of the doc readable in a text editor).
+
+The files in this distribution are:
+
+interpreter.lua - main code
+interpreter.tex - \input in plain TeX
+interpreter.sty - wrapper for LaTeX
+interpreter-doc.tex - master file for the doc
+interpreter-doc.txt - text of the doc
+interpreter-doc.pdf - typeset doc
+i-doc.lua - interpretation for the doc (because, of course, the doc
+ is typeset with Interpreter)
+
+Licensing of this package is covered by LPPL.
diff --git a/macros/luatex/generic/interpreter/i-doc.lua b/macros/luatex/generic/interpreter/i-doc.lua
new file mode 100644
index 0000000000..12bc214eda
--- /dev/null
+++ b/macros/luatex/generic/interpreter/i-doc.lua
@@ -0,0 +1,254 @@
+--[[
+Here's a description of "i-doc.lua", the file containing the interpretation
+used for Interpreter's documentation. Remember that none of the TeX macros
+used here is defined by Interpreter; instead, they are my own and should be
+adapted if necessary. Also several options taken here are far from optimal but
+are convenient examples.
+
+Shorthands for often used functions.
+--]]
+local gsub, match = string.gsub, string.match
+local add_pattern = interpreter.add_pattern
+local nomagic = interpreter.nomagic
+
+--[[
+Class 1 and 2 will be used for verbatim (thus protecting) and ``normal''
+patterns go into class 3 or higher.
+--]]
+interpreter.default_class = 3
+
+--[[
+The reader might have observed that "interpreter-doc.txt" begins with a table
+of contents. This table is useful for the source file only, and isn't typeset
+by TeX, because the following pattern suppresses it: the entire paragraph
+containing "TABLE OF CONTENTS" on a line of its own is deleted. Protecting the
+paragraph is useless, but it makes things a little bit faster because the
+paragraph won't be pointlessly searched for other patterns.
+--]]
+local function contents (buffer)
+ for n in ipairs(buffer) do
+ buffer[n] = ""
+ end
+ interpreter.protect()
+end
+add_pattern{
+ pattern = "^%s*TABLE OF CONTENTS%s*$",
+ call = contents,
+ class = 1
+}
+
+--[[
+Sections headers are typeset as
+
+ ====================================== section_tag
+ === Section title ====================
+ ======================================
+
+The first and third line are decorations and they are removed. The
+"section_tag" is meant for the source only again (linking the section to the table
+of contents). I could have used it to create PDF destinations, but that seemed
+unnecessary in such a small file. The associated pattern is: at least four
+equals signs.
+--]]
+add_pattern{
+ pattern = "^====+.*",
+ replace = ""
+}
+
+--[[
+The middle line is spotted with the tree equals sign at the beginning of the
+line (the previous pattern being longer, the decoration lines have been
+already removed and they won't be taken for section titles). The signs are
+removed and replaced with "\section{" and "}".
+--]]
+local function section (buffer, num)
+ local l = buffer[num]
+ l = gsub(l, "^===%s*", "\\section{")
+ l = gsub(l, "%s*=+%s*", "}")
+ buffer[num] = l
+end
+add_pattern{
+ pattern = "^===",
+ call = section
+}
+
+--[[
+The following pattern simply turns "Interpreter" into "\ital{Interpreter}". The
+meaning of the "\ital" command is obvious, I suppose. Note the offset:
+starting at the backslash, this leads to the _n_ in Interpreter, thus avoiding
+matching the pattern again. The Lua notation with double square brackets is
+used for strings with no escape character (hence "\ital" and not "\\ital" as
+would be necessary with a simple string).
+--]]
+add_pattern{
+ pattern = "Interpreter",
+ replace = [[\ital{Interpreter}]],
+ offset = 7
+}
+
+--[[
+Turning "TeX" into TeX. This illustrates the use of a function as "replace";
+the point is that "\TeX" should be suffixed with a space if initially followed
+by anything but a space or end of line (so as not to form a control sequence
+with the following letters), and it should be suffixed with a control space if
+initially followed by a space or end of line (so as to avoid gobbling the
+space). So the function checks the second capture. Note that simply replacing
+"TeX" with "\TeX{}" would be much simpler, but less instructive!
+--]]
+local function maketex (tex, next)
+ if next == " " or next == "" then
+ return [[\TeX\ ]]
+ else
+ return [[\TeX ]] .. next
+ end
+end
+add_pattern{
+ pattern = "(TeX)(.?)",
+ replace = maketex,
+ offset = 2
+}
+
+--[[
+The following turns "<text>" into <text> and "_text_" into _text_. Setting a
+class just so the patterns inherit the "nomagic" feature is of course an
+overkill, but that's an example.
+--]]
+interpreter.set_class(4, {nomagic = true})
+add_pattern{
+ pattern = "<...>",
+ replace = [[\arg{%1}]],
+ class = 4
+}
+add_pattern{
+ pattern = "_..._",
+ replace = [[\ital{%1}]],
+ class = 4
+}
+
+--[[
+I use double quotes as protectors; they are replaced with a "\verb" command at
+the very end of the processing (with class 0).
+--]]
+interpreter.protector('"')
+add_pattern{
+ pattern = nomagic'"..."',
+ replace = [[\verb`%1`]],
+ class = 0
+}
+
+--[[
+The description of functions (in red in the PDF file) are handled with the
+"\describe" macro, which takes the function as its first argument and
+additional information as its second one (typeset in italics in the PDF file).
+In the source, it is simply marked as
+
+ > function (arguments) [Additional information]
+
+with "[Additional information]" sometimes missing (i.e. there is no empty
+pairs of square brackets). Descriptions of entries in pattern tables follows
+the same syntax, except the line begins with ">>". So the pattern first spots
+lines beginning with ">[>]" followed by at least one space, adds an empty pair of
+brackets at the end if there isn't any, and turn the whole into "\describe".
+The number of ">" symbols sets "\describe"'s third argument, which specifies
+the level of the bookmark.
+--]]
+local function describe (buffer, num)
+ local l = buffer[num]
+ if not match (l, "%[.-%]%s*$") then
+ l = l .. " []"
+ end
+ local le = match(l, ">>") and 4 or 3
+ buffer[num] = gsub(l, ">+%s+(.-)%s+%[(.-)%]",
+ [[\describe{%1}{%2}{]] .. le .. "}")
+end
+add_pattern{
+ pattern = "^>+%s+",
+ call = describe
+}
+
+--[[
+Here's how multiline verbatim is handled; in the source it is simply marked by
+indenting the line with ten spaces; thus code is easily spotted when reading
+the source without useless and annoying "<code>"/"</code>" or anything similar
+to mark it. To be properly processed by TeX, the code should be surrounded by
+"\verbatim" and "\verbatim/" (my way of signalling blocks). Those must be on
+their own lines, so we insert a line at the beginning and at the end of the
+paragraph: for the closing "\verbatim/", we can simply replace the last line
+of the paragraph, which is the boundary line, unless we're at the end of the
+file. But for the opening "\verbatim" a line must be added at the beginning of
+the paragraph; thus line numbers in the original source file and in its
+processed version don't match anymore, and this might be annoying when TeX
+reports erros. Besides, blank verbatim lines aren't handled correctly and
+create a new verbatim block instead. So this way of marking verbatim material
+is good for small documents, but explicit marking is cleaner and more
+powerful (albeit not so good-looking in the source file).
+
+Note that the verbatim pattern belongs to class 2 and the entire paragraph is
+protected, so Interpreter leaves it alone afterward (remember the default
+class is 3). Of course, the first ten space characters are removed.
+--]]
+local function verbatim (buffer)
+ for n, l in ipairs(buffer) do
+ buffer[n] = gsub(l, "%s%s%s%s%s%s%s%s%s%s","", 1)
+ end
+ table.insert(buffer, 1, [[\verbatim]])
+ if gsub(buffer[#buffer],
+ interpreter.paragraph, "") == "" then
+ buffer[#buffer] = [[\verbatim/]]
+ else
+ table.insert(buffer, [[\verbatim/]])
+ end
+ interpreter.protect()
+end
+add_pattern{
+ pattern = "^%s%s%s%s%s%s%s%s%s%s",
+ call = verbatim,
+ class = 2
+}
+
+--[[
+And now comes the fun part. I wanted "i-doc.lua" to be self-describing. The
+source of what you're reading right now isn't "interpreter-doc.txt", but
+"i-doc.lua" itself input in the latter file with
+
+ \intepreterfile{doc}{i-doc.lua}
+
+How should code and comment be organized in "i-doc.lua"? Well, there is little
+choice, since the file is a normal Lua file: comment lines should be prefixed
+with "--" or surrounded with
+\tcode{--[{}[} and \tcode{--]{}]}. % Sorry for the braces, I can't nest Lua comments!
+I chose the latter option, which is simpler. But normal code should also be
+typeset as verbatim material; I could have begun all lines with ten spaces,
+but that would have seemed strange. Instead, \tcode{--]{}]} is turned into
+"\source" and "\source/" is added at the end of the paragraph ("\source" is
+just "\verbatim" with a different layout). Which means all paragraphs have the
+same structure: comments between
+\tcode{--[{}[} and \tcode{--]{}]}
+and code immediately following (\tcode{--[{}[} is simply removed). The pattern
+is in class 1 and the paragraph is protected, so that lines indented with ten
+spaces or more aren't touched by the previous verbatim pattern (in class 2).
+--]]
+local function autoverbatim (buffer, line)
+ buffer[line] = [[\source]]
+ for n = line + 1, #buffer do
+ interpreter.protect(n)
+ end
+ if gsub(buffer[#buffer],
+ interpreter.paragraph, "") == "" then
+ buffer[#buffer] = [[\source/]]
+ else
+ table.insert(buffer, [[\source/]])
+ end
+end
+add_pattern{
+ pattern = nomagic"%^--]]",
+ call = autoverbatim,
+ class = 1
+}
+local function remove_comment ()
+ return ""
+end
+add_pattern{
+ pattern = nomagic"%^--[[",
+ replace = remove_comment
+}
diff --git a/macros/luatex/generic/interpreter/interpreter-doc.pdf b/macros/luatex/generic/interpreter/interpreter-doc.pdf
new file mode 100644
index 0000000000..2dcc32ea48
--- /dev/null
+++ b/macros/luatex/generic/interpreter/interpreter-doc.pdf
Binary files differ
diff --git a/macros/luatex/generic/interpreter/interpreter-doc.tex b/macros/luatex/generic/interpreter/interpreter-doc.tex
new file mode 100644
index 0000000000..30a1da52ba
--- /dev/null
+++ b/macros/luatex/generic/interpreter/interpreter-doc.tex
@@ -0,0 +1,115 @@
+% This is the master file producing interpreter-doc.pdf. The version of the
+% documentation readable in a text editor is interpreter-doc.txt (input below).
+%
+% Paul Isambert - zappathustra AT free DOT fr - June 2012
+
+\input pitex
+
+\overfullrule=0pt
+
+\OutputRoutine remove {headers}{shipout}
+
+\setparameter page :
+ hsize = 25pc
+ left = 60pt
+ width = "\dimexpr 25pc + 120pt\relax"
+ lines = 35
+ height = 20cm
+
+\setparameter section :
+ font = \bf
+ link = true
+ number = none
+ numbercommand = \llap
+ beforeskip = 1
+
+\setparameter navigator :
+ title = "Interpreter documentation"
+ author = "Paul Isambert"
+ mode = outlines
+
+\setfont\mainfont:
+ name = "Chaparral Pro"
+ bold = Semibold
+ big = 18pt
+
+\setfont\codefont:
+ name = "Lucida Console"
+ slant = 15
+ bold italic = none
+ size = 8pt
+ features = "-tlig, -trep, space = mono"
+
+\parfillskip=0pt plus 1fill
+\def\describe#1#2#3{%
+ \unless\ifdim\lastskip=\baselineskip
+ \vskip\baselineskip
+ \fi
+ \needspace{2\baselineskip}%
+ \noindent\color{.8 0 0}{%
+ {\outline{#3}{\directlua{%
+ local t = string.gsub("\luaescapestring{#1}", "[ (].*", "")
+ tex.print(t)}}%
+ \codefont#1}%
+ \reverse\iffemptystring{#2}
+ {\kern1em \hfil\penalty0\hbox{\ital{(#2)}}}}%
+ \par
+ \removenextindent}
+
+\newverbatim\source{}
+ {\vskip\baselineskip
+ \hfuzz=1em
+ \codefont\parindent=0pt
+ \pdfcolorstack0 push {.8 0 0 rg}
+ \printverbatim
+ \pdfcolorstack0 pop
+ \vskip\baselineskip}
+
+% The verbatim facilities in PiTeX aren't gated yet, so I must rely on
+% this horrible hack!
+\directlua{
+function do_verbatim (name, exec)
+ if exec then
+ tex.print(pitex.verbatims[name])
+ else
+ for n, l in ipairs(pitex.verbatims[name]) do
+ if n == \string#pitex.verbatims[name] then
+ tex.print("\noexpand\\penalty\noexpand\\widowpenalty")
+ end
+ tex.print(pitex.verbatims[name].regime, l)
+ if n == 1 and \string#pitex.verbatims[name] > 2 then
+ tex.print("\noexpand\\penalty\noexpand\\widowpenalty")
+ end
+ end
+ end
+end
+}
+
+\def\arg#1{{\codefont\char"2039 #1\char"203A}}
+\pdfdef\ital#1{#1}
+\pdfdef\verb`#1`{#1}
+
+% Not optimal, but hey, with all the "intepreter.core.classes" stuff...
+\hyphenation{li-nes cla-sses}
+
+\input interpreter
+
+% Title
+\vbox to 3\baselineskip{
+\hbox to \hsize{\big Interpreter\hfil\normalsize Paul Isambert}
+\hbox to \hsize{v.1.2, June 2012 \hfil \tcode{zappathustra AT free DOT fr}}
+\vfil
+}
+
+
+
+% Bulk of the doc.
+\interpretfile{doc}{interpreter-doc.txt}
+\vskip0pt plus 1filll
+\noindent
+\bgroup\it
+Typeset with Lua\TeX\ 0.71 in Chaparral Pro and Lucida Console
+... nonetheless this documentation looks dull, I don't know why.
+\egroup
+
+\bye
diff --git a/macros/luatex/generic/interpreter/interpreter-doc.txt b/macros/luatex/generic/interpreter/interpreter-doc.txt
new file mode 100644
index 0000000000..6257aa6733
--- /dev/null
+++ b/macros/luatex/generic/interpreter/interpreter-doc.txt
@@ -0,0 +1,732 @@
+ TABLE OF CONTENTS
+ (Make a search on the tag in the right column to jump
+ to the associated section.
+ Vim users can simply type * on the tag;
+ Emacs users do that with C-s C-w (I think);
+ Other editors: I don't know!)
+===========================================================
+ Introduction intro_tag
+ Input files input_tag
+ Paragraphs paragraphs_tag
+ Declaring patterns patterns_tag
+ Classes classes_tag
+ Protecting input protect_tag
+ Technical stuff technical_tag
+ An example: i-doc.lua example_tag
+ The Gates in Interpreter gates_tag
+ The interpreter table interpreter_tag
+ The interpreter.core.tools table interpreter_tools_tag
+ The interpreter.core.reader table interpreter_reader_tag
+===========================================================
+
+
+
+================================== intro_tag
+=== Introduction =================
+==================================
+
+Interpreter preprocesses input files before their contents is fed to TeX. It
+is meant to write document with whatever markup one wishes to define while
+using normal TeX macros in the background. As a simple example, suppose you
+have a macro "\bold" to put text in boldface; then Interpreter lets you map
+"*text*", or "<strong>text</strong>", or simply "!text", or anything else, to
+"\bold{text}". Interpreter doesn't perform any trickery with active
+characters; instead, it manipulates the strings representing the lines of
+a file and search for patterns.
+
+There are two main advantages: first, TeX documents can be typeset with
+a completely non-TeX syntax; second, if one uses some lightweight markup
+language, the source file is much easier to read and might even be more useful
+than the typeset PDF file, e.g. for some technical documentation you want to
+read directly in your text editor while writing code (powerful editors
+generally have their own documentation in such a format, for a good reason).
+A third advantage, not explored in this documentation, is that while feeding
+modified lines to TeX you can also translate the original lines into, say,
+HTML, and write them to an external file, thus creating both PDF and HTML
+output at once.
+
+Interpreter has been rewritten with the Gates package (actually, only
+the Lua side) in version 1.1. That hasn't changed anything to its default
+behavior, but now it can also be customized quite deeply, since its
+code is a collection of small chunks with names that can be externally
+controlled and/or augmented. See the Gates documentation for further
+information. The last sections of this documentation describe the gates
+in Interpreter.
+
+
+
+================================== input_tag
+=== Input files ==================
+==================================
+
+Once Interpreter is loaded with
+
+ \input interpreter
+
+in plain TeX or
+
+ \usepackage{interpreter}
+
+in LaTeX, files to be processed are input as follows:
+
+ \interpretfile{<language>}{<file>}
+
+There should exist a file "i-<language>.lua" containing the language used in
+<file>. For instance, the source of this documentation is
+"interpreter-doc.txt", input in the master file "interpreter-doc.tex" with
+
+ \interpretfile{doc}{interpreter-doc.txt}
+
+and the interpretation to be used is defined in "i-doc.lua". The contents of
+such an interpretation file is the object of the rest of this documentation.
+
+
+
+================================== paragraphs_tag
+=== Paragraphs ===================
+==================================
+
+Interpreter doesn't process lines one by one. Instead, it gathers an entire
+paragraph and then processes the lines. It is important because you can
+manipulate an entire paragraph when a given pattern is detected, and modify
+several lines according to what happens in only one. A paragraph in
+Interpreter has nothing to do with what TeX considers a paragraph; instead, it
+is defined by the following string.
+
+> interpreter.paragraph [Default: blank line with spaces ignored]
+ A string to be interpreted as a paragraph boundary when Interpreter collects
+ lines before processing them. The string actually represents a pattern, so
+ magic characters are obeyed. The default is "%s*", i.e. a blank line is
+ considered a paragraph boundary, spaces notwithstanding. Of course, the end
+ of the file itself is a paragraph boundary.
+
+
+
+================================== patterns_tag
+=== Declaring patterns ===========
+==================================
+
+Once the lines of a paragraph have been collected, Interpreter searches them
+trying to match declared patterns, but it doesn't do so indiscriminately:
+patterns are searched in a given order, as explained below.
+
+Patterns are searched for in each line only, i.e. no match can occur across
+lines. However, since you can manipulate entire paragraphs based on a match in
+one line, the limitation easily vanishes.
+
+> interpreter.add_pattern(<table>)
+ This is the basic function used to defined patterns. The <table> may
+ contain the following entries, along other entries Interpreter won't use
+ but which can be useful to you, especially with "call" below. The function
+ returns a table.
+
+>> class [Default: "intepreter.default_class"]
+ The class of the pattern. See the section on classes.
+
+>> pattern
+ The pattern to match. Lua's magic characters are in force and should be
+ escaped with "%" if necessary, unless "nomagic" is "true" (or the pattern
+ itself is the result of "interpreter.nomagic").
+
+>> nomagic [Default: "false"]
+ A boolean deciding whether the pattern should be transformed with
+ "interpreter.nomagic".
+
+>> replace
+ The replacement for the pattern, applied only if there is no "call" entry.
+ This may be a string, a table or a function. Interpreter simply executes
+ something similar to "string.gsub()", hence the replacement follows this
+ function's ordinary syntax. More precisely, if "replace" is a string, the
+ pattern is replaced with it; in this string, "%n" may be used to denote the
+ _n_th capture in the pattern. If "replace" is a table, the first capture or
+ the entire match (if there is no capture) is used as the key, and the
+ associated value is used as the replacement. If "replace" is a function, it
+ is called with the captures passed as arguments, or the entire match if
+ there is no capture. For instance, the following pattern will replace all
+ "*text*" with "\bold{text}":
+
+ interpreter.add_pattern{
+ pattern = "%*(.-)%*",
+ replace = [[\bold{%1}]]
+ }
+
+>> offset [Default: 0]
+ The number of positions Interpreter should shift to the right after a match
+ has occurred. Normally, Interpreter starts searching for another occurrence
+ of the current pattern at the same position where it found the last one.
+ However, loops might easily occur: the replacement for a pattern may very
+ well contain another match for the same pattern, so Interpreter will get
+ stuck. Suppose for instance you want to replace "TeX" with "\TeX". The
+ first match will do that, but then Interpreter will start searching again
+ at the backslash, producing "\\TeX", then "\\\TeX", etc. In this case, if
+ you set "offset" to 2 in the pattern, then search will start again at the
+ "e" and no new match will occur.
+
+>> call
+ This entry shall contain a function to be called if there is a match (if
+ this entry exists, "replace" isn't applied). It is meant to perform complex
+ tasks that aren't amenable to simple string replacement. The function will
+ be executed as follows:
+
+ function (paragraph, line, index, pattern)
+
+ "paragraph" is a table representing the current paragraph; lines are stored
+ at successive indices. The last line of this paragraph is always the
+ paragraph boundary (see "interpreter.paragraph"), unless the paragraph
+ stopped at the end of the file. The second argument, "line", is a number
+ representing the index in "paragraph" containing the line where the pattern
+ was found; "index" is the position in this line where the match occurred.
+ Finally, "pattern" is the entire table declared with
+ "interpreter.add_pattern" and containing all the entries discussed here.
+
+ The function may return zero, one, or two numbers. If it returns none, the
+ search for the next occurrence of the pattern will start again on the same
+ line (rather, on the line with the same position in the paragraph), at
+ "index". If it returns one number, the search will resume at the same line
+ but at position _n_, with _n_ the returned number. Finally, if two numbers
+ are returned, the search will resume at line _m_ at position _n_, _m_ and
+ _n_ being the returned values. Specifying which line should be examined
+ when the search resumes might be necessary if the function adds new lines
+ in the paragraph _before_ the current line, since Interpreter only keeps
+ count of line numbers.
+
+ The entire paragraph can thus be modified if necessary. For instance,
+ suppose you want to declare comments in your source file with only
+ "!Comment" in the first line, i.e. TeX should ignore a paragraph such as:
+
+ !Comment
+ This should be ignored
+ by TeX
+
+ Then the following pattern will do (where the function requires only the
+ first argument):
+
+ local function comment (paragraph)
+ for n, l in ipairs(paragraph) do
+ paragraph[n] = "%" .. l
+ end
+ end
+ interpreter.add_pattern{
+ pattern = "^!Comment",
+ call = comment
+ }
+
+> interpreter.nomagic (string)
+ A function which reverses the usual Lua magic for patterns: ordinary magic
+ characters are normal characters here, unless they are prefixed with "%", in
+ which case they are magic again. For instance, a pattern like ".+" is
+ normally interpreted as ``one or more characters''. If passed to this
+ function, a pattern is returned meaning ``a dot followed by a plus sign''.
+ On the contrary, "%.%+" normally has the second interpretation, while with
+ "interpreter.nomagic" it has the first one. The function makes another
+ transformation: "..." is used to denote a capture "(.-)". Thus
+ "interpreter.nomagic('*...*')" returns a pattern matching any number of
+ characters surrounded by stars and capturing those characters; this would be
+ expressed in ordinary Lua magic as "%*(.-)%*".
+
+
+
+================================== classes_tag
+=== Classes ======================
+==================================
+
+As already alluded to, the search for patterns isn't done at random. Instead,
+patterns are organized in classes, which are applied one after the other. More
+precisely, the process is as follows: Interpreter searches the entire
+paragraph for the first pattern in class~1, then for the second pattern in the
+same class, then for the third, etc., then when there is no pattern left in
+class~1 it does the same with class~2, up to class~_n_, where _n_ is the
+highest class number such that there exists a class _n - 1_ (in other words,
+classes should be numbered consecutively). Finally, the same goes for the
+patterns in class~0 (which always exists, even if it contains no pattern).
+
+Inside a class, patterns are ordered by length from long to short, or
+alphabetically if two patterns have the same length. This means that if you
+use e.g. "/text/" for italics and "//text//" for bold, you don't need to put
+the second pattern in a class before the first to avoid "//text//" being
+interpreted as two empty arguments in italics surrounding a text in roman.
+Since the way the bold-pattern will be declared, e.g. "//(.-)//", is probably
+longer than for the italic-pattern, e.g. "/(.-)/", it will always match first.
+
+That said, the sorting isn't very clever and simply relies on the number of
+symbols, no matter what they mean; in the patterns above, the parentheses
+denote a capture but they still count in the pattern's length as understood by
+Interpreter. Alternatively, while ".*" denotes ``zero or more character'' and
+"%+" means ``a plus sign'' ("+" being magic, you have to escape it to refer to
+it), in Interpreter's eye the two patterns have the same length: two. Finally,
+one should be aware that patterns declared with a "nomagic" entry set to
+"true" are sorted after they've been transformed (so that their real length
+might not be obvious). So classes are needed when patterns need a proper
+ordering no matter their lengths. For instance, some patterns should always be
+declared first, as they protect input from Interpreter (see next section),
+while others might need to be declared last, as they rely on what previous
+patterns might have done. Besides, classes are metatables for the patterns
+they contain.
+
+> interpreter.default_class [Default: 1]
+ All patterns belong to a class, even though you may omit the "class" entry
+ when declaring one. In this case, the pattern is assigned to the class
+ denoted by this number.
+
+> interpreter.set_class(number, table)
+ Defines class "number" as "table". Classes don't need to be defined
+ beforehand for patterns to be added to them (rather, Interpreter defines
+ them implicitly when needed). However, classes are also metatables for the
+ patterns, so that if there lacks an entry in a pattern's table, the class's
+ entry is used if it exists. The function returns a table.
+
+
+
+================================== protect_tag
+=== Protecting input =============
+==================================
+
+Sometimes you want Interpreter to refrain from interpreting; that is most
+useful for verbatim code, for instance. There are various ways to do that.
+
+> interpreter.active [Default: true]
+ A boolean switching Interpreter on and off. Beware, the switching applies
+ only starting at the next paragraph.
+
+> interpreter.protect([line])
+ A function protecting all or part of the current paragraph. If "line" is
+ given, it should be a number _n_, and line _n_ in the current paragraph will
+ be protected; without "line", the entire paragraph is protected. Protecting
+ means that the patterns not yet searched for will be ignored. For instance,
+ if you want material to be read verbatim when surrounded with "<code>" and
+ "</code>", you can declare a pattern as follows:
+
+ local function verbatim (buffer)
+ buffer[1] = "\\verbatim"
+ buffer[#buffer - 1] = "\\endverbatim"
+ intepreter.protect()
+ end
+ interpreter.add_pattern{
+ pattern = "^%s*<code>%*s$",
+ call = verbatim,
+ class = 1
+ }
+
+ This code is extremely simplified : it assumes that "<code>" and "</code>"
+ starts and ends the paragraph and that "</code>" isn't the last line of the
+ file (otherwise it'd also be the last line in the paragraph, whereas here
+ the last one is the paragraph boundary). An important point is that the
+ pattern belongs to the first class, so it is called before all other
+ patterns (provided there is no shorter pattern in class~1) and prevents them
+ from doing anything, since the entire paragraph is protected. (Typesetting
+ the material as verbatim material obviously depends on the "\verbatim"
+ macro, not on Interpreter.)
+
+> interpreter.escape
+ A character which prevents patterns from being replaced if immediately
+ preceded by it. As an example, if "interpreter.escape = '_'", and "*text*"
+ denotes italic, then "*text*" will produce _text_ while "_*text*" will
+ produce *text*. Once a paragraph has been processed, Interpreter removes all
+ escape characters. Only one character can be an escape character.
+
+> interpreter.protector(left[, right]) ["right" defaults to "left"]
+ Defines two characters to protect what they surround. In other words,
+ Interpreter replaces patterns only if the match isn't found between "left"
+ and "right". Unlike the escape character, you can define as many protectors
+ as you wish; and unlike the escape character again, Interpreter _doesn't_
+ remove them once the paragraph has been processed, so you must take care of
+ them. For instance:
+
+ intepreter.protector('"')
+ interpreter.add_pattern{
+ pattern = '"(.-)"',
+ replace = '\\verb`%1`',
+ class = 0
+ }
+
+ Anything between double quotes will be left untouched; then, when the
+ paragraph has been processed for all other classes, a pattern in class~0
+ calls the "\verb" command to take care of the argument. Note that the
+ protectors should enclose what they protect without coinciding with it; this
+ is not the case here, which is why the pattern is applied.
+
+> interpreter.direct [Default: two percent signs then "I" and at least one space]
+ A string, actually a pattern, signalling that the line which it begins
+ should be processed as Lua code. The default is "%%%%I%s+", i.e. "%%I"
+ followed by at least one space. The pattern shouldn't declare itself as
+ attached to the beginning of the line (as in "^%%%%I%s+") because they will
+ be matched at the beginning of the line only anyway. The line is processed
+ with the "loadstring" function, and then turned into an empty line. For
+ instance:
+
+ %%I interpreter.active = false
+ This won't be interpreted...
+ %%I interpreter.active = true
+
+ As this example shows, lines flagged with "interpreter.direct" don't obey
+ "interpreter.active" and are always processed as described above.
+
+
+
+================================== technical_tag
+=== Technical stuff ==============
+==================================
+
+You don't have to bother with this section if you don't mind how Interpreter
+does its job; actually you won't learn much anyway.
+
+> interpreter.reset()
+ A function which resets everything to default and deletes classes. It is used
+ when calling "\interpretefile" so that new interpretetions start from zero.
+
+> interpreter.register(function)
+ A function called to put Interpreter's main function into the
+ "post_linebreak_filter" callback; you can redefine it at will. If it is
+ undefined, "callback.register()" is used, unless "luatexbase.add_to_callback()"
+ is detected. (The detection takes place at the first call to
+ "\interpretfile", so there is no need to load Interpreter after
+ "luatexbase".)
+
+> interpreter.unregister(function)
+ A function called to remove Interpreter's main function from the
+ "post_linebreak_filter" callback. It works similarly to the previous one.
+
+
+
+================================== example_tag
+=== An example: i-doc.lua ========
+==================================
+
+\interpretfile{doc}{i-doc.lua}
+
+
+
+================================== gates_tag
+=== The Gates in Interpreter =====
+==================================
+
+Interpreter is written with the Gates package (only the Lua side,
+actually). It means that it can be hacked down to the core. Here I'll
+simply list the gates involved; you should read the Gates
+documentation to learn how to use them.
+
+There are three gates families: "interpreter", associated with the main
+"interpreter" table, contains the user interface; "interpreter_tools",
+associated with "interpreter.core.tools" table, contains internal
+functions; finally "interpreter_reader", associated with the
+"interpreter.core.reader" table, contains the main functions used to
+read the file.
+
+Whenever I mention a conditional or a loop, I mean the local conditionals
+and loops, relative to the l-gate where the gate appears. Also, the
+syntax indicates the arguments a gate uses, not all the arguments that
+are passed to it (which are simply what the previous gate has returned).
+
+As an example of customizing Interpreter with Gates, you could very
+well add a bit of code which does something to all lines. Inserting a
+small gate, say "everyline", after "check_direct" in "aggregate_lines"
+below would do the trick, e.g.:
+
+
+ function interpreter.core.reader.everyline (file, line)
+ line = dosomething(line)
+ return file, line
+ end
+ interpreter.core.reader.add(
+ "everyline", "aggregate_lines", "after check_direct")
+ interpreter.core.reader.conditional(
+ "everyline", "aggregate_lines", function (f, l) return l end)
+
+(Note that it is important to check that the line really exist,
+because one might have hit the end of the file; hence the conditional,
+as with others gates in "aggregate_lines").
+
+
+
+
+================================== interpreter_tag
+=== The "interpreter" table ========
+==================================
+
+All the user functions in "interpreter" are simple m-gates, so they can
+be treated as ordinary functions, except "interpreter.add_pattern",
+which is an l-gate containing, built as:
+
+ add_pattern
+ . ensure_class
+ . apply_nomagic
+ . insert-pattern
+ . . do_insert
+ . . sort_class
+
+> ensure_class (<pattern>) [m-gate]
+ Creates the class of <pattern> if necessary, and set it as the metatable
+ for <pattern>. Classes themselves are kept in the "interpreter.core.classes"
+ table. The gate return <pattern> and the class number.
+
+> apply_nomagic (<pattern>, <class>) [m-gate]
+ Transforms the "pattern" entry in <pattern> with "intepreter.nomagic";
+ tied to a conditional that returns <pattern>'s "nomagic" entry (so
+ the gate is executed only if "nomagic" is true); autoreturns both
+ arguments.
+
+> insert_pattern (<pattern>, <class>) [l-gate]
+ An autoreturning l-gate containing the following two gates.
+
+> do_insert (<pattern>, <class>) [m-gate]
+ Adds <pattern> to <class> (i.e. "interpreter.core.classes[<class>]").
+
+> sort_class (<pattern>, <class>) [m-gate]
+ Sorts <class> with function "interpreter.core.tools.sort". This gate
+ can be skipped to apply the patterns in the order in which they were
+ declared.
+
+
+
+============================================= interpreter_tools_tag
+=== The interpreter.core.tools table ========
+=============================================
+
+All the functions in the "interpreter.core.tools" all are simple m-gates.
+
+> sort (<patt1>, <patt2>) [m-gate]
+ Returns true if the pattern in "patt1" is longer than the one in
+ "patt2", or if they have the same length and the first ranks before
+ the second with respect to alphabetical order. The gate is used in
+ the "interpreter.sort_class" m-gate.
+
+> xsub (<string>, <index>, <pattern>, <replacement>) [m-gate]
+ Returns <string> with <pattern> replaced with <replacement>, but only
+ once, and only after <index>.
+
+> protector (<string>, <index>) [m-gate]
+ Checks whether <index> in <string> isn't between characters declared
+ with "interpreter.protector". If that is the case, the function returns
+ "nil" and the index of the second protector. Otherwise, it returns
+ <index>.
+
+> get_index (<string>, <pattern>, <index>) [m-gate]
+ Checks whether <pattern> occurs in <string>, starting at <index>. If
+ it does, but if <index>-1 is "interpreter.escape", calls itself with
+ <index>+1. Otherwise, calls "interpreter.core.tools.protector" to
+ check whether <index> is in a protected part of the string. If so,
+ calls itself with <right>+1 instead of <index>, where <right> is the
+ second return value of of "interpreter.core.tools.protector", i.e.
+ it searches again after the right protector. If <index> is found,
+ end of story, returns <index>, otherwise returns nothing.
+
+The "interpreter.core.tools" table also contains
+"magic_characters", a table with an entry for each magic character in Lua except
+`"."' and `"%"'; the values to those entries are the same characters
+prefixed with `"%"'. The table is used by "interpreter.nomagic" to spot
+and replace magic characters, with the dot and the percent sign dealt
+with independantly.
+
+
+
+============================================= interpreter_reader_tag
+=== The interpreter.core.reader table =======
+=============================================
+
+Interpreter works by hooking in the "open_read_file" callback; the
+function registered there is the "interpreter.core.reader.input"
+l-gate, built as follows:
+
+ input
+ . unregister
+ . . set_unregister
+ . . use_unregister
+ . open_file
+ . set_reader
+
+> unregister (<filename>) [l-gate]
+ Contains the following two m-gates; <filename> is received from
+ "input", which itself receives it from the callback, i.e. that's the
+ file that's being input (the second argument to "\interpretfile").
+ It is also automatically returned.
+
+> set_unregister () [m-gate]
+ Sets the function to remove "input" from the callback, namely
+ "interpreter.unregister"; the gate is called only if gate
+ "interpreter.unregister" doesn't already exits. If "luatexbase" is
+ detected, the functions there are used; otherwise, "callback.register"
+ is used with "nil" as the second argument.
+
+> use_unregister () [m-gate]
+ Calls "interpreter.unregister()". (You don't want the next input file
+ to be processed with Interpreter by default, that's why you remove
+ the callback function; not that the current one is nonetheless
+ processed with the current file, of course.)
+
+> open_file (<filename>) [m-gate]
+ Returns "io.open(<filename>)".
+
+> set_reader (<file>) [m-gate]
+ Returns a table with a "reader" entry containing a function whose
+ definition is
+
+ function ()
+ return interpreter.core.reader.read_file(f)
+ end
+
+ That's the convention for the "open_read_file" callback: it should
+ return such a table, and the function will be called each time a
+ line is required from the input file.
+
+So most of the work is done by "interpreter.core.reader.read_file",
+which is why it is so heavy; it receives a file handle:
+
+ read_file
+ . make_paragraph
+ . . aggregate_lines
+ . . . read_line
+ . . . check_direct
+ . . . insert_line
+ . . apply_classes
+ . . . pass_class
+ . . . . pass_pattern
+ . . . . . process_lines
+ . . . . . . switch
+ . . . . . . call
+ . . . . . . replace
+ . . . . . . protect
+ . . . unprotect
+ . . . . undo_protected
+ . . . . unprotect_lines
+ . . . remove_escape
+ . return_line
+
+> make_paragraph (<file>) [l-gate]
+ The big l-gate that contains everything that follows, barring
+ "return_line". It is called if and only if the "interpreter.core.lines"
+ table is empty; that table is where lines of a paragraph are stored,
+ and it is emptied by "return_line".
+
+> aggregate_lines (<file>, <line>) [l-gate]
+ The main l-gate that reads line from <file> and stores them in
+ "interpreter.core.lines". It loops until <line> is "nil" or equivalent
+ to "interpreter.paragraph". (Of course, <line> is "nil" on the first
+ iteration, but the "loopuntil" conditional is evaluated after that
+ first iteration, during which the last subgate "insert_line" will
+ probably returns a line.)
+
+> read_line (<file>) [m-gate]
+ Reads the next line from <file>, and returns <file> and that line.
+
+> check_direct (<file>, <line>) [m-gate]
+ If <line> begins with "interpreter.direct", removes it and use
+ "loadstring" on the resulting string. Returns <file> and <line>, the
+ latter set to an empty string is the previous operation applied. The
+ gate depends on a conditional: <line> should be non-"nil" (of
+ course), and "interpreter.direct" should be defined.
+
+> insert_line (<file>, <line>) [m-gate]
+ Adds <line> to "interpreter.core.lines". Automatically returns the
+ two arguments (and if <line> isn't "nil" or equivalent to
+ "interpreter.paragraph", it will be executed again).
+
+> apply_classes () [l-gate]
+ The l-gate that applies transformations to the lines, once the paragraph
+ has been gathered, with the gates that follow. For each class, it
+ will apply each pattern on each line. It depends on a conditional:
+ "interpreter.core.lines" shouldn't be empty, and "interpreter.active"
+ should be true.
+
+> pass_class () [l-gate]
+ This gate iterates on all classes in "interpreter.core.classes" and
+ then on class 0. On each iteration it checks beforehand whether the
+ paragraph is protected, i.e. "interpreter.core.reader.protected" isn't
+ a boolean (see "unprotecte" below). On each iteration, the class
+ number and the class itself are returned. (This behavior is
+ implemented with a Gates iterator.)
+
+> pass_pattern (<ignored>, <class>) [l-gate]
+ Same as "pass_class", except it iterates on the patterns in "class":
+ it is executed as long as "interpreter.core.reader.protected" and
+ returns the patter number and the pattern itself. (The <ignored>
+ argument isn't used; it is for the "pass_class" iterator; the same
+ holds for the following gates.)
+
+> process_lines (<ignored>, <pattern>) [l-gate]
+ Again, this calls an iterator. It browses each line in "interpreter.core.lines"
+ and returns the line's number (provided it is valid, i.e. not a table,
+ see "protect" below), <pattern> and the current index in that line.
+ To keep track of the current line and index, two internal numbers are
+ used: "interpreter.core.reader.current_line" and
+ "interpreter.core.reader.current_index".
+
+> switch (<line>, <pattern>, <index>) [m-gate]
+ If <pattern> has a "call" entry, it sets the "call" gate below to
+ "ajar"; otherwise, if <pattern> has a "replace" entry, it sets the
+ "replace" gate to "ajar".
+
+> call (<line>, <pattern>, <index>) [m-gate]
+ This gate is closed by default and set to "ajar" by "switch" above.
+ If, starting at <index>, the <pattern>'s "pattern" entry can be
+ found in <line> with "interpreter.core.tools.get_index" (which makes
+ sure that protectors are obeyed and returns <newindex>, where the
+ pattern is found if it occurs), the <pattern>'s "call" entry is
+ applied as
+
+ <pattern>.call(interpreter.core.lines,
+ <line>, <newindex>, <pattern>)
+
+ This may returned zero, one or two values. If nothing is returned,
+ "interpreter.core.reader.current_index" is set to 0, which makes
+ the "process_lines" iterator consider the next line. If one value is
+ returned, it is the new current index and "process_lines" will not
+ update the line number. If two values are returned, the first is the
+ new current line number and the second the new current index.
+
+> replace (<line>, <pattern>, <index>) [m-gate]
+ This gate is closed by default and set to "ajar" by "switch" above.
+ This tries to find the <pattern>'s "pattern" like "call" above, and if
+ it is found, it applies "interpreter.core.tools.xsub" as:
+
+ xsub (interpreter.core.lines[<line>],
+ <newindex>, <pattern>.pattern, <pattern>.replace)
+
+ where <newindex> is defined as in "call". The return value of "xsub"
+ is assigned to "interpreter.core.lines[<line>]", and the current
+ index is set to <index> plus the <pattern>'s "offset" if any. If the
+ pattern wasn't found, the current index is set to 0, which makes
+ "process_lines" turn to the next line as explained in "call".
+
+> protect () [m-gate]
+ The "interpreter.protect()" function can either protect the whole
+ paragraph (when no argument is passed) or a single line (when a number
+ is passed). In the first case, "interpreter.core.tools.protected"
+ takes the value "true", which is checked in various gates above. In
+ the second case, "interpreter.core.tools.protected" is a table with
+ each index indicating a line to be protected. This gate implements
+ the protection in that case: it iterates on all entries in the table
+ with "pairs" and protects the line with the same index in
+ "interpreter.core.lines" by transforming it into a table (with a
+ single entry, the string representing the original line); the type
+ of the line is checked in the "process_lines" iterator above. The
+ gate's iterator doesn't take arguments, but the function itself is
+ defined as taking a number (the line).
+
+> unprotect () [l-gate]
+ Now all the patterns in all the classes have been applied to the
+ entire paragraph, and protection must be removed. This l-gate
+ contains the following two gates.
+
+> undo_protected () [m-gate]
+ Simply sets "interpreter.core.tools.protected" to "nil" so it is
+ ready for the next paragraph.
+
+> unprotect_lines () [m-gate]
+ Restores all the lines in "interpreter.core.lines" as simple strings.
+ The gate uses an iterator that simply runs "ipairs" on "interpreter.core.lines",
+ so the function's definition actually takes the line's number and
+ the line itself as arguments.
+
+> remove_escape () [m-gate]
+ If "interpreter.escape" is defined, removes all its occurrences in
+ each line of the paragraph.
+
+This is the end of the big "make_paragraph" l-gate. It won't be called
+again until the paragraph has been fully passed to TeX, i.e. when
+"interpreter.core.lines" is empty.
+
+> return_line () [m-gate]
+ Pops the first line from "interpreter.core.lines" and returns it.
+ Since this is the very last subgate of "read_file", the line is
+ passed to TeX.
+
diff --git a/macros/luatex/generic/interpreter/interpreter.lua b/macros/luatex/generic/interpreter/interpreter.lua
new file mode 100644
index 0000000000..05de6eb791
--- /dev/null
+++ b/macros/luatex/generic/interpreter/interpreter.lua
@@ -0,0 +1,450 @@
+-- This is the main Lua file for the Interpreter package.
+-- Further information in interpreter-doc.pdf or interpreter-doc.txt.
+-- Paul Isambert - zappathustra AT free DOT fr - June 2012
+--
+-- Beware, this is written with Gates. Please read the Gates doc if
+-- you want to understand something.
+
+local find, gsub, match, sub = string.find, string.gsub, string.match, string.sub
+local insert, sort, remove = table.insert, table.sort, table.remove
+local io_open = io.open
+local ipairs, pairs, type = ipairs, pairs, type
+require("gates.lua")
+if not gates.iterator then
+ tex.error("Interpreter error: Your version of Gates should be at least v.0.2. I quit. Expect chaos")
+ return
+end
+
+interpreter = gates.new("interpreter")
+
+-- *** interpreter.active ***
+-- Following paragraphs (as defined by interpreter.paragraph) are interpreted
+-- iff this is not set to false.
+interpreter.active = true
+-- *** interpreter.default_class ***
+-- Sets the default class for patterns which are added without specifying the
+-- class. Default 1.
+interpreter.default_class = 1
+
+interpreter.core = {
+ classes = {}, -- The classes of patterns.
+ lines = {}, -- The lines of the paragraph.
+ reader = gates.new("interpreter_reader"), -- The main processing functions.
+ tools = gates.new("interpreter_tools")} -- Auxiliary functions.
+
+-- Utility function sorting patterns by length (alphabetically if they are of
+-- equal length).
+function interpreter.core.tools.sort (a, b)
+ local a, b = a.pattern, b.pattern
+ return #a == #b and a < b or #a > #b
+end
+
+-- *** interpreter.add_pattern (table) ***
+-- Creates pattern <table>, which can contain the following entries:
+-- pattern [string] = The pattern to match. Magic characters are obeyed!
+-- replace [string] = The replacement for <pattern>. Can be a string, a
+-- table or a function. A simple string.gsub() is
+-- applied.
+-- call [function] = The function applied to <pattern>; <replace> is applied
+-- iff there is no <call>.
+-- offset [number] = If <pattern> is used at index n, then the search on the
+-- same line for the same pattern starts again at index n
+-- + offset. Applied only when no <call> (in this case,
+-- search starts again at the beginning of the line). By
+-- default, offset = 0. This is needed to avoid infinite
+-- loops with replacements which contain the pattern;
+-- e.g. replacing "TeX" with "\TeX" will produce an
+-- infinite loop, unless offset = 2.
+-- nomagic [boolean] = Sets whether <replace> should be transformed with interpreter.nomagic.
+-- class [number] = The pattern's <class> (classes of patterns are applied in
+-- order, e.g. all patterns in class 1 are applied, then
+-- all patterns in class 2, etc; class 0, however, is
+-- always applied last). If <class> is not given, the
+-- default_class number is used. Classes must be numbered
+-- consecutively.
+interpreter.list{"add_pattern",
+ {"ensure_class",
+ function (tb)
+ local class = tb.class or interpreter.default_class
+ interpreter.set_class(class, {})
+ setmetatable(tb, interpreter.core.classes[class].meta)
+ return tb, class
+ end},
+ {"apply_nomagic", conditional = function (tb) return tb.nomagic end,
+ autoreturn = true,
+ function (tb, class)
+ tb.pattern = interpreter.nomagic(tb.pattern)
+ end},
+ {"insert_pattern", autoreturn = true,
+ {"do_insert", autoreturn = true,
+ function (tb, class)
+ insert(interpreter.core.classes[class], tb)
+ end},
+ {"sort_class", autoreturn = true,
+ function (tb, class)
+ sort(interpreter.core.classes[class], interpreter.core.tools.sort)
+ end}}}
+
+-- *** interpreter.set_class (number, table) ***
+-- Sets default values (of the table normally specified in add_pattern) for
+-- patterns of class <number>; patterns added to this class can still specify
+-- different values, which will override defaults. In other words, this is a
+-- metatable for patterns (which are tables) of that class.
+function interpreter.set_class (num, tb)
+ interpreter.core.classes[num] = interpreter.core.classes[num] or
+ { meta = { __index = function (_, k) return interpreter.core.classes[num].meta[k] end } }
+ for a, b in pairs(tb) do
+ interpreter.core.classes[num].meta[a] = b
+ end
+ return interpreter.core.classes[num]
+end
+
+-- Class 0 must exist since it is always used at the end of the paragraph.
+interpreter.set_class(0, {})
+
+-- *** interpreter.nomagic (string) ***
+-- Turns a normal string into a string with magic characters escaped, so it
+-- can be used as a pattern.
+interpreter.core.tools.magic_characters = {
+ ["^"] = "%^",
+ ["$"] = "%$",
+ ["("] = "%(",
+ [")"] = "%)",
+ ["%"] = "%%",
+ ["."] = "%.",
+ ["["] = "%[",
+ ["]"] = "%]",
+ ["*"] = "%*",
+ ["+"] = "%+",
+ ["-"] = "%-",
+ ["?"] = "%?",
+}
+function interpreter.nomagic (str)
+ local i, s = 1, ""
+ local magic_characters = interpreter.core.tools.magic_characters
+ while i <= #str do
+ local c, c2, c3 = sub(str, i, i), sub(str, i + 1, i + 1), sub(str, i + 2, i + 2)
+ i = i + 1
+ if c == "%" and magic_characters[c2] then
+ s = s .. c2
+ i = i + 1
+ elseif c == "." and c2 == "." and c3 == "." then
+ s = s .. "(.-)"
+ i = i + 2
+ elseif magic_characters[c] then
+ s = s .. "%" .. c
+ else
+ s = s .. c
+ end
+ end
+ return s
+end
+
+-- *** interpreter.protect ([spec]) ***
+-- Protects a set of lines in a paragraph; a protected line won't be
+-- interpreted. If <spec> is a number, this protects line <spec> in the current
+-- paragraph; if <spec> is true, this protects the entire current paragraph. Of
+-- course, patterns that were applied to the line(s) or paragraph before
+-- protection happened aren't undone.
+function interpreter.protect (num)
+ if type(num) == "number" then
+ if type(interpreter.core.reader.protected) ~= "boolean" then
+ interpreter.core.reader.protected = interpreter.core.reader.protected or {}
+ interpreter.core.reader.protected[num] = true
+ end
+ else
+ interpreter.core.reader.protected = true
+ end
+end
+
+-- Utility function making a replacement in a string but only from a certain
+-- position and only once. We can't let gsub unrestricted, because some
+-- part(s) of the string might be protected.
+function interpreter.core.tools.xsub (str, num, patt, rep)
+ return sub(str, 1, num-1) .. gsub(sub(str, num), patt, rep, 1)
+end
+
+-- *** interpreter.protector (left [, right]) ***
+-- Sets <left> and <right> (set to <left> if missing) as protectors, i.e.
+-- enclosed material won't be processed even if the line is processed
+-- otherwise. For instance: after interpreter.protector ("|"), the word
+-- "little" in
+--
+-- Hello, |little| world!
+--
+-- will be left untouched; Interpreter is terribly smart (thanks to lpeg), so
+-- in "|a| b |c|", "b" isn't protected, as intended, because the "|" on its
+-- left doesn't match the one on its right but with the one before "a". An
+-- example with <right> specified: interpreter.protector("[", "]") and
+-- then:
+--
+-- Hello, [little] world!
+--
+-- achieves the same as above. Protectors AREN'T removed when the line is
+-- finally passed to TeX; and there can be several protectors. Compare with
+-- interpreter.escape.
+local P, Cf, Cg, Cp, Ct, V = lpeg.P, lpeg.Cf, lpeg.Cg, lpeg.Cp, lpeg.Ct, lpeg.V
+local _grammar
+function interpreter.core.tools.protector (str, index)
+ local protections = Cf(Ct("") * Cg{ _grammar + 1 * V(1) }^1, rawset)
+ protections = protections:match(str)
+ if protections then
+ for a, b in pairs(protections) do
+ if index > a and index < b then
+ return nil, b
+ end
+ end
+ end
+ return index
+end
+function interpreter.protector (left, right)
+ right = right or left
+ local gram = P(Cp() * P(left) * (1 - P(right))^0 * Cp() * P(right))
+ if _grammar then
+ _grammar = _grammar + gram
+ else
+ _grammar = gram
+ end
+end
+
+-- *** interpreter.escape ***
+-- A string used as an escape character: if a pattern matches, it is processed
+-- iff the character immediately to its left isn't <escape>. The escape
+-- character IS removed once the lines have been processed, so TeX never sees
+-- it; also, only one escape character is allowed, and itself can't be escaped
+-- (i.e. it doesn't mean anything to try to escape it). E.g.:
+--
+-- interpreter.escape = "|"
+-- ... this won't be |*processed*
+--
+-- Assuming you have a pattern with stars, here it won't be applied. Instead
+-- "this won't be *processed*" will be passed to TeX (note that the escape
+-- character has disappeared).
+
+function interpreter.core.tools.get_index (str, patt, index)
+ index = find(str, patt, index)
+ if index then
+ if sub(str, index-1, index-1) == interpreter.escape then
+ return interpreter.core.tools.get_index(str, patt, index + 1)
+ elseif _grammar then
+ local right
+ index, right = interpreter.core.tools.protector(str, index, patt)
+ return index or interpreter.core.tools.get_index(str, patt, right + 1)
+ else
+ return index
+ end
+ end
+end
+
+-- *** interpreter.paragraph ***
+-- The pattern that defines a line acting as a paragraph boundary,
+-- prompting Interpreter to process the lines gathered up to now. Default is a
+-- line composed of spaces at most.
+interpreter.paragraph = "%s*"
+
+-- *** interpreter.direct (pattern) ***
+-- Sets the pattern defining a line as direct Lua code: if a line begins with
+-- <pattern> (which itself shouldn't contain the beginning-of-string character "^")
+-- the code that follows is processed as Lua code, and the line is turned to
+-- an empty string; note that this empty string will be seen as a paragraph
+-- boundary if the line happened in the middle of a paragraph and
+-- interpreter.paragraph has set paragraph boundary to empty string. Default
+-- is "%%I " (two "%" followed by one "I" followed by at least one space
+-- character).
+interpreter.direct = "%%%%I%s+"
+
+-- At last, the function to be registered in open_read_file, defining the
+-- function that reads a file.
+
+interpreter.core.reader.current_line = 0
+interpreter.core.reader.current_line = 0
+
+interpreter.core.reader.list{"read_file",
+ {"make_paragraph", conditional = function () return #interpreter.core.lines == 0 end,
+ {"aggregate_lines", loopuntil = function (_, line) return not line or gsub(line, "^" .. interpreter.paragraph .. "$", "") == "" end,
+ {"read_line",
+ function (f)
+ return f, f:read()
+ end},
+ {"check_direct", conditional = function (_, line) return line and interpreter.direct end,
+ function (f, line)
+ if match(line, "^" .. interpreter.direct) then
+ loadstring(gsub(line, "^" .. interpreter.direct, ""))()
+ line = ""
+ end
+ return f, line
+ end},
+ {"insert_line", conditional = function (_, line) return line end,
+ autoreturn = true,
+ function (f, line)
+ insert(interpreter.core.lines, line)
+ end}},
+ {"apply_classes", conditional = function () return #interpreter.core.lines > 0 and interpreter.active end,
+ {"pass_class", iterator = function ()
+ local done_zero
+ local function f (t, i)
+ if type(interpreter.core.reader.protected) ~= "boolean" then
+ if not done_zero then
+ i = i+1
+ local v = t[i]
+ if v then
+ return i, v
+ else
+ done_zero = true
+ return 0, t[0]
+ end
+ end
+ end
+ end
+ return f, interpreter.core.classes, 0
+ end,
+ {"pass_pattern", iterator = function (_, class)
+ local function f (t, i)
+ if type(interpreter.core.reader.protected) ~= "boolean" then
+ i = i+1
+ local v = t[i]
+ if v then
+ return i, v
+ end
+ end
+ end
+ return f, class, 0
+ end,
+ {"process_lines", iterator = function (_, pattern)
+ interpreter.core.reader.current_line = 0
+ interpreter.core.reader.current_index = 0
+ return function ()
+ if type(interpreter.core.reader.protected) ~= "boolean" then
+ local l = interpreter.core.reader.current_line
+ local i = interpreter.core.reader.current_index
+ if i == 0 then
+ l, i = l + 1, 1
+ end
+ line = interpreter.core.lines[l]
+ -- When protected, a line is a table.
+ while type(line) == "table" do
+ l, i = l+1, 1
+ line = interpreter.core.lines[l]
+ end
+ if line then
+ interpreter.core.reader.current_line = l
+ interpreter.core.reader.current_index = i
+ return l, pattern, i
+ end
+ end
+ end
+ end,
+ {"switch", autoreturn = true,
+ function (_, pattern)
+ if pattern.call then
+ interpreter.core.reader.ajar("call", "process_lines")
+ elseif pattern.replace then
+ interpreter.core.reader.ajar("replace", "process_lines")
+ end
+ end},
+ {"call", status = "close",
+ function(i, pattern, ind)
+ local line = interpreter.core.lines[i]
+ local index = interpreter.core.tools.get_index(line, pattern.pattern, ind)
+ if index then
+ local L, O = pattern.call(interpreter.core.lines, i, index, pattern)
+ if O then
+ interpreter.core.reader.current_line = L
+ interpreter.core.reader.current_index = O
+ elseif L then
+ interpreter.core.reader.current_index = L
+ end
+ else
+ interpreter.core.reader.current_index = 0
+ end
+ end},
+ {"replace", status = "close",
+ function(i, pattern, ind)
+ local line = interpreter.core.lines[i]
+ local index = interpreter.core.tools.get_index(line, pattern.pattern, ind)
+ if index then
+ interpreter.core.lines[i] = interpreter.core.tools.xsub(line, index, pattern.pattern, pattern.replace)
+ interpreter.core.reader.current_index = index + (pattern.offset or 0)
+ else
+ interpreter.core.reader.current_index = 0
+ end
+ end},
+ {"protect", iterator = function ()
+ if type(interpreter.core.reader.protected) == "table" then
+ return pairs(interpreter.core.reader.protected)
+ end
+ end,
+ function (n)
+ if type(interpreter.core.lines[n]) == "string" then
+ interpreter.core.lines[n] = {interpreter.core.lines[n]}
+ end
+ end}}}},
+ {"unprotect",
+ {"undo_protected",
+ function ()
+ interpreter.core.reader.protected = nil
+ end},
+ {"unprotect_lines", iterator = function () return ipairs(interpreter.core.lines) end,
+ function (i, l)
+ if type(l) == "table" then
+ interpreter.core.lines[i] = l[1]
+ end
+ end}},
+ {"remove_escape", conditional = function () return interpreter.escape end,
+ function ()
+ for num, line in ipairs(interpreter.core.lines) do
+ interpreter.core.lines[num] = gsub(line, interpreter.escape, "")
+ end
+ end}}},
+ {"return_line",
+ function ()
+ return remove(interpreter.core.lines, 1)
+ end}}
+
+interpreter.core.reader.list{"input",
+ -- *** interpreter.unregister () ***
+ -- The function used to remove read_file from the "open_read_file" callback.
+ -- Uses callback.register by default, or luatexbase.remove_from_callback if
+ -- detected.
+ {"unregister", autoreturn = true,
+ {"set_unregister", conditional = function () return interpreter.type"unregister" == 0 end,
+ function ()
+ if luatexbase and luatexbase.remove_from_callback then
+ function interpreter.unregister ()
+ luatexbase.remove_from_callback("open_read_file", "interpreter")
+ end
+ else
+ function interpreter.unregister ()
+ callback.register("open_read_file", nil)
+ end
+ end
+ end},
+ {"use_unregister",
+ function () -- You can't use the `unregister' gate directly, because it isn't created yet.
+ interpreter.unregister()
+ end}},
+ {"open_file",
+ function (fname)
+ return io_open(fname)
+ end},
+ {"set_reader",
+ function (f)
+ return {reader = function () return interpreter.core.reader.read_file(f) end}
+ end}}
+
+function interpreter.reset ()
+ interpreter.active = true
+ interpreter.default_class = 1
+ interpreter.core.classes = {}
+ interpreter.set_class(0, {})
+ _grammar = nil
+ interpreter.escape = nil
+ interpreter.paragraph = "%s*"
+ interpreter.direct = "%%%%I%s+"
+end
+
+-- *** interpreter.register (function) ***
+-- The function used to register the read_file function in the
+-- "open_read_file" callback. If none is given, use callback.register, or
+-- luatexbase.add_to_callback if detected (with "interpreter" as the name).
+-- The function is defined in \interpretfile (see interpreter.tex).
diff --git a/macros/luatex/generic/interpreter/interpreter.sty b/macros/luatex/generic/interpreter/interpreter.sty
new file mode 100644
index 0000000000..ba16a86256
--- /dev/null
+++ b/macros/luatex/generic/interpreter/interpreter.sty
@@ -0,0 +1,17 @@
+% This is interpreter.sty, a style file to load
+% the Interpreter package in LaTeX. Useful information
+% can be found in interpreter-doc.pdf or interpreter-doc.txt.
+%
+% Author: Paul Isambert.
+% E-mail: zappathustra AT free DOT fr
+% Comments and suggestions are welcome.
+% Date: June 2012.
+
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesPackage{interpreter}[2012/06/10 v.1.2 Preprocessing input files on the fly.]
+% Needed to prevent LaTeX check input, which would call open_read_file and
+% thus remove Interpreter's main function (which is removed as soon as it is
+% used).
+\expandafter\let\expandafter\interpreterinput\csname @@input\endcsname
+\input interpreter.tex
+\endinput
diff --git a/macros/luatex/generic/interpreter/interpreter.tex b/macros/luatex/generic/interpreter/interpreter.tex
new file mode 100644
index 0000000000..3568fc90e9
--- /dev/null
+++ b/macros/luatex/generic/interpreter/interpreter.tex
@@ -0,0 +1,37 @@
+% This is the main TeX file for the Interpreter package.
+% Further information in interpreter-doc.pdf or interpreter-doc.txt.
+%
+% Paul Isambert - zappathustra AT free DOT fr - June 2012
+%
+\csname Oh no, Interpreter won't be loaded twice!\endcsname
+\expandafter\let\csname Oh no, Interpreter won't be loaded twice!\endcsname\endinput
+\directlua{require("interpreter.lua")}
+\def\interpretergobble#1{}
+\unless\ifdefined\interpreterinput
+ \let\interpreterinput\input
+\fi
+\def\interpretfile#1#2{%
+ \directlua{%
+ local f = kpse.find_file("i-#1.lua")
+ if f then
+ interpreter.reset()
+ dofile(f)
+ if interpreter.type"register" == 0 then
+ if luatexbase and luatexbase.add_to_callback then
+ function interpreter.register (f)
+ luatexbase.add_to_callback("open_read_file", f, "interpreter")
+ end
+ else
+ function interpreter.register (f)
+ callback.register("open_read_file", f)
+ end
+ end
+ end
+ interpreter.register(interpreter.core.reader.input)
+ tex.print([[\noexpand\interpreterinput]])
+ else
+ tex.error("I can't find file `i-#1.lua'. I won't input file `#2'")
+ tex.print([[\noexpand\interpretergobble]])
+ end
+ }{#2}%
+ }