summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/doc/context/sources/general/manuals/luametatex
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/doc/context/sources/general/manuals/luametatex')
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-callbacks.tex189
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-codes.tex22
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex172
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex1025
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex166
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-introduction.tex140
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex386
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex53
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-math.tex279
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-metapost.tex124
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex172
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex227
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-pdf.tex10
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-preamble.tex79
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-primitives.tex396
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-style.tex144
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex706
-rw-r--r--Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex.tex132
18 files changed, 2522 insertions, 1900 deletions
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-callbacks.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-callbacks.tex
index 244c06c2a5c..d8e98efa291 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-callbacks.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-callbacks.tex
@@ -15,6 +15,9 @@
\libindex{find}
\libindex{known}
+{\em The callbacks are a moving target. Don't bother me with questions about
+them.}
+
This library has functions that register, find and list callbacks. Callbacks are
\LUA\ functions that are called in well defined places. There are two kinds of
callbacks: those that mix with existing functionality, and those that (when
@@ -73,147 +76,56 @@ if callback.known("foo") then ... end
\startsection[title={File related callbacks}][library=callback]
-The behaviour documented in this subsection is considered stable in the sense that
-there will not be backward|-|incompatible changes any more.
-
-\subsection{\cbk {find_read_file}}
-
-Your callback function should have the following conventions:
-
-\startfunctioncall
-<string> actual_name =
- function (<number> id_number, <string> asked_name)
-\stopfunctioncall
-
-Arguments:
-
-\startitemize
-
-\sym{id_number}
-
-This number is zero for the log or \prm {input} files. For \TEX's \prm {read}
-or \prm {write} the number is incremented by one, so \type {\read0} becomes~1.
-
-\sym{asked_name}
-
-This is the user|-|supplied filename, as found by \prm {input}, \prm {openin}
-or \prm {openout}.
-
-\stopitemize
-
-Return value:
-
-\startitemize
-
-\sym{actual_name}
-
-This is the filename used. For the very first file that is read in by \TEX, you
-have to make sure you return an \type {actual_name} that has an extension and
-that is suitable for use as \type {jobname}. If you don't, you will have to
-manually fix the name of the log file and output file after \LUATEX\ is finished,
-and an eventual format filename will become mangled. That is because these file
-names depend on the jobname.
-
-You have to return \type {nil} if the file cannot be found.
-
-\stopitemize
-
-\subsection{\cbk {find_data_file}}
-
-\topicindex{callbacks+data files}
-
-Your callback function should have the following conventions:
-
-\startfunctioncall
-<string> actual_name =
- function (<string> asked_name)
-\stopfunctioncall
-
-Return \type {nil} if the file cannot be found.
-
-\subsection{\cbk {find_format_file}}
+\subsection{\cbk {find_format_file} and \cbk {find_log_file}}
\topicindex{callbacks+format file}
+\topicindex{callbacks+log file}
-Your callback function should have the following conventions:
+These callbacks are called as:
\startfunctioncall
-<string> actual_name =
- function (<string> asked_name)
+<string> actualname =
+ function (<string> askedname)
\stopfunctioncall
-The \type {asked_name} is a format file for reading (the format file for writing
-is always opened in the current directory).
+The \type {askedname} is a format file for reading (the format file for writing
+is always opened in the current directory) or a log file for writing.
-\subsection{\cbk {open_read_file}}
+\subsection{\cbk {open_data_file}}
\topicindex{callbacks+opening files}
-Your callback function should have the following conventions:
+This callback function gets a filename passed:
\startfunctioncall
-<table> env =
- function (<string> file_name)
+<table> env = function (<string> filename)
\stopfunctioncall
-Argument:
-
-\startitemize
-
-\sym{file_name}
-
-The filename returned by a previous \cbk {find_read_file} or the return value of
-\type {kpse.find_file()} if there was no such callback defined.
-
-\stopitemize
-
-Return value:
-
-\startitemize
-
-\sym{env}
-
-This is a table containing at least one required and one optional callback
-function for this file. The required field is \type {reader} and the associated
-function will be called once for each new line to be read, the optional one is
-\type {close} that will be called once when \LUATEX\ is done with the file.
+The return value is either the boolean value false or a table with two functions.
+A mandate \type {reader} function fill be called once for each new line to be
+read, the optional \type {close} function will be called once \LUATEX\ is done
+with the file.
\LUATEX\ never looks at the rest of the table, so you can use it to store your
private per|-|file data. Both the callback functions will receive the table as
their only argument.
-\stopitemize
-
-\subsubsection{\type {reader}}
-
-\topicindex{callbacks+reader}
-
-\LUATEX\ will run this function whenever it needs a new input line from the file.
-
-\startfunctioncall
-function(<table> env)
- return <string> line
-end
-\stopfunctioncall
-
-Your function should return either a string or \type {nil}. The value \type {nil}
-signals that the end of file has occurred, and will make \TEX\ call the optional
-\type {close} function next.
-
-\subsubsection{\type {close}}
-
-\topicindex{callbacks+closing files}
-
-\LUATEX\ will run this optional function when it decides to close the file.
-
-\startfunctioncall
-function(<table> env)
-end
-\stopfunctioncall
-
-Your function should not return any value.
-
-\stopsection
+% No longer needed anyway.
+%
+% \subsection{\cbk {if_end_of_file}}
+%
+% \topicindex{callbacks+checking files}
+%
+% This callback has no arguments and your function should return true or false. The
+% callback is triggered by \type {\ifeof}. It's up to the macro package to come up
+% with a reasonable implementation. By default the test is always true.
+%
+% \startfunctioncall
+% <boolean> eof =
+% function ()
+% \stopfunctioncall
+%
+% \stopsection
\startsection[title={Data processing callbacks}][library=callback]
@@ -339,7 +251,7 @@ into a stack of \prm {hbox}es, after the addition of \prm {parfillskip}.
\startfunctioncall
function(<node> head, <string> groupcode)
- return true | false | <node> newhead
+ return <node> newhead
end
\stopfunctioncall
@@ -441,7 +353,7 @@ stack of \prm {hbox}es.
\startfunctioncall
function(<node> head, <string> groupcode)
- return true | false | <node> newhead
+ return <node> newhead
end
\stopfunctioncall
@@ -457,7 +369,7 @@ material. Math items and line boxes are ignored at the moment.
\startfunctioncall
function(<node> head, <string> groupcode, <number> size,
<string> packtype [, <string> direction] [, <node> attributelist])
- return true | false | <node> newhead
+ return <node> newhead
end
\stopfunctioncall
@@ -485,7 +397,7 @@ that it is called at different moments, there is an extra variable that matches
\startfunctioncall
function(<node> head, <string> groupcode, <number> size, <string> packtype,
<number> maxdepth [, <string> direction] [, <node> attributelist]))
- return true | false | <node> newhead
+ return <node> newhead
end
\stopfunctioncall
@@ -550,7 +462,7 @@ This callback is called when \TEX\ is ready to start boxing the box 255 for \prm
\startfunctioncall
function(<node> head, <string> groupcode, <number> size, <string> packtype,
<number> maxdepth [, <string> direction])
- return true | false | <node> newhead
+ return <node> newhead
end
\stopfunctioncall
@@ -619,13 +531,13 @@ pass.
You must not ruin the node list. For instance, the head normally is a local par node,
and the tail a glue. Messing too much can push \LUATEX\ into panic mode.
-\subsection{\type {insert_local_par}}
+\subsection{\type {insert_par}}
Each paragraph starts with a local par node that keeps track of for instance
the direction. You can hook a callback into the creator:
\startfunctioncall
-function(<node> local_par, <string> location)
+function(<node> par, <string> location)
end
\stopfunctioncall
@@ -694,7 +606,7 @@ This callback replaces the code that prints \LUATEX's statistics and \quote
{output written to} messages. The engine can still do housekeeping and therefore
you should not rely on this hook for postprocessing the \PDF\ or log file.
-\subsection{\cbk {show_error_hook}}
+\subsection{\cbk {intercept_tex_error}, \cbk {intercept_lua_error}}
\topicindex{callbacks+errors}
@@ -706,31 +618,22 @@ end
This callback is run from inside the \TEX\ error function, and the idea is to
allow you to do some extra reporting on top of what \TEX\ already does (none of
the normal actions are removed). You may find some of the values in the \type
-{status} table useful. This callback does not replace any internal code.
+{status} table useful. The \TEX\ related callback gets two arguments: the current
+processing mode and a boolean indicating if there was a runaway.
-\subsection{\cbk {show_error_message}}
+\subsection{\cbk {show_error_message} and \cbk {show_warning_message}}
\topicindex{callbacks+errors}
+\topicindex{callbacks+warnings}
\startfunctioncall
function()
end
\stopfunctioncall
-This callback replaces the code that prints the error message. The usual
+These callback replaces the code that prints the error message. The usual
interaction after the message is not affected.
-\subsection{\cbk {show_lua_error_hook}}
-
-\topicindex{callbacks+errors}
-
-\startfunctioncall
-function()
-end
-\stopfunctioncall
-
-This callback replaces the code that prints the extra \LUA\ error message.
-
\subsection{\cbk {start_file}}
\topicindex{callbacks+files}
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-codes.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-codes.tex
new file mode 100644
index 00000000000..103bb247310
--- /dev/null
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-codes.tex
@@ -0,0 +1,22 @@
+\environment luametatex-style
+
+\startcomponent luametatex-codes
+
+\startchapter[title=Primitive codes]
+
+here follows a list with all primitives and their category is shown. When the
+engine starts up in ini mode all primitives get defined along with some
+properties that makes it possible to do a reverse lookup of a combination of
+command code and char code. But, a primitive, being also a regular command can be
+redefined later on. The table below shows the original pairs but in \CONTEXT\
+some of these primitives are redefined. However, any macro that fits a command
+and char pair is (reported as) a primitive in logs and error messages. In the end
+all tokens are such a combination, The first 16 command codes are reserved for
+characters (the whole \UNICODE\ range can be used as char code) with specific
+catcodes and not mentioned in the list.
+
+\ctxlua{document.allprimitives()}
+
+\stopchapter
+
+\stopcomponent
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex
index 35812b13fcf..5be5463e97e 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex
@@ -10,7 +10,7 @@ As \LUAMETATEX\ is a leaner and meaner \LUATEX, this chapter will discuss
what is gone. We start with the primitives that were dropped.
\starttabulate[|l|pl|]
-\NC fonts \NC \type {\letterspacefont}
+\BC fonts \NC \type {\letterspacefont}
\type {\copyfont}
\type {\expandglyphsinfont}
\type {\ignoreligaturesinfont}
@@ -18,7 +18,7 @@ what is gone. We start with the primitives that were dropped.
\type {\leftghost}
\type {\rightghost}
\NC \NR
-\NC backend \NC \type {\dviextension}
+\BC backend \NC \type {\dviextension}
\type {\dvivariable }
\type {\dvifeedback}
\type {\pdfextension}
@@ -28,14 +28,14 @@ what is gone. We start with the primitives that were dropped.
\type {\draftmode}
\type {\outputmode}
\NC \NR
-\NC dimensions \NC \type {\pageleftoffset}
+\BC dimensions \NC \type {\pageleftoffset}
\type {\pagerightoffset}
\type {\pagetopoffset}
\type {\pagebottomoffset}
\type {\pageheight}
\type {\pagewidth}
\NC \NR
-\NC resources \NC \type {\saveboxresource}
+\BC resources \NC \type {\saveboxresource}
\type {\useboxresource}
\type {\lastsavedboxresourceindex}
\type {\saveimageresource}
@@ -43,11 +43,11 @@ what is gone. We start with the primitives that were dropped.
\type {\lastsavedimageresourceindex}
\type {\lastsavedimageresourcepages}
\NC \NR
-\NC positioning \NC \type {\savepos}
+\BC positioning \NC \type {\savepos}
\type {\lastxpos}
\type {\lastypos}
\NC \NR
-\NC directions \NC \type {\textdir}
+\BC directions \NC \type {\textdir}
\type {\linedir}
\type {\mathdir}
\type {\pardir}
@@ -56,21 +56,25 @@ what is gone. We start with the primitives that were dropped.
\type {\pagedirection}
\type {\bodydirection}
\NC \NR
-\NC randomizer \NC \type {\randomseed}
+\BC randomizer \NC \type {\randomseed}
\type {\setrandomseed}
\type {\normaldeviate}
\type {\uniformdeviate}
\NC \NR
-\NC utilities \NC \type {\synctex}
+\BC utilities \NC \type {\synctex}
\NC \NR
-\NC extensions \NC \type {\latelua}
+\BC extensions \NC \type {\latelua}
\type {\lateluafunction}
- \type {\immediate}
\type {\openout}
\type {\write}
\type {\closeout}
+ \type {\openin}
+ \type {\read}
+ \type {\readline}
+ \type {\closein}
+ \type {\ifeof}
\NC \NR
-\NC control \NC \type {\suppressfontnotfounderror}
+\BC control \NC \type {\suppressfontnotfounderror}
\type {\suppresslongerror}
\type {\suppressprimitiveerror}
\type {\suppressmathparerror}
@@ -78,10 +82,11 @@ what is gone. We start with the primitives that were dropped.
\type {\suppressoutererror}
\type {\mathoption}
\NC \NR
-\NC whatever \NC \type {\primitive}
+\BC system \NC \type {\primitive}
\type {\ifprimitive}
+ \type {\formatname}
\NC \NR
-\NC ignored \NC \type {\long}
+\BC ignored \NC \type {\long}
\type {\outer}
\type {\mag}
\NC \NR
@@ -96,50 +101,68 @@ and they don't work well in \LUATEX\ anyway. We only have two directions left.
The primitive related extensions were not that useful and reliable so they have
been removed. There are some new variants that will be discussed later. The \type
{\outer} and \type {\long} prefixes are gone as they don't make much sense
-nowadays and them becoming dummies opened the way to something new, again to be
-discussed elsewhere. I don't think that (\CONTEXT) users will notice it. The
-\type {\suppress..} features are now default.
+nowadays and them becoming dummies opened the way to something new: control
+sequence properties that permit protection against as well as controlled
+overloading of definitions. I don't think that (\CONTEXT) users will notice these
+prefixed being gone. The definition and parsing related \type {\suppress..}
+features are now default and can't be changed so related primitives are gone.
The \type {\shipout} primitive does no ship out but just erases the content of
-the box, if that hasn't happened already in another way.
-
-The extension primitives relate to the backend (when not immediate) and can be
-implemented as part of a backend design using generic whatsits. There is only one
-type of whatsit now. In fact we're now closer to original \TEX\ with respect to
-the extensions.
+the box, if that hasn't happened already in another way. A macropackage should
+implement its own backend and related shipout. Talking of backend, the extension
+primitives that relate to backends can be implemented as part of a backend design
+using generic whatsits. There is only one type of whatsit now. In fact we're now
+closer to original \TEX\ with respect to the extensions.
The \type {img} library has been removed as it's rather bound to the backend. The
\type {slunicode} library is also gone. There are some helpers in the string
library that can be used instead and one can write additional \LUA\ code if
-needed. There is no longer a \type {pdf} backend library.
+needed. There is no longer a \type {pdf} backend library but we have an up to
+date \PDF\ parsing library on board.
In the \type {node}, \type {tex} and \type {status} library we no longer have
helpers and variables that relate to the backend. The \LUAMETATEX\ engine is in
-principle \DVI\ and \PDF\ unaware. There are only generic whatsit nodes that can
-be used for some management related tasks. For instance you can use them to
-implement user nodes.
+principle \DVI\ and \PDF\ unaware. There are, as mentioned, only generic whatsit
+nodes that can be used for some management related tasks. For instance you can
+use them to implement user nodes. More extensive status information is provided
+in the overhauled status library.
The margin kern nodes are gone and we now use regular kern nodes for them. As a
consequence there are two extra subtypes indicating the injected left or right
kern. The glyph field served no real purpose so there was no reason for a special
kind of node.
-The \KPSE\ library is no longer built|-|in. Because there is no backend, quite
-some file related callbacks could go away. The following file related callbacks
+The \KPSE\ library is no longer built|-|in, but one can use an external \KPSE\
+library, assuming that it is present on the system, because the engine has a so
+called optional library interface to it. Because there is no backend, quite some
+file related callbacks could go away. The following file related callbacks
remained (till now):
\starttyping
-find_write_file find_data_file find_format_file
-open_data_file read_data_file
+find_write_file find_format_file open_data_file
\stoptyping
-Also callbacks related to errors stay:
+The callbacks related to errors are changed:
\starttyping
-show_error_hook show_lua_error_hook,
+intercept_tex_error intercept_lua_error
show_error_message show_warning_message
\stoptyping
+There is a hook that gets called when one of the fundamental memory structures
+gets reallocated.
+
+\starttyping
+trace_memory
+\stoptyping
+
+When you use the overload protect mechanisms, a callback can be plugged in to handle
+exceptions:
+
+\starttyping
+handle_overload
+\stoptyping
+
The (job) management hooks are kept:
\starttyping
@@ -158,19 +181,9 @@ show_whatsit
Being the core of extensibility, the typesetting callbacks of course stayed. This
is what we ended up with:
-% \ctxlua{inspect(table.sortedkeys(callbacks.list))}
-
-\starttyping
-find_log_file, find_data_file, find_format_file, open_data_file, read_data_file,
-process_jobname, start_run, stop_run, define_font, pre_output_filter,
-buildpage_filter, hpack_filter, vpack_filter, hyphenate, ligaturing, kerning,
-pre_linebreak_filter, linebreak_filter, post_linebreak_filter,
-append_to_vlist_filter, mlist_to_hlist, pre_dump, start_file, stop_file,
-handle_error_hook, show_error_hook, show_lua_error_hook, show_error_message,
-show_warning_message, hpack_quality, vpack_quality, insert_local_par,
-contribute_filter, build_page_insert, wrapup_run, new_graf, make_extensible,
-show_whatsit, terminal_input,
-\stoptyping
+\startalign[flushleft,nothyphenated]
+\tt \cldcontext{table.concat(table.sortedkeys(callbacks.list), ", ")}
+\stopalign
As in \LUATEX\ font loading happens with the following callback. This time it
really needs to be set because there is no built|-|in font loader.
@@ -182,10 +195,10 @@ define_font
There are all kinds of subtle differences in the implementation, for instance we
no longer intercept \type {*} and \type {&} as these were already replaced long
ago in \TEX\ engines by command line options. Talking of options, only a few are
-left.
+left. All input goes via \LUA, even the console.
We took our time for reaching a stable state in \LUATEX. Among the reasons is the
-fact that most was experimented with in \CONTEXT. It took many man|-|years to
+fact that most was experimented with in \CONTEXT. It took many years of work to
decide what to keep and how to do things. Of course there are places when things
can be improved and it might happen in \LUAMETATEX. Contrary to what is sometimes
suggested, the \LUATEX|-|\CONTEXT\ \MKIV\ combination (assuming matched versions)
@@ -207,10 +220,71 @@ might be noticed that it is not possible to backtrack or inject something. Of
course it is no big deal to implement all that in \LUA\ if needed. It removes a
system dependency and makes for a bit cleaner code.
-There are new primitives too as well as some extensions to existing primitive
+There are new primitives as well as some extensions to existing primitive
functionality. These are described in following chapters but there might be
hidden treasures in the binary. If you locate them, don't automatically assume
-them to stay, some might be part of experiments!
+them to stay, some might be part of experiments! There are for instance a few
+csname related definers, we have integer and dimension constants, the macro
+argument parser can be brought in tolerant mode, the repertoire of conditionals
+has been extended, some internals can be controlled (think of normalization of
+lines, hyphenation etc.), and macros can be protected against user overload. Not
+all is discussed in detail in this manual but there are introductions in the
+\CONTEXT\ distribution that explain them. But the \TEX\ kernel is of course
+omnipresent.
+
+\startluacode
+
+local luametatex = tex.primitives()
+local luatex = table.load("luatex-primitives.lua")
+
+if not luatex then
+ local tex = "\\starttext \\ctxlua {table.save(tex.jobname .. '.lua',tex.primitives())} \\stoptext"
+
+ io.savedata("luatex-primitives.tex", tex)
+
+ os.execute("context --luatex --once luatex-primitives")
+
+ luatex = table.load("luatex-primitives.lua")
+end
+
+
+if luatex and luametatex then
+
+ luatex = table.tohash(luatex)
+ luametatex = table.tohash(luametatex)
+
+ -- context.page()
+
+ context("The following primitives are available in \\LUATEX\\ but not in \\LUAMETATEX. ")
+ context("Some of these are emulated in \\CONTEXT.")
+
+ context.startcolumns { n = 2 }
+ for k, v in table.sortedhash(luatex) do
+ if not luametatex[k] then
+ context.type(k)
+ context.crlf()
+ end
+ end
+ context.stopcolumns()
+
+
+ -- context.page()
+
+ context("The following primitives are available in \\LUAMETATEX\\ only. ")
+ context("At some point in time some might be added to \\LUATEX.")
+
+ context.startcolumns { n = 2 }
+ for k, v in table.sortedhash(luametatex) do
+ if not luatex[k] then
+ context.type(k)
+ context.crlf()
+ end
+ end
+ context.stopcolumns()
+
+end
+
+\stopluacode
\stopchapter
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
index f295b34643a..947b2320a13 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
@@ -19,7 +19,7 @@ and there will be no \type {luametatex} variants. This is because we consider
\LUAMETATEX\ to be \LUATEX 2\high{+}.
Contrary to the \LUATEX\ engine \LUAMETATEX\ enables all its primitives. You can
-clone (a selection of) primitives with a different prefix, like:
+clone (a selection of) primitives with a different prefix, like this:
\starttyping
\directlua { tex.enableprimitives('normal',tex.extraprimitives()) }
@@ -27,38 +27,26 @@ clone (a selection of) primitives with a different prefix, like:
The \type {extraprimitives} function returns the whole list or a subset,
specified by one or more keywords \type {core}, \type {tex}, \type {etex} or
-\type {luatex}. \footnote {At some point this function might be changed to return
-the whole list always}.
+\type {luatex}. When you clone all primitives you can also do this:
+
+\starttyping
+\directlua { tex.enableprimitives('normal',true) }
+\stoptyping
But be aware that the curly braces may not have the proper \prm {catcode}
assigned to them at this early time (giving a \quote {Missing number} error), so
it may be needed to put these assignments before the above line:
\starttyping
-\catcode `\{=1
-\catcode `\}=2
+\catcode `\{ = 1
+\catcode `\} = 2
\stoptyping
More fine|-|grained primitives control is possible and you can look up the
details in \in {section} [luaprimitives]. There are only three kinds of
primitives: \type {tex}, \type {etex} and \type {luatex} but a future version
-might drop this and no longer make that distinction as it no longer serves
-a purpose.
-
-\stopsubsection
-
-\startsubsection[title={Experiments}]
-
-There are a few extensions to the engine regarding the macro machinery. Some are
-already well tested but others are (still) experimental. Although they are likely
-to stay, their exact behaviour might evolve. Because \LUAMETATEX\ is also used
-for experiments, this is not a problem. We can always decide to also add some of
-what is discussed here to \LUATEX, but it will happen with a delay.
-
-There are all kinds of small improvements that might find their way into stock
-\LUATEX: a few more helpers, some cleanup of code, etc. We'll see. In any case,
-if you play with these before they are declared stable, unexpected side effects
-are what you have to accept.
+might drop this and no longer make that distinction as it no longer serves a
+purpose apart from the fact that it reveals some history.
\stopsubsection
@@ -83,9 +71,9 @@ There are three primitives to test the version of \LUATEX\ (and \LUAMETATEX):
\BC explanation \NC \NR
\TB
\NC \lpr {luatexbanner} \NC \VersionHack{\luatexbanner}
- \NC the banner reported on the command line \NC \NR
+ \NC the banner reported on the console \NC \NR
\NC \lpr {luatexversion} \NC \the\luatexversion
- \NC a combination of major and minor number \NC \NR
+ \NC major and minor number combined \NC \NR
\NC \lpr {luatexrevision} \NC \the\luatexrevision
\NC the revision number \NC \NR
\LL
@@ -123,17 +111,6 @@ The \LUAMETATEX\ version number starts at~2 in order to prevent a clash with
\LUATEX, and the version commands are the same. This is a way to indicate that
these projects are related.
-\startsubsubsection[title={\lpr {formatname}}]
-
-\topicindex{format}
-
-The \lpr {formatname} syntax is identical to \prm {jobname}. In \INITEX, the
-expansion is empty. Otherwise, the expansion is the value that \prm {jobname} had
-during the \INITEX\ run that dumped the currently loaded format. You can use this
-token list to provide your own version info.
-
-\stopsubsubsection
-
\stopsubsection
\stopsection
@@ -178,7 +155,9 @@ the time comes to print a character $c>=1{,}114{,}112$, \LUATEX\ will actually
print the single byte corresponding to $c$ minus 1{,}114{,}112.
Contrary to other \TEX\ engines, the output to the terminal is as|-|is so there
-is no escaping with \type {^^}. We operate in a \UTF\ universe.
+is no escaping with \type {^^}. We operate in a \UTF\ universe. Because we
+operate in a \CCODE\ universum, zero characters are special but because we also
+live in a \UNICODE\ galaxy that is no real problem.
\stopsubsection
@@ -229,6 +208,34 @@ Fonts are loaded via \LUA\ and a minimal amount of information is kept at the
about what a character (or glyph) number represents (a \UNICODE\ or index) as it
only is interested in dimensions.
+In \TEX\ the number of registers is 256 and \ETEX\ bumped that to 32K. One reason
+for a fixed number is that these registers are fast ways to store data and
+therefore are part of the main lookup table (used for data and pointers to data
+as well as save and restore housekeeping). In \LUATEX\ the number was bumped to
+64K but one can argue that less would also do. In order to keep the default
+memory footprint reasonable, in \LUAMETATEX\ the number of languages, fonts and
+marks is limited. The size of some tables can be limited by configuration
+settings, so they can start out small and grow till configured maximum which is
+smaller than the absolute maximum. The following table shows all kind of defaults
+as reported by \typ {status.getconstants()}.
+
+\startluacode
+ context.starttabulate { "|T|r|" }
+ for k, v in table.sortedhash(status.getconstants()) do
+ context.NC() context(k) context.NC() context(v) context.NC() context.NR()
+ end
+ context.stoptabulate()
+\stopluacode
+
+Because we have additional ways to store integers, dimensions and glue, we might
+actually decide to decrease the maximum of the registers: if 64K is not enough,
+and you work around it, then likely 32K might do as well. Also, we have \LUA\ to
+store massive amounts of data. One can argue that saving some 1.5MB memory (when
+we go halfway) is not worth the effort in a time when you have to close a browser
+in order to free the gigabytes it consumes, but there is no reason not to be lean
+and mean: a more conservative approach to start with creates headroom for going
+wild later.
+
\stopsubsection
\stopsection
@@ -274,6 +281,12 @@ language related information, the expansion factor, etc. Now that we have access
to these nodes from \LUA\ it makes sense to be able to carry more information
with a node and this is where attributes kick in.
+It is important to keep in mind that there are situations where nodes get created
+in the current context. For instance, when \TEX\ builds a paragraph or page or
+constructs math formulas, it does add nodes and giving these the current
+attributes makes no sense and can even give weird side effects. In these cases,
+the attributes are inherited from neighbouring nodes.
+
\stopsubsection
\startsubsection[title={Attribute registers}]
@@ -320,6 +333,7 @@ experimental.
\topicindex {attributes}
\topicindex {boxes}
+\topicindex {vcentering}
Nodes typically receive the list of attributes that is in effect when they are
created. This moment can be quite asynchronous. For example: in paragraph
@@ -343,7 +357,7 @@ already boxed material will have no effect. Keep in mind that this
incompatibility is mostly due to the fact that separate specials and literals are
a more unnatural approach to colors than attributes.
-It is possible to fine-tune the list of attributes that are applied to a \type
+It is possible to fine|-|tune the list of attributes that are applied to a \type
{hbox}, \type {vbox} or \type {vtop} by the use of the keyword \type {attr}. The
\type {attr} keyword(s) should come before a \type {to} or \type {spread}, if
that is also specified. An example is:
@@ -392,6 +406,19 @@ indexing the node (here we do that with \type {[a]}). Running this snippet gives
Because some values are not set we need to apply the \type {tostring} function
here so that we get the word \type {nil}.
+A special kind of box is \prm {vcenter}. This one also can have attributes. When
+one or more are set these plus the currently set attributes are bound to the
+resulting box. In regular \TEX\ these centered boxes are only permitted in math
+mode, but in \LUAMETATEX\ there is no error message and the box the height and
+depth are equally divided. Of course in text mode there is no math axis related
+offset applied.
+
+It is possible to change or add to the attributes assigned to a box:
+
+\starttyping
+\boxattr 0 123 456
+\stoptyping
+
\stopsubsection
\stopsection
@@ -467,20 +494,22 @@ will result in
Note that the expansion of \prm {directlua} is a sequence of characters, not of
tokens, contrary to all \TEX\ commands. So formally speaking its expansion is
-null, but it places material on a pseudo-file to be immediately read by \TEX, as
-\ETEX's \prm {scantokens}. For a description of print functions look at \in
-{section} [sec:luaprint].
+null, but it collects material in a new level on the input stack to be
+immediately read by \TEX\ after the \LUA\ call as finished. It is a bit like
+\ETEX's \prm {scantokens}, which now uses the same mechanism. For a description
+of print functions look at \in {section} [sec:luaprint].
Because the \syntax {<general text>} is a chunk, the normal \LUA\ error handling
is triggered if there is a problem in the included code. The \LUA\ error messages
-should be clear enough, but the contextual information is still pretty bad.
-Often, you will only see the line number of the right brace at the end of the
-code.
+should be clear enough, but the contextual information is often suboptimal
+because it can come from deep down, and \TEX\ has no knowledge about what you do
+in \LUA. Often, you will only see the line number of the right brace at the end
+of the code.
While on the subject of errors: some of the things you can do inside \LUA\ code
can break up \LUAMETATEX\ pretty bad. If you are not careful while working with
-the node list interface, you may even end up with assertion errors from within
-the \TEX\ portion of the executable.
+the node list interface, you may even end up with errors or even crashes from
+within the \TEX\ portion of the executable.
\stopsubsection
@@ -557,6 +586,8 @@ tokens and assume that the function is available when that token expands. On the
other hand, as we have tested this functionality in relative complex situations
normal usage should not give problems.
+{\em It makes sense to delegate the implementation of the primitives to \LUA.}
+
\stopsubsection
\startsubsection[title={\lpr {luabytecode} and \lpr {luabytecodecall}}]
@@ -586,6 +617,8 @@ The variable \type {s} in the code is the number of the byte code register that
can be used for diagnostic purposes. The advantage of bytecode registers over
function calls is that they are stored in the format (but without upvalues).
+{\em It makes sense to delegate the implementation of the primitives to \LUA.}
+
\stopsubsection
\stopsection
@@ -608,14 +641,14 @@ contents is stored and retrieved from the format file.
\startsubsection[title={\lpr {catcodetable}}]
-\startsyntax
-\catcodetable <15-bit number>
-\stopsyntax
-
The primitive \lpr {catcodetable} switches to a different catcode table. Such a
table has to be previously created using one of the two primitives below, or it
has to be zero. Table zero is initialized by \INITEX.
+\startsyntax
+\catcodetable <15-bit number>
+\stopsyntax
+
\stopsubsection
\startsubsection[title={\lpr {initcatcodetable}}]
@@ -655,11 +688,9 @@ initial values are:
\lpr {savecatcodetable} copies the current set of catcodes to a new table with
the requested number. The definitions in this new table are all treated as if
-they were made in the outermost level.
-
-The new table is allocated globally: it will not go away after the current group
-has ended. If the supplied number is the currently active table, an error is
-raised.
+they were made in the outermost level. Again, the new table is allocated globally:
+it will not go away after the current group has ended. If the supplied number is
+the currently active table, an error is raised.
\stopsubsection
@@ -667,7 +698,7 @@ raised.
\startsection[title={Tokens, commands and strings}]
-\startsubsection[title={\lpr {scantextokens}}]
+\startsubsection[title={\lpr {scantextokens} and \lpr {tokenized}}]
\topicindex {tokens+scanning}
@@ -677,8 +708,7 @@ differences are:
\startitemize
\startitem
- The last (and usually only) line does not have a \prm {endlinechar}
- appended.
+ The last (and usually only) line does not have a \prm {endlinechar} appended.
\stopitem
\startitem
\lpr {scantextokens} never raises an EOF error, and it does not execute
@@ -691,6 +721,18 @@ differences are:
\stopitem
\stopitemize
+The implementation in \LUAMETATEX\ is different in the sense that it uses the same
+methods as printing from \LUA\ to \TEX\ does. Therefore, in addition to the two
+commands we also have this expandable command:
+
+\startsyntax
+\tokenized {...}
+\tokenized catcodetable <number> {...}
+\stopsyntax
+
+The \ETEX\ command \type {\tracingscantokens} has been dropped in the process as
+that was interwoven with the old code.
+
\stopsubsection
\startsubsection[title={\lpr {toksapp}, \lpr {tokspre}, \lpr {etoksapp}, \lpr {etokspre},
@@ -750,7 +792,7 @@ that it saves a few tokens and can make code a bit more readable.
This primitive complements the \ETEX\ mark primitives and clears a mark class
completely, resetting all three connected mark texts to empty. It is an
-immediate command.
+immediate command (no synchronization node is used).
\startsyntax
\clearmarks <16-bit number>
@@ -792,103 +834,135 @@ but faster (only measurable with millions of calls) and probably more convenient
\stopsubsection
-\startsubsection[title={\lpr {expanded}, \lpr {immediateassignment} and \lpr {immediateassigned}}]
-
-\topicindex {expansion}
-
-The \lpr {expanded} primitive takes a token list and expands its content which can
-come in handy: it avoids a tricky mix of \prm {expandafter} and \prm {noexpand}.
-You can compare it with what happens inside the body of an \prm {edef}. But this
-kind of expansion still doesn't expand some primitive operations.
-
-\startbuffer
-\newcount\NumberOfCalls
-
-\def\TestMe{\advance\NumberOfCalls1 }
-
-\edef\Tested{\TestMe foo:\the\NumberOfCalls}
-\edef\Tested{\TestMe foo:\the\NumberOfCalls}
-\edef\Tested{\TestMe foo:\the\NumberOfCalls}
-
-\meaning\Tested
-\stopbuffer
-
-\typebuffer
-
-The result is a macro that has the not expanded code in its body:
-
-\getbuffer
-
-Instead we can define \tex {TestMe} in a way that expands the assignment
-immediately. You need of course to be aware of preventing look ahead interference
-by using a space or \tex {relax} (often an expression works better as it doesn't
-leave an \tex {relax}).
-
-\startbuffer
-\def\TestMe{\immediateassignment\advance\NumberOfCalls1 }
+\startsubsection[title={\lpr {defcsname}, \lpr {edefcsname}, \lpr {edefcsname} and \lpr {xdefcsname}}]
-\edef\Tested{\TestMe foo:\the\NumberOfCalls}
-\edef\Tested{\TestMe foo:\the\NumberOfCalls}
-\edef\Tested{\TestMe foo:\the\NumberOfCalls}
+Although we can implement these primitives easily using macros it makes sense,
+given the popularity of \prm {csname} to have these as primitives. It also saves
+some \prm {expandafter} usage and it looks a bit better in the source.
-\meaning\Tested
-\stopbuffer
-
-\typebuffer
-
-This time the counter gets updates and we don't see interference in the
-resulting \tex {Tested} macro:
-
-\getbuffer
-
-Here is a somewhat silly example of expanded comparison:
-
-\startbuffer
-\def\expandeddoifelse#1#2#3#4%
- {\immediateassignment\edef\tempa{#1}%
- \immediateassignment\edef\tempb{#2}%
- \ifx\tempa\tempb
- \immediateassignment\def\next{#3}%
- \else
- \immediateassignment\def\next{#4}%
- \fi
- \next}
-
-\edef\Tested
- {(\expandeddoifelse{abc}{def}{yes}{nop}/%
- \expandeddoifelse{abc}{abc}{yes}{nop})}
-
-\meaning\Tested
-\stopbuffer
+\starttyping
+\def\gdefcsname foo\endcsname{oof}
+\stoptyping
-\typebuffer
+\stopsubsection
-It gives:
+\startsubsection[title={\lpr {expanded}}]
-\getbuffer
+\topicindex {expansion}
-A variant is:
+The \lpr {expanded} primitive takes a token list and expands its content which
+can come in handy: it avoids a tricky mix of \prm {expandafter} and \prm
+{noexpand}. You can compare it with what happens inside the body of an \prm
+{edef}. The \tex {immediateassignment} and \tex {immediateassigned} commands are
+gone because we have the more powerful local control commands. They are a tad
+slower but this mechanism isn't used that much anyway. Inside an \prm {edef} you
+can use the \type {\immediate} prefix anyway, so if you really want these
+primitives back you can say:
\starttyping
-\def\expandeddoifelse#1#2#3#4%
- {\immediateassigned{
- \edef\tempa{#1}%
- \edef\tempb{#2}%
- }%
- \ifx\tempa\tempb
- \immediateassignment\def\next{#3}%
- \else
- \immediateassignment\def\next{#4}%
- \fi
- \next}
+\let\immediateassignment\immediate
+\let\immediateassigned \localcontrolled
\stoptyping
-The possible error messages are the same as using assignments in preambles of
-alignments and after the \prm {accent} command. The supported assignments are the
-so called prefixed commands (except box assignments).
-
\stopsubsection
+% \startsubsection[title={\lpr {expanded}, \lpr {immediateassignment} and \lpr {immediateassigned}}]
+%
+% \topicindex {expansion}
+%
+% The \lpr {expanded} primitive takes a token list and expands its content which can
+% come in handy: it avoids a tricky mix of \prm {expandafter} and \prm {noexpand}.
+% You can compare it with what happens inside the body of an \prm {edef}. But this
+% kind of expansion still doesn't expand some primitive operations.
+%
+% \startbuffer
+% \newcount\NumberOfCalls
+%
+% \def\TestMe{\advance\NumberOfCalls1 }
+%
+% \edef\Tested{\TestMe foo:\the\NumberOfCalls}
+% \edef\Tested{\TestMe foo:\the\NumberOfCalls}
+% \edef\Tested{\TestMe foo:\the\NumberOfCalls}
+%
+% \meaning\Tested
+% \stopbuffer
+%
+% \typebuffer
+%
+% The result is a macro that has the not expanded code in its body:
+%
+% \getbuffer
+%
+% Instead we can define \tex {TestMe} in a way that expands the assignment
+% immediately. You need of course to be aware of preventing look ahead interference
+% by using a space or \tex {relax} (often an expression works better as it doesn't
+% leave an \tex {relax}).
+%
+% \startbuffer
+% \def\TestMe{\immediateassignment\advance\NumberOfCalls1 }
+%
+% \edef\Tested{\TestMe foo:\the\NumberOfCalls}
+% \edef\Tested{\TestMe foo:\the\NumberOfCalls}
+% \edef\Tested{\TestMe foo:\the\NumberOfCalls}
+%
+% \meaning\Tested
+% \stopbuffer
+%
+% \typebuffer
+%
+% This time the counter gets updates and we don't see interference in the
+% resulting \tex {Tested} macro:
+%
+% \getbuffer
+%
+% Here is a somewhat silly example of expanded comparison:
+%
+% \startbuffer
+% \def\expandeddoifelse#1#2#3#4%
+% {\immediateassignment\edef\tempa{#1}%
+% \immediateassignment\edef\tempb{#2}%
+% \ifx\tempa\tempb
+% \immediateassignment\def\next{#3}%
+% \else
+% \immediateassignment\def\next{#4}%
+% \fi
+% \next}
+%
+% \edef\Tested
+% {(\expandeddoifelse{abc}{def}{yes}{nop}/%
+% \expandeddoifelse{abc}{abc}{yes}{nop})}
+%
+% \meaning\Tested
+% \stopbuffer
+%
+% \typebuffer
+%
+% It gives:
+%
+% \getbuffer
+%
+% A variant is:
+%
+% \starttyping
+% \def\expandeddoifelse#1#2#3#4%
+% {\immediateassigned{
+% \edef\tempa{#1}%
+% \edef\tempb{#2}%
+% }%
+% \ifx\tempa\tempb
+% \immediateassignment\def\next{#3}%
+% \else
+% \immediateassignment\def\next{#4}%
+% \fi
+% \next}
+% \stoptyping
+%
+% The possible error messages are the same as using assignments in preambles of
+% alignments and after the \prm {accent} command. The supported assignments are the
+% so called prefixed commands (except box assignments).
+%
+% \stopsubsection
+
\startsubsection[title={\lpr {ignorepars}}]
This primitive is like \prm {ignorespaces} but also skips paragraph ending
@@ -1034,6 +1108,55 @@ or more than zero.
\typebuffer \blank {\tt \getbuffer} \blank
+The last checked values are available in \lpr {lastchknum} and \lpr {lastchkdim}.
+These don't obey grouping.
+
+\stopsubsection
+
+\startsubsection[title={\lpr {ifmathstyle} and \lpr {ifmathparameter}}]
+
+These two are variants on \prm {ifcase} where the first one operates with values
+in ranging from zero (display style) to seven (cramped script script style) and
+the second one can have three values: a parameter is zero, has a value or is
+unset. The \type {\ifmathparameter} primitive takes a proper parameter name and a
+valid style identifier (a primitive identifier or number). The \type
+{\ifmathstyle} primitive is equivalent to \type {\ifcase \mathstyle}.
+
+\stopsubsection
+
+\startsubsection[title={\lpr {ifempty}}]
+
+This primitive tests for the following token (control sequence) having no
+content. Assuming that \type {\empty} is indeed empty, the following two are
+equivalent:
+
+\starttyping
+\ifempty\whatever
+\ifx\whatever\empty
+\stoptyping
+
+There is no real performance gain here, it's more one of these extensions that
+lead to less clutter in tracing.
+
+\stopsubsection
+
+\startsubsection[title={\lpr {ifrelax}}]
+
+This primitive complements \type {\ifdefined}, \type {\ifempty} and \type
+{\ifcsname} so that we have all reasonable tests directly available.
+
+\stopsubsection
+
+\startsubsection[title={\lpr {ifboolean}}]
+
+This primitive tests for non|-|zero, so the next variants are similar
+
+\starttyping
+ \ifcase <integer>.F.\else .T.\fi
+\unless\ifcase <integer>.T.\else .F.\fi
+ \ifboolean<integer>.T.\else .F.\fi
+\stoptyping
+
\stopsubsection
\startsubsection[title={\lpr {iftok} and \lpr {ifcstok}}]
@@ -1058,13 +1181,40 @@ provided in \LUAMETATEX:
\typebuffer \startpacked[blank] {\tt\nospacing\getbuffer} \stoppacked
-You can check if a macro is is defined as protected with \type {\ifprotected}
-while frozen macros can be tested with \type {\iffrozen}. A provisional \type
+You can check if a macro is defined as protected with \type {\ifprotected} while
+frozen macros can be tested with \type {\iffrozen}. A provisional \type
{\ifusercmd} tests will check if a command is defined at the user level (and this
one might evolve).
\stopsubsection
+\startsubsection[title={\lpr {ifarguments}, \lpr {ifparameters} and \lpr {ifparameter}}]
+
+These are part of the extended macro argument parsing features. The \type
+{\ifarguments} condition is like an \type {\ifcase} where the number is the
+picked up number of arguments. The number reflects the {\em last} count, so
+successive macro expansions will adapt the value. The \type {\ifparameters}
+counts till the first empty parameter and the \type {\ifparameter} (singular)
+takes a parameter reference (like \type {#2}) and again is an \type {\ifcase}
+where zero means a bad reference, one a non|-|empty argument and two an empty
+one. A typical usage is:
+
+\starttyping
+\def\foo#1#2%
+ {\ifparameter#1\or one\fi
+ \ifparameter#2\or two\fi}
+\stoptyping
+
+No expansion of arguments takes place here but you can use a test like this:
+
+\starttyping
+\def\foo#1#2%
+ {\iftok{#1}{}\else one\fi
+ \iftok{#2}{}\else two\fi}
+\stoptyping
+
+\stopsubsection
+
\startsubsection[title={\lpr {ifcondition}}]
\topicindex {conditions}
@@ -1123,7 +1273,7 @@ we use \type {\unless} to negate the result.
\stopsubsection
-\startsubsection[title={\lpr {orelse}}]
+\startsubsection[title={\lpr {orelse} and \lpr {orunless}}]
Sometimes you have successive tests that, when laid out in the source lead to
deep trees. The \type {\ifcase} test is an exception. Experiments with \type
@@ -1189,6 +1339,12 @@ Of course it is only useful at the right level, so you might end up with cases l
\fi
\stoptyping
+The \lpr {orunless} variant negates the next test, just like \prm {unless}. In
+some cases these commands look at the next token to see if it is an if|-|test so
+a following negation will not work (read: making that work would complicate the
+code and hurt efficiency too). Side note: interesting is that in \CONTEXT\ we
+hardly use this kind of negation.
+
\stopsubsection
\startsubsection[title={\lpr {ifprotected}, \lpr {frozen}, \lpr {iffrozen} and \lpr {ifusercmd}}]
@@ -1219,10 +1375,18 @@ the acceptable range is from 0 to 65535.
\stopsubsection
-\startsubsection[title={\prm {vpack}, \prm {hpack} and \prm {tpack}}]
+\startsubsection[title={\prm {hrule}, \prm {vrule}, \lpr {nohrule} and \lpr {novrule}}]
+
+\topicindex {rules}
+
+Both rule drawing commands take an optional \type {xoffset} and \type {yoffset}
+parameter. The displacement is virtual and not taken into account when the
+dimensions are calculated.
-These three primitives are like \prm {vbox}, \prm {hbox} and \prm {vtop}
-but don't apply the related callbacks.
+Two new primitives were introduced: \lpr {nohrule} and \lpr {novrule}. These can
+be used to reserve space. This is often more efficient than creating an empty box
+with fake dimensions. Of course this assumes that the backend implements them
+being invisible but still taking space.
\stopsubsection
@@ -1236,7 +1400,7 @@ a split of the given size but result has the natural dimensions then.
\stopsubsection
-\startsubsection[title={Images and reused box objects},reference=sec:imagedandforms]
+\startsubsection[title={Images and reused box objects},reference=sec:imagesandforms]
In original \TEX\ image support is dealt with via specials. It's not a native
feature of the engine. All that \TEX\ cares about is dimensions, so in practice
@@ -1252,12 +1416,12 @@ point this approach was abandoned and a more natural trick was used: images (and
box resources) became a special kind of rules, and as rules already have
dimensions, the code could be simplified.
-When direction nodes and localpar nodes also became first class nodes, whatsits
-again became just that: nodes representing whatever you want, but without
-dimensions, and therefore they could again be ignored when dimensions mattered.
-And, because images were disguised as rules, as mentioned, their dimensions
-automatically were taken into account. This seperation between front and backend
-cleaned up the code base already quite a bit.
+When direction nodes and (formerly local) par nodes also became first class
+nodes, whatsits again became just that: nodes representing whatever you want, but
+without dimensions, and therefore they could again be ignored when dimensions
+mattered. And, because images were disguised as rules, as mentioned, their
+dimensions automatically were taken into account. This separation between front
+and backend cleaned up the code base already quite a bit.
In \LUAMETATEX\ we still have the image specific subtypes for rules, but the
engine never looks at subtypes of rules. That was up to the backend. This means
@@ -1313,21 +1477,12 @@ packages.
\startsubsection[title={\lpr {hpack}, \lpr {vpack} and \lpr {tpack}}]
-These three primitives are the equivalents of \type {\hbox}, \type {\vbox} and
-\type {\vtop} but they don't trigger the packaging related callbacks. Of course
+These three primitives are the equivalents of \prm {hbox}, \prm {vbox} and
+\prm {vtop} but they don't trigger the packaging related callbacks. Of course
one never know if content needs a treatment so using them should be done with
-care.
-
-\stopsubsection
-
-\startsubsection[title={\lpr {nohrule} and \lpr {novrule}}]
-
-\topicindex {rules}
-
-Because introducing a new keyword can cause incompatibilities, two new primitives
-were introduced: \lpr {nohrule} and \lpr {novrule}. These can be used to
-reserve space. This is often more efficient than creating an empty box with fake
-dimensions.
+care. Apart from accepting more keywords (and therefore options) the normal
+box behave the same as before. The \prm {vcenter} builder also works in text
+mode.
\stopsubsection
@@ -1390,14 +1545,29 @@ the node number of the nodes.
\stopsubsection
+% \startsubsection[title={\lpr {lastnodetype}, \lpr {lastnodesubtype}, \lpr
+% {currentiftype} and \lpr {internalcodesmode}.}]
+%
+% The \ETEX\ command \type {\lastnodetype} is limited to some nodes. When the
+% parameter \type {\internalcodesmode} is set to a non|-|zero value the normal
+% (internally used) numbers are reported. The same is true for \type
+% {\currentiftype}, as we have more conditionals and also use a different order.
+% The \type {\lastnodesubtype} is a bonus.
+%
+% \stopsubsection
+
\startsubsection[title={\lpr {lastnodetype}, \lpr {lastnodesubtype}, \lpr
-{currentiftype} and \lpr {internalcodesmode}.}]
+{currentiftype}}]
-The \ETEX\ command \type {\lastnodetype} is limited to some nodes. When the
-parameter \type {\internalcodesmode} is set to a non|-|zero value the normal
-(internally used) numbers are reported. The same is true for \type
+The \ETEX\ command \type {\lastnodetype} returns the node codes as used in the
+engine. You can query the numbers at the \LUA\ end if you need the actual values.
+The parameter \type {\internalcodesmode} is no longer provided as compatibility
+switch because \LUATEX\ has more cq. some different nodes and it makes no sense
+to be incompatible with the \LUA\ end of the engine. The same is true for \type
{\currentiftype}, as we have more conditionals and also use a different order.
-The \type {\lastnodesubtype} is a bonus.
+The \type {\lastnodesubtype} is a bonus and again reports the codes used
+internally. During development these might occasionally change, but eventually
+they will be stable.
\stopsubsection
@@ -1417,7 +1587,10 @@ The \type {\lastnodesubtype} is a bonus.
\stoptyping
This allows for embedded spaces, without the need for double quotes. Macro
-expansion takes place inside the argument.
+expansion takes place inside the argument. Keep in mind that as side effect of
+delegating \IO\ to \LUA\ the \prm {openin} primitive is nor provided by the
+engine and has to be implemented by the macro package. This also means that the
+limit on the number of open files is not enforced by the engine.
The \lpr {tracingfonts} primitive that has been inherited from \PDFTEX\ has
been adapted to support variants in reporting the font. The reason for this
@@ -1442,10 +1615,12 @@ extension is that a csname not always makes sense. The zero case is the default.
\topicindex {files+writing}
-You can now open upto 127 files with \prm {openout}. When no file is open writes
-will go to the console and log. The \type {write} related primitives have to be
-implemented as part of a backend! As a consequence a system command is no longer
-possible but one can use \type {os.execute} to do the same.
+Writing to a file in \TEX\ has two forms: delayed and immediate. Delayed writing
+means that the to be written text is anchored in the node list and flushed by the
+backend. As all \IO\ is delegated to \LUA, this also means that it has to deal
+with distinction. In \LUATEX\ the number of open files was already bumped to 127,
+but in \LUAMETATEX\ it depends on the macro package. The special meaning of
+channel 18 was already dropped in \LUATEX\ because we have \type {os.execute}.
\stopsubsection
@@ -1617,106 +1792,143 @@ convenience extra which could also have been implemented using node list parsing
\stopsubsection
-\startsubsection[title={Controlling glue with \lpr {breakafterdirmode}}]
+\startsubsection[title={Normalizing lines}]
-Glue after a dir node is ignored in the linebreak decision but you can bypass that
-by setting \lpr {breakafterdirmode} to~\type {1}. The following table shows the
-difference. Watch your spaces.
+The original \TEX\ machinery was never meant to be opened up. As a consequence a
+constructed line can have different layouts. There can be left- and/or right
+skips and hanging indentation or parshape can result in a shift and adapted
+width. In \LUATEX\ glue got subtypes so we can recognize the left-, right and
+parfill skips, but still there is no hundred percent certainty about the shape.
-\def\ShowSome#1{%
- \BC \type{#1}
- \NC \breakafterdirmode\zerocount\hsize\zeropoint#1
- \NC
- \NC \breakafterdirmode\plusone\hsize\zeropoint#1
- \NC
- \NC \NR
-}
+In \LUAMETATEX\ lines can be normalized. This is optional because we want to
+preserve the original (for comparison) and is controlled by \lpr
+{normalizelinemode}. That variable actually drives some more. An earlier version
+provided a few more granular options (for instance: does a leftskip comes before
+or after a left hanging indentation) but in the end that was dropped. Because
+this normalization only is seen at the \LUA\ end there is no need to go into much
+detail here.
-\starttabulate[|l|Tp(1pt)|w(5em)|Tp(1pt)|w(5em)|]
- \DB
- \BC \type{0}
- \NC
- \BC \type{1}
- \NC
- \NC \NR
- \TB
- \ShowSome{pre {\textdirection 0 xxx} post}
- \ShowSome{pre {\textdirection 0 xxx }post}
- \ShowSome{pre{ \textdirection 0 xxx} post}
- \ShowSome{pre{ \textdirection 0 xxx }post}
- \ShowSome{pre { \textdirection 0 xxx } post}
- \ShowSome{pre {\textdirection 0\relax\space xxx} post}
- \LL
-\stoptabulate
+At this moment a line has this pattern: left parfill, left hang, left skip,
+indentation, content, right hang, right skip, right parfill. Of course the
+indentation and fill skips are not present in every line.
-\stopsubsection
+Control over normalization happens via the mentioned mode variable and here is
+what the engine provides right now. We use a bitmap:
-\startsubsection[title={Controling parshapes with \lpr {shapemode}}]
-
-Another adaptation to the \ALEPH\ directional model is control over shapes driven
-by \prm {hangindent} and \prm {parshape}. This is controlled by a new parameter
-\lpr {shapemode}:
-
-\starttabulate[|c|l|l|]
-\DB value \BC \prm {hangindent} \BC \prm {parshape} \NC \NR
+\starttabulate[|l|l|]
+\DB value \BC reported \NC \NR
\TB
-\BC \type{0} \NC normal \NC normal \NC \NR
-\BC \type{1} \NC mirrored \NC normal \NC \NR
-\BC \type{2} \NC normal \NC mirrored \NC \NR
-\BC \type{3} \NC mirrored \NC mirrored \NC \NR
+\NC \type{0x0001} \NC normalize line as described above \NC \NR
+\NC \type{0x0002} \NC use a skip for parindent instead of a box \NC \NR
+\NC \type{0x0004} \NC swap hangindent in l2r mode \NC \NR
+\NC \type{0x0008} \NC swap parshape in l2r mode \NC \NR
+\NC \type{0x0010} \NC put breaks after dir in l2r mode \NC \NR
+\NC \type{0x0020} \NC remove margin kerns (\PDFTEX\ left-over) \NC \NR
+\NC \type{0x0040} \NC if needed clip width and use correction kern \NC \NR
\LL
\stoptabulate
-The value is reset to zero (like \prm {hangindent} and \prm {parshape})
-after the paragraph is done with. You can use negative values to prevent
-this. In \in {figure} [fig:shapemode] a few examples are given.
-
-\startplacefigure[reference=fig:shapemode,title={The effect of \type {shapemode}.}]
- \startcombination[2*3]
- {\ruledvbox \bgroup \setuptolerance[verytolerant]
- \hsize .45\textwidth \switchtobodyfont[6pt]
- \pardirection 0 \textdirection 0
- \hangindent 40pt \hangafter -3
- \leftskip10pt \input tufte \par
- \egroup} {TLT: hangindent}
- {\ruledvbox \bgroup \setuptolerance[verytolerant]
- \hsize .45\textwidth \switchtobodyfont[6pt]
- \pardirection 0 \textdirection 0
- \parshape 4 0pt .8\hsize 10pt .8\hsize 20pt .8\hsize 0pt \hsize
- \input tufte \par
- \egroup} {TLT: parshape}
- {\ruledvbox \bgroup \setuptolerance[verytolerant]
- \hsize .45\textwidth \switchtobodyfont[6pt]
- \pardirection 1 \textdirection 1
- \hangindent 40pt \hangafter -3
- \leftskip10pt \input tufte \par
- \egroup} {TRT: hangindent mode 0}
- {\ruledvbox \bgroup \setuptolerance[verytolerant]
- \hsize .45\textwidth \switchtobodyfont[6pt]
- \pardirection 1 \textdirection 1
- \parshape 4 0pt .8\hsize 10pt .8\hsize 20pt .8\hsize 0pt \hsize
- \input tufte \par
- \egroup} {TRT: parshape mode 0}
- {\ruledvbox \bgroup \setuptolerance[verytolerant]
- \hsize .45\textwidth \switchtobodyfont[6pt]
- \shapemode=3
- \pardirection 1 \textdirection 1
- \hangindent 40pt \hangafter -3
- \leftskip10pt \input tufte \par
- \egroup} {TRT: hangindent mode 1 & 3}
- {\ruledvbox \bgroup \setuptolerance[verytolerant]
- \hsize .45\textwidth \switchtobodyfont[6pt]
- \shapemode=3
- \pardirection 1 \textdirection 1
- \parshape 4 0pt .8\hsize 10pt .8\hsize 20pt .8\hsize 0pt \hsize
- \input tufte \par
- \egroup} {TRT: parshape mode 2 & 3}
- \stopcombination
-\stopplacefigure
-
-We have \type {\pardirection}, \type {\textdirection}, \type {\mathdirection} and
-\type {\linedirection} that is like \type {\textdirection} but with some
-additional (inline) glue checking.
+Setting the bit enables the related normalization. More features might be added
+in future releases.
+
+% Swapping shapes
+%
+% Another adaptation to the \ALEPH\ directional model is control over shapes driven
+% by \prm {hangindent} and \prm {parshape}. This is controlled by a new parameter
+% \lpr {shapemode}:
+%
+% \starttabulate[|c|l|l|]
+% \DB value \BC \prm {hangindent} \BC \prm {parshape} \NC \NR
+% \TB
+% \BC \type{0} \NC normal \NC normal \NC \NR
+% \BC \type{1} \NC mirrored \NC normal \NC \NR
+% \BC \type{2} \NC normal \NC mirrored \NC \NR
+% \BC \type{3} \NC mirrored \NC mirrored \NC \NR
+% \LL
+% \stoptabulate
+%
+% The value is reset to zero (like \prm {hangindent} and \prm {parshape})
+% after the paragraph is done with. You can use negative values to prevent
+% this. In \in {figure} [fig:shapemode] a few examples are given.
+%
+% \startplacefigure[reference=fig:shapemode,title={The effect of \type {shapemode}.}]
+% \startcombination[2*3]
+% {\ruledvbox \bgroup \setuptolerance[verytolerant]
+% \hsize .45\textwidth \switchtobodyfont[6pt]
+% \pardirection 0 \textdirection 0
+% \hangindent 40pt \hangafter -3
+% \leftskip10pt \input tufte \par
+% \egroup} {TLT: hangindent}
+% {\ruledvbox \bgroup \setuptolerance[verytolerant]
+% \hsize .45\textwidth \switchtobodyfont[6pt]
+% \pardirection 0 \textdirection 0
+% \parshape 4 0pt .8\hsize 10pt .8\hsize 20pt .8\hsize 0pt \hsize
+% \input tufte \par
+% \egroup} {TLT: parshape}
+% {\ruledvbox \bgroup \setuptolerance[verytolerant]
+% \hsize .45\textwidth \switchtobodyfont[6pt]
+% \pardirection 1 \textdirection 1
+% \hangindent 40pt \hangafter -3
+% \leftskip10pt \input tufte \par
+% \egroup} {TRT: hangindent mode 0}
+% {\ruledvbox \bgroup \setuptolerance[verytolerant]
+% \hsize .45\textwidth \switchtobodyfont[6pt]
+% \pardirection 1 \textdirection 1
+% \parshape 4 0pt .8\hsize 10pt .8\hsize 20pt .8\hsize 0pt \hsize
+% \input tufte \par
+% \egroup} {TRT: parshape mode 0}
+% {\ruledvbox \bgroup \setuptolerance[verytolerant]
+% \hsize .45\textwidth \switchtobodyfont[6pt]
+% \shapemode=3
+% \pardirection 1 \textdirection 1
+% \hangindent 40pt \hangafter -3
+% \leftskip10pt \input tufte \par
+% \egroup} {TRT: hangindent mode 1 & 3}
+% {\ruledvbox \bgroup \setuptolerance[verytolerant]
+% \hsize .45\textwidth \switchtobodyfont[6pt]
+% \shapemode=3
+% \pardirection 1 \textdirection 1
+% \parshape 4 0pt .8\hsize 10pt .8\hsize 20pt .8\hsize 0pt \hsize
+% \input tufte \par
+% \egroup} {TRT: parshape mode 2 & 3}
+% \stopcombination
+% \stopplacefigure
+%
+% We have \type {\pardirection}, \type {\textdirection}, \type {\mathdirection} and
+% \type {\linedirection} that is like \type {\textdirection} but with some
+% additional (inline) glue checking.
+
+% Controlling glue with \lpr {breakafterdirmode}
+%
+% Glue after a dir node is ignored in the linebreak decision but you can bypass that
+% by setting \lpr {breakafterdirmode} to~\type {1}. The following table shows the
+% difference. Watch your spaces.
+%
+% \def\ShowSome#1{%
+% \BC \type{#1}
+% \NC \breakafterdirmode\zerocount\hsize\zeropoint#1
+% \NC
+% \NC \breakafterdirmode\plusone\hsize\zeropoint#1
+% \NC
+% \NC \NR
+% }
+%
+% \starttabulate[|l|Tp(1pt)|w(5em)|Tp(1pt)|w(5em)|]
+% \DB
+% \BC \type{0}
+% \NC
+% \BC \type{1}
+% \NC
+% \NC \NR
+% \TB
+% \ShowSome{pre {\textdirection 0 xxx} post}
+% \ShowSome{pre {\textdirection 0 xxx }post}
+% \ShowSome{pre{ \textdirection 0 xxx} post}
+% \ShowSome{pre{ \textdirection 0 xxx }post}
+% \ShowSome{pre { \textdirection 0 xxx } post}
+% \ShowSome{pre {\textdirection 0\relax\space xxx} post}
+% \LL
+% \stoptabulate
\stopsubsection
@@ -1751,17 +1963,286 @@ shows that is't okay, they will become official, so we just mention them: \type
{\boxdirection}, \type {\boxattr}, \type {\boxorientation}, \type {\boxxoffset},
\type {\boxyoffset}, \type {\boxxmove}, \type {\boxymove} and \type {\boxtotal}.
+{\em This is still somewhat experimental and will be documented in more detail
+when I've used it more in \CONTEXT\ and the specification is frozen. This might
+take some time (and user input).}
+
\stopsubsection
\stopsection
-\startsection[title=Expressions]
+\startsection[title=Keywords]
+
+Some primitives accept one or more keywords and \LUAMETATEX\ adds some more. In
+order to deal with this efficiently the keyword scanner has been optimized, where
+even the context was taken into account. As a result the scanner was quite a bit
+faster. This kind of optimization was a graduate process the eventually ended up
+in what we have now. In traditional \TEX\ (and also \LUATEX) the order of
+keywords is sometimes mixed and sometimes prescribed. In most cases only one
+occurrence is permitted. So, for instance, this is valid in \LUATEX:
+
+\starttyping
+\hbox attr 123 456 attr 123 456 spread 10cm { }
+\hrule width 10cm depth 3mm
+\hskip 3pt plus 2pt minus 1pt
+\stoptyping
+
+The \type {attr} comes before the \type {spread}, rules can have multiple mixed
+dimension specifiers, and in glue the optional \type {minus} part always comes
+last. The last two commands are famous for look ahead side effects which is why
+macro packages will end them with something not keyword, like \type {\relax},
+when needed.
+
+In \LUAMETATEX\ the following is okay. Watch the few more keywords in box and
+rule specifications.
+
+\starttyping
+\hbox reverse to 10cm attr 123 456 orientation 4 xoffset 10pt spread 10cm { }
+\hrule xoffset 10pt width 10cm depth 3mm
+\hskip 3pt minus 1pt plus 2pt
+\stoptyping
+
+Here the order is not prescribed and, as demonstrated with the box specifier, for
+instance dimensions (specified by \type {to} or \type {spread} can be overloaded
+by later settings. In case you wonder if that breaks compatibility: in some way
+it does but bad or sloppy keyword usage breaks a run anyway. For instance \type
+{minuscule} results in \type {minus} with no dimension being seen. So, in the end
+the user should not noticed it and when a user does, the macro package already
+had an issue that had to be fixed.
+
+\stopsection
+
+\startsection[title=Expressions and \lpr {numericscale}]
The \type {*expr} parsers now accept \type {:} as operator for integer division
(the \type {/} operators does rounding. This can be used for division compatible
with \type {\divide}. I'm still wondering if adding a couple of bit operators
makes sense (for integers).
+The \lpr{numericscale} parser is kind of special (and might evolve). For now it
+converts a following number in a scale value as often used in \TEX, where 1000
+means scaling by~1.0. The trick is in the presence of a digit (or comma): 1.234
+becomes 1234 but 1234 stays 1234 and from this you can deduce that 12.34 becomes
+123400. Internally \TEX\ calculates with integers, but this permits the macro
+package to provide an efficient mix.
+
+\stopsection
+
+\startsection[title=Macro arguments]
+
+Again this is experimental and (used and) discussed in document that come with the
+\CONTEXT\ distribution. When defining a macro you can do this:
+
+\starttyping
+\def\foo(#1)#2{...}
+\stoptyping
+
+Here the first argument between parentheses is mandate. But the magic
+prefix \lpr {tolerant} makes that limitation go away:
+
+\starttyping
+\tolerant\def\foo(#1)#2{...}
+\stoptyping
+
+A variant is this:
+
+\starttyping
+\tolerant\def\foo(#1)#*(#2){...}
+\stoptyping
+
+Here we have two optional arguments, possibly be separated by spaces. There are
+more parsing options:
+
+\starttabulate[|T|i2l|]
+\FL
+\NC + \NC keep the braces \NC \NR
+\NC - \NC discard and don't count the argument \NC \NR
+\NC / \NC remove leading an trailing spaces and pars \NC \NR
+\NC = \NC braces are mandate \NC \NR
+\NC _ \NC braces are mandate and kept \NC \NR
+\NC ^ \NC keep leading spaces \NC \NR
+\ML
+\NC 1-9 \NC an argument \NC \NR
+\NC 0 \NC discard but count the argument \NC \NR
+\ML
+\NC * \NC ignore spaces \NC \NR
+\NC : \NC pick up scanning here \NC \NR
+\NC ; \NC quit scanning \NC \NR
+\LL
+\stoptabulate
+
+For the moment we leave it to your fantasy what these options do. Most probably
+only make sense when you write a bit more complex macros. Just try to imagine
+what this does:
+
+\starttyping
+\permanent\tolerant\global\protected\def\foo(#1)#*#;[#2]#:#3{...}
+\stoptyping
+
+Of course complex combinations can be confusing because after all \TEX\ is
+parsing for (multi|-|token) delimiters and will happily gobble the whole file if
+you are not careful. You can quit scanning if you want:
+
+\starttyping
+\mymacro 123\ignorearguments
+\stoptyping
+
+which of course only makes sense when used in a nested call where an already
+picked up arguments is processed further. A not (yet) discussed feature of the
+parser is that it will happily skip tokens that have the (probably seldom used)
+ignored characters property.
+
+When you use tracing or see error messages arguments defined using for instance
+\type {#=} will have their usual number in the macro body, so you need to keep
+track of the numbers.
+
+All this is rather easy on the engine and although it might have a little impact
+on performance this has been compensated by some more efficiency in the macro
+parser and engine in general and of course you can gain back some by using these
+features.
+
+\stopsection
+
+\startsection[title=Overload protection]
+
+There is an experimental overload protection mechanism that we will test for a
+while before declaring it stable. The reason for that is that we need to adapt
+the \CONTEXT\ code base in order to test its usefulness. Protection is achieved
+via prefixes. Depending on the value of the \lpr {overloadmode} variable
+warnings or errors will be triggered. Examples of usage can be found in some
+documents that come with \CONTEXT, so here we just stick to the basics.
+
+\starttyping
+\mutable \def\foo{...}
+\immutable\def\foo{...}
+\permanent\def\foo{...}
+\frozen \def\foo{...}
+\aliased \def\foo{...}
+\stoptyping
+
+A \lpr {mutable} macro can always be changed contrary to an \lpr {immutable} one.
+For instance a macro that acts as a variable is normally \lpr {mutable}, while a
+constant can best be immutable. It makes sense to define a public core macro as
+\lpr {permanent}. Primives start out a \lpr {permanent} ones but with a primitive
+property instead.
+
+\startbuffer
+ \let\relaxone \relax 1: \meaningfull\relaxone
+\aliased \let\relaxtwo \relax 2: \meaningfull\relaxtwo
+\permanent\let\relaxthree\relax 3: \meaningfull\relaxthree
+\stopbuffer
+
+\typebuffer
+
+The \lpr {meaningfull} primitive is like \prm {meaning} but report the
+properties too. The \lpr {meaningless} companion reports the body of a macro.
+Anyway, this typesets:
+
+\startlines \tttf \getbuffer \stoplines
+
+So, the \lpr {aliased} prefix copies the properties. Keep in mind that a macro
+package can redefine primitives, but \prm {relax} is an unlikely candidate.
+
+There is an extra prefix \lpr {noaligned} that flags a macro as being valid
+for \prm {noalign} compatible usage (which means that the body must contain that
+one. The idea is that we then can do this:
+
+\starttyping
+\permanent\protected\noaligned\def\foo{\noalign{...}} % \foo is unexpandable
+\stoptyping
+
+that is: we can have protected macros that don't trigger an error in the parser
+where there is a look ahead for \prm {noalign} which is why normally protection
+doesn't work well. So: we have macro flagged as permanent (overload protection),
+being protected (that is, not expandable by default) and a valid equivalent of
+the noalign primitive. Of course we can also apply the \prm {global} and \lpr
+{tolerant} prefixes here. The complete repertoire of extra prefixes is:
+
+\starttabulate
+\HL
+\NC \type {frozen} \NC a macro that has to be redefined in a managed way \NC \NR
+\NC \type {permanent} \NC a macro that had better not be redefined \NC \NR
+\NC \type {primitive} \NC a primitive that normally will not be adapted \NC \NR
+\NC \type {immutable} \NC a macro or quantity that cannot be changed, it is a constant \NC \NR
+\NC \type {mutable} \NC a macro that can be changed no matter how well protected it is \NC \NR
+\HL
+\NC \type {instance} \NC a macro marked as (for instance) be generated by an interface \NC \NR
+\HL
+\NC \type {noaligned} \NC the macro becomes acceptable as \type {\noalign} alias \NC \NR
+\HL
+\NC \type {overloaded} \NC when permitted the flags will be adapted \NC \NR
+\NC \type {enforced} \NC all is permitted (but only in zero mode or ini mode) \NC \NR
+\NC \type {aliased} \NC the macro gets the same flags as the original \NC \NR
+\HL
+\stoptabulate
+
+The not yet discussed \lpr {instance} is just a flag with no special meaning
+which can be used as classifier. The \lpr {frozen} also protects against overload
+which brings amount of blockers to four.
+
+To what extent the engine will complain when a property is changed in a way that
+violates the flags depends on the parameter \lpr {overloadmode}. When this
+parameter is set to zero no checking takes place. More interesting are values
+larger than zero. If that is the case, when a control sequence is flagged as
+mutable, it is always permitted to change. When it is set to immutable one can
+never change it. The other flags determine the kind of checking done. Currently
+the following overload values are used:
+
+\starttabulate[|l|l|c|c|c|c|c|]
+ \NC \NC \BC immutable \BC permanent \BC primitive \BC frozen \BC instance \NC \NR
+ \NC 1 \NC warning \NC \star \NC \star \NC \star \NC \NC \NC \NR
+ \NC 2 \NC error \NC \star \NC \star \NC \star \NC \NC \NC \NR
+ \NC 3 \NC warning \NC \star \NC \star \NC \star \NC \star \NC \NC \NR
+ \NC 4 \NC error \NC \star \NC \star \NC \star \NC \star \NC \NC \NR
+ \NC 5 \NC warning \NC \star \NC \star \NC \star \NC \star \NC \star \NC \NR
+ \NC 6 \NC error \NC \star \NC \star \NC \star \NC \star \NC \star \NC \NR
+\stoptabulate
+
+The even values (except zero) will abort the run. A value of 255 will freeze this
+parameter. At level five and above the \lpr {instance} flag is also checked but
+no drastic action takes place. We use this to signal to the user that a specific
+instance is redefined (of course the definition macros can check for that too).
+
+The \lpr {overloaded} prefix can be used to overload a frozen macro. The \lpr
+{enforced} is more powerful and forces an overload but that prefix is only
+effective in ini mode or when it's embedded in the body of a macro or token list
+at ini time unless of course at runtime the mode is zero.
+
+So far for a short explanation. More details can be found in the \CONTEXT\
+documentation where we can discuss it in a more relevant perspective. It must be
+noted that this feature only makes sense a controlled situation, that is: user
+modules or macros of unpredictable origin will probably suffer from warnings and
+errors when de mode is set to non zero. In \CONTEXT\ we're okay unless of course
+users redefine instances but there a warning or error is kind of welcome.
+
+\stopsection
+
+\startsection[title=Constants]
+
+It is rather common to store constant values in a register or character
+definition.
+
+\starttyping
+\newcount\MyConstantA \MyConstantA 123
+\newdimen\MyConstantB \MyConstantB 123pt
+\chardef \MyConstantC \MyConstantC 123
+\stoptyping
+
+But in \LUAMETATEX\ we also can do this:
+
+\starttyping
+\integerdef\MyConstantC 456
+\dimendef \MyConstantD 456pt
+\stoptyping
+
+These two are stored as efficient as a register but don't occupy a register slot.
+They can be set as above, need \prm {the} for serializations and are seen as
+valid number or dimension when needed.
+
+Experiments with constant strings made the engine source more complex than I
+wanted so that features was rejected. Of course we can use the prefixes mentioned
+in a previous section.
+
\stopsection
\startsection[title=Nodes]
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex
index 917ebfeb86f..9a09f705c10 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex
@@ -140,7 +140,6 @@ For example, here is the character \quote {f} (decimal 102) in the font \type
}
\stoptyping
-
Two very special string indexes can be used also: \type {left_boundary} is a
virtual character whose ligatures and kerns are used to handle word boundary
processing. \type {right_boundary} is similar but not actually used for anything
@@ -269,9 +268,25 @@ third one is a previously defined font that is known to \LUATEX\ as font id~38.
The array index numbers are used by the character command definitions that are
part of each character.
-The \type {commands} array is a hash where each item is another small array,
-with the first entry representing a command and the extra items being the
-parameters to that command. The allowed commands and their arguments are:
+The \type {commands} array is a hash where each item is another small array, with
+the first entry representing a command and the extra items being the parameters
+to that command. The frontend is only interested in the dimensions, ligatures and
+kerns of a font, which is the reason why the \TEX\ engine didn't have to be
+extended when virtual fonts showed up: dealing with it is up to the driver that
+comes after the backend. In \PDFTEX\ and \LUATEX\ that driver is integrated so
+there the backend also deals with virtual fonts. The first block in the next
+table is what the standard mentions. The \type {special} command is indeed
+special because it is an extension container. The mentioned engines only support
+pseudo standards where the content starts with \type {pdf:}. The last block is
+\LUATEX\ specific and will not be found in native fonts. These entries can be
+used in virtual fonts that are constructed in \LUA.
+
+But \unknown\ in \LUAMETATEX\ there is no backend built in but we might assume
+that the one provided deals with these entries. However, a provided backend can
+provide more and that is indeed what happens in \CONTEXT. There, because we no
+longer have compacting (of passed tables) and unpacking (when embedding) of these
+tables going on we stay in the \LUA\ domain. None of the virtual specification is
+ever seen in the engine.
\starttabulate[|l|l|l|p|]
\DB command \BC arguments \BC type \BC description \NC \NR
@@ -279,16 +294,18 @@ parameters to that command. The allowed commands and their arguments are:
\NC \type{font} \NC 1 \NC number \NC select a new font from the local \type {fonts} table \NC \NR
\NC \type{char} \NC 1 \NC number \NC typeset this character number from the current font,
and move right by the character's width \NC \NR
-\NC \type{node} \NC 1 \NC node \NC output this node (list), and move right
- by the width of this list\NC \NR
-\NC \type{slot} \NC 2 \NC 2 numbers \NC a shortcut for the combination of a font and char command\NC \NR
\NC \type{push} \NC 0 \NC \NC save current position\NC \NR
-\NC \type{nop} \NC 0 \NC \NC do nothing \NC \NR
\NC \type{pop} \NC 0 \NC \NC pop position \NC \NR
\NC \type{rule} \NC 2 \NC 2 numbers \NC output a rule $ht*wd$, and move right. \NC \NR
\NC \type{down} \NC 1 \NC number \NC move down on the page \NC \NR
\NC \type{right} \NC 1 \NC number \NC move right on the page \NC \NR
+\HL
\NC \type{special} \NC 1 \NC string \NC output a \prm {special} command \NC \NR
+\HL
+\NC \type{nop} \NC 0 \NC \NC do nothing \NC \NR
+\NC \type{slot} \NC 2 \NC 2 numbers \NC a shortcut for the combination of a font and char command\NC \NR
+\NC \type{node} \NC 1 \NC node \NC output this node (list), and move right
+ by the width of this list\NC \NR
\NC \type{pdf} \NC 2 \NC 2 strings \NC output a \PDF\ literal, the first string is one of \type {origin},
\type {page}, \type {text}, \type {font}, \type {direct} or \type {raw}; if you
have one string only \type {origin} is assumed \NC \NR
@@ -363,10 +380,6 @@ The special can have a \type {pdf:}, \type {pdf:origin:}, \type {pdf:page:},
\type {pdf:direct:} or \type {pdf:raw:} prefix. When you have to concatenate
strings using the \type {pdf} command might be more efficient.
-The fields mentioned above can be found in external fonts. It is good to keep in
-mind that we can extend this model, given that the backend knows what to do with
-it.
-
\stopsection
\startsection[title={Additional \TEX\ commands}]
@@ -392,17 +405,19 @@ expansion takes place inside the argument.
\fontid\font
\stopsyntax
-This primitive expands into a number. It is not a register so there is no need to
-prefix with \prm {number} (and using \prm {the} gives an error). The currently
-used font id is \fontid\font. Here are some more:
+This primitive expands into a number. The currently used font id is
+\number\fontid\font. Here are some more: \footnote {Contrary to \LUATEX\ this is
+now a number so you need to use \type {\number} or \type {\the}. The same is true
+for some other numbers and dimensions that for some reason ended up in the
+serializer that produced a sequence of tokens.}
\starttabulate[|l|c|c|]
\DB style \BC command \BC font id \NC \NR
\TB
-\NC normal \NC \type {\tf} \NC \tf \fontid\font \NC \NR
-\NC bold \NC \type {\bf} \NC \bf \fontid\font \NC \NR
-\NC italic \NC \type {\it} \NC \it \fontid\font \NC \NR
-\NC bold italic \NC \type {\bi} \NC \bi \fontid\font \NC \NR
+\NC normal \NC \type {\tf} \NC \tf \number\fontid\font \NC \NR
+\NC bold \NC \type {\bf} \NC \bf \number\fontid\font \NC \NR
+\NC italic \NC \type {\it} \NC \it \number\fontid\font \NC \NR
+\NC bold italic \NC \type {\bi} \NC \bi \number\fontid\font \NC \NR
\LL
\stoptabulate
@@ -417,29 +432,92 @@ which of course needs to be a valid one.
\stopsubsection
-\startsubsection[title={\lpr {noligs} and \lpr {nokerns}}]
+\startsubsection[title={\lpr {glyphoptions}}]
\topicindex {ligatures+suppress}
\topicindex {kerns+suppress}
+\topicindex {expansion+suppress}
+\topicindex {protrusion+suppress}
-These primitives prohibit ligature and kerning insertion at the time when the
-initial node list is built by \LUATEX's main control loop. You can enable these
-primitives when you want to do node list processing of \quote {characters}, where
-\TEX's normal processing would get in the way.
+In \LUATEX\ the \type {\noligs} and \type {\nokerns} primitives suppress these
+features but in \LUAMETATEX\ these primitives are gone. They are replace by a more
+generic control primitive \lpr {glyphoptions}. This numerical parameter is a
+bitset with the following fields:
-\startsyntax
-\noligs <integer>!crlf
-\nokerns <integer>
-\stopsyntax
+\starttabulate[|l|pl|]
+\DB value \BC effect \NC\NR
+\TB
+\NC \type{0x01} \NC prevent left ligature \NC \NR
+\NC \type{0x02} \NC prevent right ligature \NC \NR
+\NC \type{0x04} \NC block left kern \NC \NR
+\NC \type{0x08} \NC block right kern \NC \NR
+\NC \type{0x10} \NC don't apply expansion \NC \NR
+\NC \type{0x20} \NC don't apply protrusion \NC \NR
+\NC \type{0x40} \NC apply xoffset to width \NC \NR
+\NC \type{0x80} \NC apply yoffset to height and depth \NC \NR
+\LL
+\stoptabulate
-These primitives can also be implemented by overloading the ligature building and
-kerning functions, i.e.\ by assigning dummy functions to their associated
-callbacks. Keep in mind that when you define a font (using \LUA) you can also
-omit the kern and ligature tables, which has the same effect as the above.
+The effects speak for themselves. They provide detailed control over individual
+glyph, this because the current value of this option is stored with glyphs.
\stopsubsection
-\startsubsection[title={\type{\nospaces}}]
+\startsubsection[title={\lpr {glyphxscale}, \lpr {glyphyscale} and \lpr {scaledfontdimen}}]
+
+The two scale parameters control the current scaling. They are traditional \TEX\
+integer parameters that operate independent of each other. The scaling is
+reflected in the dimensions of glyphs as well as in the related font dimensions,
+which means that units like \type {ex} and \type {em} work as expected. If you
+query a font dimensions with \prm {fontdimen} you get the raw value but with \lpr
+{scaledfontdimen} you get the useable value.
+
+\stopsubsection
+
+\startsubsection[title={\lpr {glyphxoffset}, \lpr {glyphyoffset}}]
+
+These two parameters control the horizontal and vertical shift of glyphs with,
+when applied to a stretch of them, the horizontal offset probably being the least
+useful.
+
+\stopsubsection
+
+\startsubsection[title={\lpr {glyph}}]
+
+This command is a variation in \prm {char} that takes keywords:
+
+\starttabulate[|l|p|]
+\DB keyword \BC effect \NC type \NC \NR
+\TB
+\NC \type {xoffset} \NC (virtual) horizontal shift \NC dimension \NC \NR
+\NC \type {yoffset} \NC (virtual) vertical shift \NC dimension \NC \NR
+\NC \type {xscale} \NC horizontal scaling \NC integer \NC \NR
+\NC \type {yscale} \NC vertical scaling \NC integer \NC \NR
+\NC \type {options} \NC glyph options \NC bitset \NC \NR
+\NC \type {font} \NC font \NC identifier \NC \NR
+\NC \type {id} \NC font \NC integer \NC \NR
+\LL
+\stoptabulate
+
+The values default to the currently set values. Here is a \CONTEXT\ example:
+
+\startbuffer
+\ruledhbox{
+ \ruledhbox{\glyph yoffset 1ex options 0 123}
+ \ruledhbox{\glyph xoffset .5em yoffset 1ex options "C0 125}
+ \ruledhbox{baseline\glyphyoffset 1ex \glyphxscale 800 \glyphyscale\glyphxscale raised}
+}
+\stopbuffer
+
+\typebuffer
+
+Visualized:
+
+\getbuffer
+
+\stopsubsection
+
+\startsubsection[title={\lpr{nospaces}}]
\topicindex {spaces+suppress}
@@ -462,7 +540,7 @@ space.
\stopsubsection
-\startsubsection[title={\type{\protrusionboundary}}]
+\startsubsection[title={\lpr{protrusionboundary}}]
\topicindex {protrusion}
\topicindex {boundaries}
@@ -581,13 +659,19 @@ font.current(<number> i)
\stopsubsection
-% \startsubsection[title={Glyph data: \lpr {glyphdata}]
-%
-% This primitive can be used to set an additional glyph property. Of course it's very
-% macro package dependant what is done with that. Consider it an experiment (we had
-% some room left in the glyphs data structure). It's basically an single attribute.
-%
-% \stopsubsection
+\startsubsection[title={Glyph data: \lpr {glyphdata}, \lpr {glyphscript}, \lpr {glyphstate}}]
+
+These primitives can be used to set an additional glyph properties. Of course
+it's very macro package dependant what is done with that. It started with just
+the first one as experiment, simply because we had some room left in the glyph
+data structure. It's basically an single attribute. Then, when we got rid of the
+ligature pointer we could either drop it or use that extra field for some more,
+and because \CONTEXT\ already used the data field, that is what happened. The
+script and state fields are shorts, that is, they run from zero to \type {0xFFFF}
+where we assume that zero means \quote {unset}. Although they can be used for
+whatever purpose their use in \CONTEXT\ is fixed.
+
+\stopsubsection
\stopsection
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-introduction.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-introduction.tex
index 774f3d8d9a1..2339bd34201 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-introduction.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-introduction.tex
@@ -28,42 +28,60 @@ happened in a series of documents, parts of which were published as articles in
user group journals, but all are in the \CONTEXT\ distribution. I did the same
with the development of \LUAMETATEX.
-The \LUAMETATEX\ engine is, as said, a lightweight version of \LUATEX, that for
-now targets \CONTEXT. We will use it for possibly drastic experiments but without
-affecting \LUATEX. As we can easily adapt \CONTEXT\ to support both, no other
-macro package will be harmed when (for instance) interfaces change as part of an
-experiment. Of course, when we consider something to be useful, it can be ported
-back to \LUATEX, but only when there are good reasons for doing so and when no
-compatibility issues are involved. When considering this follow up one
-consideration was that a lean and mean version with an extension mechanism is a
-bit closer to original \TEX. Of course, because we also have new primitives, this
-is not entirely true. The move to \LUA\ already meant that some aspects,
-especially system dependent ones, no longer made sense and therefore had
-consequences for the interface at the system level.
-
-This manual currently has quite a bit of overlap with the \LUATEX\ manual but
-some chapters are removed, others added and the rest has been (and will be
-further) adapted. It also discusses the (main) differences. Some of the new
-primitives or functions that show up in \LUAMETATEX\ might show up in \LUATEX\ at
-some point, others might not, so don't take this manual as reference for \LUATEX
-! For now it is an experimental engine in which we can change things at will but
-with \CONTEXT\ in tandem so that this macro package will keep working.
-
-For \CONTEXT\ users the \LUAMETATEX\ engine will become the default. The
-\CONTEXT\ variant for this engine is tagged \LMTX. The pair can be used in
-production, just as with \LUATEX\ and \MKIV. In fact, most users will probably
-not really notice the difference. In some cases there will be a drop in
+The \LUAMETATEX\ engine is, as said, a follow up on \LUATEX. Just as we have
+\CONTEXT\ \MKII\ for \PDFTEX\ and \XETEX, we have \MKIV\ for \LUATEX. For
+\LUAMETATEX\ we have yet another version of \CONTEXT: \LMTX. By freezing \MKII,
+and at some point freezing \MKIV, we can move on as we like, but we try to remain
+downward compatible where possible, something that the user interface makes
+possible. Although \LUAMETATEX\ can be used for production we can also use it for
+possibly drastic experiments but without affecting \LUATEX. Because we can easily
+adapt \CONTEXT\ to support both, no other macro package will be harmed when (for
+instance) the interface that the engine provides change as part of an experiment
+or cleanup of code. Of course, when we consider something to be useful, it can be
+ported back to \LUATEX, but only when there are good reasons for doing so and
+when no compatibility issues are involved.
+
+By now the code of these two related engines differs a lot so in retrospect it
+makes less sense to waste time on backporting anyway. When considering this
+follow up one consideration was that a lean and mean version with an extension
+mechanism is a bit closer to original \TEX. Of course, because we also have new
+primitives, this is not entirely true. The basic algorithms remain the same but
+code got reshuffled and because we expose internals names of variables and such
+are sometimes changed, something that is noticeable in the token and node
+interfaces. Delegating tasks to \LUA\ already meant that some aspects, especially
+system dependent ones, no longer made sense and therefore had consequences for
+the interface at the system level. In \LUAMETATEX\ more got delegated, like all
+file related operations. The penalty of moving more responsibility to \LUA\ has
+been compensated by (hopefully) harmless optimization of code in the engine and
+some more core functionality.
+
+This manual started as an adaptation of the \LUATEX\ manual and therefore looks
+similar. Some chapters are removed, others were added and the rest has been (and
+will be further) adapted. It also discusses the (main) differences. Some of the
+new primitives or functions that show up in \LUAMETATEX\ might show up in
+\LUATEX\ at some point, but most will be exclusive to \LUAMETATEX, so don't take
+this manual as reference for \LUATEX ! As long as we're experimenting we can
+change things at will but as we keep \CONTEXT\ \LMTX\ synchronized users normally
+won't notice this. Often you can find examples of usage in \CONTEXT\ related
+documents and the source code so that serves a reference too.
+
+For \CONTEXT\ users the \LUAMETATEX\ engine will become the default. As
+mentioned, the \CONTEXT\ variant for this engine is tagged \LMTX. The pair can be
+used in production, just as with \LUATEX\ and \MKIV. In fact, most users will
+probably not really notice the difference. In some cases there will be a drop in
performance, due to more work being delegated to \LUA, but on the average
performance will be better, also due to some changes below the hood of the
-engine.
+engine. Memory consumption is also less. The timeline of development is roughly:
+from 2018 upto 2020 engine development, 2019 upto 2021 the stepwise code split
+between \MKIV\ and \LMTX, while in 2020 we will (mostly) freeze \MKIV\ and \LMTX\
+will be the default.
As this follow up is closely related to \CONTEXT\ development, and because we
expect stock \LUATEX\ to be used outside the \CONTEXT\ proper, there will be no
special mailing list nor coverage (or pollution) on the \LUATEX\ related mailing
lists. We have the \CONTEXT\ mailing lists for that. In due time the source code
-will be part of the regular \CONTEXT\ distribution.
-
-% \testpage[8]
+will be part of the regular \CONTEXT\ distribution so that is then also the
+reference implementation: if needed users can compile the binary themselves.
This manual sometimes refers to \LUATEX, especially when we talk of features
common to both engine, as well as to \LUAMETATEX, when it is more specific to the
@@ -100,43 +118,55 @@ Hans Hagen
\vfilll
-{\bf remark:} \LUAMETATEX\ development is mostly done by Hans Hagen and Alan
-Braslau, who love playing with the three languages involved. And as usual Mojca
-Miklavec make sure all compiles well on the buildbot infrastructure. Testing is
-done by \CONTEXT\ developers and users. Many thanks for their patience!
+{\bf remark:} \LUAMETATEX\ development is mostly done by Hans Hagen and in
+adapting the macros to the new features Wolfgang Schuster, who knows the code
+inside||out is a instrumental. In the initial phase Alan Braslau, who love
+playing with the three languages did extensive testing and compiled for several
+platforms. Later Mojca Miklavec make sure all compiles well on the buildbot
+infrastructure. After the first release more users got involved in testing. Many
+thanks for their patience! The development also triggered upgrading of the wiki
+support infrastructure where Taco Hoekwater and Paul Mazaitis have teamed up. So,
+progress all around.
{\bf remark:} When there are non|-|intrusive features that also make sense in
\LUATEX, these will be applied in the experimental branch first, so that there is
-no interference with the stable release.
+no interference with the stable release. However, given that in the meantime the
+code bases differs a lot, it is unlikely that much will trickle back. This is no
+real problem as there's not much demand for that anyway.
{\bf remark:} Most \CONTEXT\ users seem always willing to keep up with the latest
versions which means that \LMTX\ is tested well. We can therefore safely claim
that end of 2019 the code has become quite stable. There are no complaints about
-performance (on my laptop this manual compiles at 22.5 pps with \LMTX\ versus
-20.7 pps for the \LUATEX\ manual with \MKIV). Probably no one notices it, but
-memory consumption stepwise got reduced too. And \unknown\ the binary is still
-below 3~MegaBytes on all platforms.
+performance (on my 2013 laptop this manual compiles at 24.5 pps with \LMTX\
+versus 20.7 pps for the \LUATEX\ manual with \MKIV). After updating some of the
+\CONTEXT\ code to use recently added features by the end of 2020 I could do more
+than 25.5 pps but don't expect spectacular bumps in performance (I need a new
+machine for that to happen). Probably no one notices it, but memory consumption
+stepwise got reduced too. And \unknown\ the binary is still below 3~MegaBytes on
+all platforms.
\stopchapter
\stopcomponent
-% I'm not that strict with incrementing numbers, but let's occasionally bump
-% the number. Once we're stable it might happen more systematically. For sure
-% I don't want to end up with these redicoulous 0.99999999 kind of numbers.
-%
+% I'm not that strict with incrementing numbers, but let's occasionally bump the
+% number. Once we're stable it might happen more systematically. For sure I don't
+% want to end up with these 0.99999999 kind of numbers.
+
% We started with 2.00.0 and kept that number till November 2019, after Alan
-% Braslau and I did the initial beta release at April 1, 2019. After more than
-% a year working on the code base after the \CONTEXT\ 2019 meeting a state was
-% reached where nothing fundamental got added for a while. When \LUATEX\ needs
-% a patch, I check the \LUAMETATEX\ code base as the same patch might be needed
-% there. On the other hand, we don't need a strict compatibility, so some
-% patched in \LUATEX\ are not applied here.
+% Braslau and I did the initial beta release at April 1, 2019. After more than a
+% year working on the code base after the \CONTEXT\ 2019 meeting a state was
+% reached where nothing fundamental got added for a while. When \LUATEX\ needs a
+% patch, I check the \LUAMETATEX\ code base as the same patch might be needed
+% there. On the other hand, we don't need a strict compatibility, so some patched
+% in \LUATEX\ are not applied here.
+%
+% In November 2019 I started wondering if we should bump the number, just for the
+% sake of showing that there's still some progress. So I decided to bump to 2.01.0
+% then. Just as a reminder for myself: it was the day when I watched Jacob Collier
+% perform \LUA\ (feat. MARO) live on YouTube (of course that is not about the
+% language at all, but still a nice coincidence). Just for the fun of it the number
+% bumped a few more times, just to catch up, so end 2019 we're at 2.03.5.
%
-% In November 2019 I started wondering if we should bump the number, just for
-% the sake of showing that there's still some progress. So I decided to bump to
-% 2.01.0 then. Just as a reminder for myself: it was the day when I watched
-% Jacob Collier perform Lua (feat. MARO) live on YouTube (of course that is not
-% about the language at all, but still a nice coincidence). Just for the fun of
-% it the number bumped a few more times, just to catch up, so end 2019 we're at
-% 2.03.5.
+% Thanks to the patient \CONTEXT\ users we were able to apply the new macro scanner
+% and protection mechanisms that were introduced mid 2020.
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
index 78d73985174..77c2d93d834 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
@@ -190,17 +190,17 @@ kerning steps.
The start and end of a sequence of characters is signalled by a \nod {glue}, \nod
{penalty}, \nod {kern} or \nod {boundary} node. But by default also a \nod
-{hlist}, \nod {vlist}, \nod {rule}, \nod {dir}, \nod {whatsit}, \nod {ins}, and
+{hlist}, \nod {vlist}, \nod {rule}, \nod {dir}, \nod {whatsit}, \nod {insert}, and
\nod {adjust} node indicate a start or end. You can omit the last set from the
-test by setting \lpr {hyphenationbounds} to a non|-|zero value:
+test by setting flags in \lpr {hyphenationmode}:
\starttabulate[|c|l|]
-\DB value \BC behaviour \NC \NR
+\DB value \BC behaviour \NC \NR
\TB
-\NC \type{0} \NC not strict \NC \NR
-\NC \type{1} \NC strict start \NC \NR
-\NC \type{2} \NC strict end \NC \NR
-\NC \type{3} \NC strict start and strict end \NC \NR
+\NC \NC not strict \NC \NR
+\NC \type{64} \NC strict start \NC \NR
+\NC \type{128} \NC strict end \NC \NR
+\NC \type{192} \NC strict start and strict end \NC \NR
\LL
\stoptabulate
@@ -210,11 +210,11 @@ The word start is determined as follows:
\DB node \BC behaviour \NC \NR
\TB
\BC boundary \NC yes when wordboundary \NC \NR
-\BC hlist \NC when hyphenationbounds 1 or 3 \NC \NR
-\BC vlist \NC when hyphenationbounds 1 or 3 \NC \NR
-\BC rule \NC when hyphenationbounds 1 or 3 \NC \NR
-\BC dir \NC when hyphenationbounds 1 or 3 \NC \NR
-\BC whatsit \NC when hyphenationbounds 1 or 3 \NC \NR
+\BC hlist \NC when the start bit is set \NC \NR
+\BC vlist \NC when the start bit is set \NC \NR
+\BC rule \NC when the start bit is set \NC \NR
+\BC dir \NC when the start bit is set \NC \NR
+\BC whatsit \NC when the start bit is set \NC \NR
\BC glue \NC yes \NC \NR
\BC math \NC skipped \NC \NR
\BC glyph \NC exhyphenchar (one only) : yes (so no -- ---) \NC \NR
@@ -232,13 +232,13 @@ The word end is determined as follows:
\BC glue \NC yes \NC \NR
\BC penalty \NC yes \NC \NR
\BC kern \NC yes when not italic (for some historic reason) \NC \NR
-\BC hlist \NC when hyphenationbounds 2 or 3 \NC \NR
-\BC vlist \NC when hyphenationbounds 2 or 3 \NC \NR
-\BC rule \NC when hyphenationbounds 2 or 3 \NC \NR
-\BC dir \NC when hyphenationbounds 2 or 3 \NC \NR
-\BC whatsit \NC when hyphenationbounds 2 or 3 \NC \NR
-\BC ins \NC when hyphenationbounds 2 or 3 \NC \NR
-\BC adjust \NC when hyphenationbounds 2 or 3 \NC \NR
+\BC hlist \NC when the end bit is set \NC \NR
+\BC vlist \NC when the end bit is set \NC \NR
+\BC rule \NC when the end bit is set \NC \NR
+\BC dir \NC when the end bit is set \NC \NR
+\BC whatsit \NC when the end bit is set \NC \NR
+\BC ins \NC when the end bit is set \NC \NR
+\BC adjust \NC when the end bit is set \NC \NR
\LL
\stoptabulate
@@ -256,131 +256,80 @@ min values to 1 and make sure that the words hyphenate at each character.
\hbox to 2cm {%
\vtop {%
\hsize 1pt
- \hyphenationbounds#1
+ \advance\hyphenationmode#1\relax
#2
\par}}}
\startplacefigure[reference=hb:1,title={\type{one}}]
\startcombination[4*1]
- {\SomeTest{0}{one}} {\type{0}}
- {\SomeTest{1}{one}} {\type{1}}
- {\SomeTest{2}{one}} {\type{2}}
- {\SomeTest{3}{one}} {\type{3}}
+ {\SomeTest {0}{one}} {\type {0}}
+ {\SomeTest {64}{one}} {\type {64}}
+ {\SomeTest{128}{one}} {\type{128}}
+ {\SomeTest{192}{one}} {\type{192}}
\stopcombination
\stopplacefigure
\startplacefigure[reference=hb:2,title={\type{one\null two}}]
\startcombination[4*1]
- {\SomeTest{0}{one\null two}} {\type{0}}
- {\SomeTest{1}{one\null two}} {\type{1}}
- {\SomeTest{2}{one\null two}} {\type{2}}
- {\SomeTest{3}{one\null two}} {\type{3}}
+ {\SomeTest {0}{one\null two}} {\type {0}}
+ {\SomeTest {64}{one\null two}} {\type {64}}
+ {\SomeTest{128}{one\null two}} {\type{128}}
+ {\SomeTest{192}{one\null two}} {\type{192}}
\stopcombination
\stopplacefigure
\startplacefigure[reference=hb:3,title={\type{\null one\null two}}]
\startcombination[4*1]
- {\SomeTest{0}{\null one\null two}} {\type{0}}
- {\SomeTest{1}{\null one\null two}} {\type{1}}
- {\SomeTest{2}{\null one\null two}} {\type{2}}
- {\SomeTest{3}{\null one\null two}} {\type{3}}
+ {\SomeTest {0}{\null one\null two}} {\type {0}}
+ {\SomeTest {64}{\null one\null two}} {\type {64}}
+ {\SomeTest{128}{\null one\null two}} {\type{128}}
+ {\SomeTest{192}{\null one\null two}} {\type{192}}
\stopcombination
\stopplacefigure
\startplacefigure[reference=hb:4,title={\type{one\null two\null}}]
\startcombination[4*1]
- {\SomeTest{0}{one\null two\null}} {\type{0}}
- {\SomeTest{1}{one\null two\null}} {\type{1}}
- {\SomeTest{2}{one\null two\null}} {\type{2}}
- {\SomeTest{3}{one\null two\null}} {\type{3}}
+ {\SomeTest {0}{one\null two\null}} {\type {0}}
+ {\SomeTest {64}{one\null two\null}} {\type {64}}
+ {\SomeTest{128}{one\null two\null}} {\type{128}}
+ {\SomeTest{192}{one\null two\null}} {\type{192}}
\stopcombination
\stopplacefigure
\startplacefigure[reference=hb:5,title={\type{\null one\null two\null}}]
\startcombination[4*1]
- {\SomeTest{0}{\null one\null two\null}} {\type{0}}
- {\SomeTest{1}{\null one\null two\null}} {\type{1}}
- {\SomeTest{2}{\null one\null two\null}} {\type{2}}
- {\SomeTest{3}{\null one\null two\null}} {\type{3}}
+ {\SomeTest {0}{\null one\null two\null}} {\type {0}}
+ {\SomeTest {64}{\null one\null two\null}} {\type {64}}
+ {\SomeTest{128}{\null one\null two\null}} {\type{128}}
+ {\SomeTest{192}{\null one\null two\null}} {\type{192}}
\stopcombination
\stopplacefigure
-% (Future versions of \LUATEX\ might provide more granularity.)
-
In traditional \TEX\ ligature building and hyphenation are interwoven with the
line break mechanism. In \LUATEX\ these phases are isolated. As a consequence we
deal differently with (a sequence of) explicit hyphens. We already have added
some control over aspects of the hyphenation and yet another one concerns
automatic hyphens (e.g.\ \type {-} characters in the input).
-When \lpr {automatichyphenmode} has a value of 0, a hyphen will be turned into
-an automatic discretionary. The snippets before and after it will not be
-hyphenated. A side effect is that a leading hyphen can lead to a split but one
-will seldom run into that situation. Setting a pre and post character makes this
-more prominent. A value of 1 will prevent this side effect and a value of 2 will
-not turn the hyphen into a discretionary. Experiments with other options, like
-permitting hyphenation of the words on both sides were discarded.
-
-\startbuffer[a]
-before-after \par
-before--after \par
-before---after \par
-\stopbuffer
-
-\startbuffer[b]
--before \par
-after- \par
---before \par
-after-- \par
----before \par
-after--- \par
-\stopbuffer
-
-\startbuffer[c]
-before-after \par
-before--after \par
-before---after \par
-\stopbuffer
-
-\startbuffer[demo]
-\startcombination[nx=4,ny=3,location=top]
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize6em \getbuffer[a]}} {A~0~6em}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize2pt \getbuffer[a]}} {A~0~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plusone \hsize2pt \getbuffer[a]}} {A~1~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plustwo \hsize2pt \getbuffer[a]}} {A~2~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize6em \getbuffer[b]}} {B~0~6em}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize2pt \getbuffer[b]}} {B~0~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plusone \hsize2pt \getbuffer[b]}} {B~1~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plustwo \hsize2pt \getbuffer[b]}} {B~2~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize6em \getbuffer[c]}} {C~0~6em}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize2pt \getbuffer[c]}} {C~0~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plusone \hsize2pt \getbuffer[c]}} {C~1~2pt}
- {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plustwo \hsize2pt \getbuffer[c]}} {C~2~2pt}
-\stopcombination
-\stopbuffer
-
-\startplacefigure[reference=automatichyphenmode:1,title={The automatic modes \type {0} (default), \type {1} and \type {2}, with a \prm {hsize}
-of 6em and 2pt (which triggers a linebreak).}]
- \dontcomplain \tt \getbuffer[demo]
-\stopplacefigure
-
-\startplacefigure[reference=automatichyphenmode:2,title={The automatic modes \type {0} (default), \type {1} and \type {2}, with \lpr {preexhyphenchar} and \lpr {postexhyphenchar} set to characters \type {A} and \type {B}.}]
- \postexhyphenchar`A\relax
- \preexhyphenchar `B\relax
- \dontcomplain \tt \getbuffer[demo]
-\stopplacefigure
-
-In \in {figure} [automatichyphenmode:1] \in {and} [automatichyphenmode:2] we show
-what happens with three samples:
-
-Input A: \typebuffer[a]
-Input B: \typebuffer[b]
-Input C: \typebuffer[c]
-
-As with primitive companions of other single character commands, the \prm {-}
-command has a more verbose primitive version in \lpr {explicitdiscretionary}
-and the normally intercepted in the hyphenator character \type {-} (or whatever
-is configured) is available as \lpr {automaticdiscretionary}.
+Hyphenation and discretionary injection is driven by a mode parameter which is
+a bitset made from the following values, some of which we saw in the previous
+examples.
+
+\starttabulate[|l|p|]
+\NC \number \normalhyphenationmodecode \NC honour (normal) \prm{discretionary}'s \NC \NR
+\NC \number \automatichyphenationmodecode \NC turn \type {-} into (automatic) discretionaries \NC \NR
+\NC \number \explicithyphenationmodecode \NC turn \type {\-} into (explicit) discretionaries \NC \NR
+\NC \number \syllablehyphenationmodecode \NC hyphenate (syllable) according to language \NC \NR
+\NC \number \uppercasehyphenationmodecode \NC hyphenate uppercase characters too (replaces \prm {uchyph} \NC \NR
+\NC \number \compoundhyphenationmodecode \NC permit break at an explicit hyphen (border cases) \NC \NR
+\NC \number \strictstarthyphenationmodecode \NC traditional \TEX\ compatibility wrt the start of a word \NC \NR
+\NC \number \strictendhyphenationmodecode \NC traditional \TEX\ compatibility wrt the end of a word \NC \NR
+\NC \number \automaticpenaltyhyphenationmodecode \NC use \lpr {automatichyphenpenalty} \NC \NR
+\NC \number \explicitpenaltyhyphenationmodecode \NC use \lpr {explicithyphenpenalty} \NC \NR
+\NC \number \permitgluehyphenationmodecode \NC turn glue in discretionaries into kerns \NC \NR
+\NC \number \lazyligatureshyphenationmodecode \NC controls how successive explicit discretionaries are handled in base mode \NC \NR
+\NC \number \forcecheckhyphenationmodecode \NC treat all discretionaries equal when breaking lines (in all three passes) \NC \NR
+\stoptabulate
\stopsection
@@ -493,25 +442,10 @@ have been added:
\explicithyphenpenalty
\stoptyping
-The first parameter has the following consequences for automatic discs (the ones
-resulting from an \prm {exhyphenchar}):
-
-\starttabulate[|c|l|l|]
-\DB mode \BC automatic disc \type {-} \BC explicit disc \prm{-} \NC \NR
-\TB
-\NC \type{0} \NC \prm {exhyphenpenalty} \NC \prm {exhyphenpenalty} \NC \NR
-\NC \type{1} \NC \prm {hyphenpenalty} \NC \prm {hyphenpenalty} \NC \NR
-\NC \type{2} \NC \prm {exhyphenpenalty} \NC \prm {hyphenpenalty} \NC \NR
-\NC \type{3} \NC \prm {hyphenpenalty} \NC \prm {exhyphenpenalty} \NC \NR
-\NC \type{4} \NC \lpr {automatichyphenpenalty} \NC \lpr {explicithyphenpenalty} \NC \NR
-\NC \type{5} \NC \prm {exhyphenpenalty} \NC \lpr {explicithyphenpenalty} \NC \NR
-\NC \type{6} \NC \prm {hyphenpenalty} \NC \lpr {explicithyphenpenalty} \NC \NR
-\NC \type{7} \NC \lpr {automatichyphenpenalty} \NC \prm {exhyphenpenalty} \NC \NR
-\NC \type{8} \NC \lpr {automatichyphenpenalty} \NC \prm {hyphenpenalty} \NC \NR
-\LL
-\stoptabulate
-
-other values do what we always did in \LUATEX: insert \prm {exhyphenpenalty}.
+The usage of these penalties is controlled by the \lpr {hyphenationmode} flags
+\number\automaticpenaltyhyphenationmodecode\space and
+\number\explicitpenaltyhyphenationmodecode\space and when these are not set \prm
+{exhyphenpenalty} is used.
\stopsection
@@ -575,7 +509,7 @@ In the last line of the table, you see there is no \prm {discretionary} command
in the value: the command is optional in the \TEX-based input syntax. The
underlying reason for that is that it is conceivable that a whole dictionary of
words is stored as a plain text file and loaded into \LUATEX\ using one of the
-functions in the \LUA\ \type {lang} library. This loading method is quite a bit
+functions in the \LUA\ \type {language} library. This loading method is quite a bit
faster than going through the \TEX\ language primitives, but some (most?) of that
speed gain would be lost if it had to interpret command sequences while doing so.
@@ -666,7 +600,7 @@ of the implementation:
\startitem
Because there is no \quote {trie preparation} stage, language patterns never
become frozen. This means that the primitive \prm {patterns} (and its \LUA\
- counterpart \type {lang.patterns}) can be used at any time, not only in
+ counterpart \type {language.patterns}) can be used at any time, not only in
ini\TEX.
\stopitem
\startitem
@@ -690,7 +624,7 @@ of the implementation:
permits a local overload for explicit \prm {discretionary} commands. The
value current when the hyphenation pass is applied is used. When no callbacks
are used this is compatible with traditional \TEX. When you apply the \LUA\
- \type {lang.hyphenate} function the current values are used.
+ \type {language.hyphenate} function the current values are used.
\stopitem
\startitem
The hyphenation exception dictionary is maintained as key|-|value hash, and
@@ -736,7 +670,7 @@ eventually either the limitation will be removed or perhaps it will become
possible to silently ignore the excess characters (this is what happens in
\TEX82, but there the behaviour cannot be controlled).
-If you are using the \LUA\ function \type {lang.hyphenate}, you should be aware
+If you are using the \LUA\ function \type {language.hyphenate}, you should be aware
that this function expects to receive a list of \quote {character} nodes. It will
not operate properly in the presence of \quote {glyph}, \quote {ligature}, or
\quote {ghost} nodes, nor does it know how to deal with kerning.
@@ -774,113 +708,15 @@ that we provide (which is part of \CONTEXT) distinguishes between base mode
(which essentially is what we describe here and which delegates the task to \TEX)
and node mode (which deals with more complex fonts.
-Let's look at an example. Take the word \type {office}, hyphenated \type
-{of-fice}, using a \quote {normal} font with all the \type {f}-\type {f} and
-\type {f}-\type {i} type ligatures:
-
-\starttabulate[|l|l|]
-\NC initial \NC \type {{o}{f}{f}{i}{c}{e}} \NC\NR
-\NC after hyphenation \NC \type {{o}{f}{{-},{},{}}{f}{i}{c}{e}} \NC\NR
-\NC first ligature stage \NC \type {{o}{{f-},{f},{<ff>}}{i}{c}{e}} \NC\NR
-\NC final result \NC \type {{o}{{f-},{<fi>},{<ffi>}}{c}{e}} \NC\NR
-\stoptabulate
-
-That's bad enough, but let us assume that there is also a hyphenation point
-between the \type {f} and the \type {i}, to create \type {of-f-ice}. Then the
-final result should be:
-
-\starttyping
-{o}{{f-},
- {{f-},
- {i},
- {<fi>}},
- {{<ff>-},
- {i},
- {<ffi>}}}{c}{e}
-\stoptyping
-
-with discretionaries in the post-break text as well as in the replacement text of
-the top-level discretionary that resulted from the first hyphenation point.
-
-Here is that nested solution again, in a different representation:
-
-\testpage[4]
-
-\starttabulate[|l|c|c|c|c|c|c|]
-\DB \BC pre \BC \BC post \BC \BC replace \BC \NC \NR
-\TB
-\NC topdisc \NC \type {f-} \NC (1) \NC sub 1 \NC \NC sub 2 \NC \NC \NR
-\NC sub 1 \NC \type {f-} \NC (2) \NC \type {i} \NC (3) \NC \type {<fi>} \NC (4) \NC \NR
-\NC sub 2 \NC \type {<ff>-} \NC (5) \NC \type {i} \NC (6) \NC \type {<ffi>} \NC (7) \NC \NR
-\LL
-\stoptabulate
-
-When line breaking is choosing its breakpoints, the following fields will
-eventually be selected:
-
-\starttabulate[|l|c|c|]
-\NC \type {of-f-ice} \NC \type {f-} \NC (1) \NC \NR
-\NC \NC \type {f-} \NC (2) \NC \NR
-\NC \NC \type {i} \NC (3) \NC \NR
-\NC \type {of-fice} \NC \type {f-} \NC (1) \NC \NR
-\NC \NC \type {<fi>} \NC (4) \NC \NR
-\NC \type {off-ice} \NC \type {<ff>-} \NC (5) \NC \NR
-\NC \NC \type {i} \NC (6) \NC \NR
-\NC \type {office} \NC \type {<ffi>} \NC (7) \NC \NR
-\stoptabulate
-
-The current solution in \LUATEX\ is not able to handle nested discretionaries,
-but it is in fact smart enough to handle this fictional \type {of-f-ice} example.
-It does so by combining two sequential discretionary nodes as if they were a
-single object (where the second discretionary node is treated as an extension of
-the first node).
-
-One can observe that the \type {of-f-ice} and \type {off-ice} cases both end with
-the same actual post replacement list (\type {i}), and that this would be the
-case even if \type {i} was the first item of a potential following ligature like
-\type {ic}. This allows \LUATEX\ to do away with one of the fields, and thus make
-the whole stuff fit into just two discretionary nodes.
-
-The mapping of the seven list fields to the six fields in this discretionary node
-pair is as follows:
-
-\starttabulate[|l|c|c|]
-\DB field \BC description \NC \NC \NR
-\TB
-\NC \type {disc1.pre} \NC \type {f-} \NC (1) \NC \NR
-\NC \type {disc1.post} \NC \type {<fi>} \NC (4) \NC \NR
-\NC \type {disc1.replace} \NC \type {<ffi>} \NC (7) \NC \NR
-\NC \type {disc2.pre} \NC \type {f-} \NC (2) \NC \NR
-\NC \type {disc2.post} \NC \type {i} \NC (3,6) \NC \NR
-\NC \type {disc2.replace} \NC \type {<ff>-} \NC (5) \NC \NR
-\LL
-\stoptabulate
-
-What is actually generated after ligaturing has been applied is therefore:
-
-\starttyping
-{o}{{f-},
- {<fi>},
- {<ffi>}}
- {{f-},
- {i},
- {<ff>-}}{c}{e}
-\stoptyping
-
-The two discretionaries have different subtypes from a discretionary appearing on
-its own: the first has subtype 4, and the second has subtype 5. The need for
-these special subtypes stems from the fact that not all of the fields appear in
-their \quote {normal} location. The second discretionary especially looks odd,
-with things like the \type {<ff>-} appearing in \type {disc2.replace}. The fact
-that some of the fields have different meanings (and different processing code
-internally) is what makes it necessary to have different subtypes: this enables
-\LUATEX\ to distinguish this sequence of two joined discretionary nodes from the
-case of two standalone discretionaries appearing in a row.
-
-Of course there is still that relationship with fonts: ligatures can be implemented by
-mapping a sequence of glyphs onto one glyph, but also by selective replacement and
-kerning. This means that the above examples are just representing the traditional
-approach.
+In so called base mode, where \TEX\ does the work, the ligature construction
+(normally) goes in small steps. An \type {f} followed by an \type {f} becomes an
+\type {ff} ligatures and that one followed by an \type {i} can become a \type
+{ffi} ligature. The situation can be complicated by hyphenation points between
+these characters. When there are several in a ligature collapsing happens. Flag
+{\tttf "\uchexnumbers {\lazyligatureshyphenationmodecode}} in the \lpr
+{hyphenationmode} variable determines if this happens lazy or greedy, i.e.\ the
+first hyphen wins or the last one does. In practice a \CONTEXT\ user won't have
+to deal with this because most fonts are processed in node mode.
\stopsection
@@ -912,7 +748,7 @@ ligatures are used. Of course kerning also complicates matters here.
\stopsection
-\startsection[title={The \type {lang} library}][library=lang]
+\startsection[title={The \type {language} library}][library=lang]
\subsection {\type {new} and \type {id}}
@@ -925,34 +761,38 @@ This library provides the interface to \LUATEX's structure representing a
language, and the associated functions.
\startfunctioncall
-<language> l = lang.new()
-<language> l = lang.new(<number> id)
+<language> l = language.new()
+<language> l = language.new(<number> id)
\stopfunctioncall
This function creates a new userdata object. An object of type \type {<language>}
-is the first argument to most of the other functions in the \type {lang} library.
-These functions can also be used as if they were object methods, using the colon
-syntax. Without an argument, the next available internal id number will be
-assigned to this object. With argument, an object will be created that links to
-the internal language with that id number.
+is the first argument to most of the other functions in the \type {language}
+library. These functions can also be used as if they were object methods, using
+the colon syntax. Without an argument, the next available internal id number will
+be assigned to this object. With argument, an object will be created that links
+to the internal language with that id number.
\startfunctioncall
-<number> n = lang.id(<language> l)
+<number> n = language.id(<language> l)
\stopfunctioncall
-The number returned is the internal \prm {language} id number this object refers to.
+The number returned is the internal \prm {language} id number this object refers
+to.
\subsection {\type {hyphenation}}
\libindex {hyphenation}
-You can hyphenate a string directly with:
+You can load exceptions with:
\startfunctioncall
-<string> n = lang.hyphenation(<language> l)
-lang.hyphenation(<language> l, <string> n)
+<string> n = language.hyphenation(<language> l)
+language.hyphenation(<language> l, <string> n)
\stopfunctioncall
+When no string is given (the first example) a string with all exceptions is
+returned.
+
\subsection {\type {clear_hyphenation} and \type {clean}}
\libindex {clear_hyphenation}
@@ -963,14 +803,14 @@ new ones. The syntax of the string is explained in~\in {section}
[patternsexceptions].
\startfunctioncall
-lang.clear_hyphenation(<language> l)
+language.clear_hyphenation(<language> l)
\stopfunctioncall
This call clears the exception dictionary (string) for this language.
\startfunctioncall
-<string> n = lang.clean(<language> l, <string> o)
-<string> n = lang.clean(<string> o)
+<string> n = language.clean(<language> l, <string> o)
+<string> n = language.clean(<string> o)
\stopfunctioncall
This function creates a hyphenation key from the supplied hyphenation value. The
@@ -984,8 +824,8 @@ dictionary file, like spell|-|checking.
\libindex {clear_patterns}
\startfunctioncall
-<string> n = lang.patterns(<language> l)
-lang.patterns(<language> l, <string> n)
+<string> n = language.patterns(<language> l)
+language.patterns(<language> l, <string> n)
\stopfunctioncall
This adds additional patterns for this language object, or returns the current
@@ -993,7 +833,7 @@ set. The syntax of this string is explained in \in {section}
[patternsexceptions].
\startfunctioncall
-lang.clear_patterns(<language> l)
+language.clear_patterns(<language> l)
\stopfunctioncall
This can be used to clear the pattern dictionary for a language.
@@ -1006,8 +846,8 @@ This function sets (or gets) the value of the \TEX\ parameter
\type {\hyphenationmin}.
\startfunctioncall
-n = lang.hyphenationmin(<language> l)
-lang.hyphenationmin(<language> l, <number> n)
+n = language.hyphenationmin(<language> l)
+language.hyphenationmin(<language> l, <number> n)
\stopfunctioncall
\subsection {\type {[pre|post][ex|]hyphenchar}}
@@ -1018,11 +858,11 @@ lang.hyphenationmin(<language> l, <number> n)
\libindex {postexhyphenchar}
\startfunctioncall
-<number> n = lang.prehyphenchar(<language> l)
-lang.prehyphenchar(<language> l, <number> n)
+<number> n = language.prehyphenchar(<language> l)
+language.prehyphenchar(<language> l, <number> n)
-<number> n = lang.posthyphenchar(<language> l)
-lang.posthyphenchar(<language> l, <number> n)
+<number> n = language.posthyphenchar(<language> l)
+language.posthyphenchar(<language> l, <number> n)
\stopfunctioncall
These two are used to get or set the \quote {pre|-|break} and \quote
@@ -1030,11 +870,11 @@ These two are used to get or set the \quote {pre|-|break} and \quote
intial values are decimal 45 (hyphen) and decimal~0 (indicating emptiness).
\startfunctioncall
-<number> n = lang.preexhyphenchar(<language> l)
-lang.preexhyphenchar(<language> l, <number> n)
+<number> n = language.preexhyphenchar(<language> l)
+language.preexhyphenchar(<language> l, <number> n)
-<number> n = lang.postexhyphenchar(<language> l)
-lang.postexhyphenchar(<language> l, <number> n)
+<number> n = language.postexhyphenchar(<language> l)
+language.postexhyphenchar(<language> l, <number> n)
\stopfunctioncall
These gets or set the \quote {pre|-|break} and \quote {post|-|break} hyphen
@@ -1051,8 +891,8 @@ The next call inserts hyphenation points (discretionary nodes) in a node list. I
are proper nodes, regardless of possible other errors.
\startfunctioncall
-<boolean> success = lang.hyphenate(<node> head)
-<boolean> success = lang.hyphenate(<node> head, <node> tail)
+<boolean> success = language.hyphenate(<node> head)
+<boolean> success = language.hyphenate(<node> head, <node> tail)
\stopfunctioncall
Hyphenation works only on \quote {characters}, a special subtype of all the glyph
@@ -1068,8 +908,8 @@ more details.
The following two commands can be used to set or query hj codes:
\startfunctioncall
-lang.sethjcode(<language> l, <number> char, <number> usedchar)
-<number> usedchar = lang.gethjcode(<language> l, <number> char)
+language.sethjcode(<language> l, <number> char, <number> usedchar)
+<number> usedchar = language.gethjcode(<language> l, <number> char)
\stopfunctioncall
When you set a hjcode the current sets get initialized unless the set was already
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex
index a126a95dcea..ed4b269b6fb 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex
@@ -31,9 +31,10 @@ In this mode, it will set \LUA's \type {arg[0]} to the found script name, pushin
preceding options in negative values and the rest of the command line in the
positive values, just like the \LUA\ interpreter does.
-\LUAMETATEX\ will exit immediately after executing the specified \LUA\ script and is,
-in effect, a somewhat bulky stand alone \LUA\ interpreter with a bunch of extra
-preloaded libraries.
+\LUAMETATEX\ will exit immediately after executing the specified \LUA\ script and
+is, in effect, a somewhat bulky stand alone \LUA\ interpreter with a bunch of
+extra preloaded libraries. But we really want to keep the binary small, if
+possible below the 3MB which is okay for a script engine.
When no argument is given, \LUAMETATEX\ will look for a \LUA\ file with the same
name as the binary and run that one when present. This makes it possible to use
@@ -69,6 +70,20 @@ There are less options than with \LUATEX, because one has to deal with them in
\LUA\ anyway. There are no options to enter a safer mode or control executing
programs. This can easily be achieved with a startup \LUA\ script.
+Next the initialization script is loaded and executed. From within the script,
+the entire command line is available in the \LUA\ table \type {arg}, beginning
+with \type {arg[0]}, containing the name of the executable. As consequence
+warnings about unrecognized options are suppressed.
+
+Command line processing happens very early on. So early, in fact, that none of
+\TEX's initializations have taken place yet. The \LUA\ libraries that don't deal
+with \TEX\ are initialized early.
+
+\LUAMETATEX\ allows some of the command line options to be overridden by reading
+values from the \type {texconfig} table at the end of script execution (see the
+description of the \type {texconfig} table later on in this document for more
+details on which ones exactly).
+
The value to use for \prm {jobname} is decided as follows:
\startitemize
@@ -91,21 +106,7 @@ The value to use for \prm {jobname} is decided as follows:
\stopitem
\stopitemize
-Next the initialization script is loaded and executed. From within the script,
-the entire command line is available in the \LUA\ table \type {arg}, beginning
-with \type {arg[0]}, containing the name of the executable. As consequence
-warnings about unrecognized options are suppressed.
-
-Command line processing happens very early on. So early, in fact, that none of
-\TEX's initializations have taken place yet. The \LUA\ libraries that don't deal
-with \TEX\ are initialized early.
-
-\LUAMETATEX\ allows some of the command line options to be overridden by reading
-values from the \type {texconfig} table at the end of script execution (see the
-description of the \type {texconfig} table later on in this document for more
-details on which ones exactly).
-
-So let's summarize this. The handling of when is called jobname is a bit complex.
+So let's summarize this. The handling of what is called jobname is a bit complex.
There can be explicit names set on the command line but when not set they can be
taken from the \type {texconfig} table.
@@ -117,7 +118,9 @@ taken from the \type {texconfig} table.
These names are initialized according to \type {--luaonly} or the first filename
seen in the list of options. Special treatment of \type {&} and \type {*} as well
-as interactive startup is gone.
+as interactive startup is gone but we still enter \TEX\ via an forced \type {\input}
+into the input buffer. \footnote {This might change at some point into an explicit
+loading triggered via \LUA.}
When we are in \TEX\ mode at some point the engine needs a filename, for instance
for opening a log file. At that moment the set jobname becomes the internal one
@@ -133,6 +136,12 @@ access etc.\ internally uses the current code page but to the user is exposed as
% engine_state .startup_jobname : handles by option parser
% environment_state.input_name : temporary interceptor
+There is an extra options \type{--permitloadlib} that needs to be given when you
+load external libraries via \LUA. Although you could manage this via \LUA\ itself
+in a startup script, the reason for having this as option is the wish for
+security (at some point that became a demand for \LUATEX), so this might give an
+extra feeling of protection.
+
\stopsubsection
\stopsection
@@ -199,8 +208,10 @@ some care you can deal with \UNICODE\ just fine.
There are some more libraries present. These are discussed on a later chapter.
For instance we embed \type {luasocket} but contrary to \LUATEX\ don't embed the
related \LUA\ code. An adapted version of \type {luafilesystem} is also included.
-There is a more extensive math library and there are libraries that deal with
-encryption and compression.
+There are more extensive math libraries and there are libraries that deal with
+encryption and compression. At some point some of these might become so called
+optional libraries (read: the handful that we provide interfaces for but that get
+loaded on demand).
\stopsection
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-math.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-math.tex
index 7f77bbd6d39..02f5f8c71da 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-math.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-math.tex
@@ -143,7 +143,8 @@ It is possible to discover the math style that will be used for a formula in an
expandable fashion (while the math list is still being read). To make this
possible, \LUATEX\ adds the new primitive: \lpr {mathstyle}. This is a \quote
{convert command} like e.g. \prm {romannumeral}: its value can only be read,
-not set.
+not set. Beware that contrary to \LUATEX\ this is now a proper number so you need
+to use \type {\number} o r\type {\the} in order to serialize it.
The returned value is between 0 and 7 (in math mode), or $-1$ (all other modes).
For easy testing, the eight math style commands have been altered so that they can
@@ -178,55 +179,55 @@ differ from the used style (second pass). In the case of a math choice used
ungrouped, the chosen style is used after the choice too, unless you group.
\startbuffer[1]
- [a:\mathstyle]\quad
+ [a:\number\mathstyle]\quad
\bgroup
\mathchoice
- {\bf \scriptstyle (x:d :\mathstyle)}
- {\bf \scriptscriptstyle (x:t :\mathstyle)}
- {\bf \scriptscriptstyle (x:s :\mathstyle)}
- {\bf \scriptscriptstyle (x:ss:\mathstyle)}
+ {\bf \scriptstyle (x:d :\number\mathstyle)}
+ {\bf \scriptscriptstyle (x:t :\number\mathstyle)}
+ {\bf \scriptscriptstyle (x:s :\number\mathstyle)}
+ {\bf \scriptscriptstyle (x:ss:\number\mathstyle)}
\egroup
- \quad[b:\mathstyle]\quad
+ \quad[b:\number\mathstyle]\quad
\mathchoice
- {\bf \scriptstyle (y:d :\mathstyle)}
- {\bf \scriptscriptstyle (y:t :\mathstyle)}
- {\bf \scriptscriptstyle (y:s :\mathstyle)}
- {\bf \scriptscriptstyle (y:ss:\mathstyle)}
- \quad[c:\mathstyle]\quad
+ {\bf \scriptstyle (y:d :\number\mathstyle)}
+ {\bf \scriptscriptstyle (y:t :\number\mathstyle)}
+ {\bf \scriptscriptstyle (y:s :\number\mathstyle)}
+ {\bf \scriptscriptstyle (y:ss:\number\mathstyle)}
+ \quad[c:\number\mathstyle]\quad
\bgroup
\mathchoice
- {\bf \scriptstyle (z:d :\mathstyle)}
- {\bf \scriptscriptstyle (z:t :\mathstyle)}
- {\bf \scriptscriptstyle (z:s :\mathstyle)}
- {\bf \scriptscriptstyle (z:ss:\mathstyle)}
+ {\bf \scriptstyle (z:d :\number\mathstyle)}
+ {\bf \scriptscriptstyle (z:t :\number\mathstyle)}
+ {\bf \scriptscriptstyle (z:s :\number\mathstyle)}
+ {\bf \scriptscriptstyle (z:ss:\number\mathstyle)}
\egroup
- \quad[d:\mathstyle]
+ \quad[d:\number\mathstyle]
\stopbuffer
\startbuffer[2]
- [a:\mathstyle]\quad
+ [a:\number\mathstyle]\quad
\begingroup
\mathchoice
- {\bf \scriptstyle (x:d :\mathstyle)}
- {\bf \scriptscriptstyle (x:t :\mathstyle)}
- {\bf \scriptscriptstyle (x:s :\mathstyle)}
- {\bf \scriptscriptstyle (x:ss:\mathstyle)}
+ {\bf \scriptstyle (x:d :\number\mathstyle)}
+ {\bf \scriptscriptstyle (x:t :\number\mathstyle)}
+ {\bf \scriptscriptstyle (x:s :\number\mathstyle)}
+ {\bf \scriptscriptstyle (x:ss:\number\mathstyle)}
\endgroup
- \quad[b:\mathstyle]\quad
+ \quad[b:\number\mathstyle]\quad
\mathchoice
- {\bf \scriptstyle (y:d :\mathstyle)}
- {\bf \scriptscriptstyle (y:t :\mathstyle)}
- {\bf \scriptscriptstyle (y:s :\mathstyle)}
- {\bf \scriptscriptstyle (y:ss:\mathstyle)}
- \quad[c:\mathstyle]\quad
+ {\bf \scriptstyle (y:d :\number\mathstyle)}
+ {\bf \scriptscriptstyle (y:t :\number\mathstyle)}
+ {\bf \scriptscriptstyle (y:s :\number\mathstyle)}
+ {\bf \scriptscriptstyle (y:ss:\number\mathstyle)}
+ \quad[c:\number\mathstyle]\quad
\begingroup
\mathchoice
- {\bf \scriptstyle (z:d :\mathstyle)}
- {\bf \scriptscriptstyle (z:t :\mathstyle)}
- {\bf \scriptscriptstyle (z:s :\mathstyle)}
- {\bf \scriptscriptstyle (z:ss:\mathstyle)}
+ {\bf \scriptstyle (z:d :\number\mathstyle)}
+ {\bf \scriptscriptstyle (z:t :\number\mathstyle)}
+ {\bf \scriptscriptstyle (z:s :\number\mathstyle)}
+ {\bf \scriptscriptstyle (z:ss:\number\mathstyle)}
\endgroup
- \quad[d:\mathstyle]
+ \quad[d:\number\mathstyle]
\stopbuffer
\typebuffer[1]
@@ -423,6 +424,7 @@ has resulted in turning some hard codes properties into parameters.
\NC \lpr {Umathsupsubbottommax} \NC the bottom of the superscript of a combined super- and subscript
be at least as high as this above the baseline \NC \NR
\NC \lpr {Umathsubsupvgap} \NC vertical clearance between super- and subscript \NC \NR
+\NC \lpr {Umathspacebeforescript} \NC additional space added before a super- or subprescript (bonus setting) \NC \NR
\NC \lpr {Umathspaceafterscript} \NC additional space added after a super- or subscript \NC \NR
\NC \lpr {Umathconnectoroverlapmin}\NC minimum overlap between parts in an extensible recipe \NC \NR
\LL
@@ -734,7 +736,7 @@ marked \type {*} in the \TEX book. These will not actually be used as those
combinations of atoms cannot actually happen, but it seemed better not to break
orthogonality. They are initialized to zero.
-\subsection{Local settings}
+\subsection{Local \lpr {frozen} settings with}
Math is processed in two passes. The first pass is needed to intercept for
instance \type {\over}, one of the few \TEX\ commands that actually has a
@@ -750,7 +752,7 @@ front.
\startbuffer
$
- x \Usubscript {-}
+ x \Usubscript {-}
\frozen\Umathsubshiftdown\textstyle 0pt x \Usubscript {0}
{\frozen\Umathsubshiftdown\textstyle 5pt x \Usubscript {5}}
x \Usubscript {0}
@@ -775,7 +777,22 @@ unprocessed math list. The result looks as follows:
\blank \getbuffer \blank
+\subsection{Checking a state with \lpr {ifmathparameter}}
+When you adapt math parameters it might make sense to see if they are set at all.
+When a parameter is unset its value has the maximum dimension value and you might
+for instance mistakenly multiply that value to open up things a bit, which gives
+unexpected side effects. For that reason there is a convenient checker: \lpr
+{ifmathparameter}. This test primitive behaves like an \prm {ifcase}, with:
+
+\starttabulate[|c|l|]
+\DB value \BC meaning \NC \NR
+\TB
+\NC 0 \NC the parameter value is zero \NC \NR
+\NC 1 \NC the parameter is set \NC \NR
+\NC 2 \NC the parameter is unset \NC \NR
+\LL
+\stoptabulate
\subsection{Skips around display math and \lpr {mathdisplayskipmode}}
@@ -788,7 +805,7 @@ to fully control spacing. Therefore \LUATEX\ comes with a new directive: \lpr
{mathdisplayskipmode}. The following values apply:
\starttabulate[|c|l|]
-\DB value \BC meaning \NC \NR
+\DB value \BC meaning \NC \NR
\TB
\NC 0 \NC normal \TEX\ behaviour \NC \NR
\NC 1 \NC always (same as 0) \NC \NR
@@ -1096,7 +1113,34 @@ with in the engine). The full list of flags is given in the next table:
\LL
\stoptabulate
-The effect can depend on the font (and for Cambria one can use for instance \type {"16}).
+The effect can depend on the font (and for Cambria one can use for instance \type
+{"16}).
+
+Sometimes you might want to act upon the size of a delimiter, something that is
+not really possible because of the fact that they are calculated {\em after} most
+has been typeset already. For this we have two keyword: \type {phantom} and
+\type {void}. In both cases the symbol is replaced by an empty rule, in the first
+case all three dimensions are preserved in the last case only the height and depth.
+
+\startbuffer
+\startformula
+ x\mathlimop{\Uvextensible \Udelimiter 5 0 "222B}_1^2 x
+\stopformula
+\vskip-9ex
+\startformula \red
+ x\mathlimop{\Uvextensible phantom \Udelimiter 5 0 "222B}_1^2 x
+\stopformula
+\vskip-9ex
+\startformula \blue
+ x\mathlimop{\Uvextensible void \Udelimiter 5 0 "222B}_1^2 x
+\stopformula
+\stopbuffer
+
+\typebuffer
+
+In typeset form this looks like:
+
+\getbuffer
\subsection[mathacc]{Accent handling with \lpr {Umathaccent}}
@@ -1350,7 +1394,80 @@ For Latin Modern the result looks as follows:
\stoptabulate
\stop
-\subsection {Delimiters: \type{\Uleft}, \prm {Umiddle} and \prm {Uright}}
+The \type {\over} and related primitives have the form:
+
+\starttyping
+{{top}\over{bottom}}
+\stoptyping
+
+For convenience, which also avoids some of the trickery that makes this
+\quote {looking back} possible, the \LUAMETATEX\ also provides this variant:
+
+\starttyping
+\Uover{top}{bottom}
+\stoptyping
+
+The optional arguments are also supported but we have one extra option: \type
+{style}. The style is applied to the numerator and denominator.
+
+\starttyping
+\Uover style \scriptstyle {top} {bottom}
+\stoptyping
+
+The complete list of these commands is: \lpr {Uabove}, \lpr {Uatop}, \lpr
+{Uover}, \lpr {Uabovewithdelims}, \lpr {Uatopwithdelims}, \lpr {Uoverwithdelims},
+\lpr {UUskewed}, \lpr {UUskewedwithdelims}. As with other extensions we use a
+leading \type {U} and because we already had extra skew related primitives we end
+up with a \type {UU} there. This obscurity is not that big an issue because
+normally such primitives are wrapped in a macro. Here are a few examples:
+
+\startbuffer
+$\Uover { 1234} { 5678} $\quad
+$\Uover {\textstyle 1234} {\textstyle 5678} $\quad
+$\Uover {\scriptstyle 1234} {\scriptstyle 5678} $\quad
+$\Uover {\scriptscriptstyle 1234} {\scriptscriptstyle 5678} $\blank
+
+$\Uover {1234} {5678} $\quad
+$\Uover style \textstyle {1234} {5678} $\quad
+$\Uover style \scriptstyle {1234} {5678} $\quad
+$\Uover style \scriptscriptstyle {1234} {5678} $\blank
+\stopbuffer
+
+\typebuffer
+
+These render as: \getbuffer
+
+\subsection {Math styles: \lpr {Ustyle}}
+
+This primitive accepts a style identifier:
+
+\starttyping
+\Ustyle \displaystyle
+\stoptyping
+
+This in itself is not spectacular because it is equivalent to
+
+\starttyping
+\displaystyle
+\stoptyping
+
+Both commands inject a style node and change the current style. However, as in other
+places where \LUAMETATEX\ expects a style you can also pass a number in the range
+zero upto seven (like the ones reported by the primitive \lpr {mathstyle}). So, the
+next few lines give identical results:
+
+\startbuffer
+$\Ustyle0 \number\mathstyle \Ustyle7 \number\mathstyle$
+$\Ustyle\displaystyle \number\mathstyle \Ustyle\crampedscriptscriptstyle \number\mathstyle$
+$ \displaystyle \number\mathstyle \crampedscriptscriptstyle \number\mathstyle$
+\stopbuffer
+
+Like: \inlinebuffer . Values outside the valid range are ignored.
+
+There is an extra option \type {norule} that can be used to suppress the rule while
+keeping the spacing compatible.
+
+\subsection {Delimiters: \type{\Uleft}, \lpr {Umiddle} and \lpr {Uright}}
\topicindex {math+delimiters}
@@ -1430,9 +1547,15 @@ These commands are provided as convenience. Before they come available you could
do the following:
\starttyping
-\def\Umathcharclass{\directlua{tex.print(tex.getmathcode(token.scan_int())[1])}}
-\def\Umathcharfam {\directlua{tex.print(tex.getmathcode(token.scan_int())[2])}}
-\def\Umathcharslot {\directlua{tex.print(tex.getmathcode(token.scan_int())[3])}}
+\def\Umathcharclass{\numexpr
+ \directlua{tex.print(tex.getmathcode(token.scan_int())[1])}
+\relax}
+\def\Umathcharfam{\numexpr
+ \directlua{tex.print(tex.getmathcode(token.scan_int())[2])}
+\relax}
+\def\Umathcharslot{\numexpr
+ \directlua{tex.print(tex.getmathcode(token.scan_int())[3])}
+\relax}
\stoptyping
\subsection {Last lines and \lpr{predisplaygapfactor}}
@@ -1569,6 +1692,7 @@ mode parameter. The default value is~1.
\LL
\stoptabulate
+
\subsection {Less Tracing}
\topicindex {math+tracing}
@@ -1579,6 +1703,79 @@ to limit tracing. Only when \type {tracingassigns} and|/|or \type
\stopsection
+\startsection[title={Experiments}]
+
+There are a couple of experimental features. They will stay but details might
+change, for instance more control over spacing. We just show some examples and
+let your imagination work it out. First we have prescripts:
+
+\subsection {Prescripts with \lpr {Usuperprescript} and {Usubprescript}}
+
+\startbuffer
+\hbox{$
+ {\tf X}^1_2^^3__4 \quad
+ {\tf X}^1 ^^3 \quad
+ {\tf X} _1 __4 \quad
+ {\tf X} ^^3 \quad
+ {\tf X} __4 \quad
+ {\tf X}^^3 __4
+$}
+\stopbuffer
+
+\typebuffer
+
+The question is: are these double super and subscript triggers the way to go?
+Anyway, you need to have them either being active (which in \CONTEXT\ then boils
+down to them being other characters), or say \type {\supmarkmode = 1} to disable
+the normal multiple \type {^} treatment (a value larger than 1 will also disable
+that in text mode).
+
+\blank \getbuffer \blank
+
+The more explicit commands are:
+
+\startbuffer
+\hbox{$
+{\tf X}\Usuperscript{1} \quad
+{\tf X} \Usubscript{2} \quad
+{\tf X}\Usuperscript{1}\Usubscript{2} \quad
+{\tf X}\Usuperscript{1} \Usuperprescript{3} \quad
+{\tf X} \Usubscript{2} \Usubprescript{4}\quad
+{\tf X}\Usuperscript{1}\Usubscript{2}\Usuperprescript{3}\Usubprescript{4}\quad
+{\tf X} \Usuperprescript{3} \quad
+{\tf X} \Usubprescript{4}\quad
+{\tf X} \Usuperprescript{3}\Usubprescript{4}
+$}
+\stopbuffer
+
+\typebuffer
+
+These more verbose triggers can be used to build interfaces:
+
+\blank \getbuffer \blank
+
+\subsection {Prescripts with \lpr {Usuperprescript} and {Usubprescript}}
+
+You can change the class of a math character on the fly:
+
+\startbuffer
+$x\mathopen {!}+123+\mathclose {!}x$
+$x\Umathclass4 ! +123+\Umathclass5 ! x$
+$x ! +123+ ! x$
+$x\mathclose {!}+123+\mathopen {!}x$
+$x\Umathclass5 ! +123+\Umathclass4 ! x$
+\stopbuffer
+
+\typebuffer
+
+Watch how the spacing changes:
+
+\startlines
+\getbuffer
+\stoplines
+
+\stopsection
+
\stopchapter
\stopcomponent
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-metapost.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-metapost.tex
index bd918995670..c687a5a46d3 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-metapost.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-metapost.tex
@@ -34,26 +34,28 @@ This creates the \type {mp} instance object. The argument is a hash table that
can have a number of different fields, as follows:
\starttabulate[|l|l|pl|pl|]
-\DB name \BC type \BC description \BC default \NC \NR
+\DB name \BC type \BC description \BC default \NC \NR
\TB
-\NC \type{error_line} \NC number \NC error line width \NC 79 \NC \NR
-\NC \type{print_line} \NC number \NC line length in ps output \NC 100 \NC \NR
-\NC \type{random_seed} \NC number \NC the initial random seed \NC variable \NC \NR
+\NC \type{error_line} \NC number \NC error line width \NC 79 \NC \NR
+\NC \type{print_line} \NC number \NC line length in ps output \NC 100 \NC \NR
+\NC \type{random_seed} \NC number \NC the initial random seed \NC variable \NC \NR
\NC \type{math_mode} \NC string \NC the number system to use:
\type {scaled},
\type {double} or
% \type {binary} or
- \type {decimal} \NC \type {scaled} \NC \NR
+ \type {decimal} \NC \type {scaled} \NC \NR
\NC \type{interaction} \NC string \NC the interaction mode:
\type {batch},
\type {nonstop},
\type {scroll} or
- \type {errorstop} \NC \type {errorstop} \NC \NR
-\NC \type{job_name} \NC string \NC a compatibility value \NC \type {mpout} \NC \NR
-\NC \type{find_file} \NC function \NC a function to find files \NC only local files \NC \NR
-\NC \type{utf8} \NC boolean \NC permit characters in the
+ \type {errorstop} \NC \type {errorstop} \NC \NR
+\NC \type{job_name} \NC string \NC a compatibility value \NC \NC \NR
+\NC \type{find_file} \NC function \NC a function to find files \NC only local files \NC \NR
+\NC \type{utf8_mode} \NC boolean \NC permit characters in the
range 128 upto 255 to be
- part of names \NC \type {false} \NC \NR
+ part of names \NC \type {false} \NC \NR
+\NC \type{text_mode} \NC boolean \NC permit characters 2 and 3
+ as fencing string literals \NC \type {false} \NC \NR
\LL
\stoptabulate
@@ -427,10 +429,6 @@ a bunch of vital characteristics of the used pen (all values are floats):
\LL
\stoptabulate
-\stopsection
-
-\startsection[title=Acessors]
-
\subsection[title={Character size information}]
\libindex{char_width}
@@ -441,13 +439,109 @@ These functions find the size of a glyph in a defined font. The \type {fontname}
is the same name as the argument to \type {infont}; the \type {char} is a glyph
id in the range 0 to 255; the returned \type {w} is in AFM units.
-
\startfunctioncall
<number> w = char_width(mp,<string> fontname, <number> char)
<number> h = char_height(mp,<string> fontname, <number> char)
<number> d = char_depth(mp,<string> fontname, <number> char)
\stopfunctioncall
+\stopsection
+
+\startsection[title=Scanners]
+
+After a relative long period of testing the scanners are now part of the
+interface. That doesn't mean that there will be no changes: depending on the
+needs and experiences details might evolve. The summary below is there
+still preliminary and mostly provided as reminder.
+
+\starttabulate[|l|l|p|]
+\DB scanner \BC argument \BC returns \NC \NR
+\TB
+\NC \type{scan_next} \NC instance, keep \NC token, mode, type \NC \NR
+\NC \type{scan_expression} \NC instance, keep \NC type \NC \NR
+\NC \type{scan_token} \NC instance, keep \NC token, mode, kind \NC \NR
+\NC \type{scan_symbol} \NC instance, keep, expand \NC string \NC \NR
+\NC \type{scan_numeric} \NC instance, type \NC number \NC \NR
+\NC \type{scan_integer} \NC instance, type \NC integer \NC \NR
+\NC \type{scan_boolean} \NC instance, type \NC boolean \NC \NR
+\NC \type{scan_string} \NC instance, type \NC string \NC \NR
+\NC \type{scan_pair} \NC instance, hashed, type \NC table or two numbers \NC \NR
+\NC \type{scan_color} \NC instance, hashed, type \NC table or three numbers \NC \NR
+\NC \type{scan_cmykcolor} \NC instance, hashed, type \NC table or four numbers \NC \NR
+\NC \type{scan_transform} \NC instance, hashed, type \NC table or six numbers \NC \NR
+\NC \type{scan_path} \NC instance, hashed, type \NC table with hashes or arrays \NC \NR
+\NC \type{scan_pen} \NC instance, hashed, type \NC table with hashes or arrays \NC \NR
+\LL
+\stoptabulate
+
+The types and token codes are numbers but they actually depend on the
+implementation (although changes are unlikely). The types of data structures can
+be queried with \type {mplib.gettypes()}:
+
+\dontleavehmode \startluacode
+ local t = mplib.gettypes()
+ for i=0,#t do
+ if i > 0 then
+ context(", ")
+ end
+ context("%i: %s",i,t[i])
+ end
+\stopluacode .
+
+The command codes are available with \type {mplib.getcodes()}:
+
+\dontleavehmode \startluacode
+ local t = mplib.getcodes()
+ for i=0,#t do
+ if i > 0 then
+ context(", ")
+ end
+ context("%i: %s",i,t[i])
+ end
+\stopluacode .
+
+Now, if you really want to use these, keep in mind that the internals of
+\METAPOST\ are not trivial, especially because expression scanning can be
+complex. So you need to experiment a bit. In \CONTEXT\ all is (and will be)
+hidden below an abstraction layer so users are not bothered by all these
+look|-|ahead and push|-|back issues that originate in the way \METAPOST\ scans
+its input.
+
+\stopsection
+
+\startsection[title=Injectors]
+
+It is important to know that piping code into the library is pretty fast and
+efficient. Most processing time relates to memory management, calculations and
+generation of output can not be neglected either. Out of curiousity I added some
+functions that directly push data into the library but the gain is not that
+large. \footnote {The main motivation was checking of huge paths could be
+optimized. The other data structures were then added for completeness.}
+
+\starttabulate[|l|l|]
+\DB scanner \BC argument \NC \NR
+\TB
+\NC \type{inject_numeric} \NC instance, number \NC \NR
+\NC \type{inject_integer} \NC instance, number \NC \NR
+\NC \type{inject_boolean} \NC instance, boolean \NC \NR
+\NC \type{inject_string} \NC instance, string \NC \NR
+\NC \type{inject_pair} \NC instance, (table with) two numbers \NC \NR
+\NC \type{inject_color} \NC instance, (table with) three numbers \NC \NR
+\NC \type{inject_cmykcolor} \NC instance, (table with) four numbers \NC \NR
+\NC \type{inject_transform} \NC instance, (table with) six numbers \NC \NR
+\NC \type{inject_path} \NC instance, table with hashes or arrays, cycle, variant \NC \NR
+\LL
+\stoptabulate
+
+The path injector takes a table with subtables that are either hashed (like the
+path solver) or arrays with two, four or six entries. When the third argument has
+the value \type {true} the path is closed. When the fourth argument is \type
+{true} the path is constructed out of straight lines (as with \type {--}) by
+setting the \type {curl} values to~1 automatically. \footnote {This is all
+experimental so future versions might provide more control.}
+
+\stopsection
+
\stopchapter
\stopcomponent
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex
index 6448f2b01c1..ac283bafd76 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex
@@ -16,14 +16,15 @@
The first version of \LUATEX, made by Hartmut after we discussed the possibility
of an extension language, only had a few extra primitives and it was largely the
same as \PDFTEX. It was presented to the public in 2005. As part of the Oriental
-\TEX\ project, Taco merged substantial parts of \ALEPH\ into the code and some
-more primitives were added. Then we started more fundamental experiments. After
-many years, when the engine had become more stable, the decision was made to
-clean up the rather hybrid nature of the program. This means that some primitives
-were promoted to core primitives, often with a different name, and that others
-were removed. This also made it possible to start cleaning up the code base. In
-\in {chapter} [enhancements] we discuss some new primitives, here we will cover
-most of the adapted ones.
+\TEX\ project, Taco merged some parts of \ALEPH\ into the code and some more
+primitives were added. Then we started more fundamental experiments. After many
+years, when the engine had become more stable, the decision was made to clean up
+the rather hybrid nature of the program. This means that some primitives were
+promoted to core primitives, often with a different name, and that others were
+removed. This also made it possible to start cleaning up the code base, which
+showed decades of stepwise additions to original \TEX. In \in {chapter}
+[enhancements] we discuss some new primitives, here we will cover most of the
+adapted ones.
During more than a decade stepwise new functionality was added and after 10 years
the more of less stable version 1.0 was presented. But we continued and after
@@ -38,9 +39,18 @@ not|-|so|-|expected changes. These are sometimes a side|-|effect of a new
(conflicting) feature, or, more often than not, a change necessary to clean up
the internal interfaces. These will also be mentioned.
+Again we stress that {\em this is not a \TEX\ manual, nor a tutorial}. If you are
+unfamiliar with \TEX\ first play a little with a macro package, take a look at
+the \TEX\ book, make yourself familiar with the concepts and macro language. That
+will likely take days and not hours. Also, many of the new concepts introduced in
+\LUATEX\ and \LUAMETATEX\ are explained in documents that come with the \CONTEXT\
+distribution, articles and presentations. It doesn't pay of to repeat that here,
+especially not in a time when users often search instead of read from cover to
+cover.
+
\stopsubsection
-\startsubsection[title=Changes from \TEX\ 3.1415926]
+\startsubsection[title=Changes from \TEX\ 3.1415926...]
\topicindex {\TEX}
@@ -50,19 +60,21 @@ most still comes from original Knuthian \TEX. But we divert a bit.
\startitemize
\startitem
- The current code base is written in \CCODE, not \PASCAL. The original \CWEB\
+ The current code base is written in \CCODE, not \PASCAL. The original \WEB\
documentation is kept when possible and not wrapped in tagged comments. As a
consequence instead of one large file plus change files, we now have multiple
- files organized in categories like \type {tex}, \type {luaf}, \type
+ files organized in categories like \type {tex}, \type {lua}, \type
{languages}, \type {fonts}, \type {libraries}, etc. There are some artifacts
of the conversion to \CCODE, but these got (and get) removed stepwise. The
documentation, which actually comes from the mix of engines (via so called
- change files), is kept as much as possible. Of course we want to stay as
- close as possible to the original so that the documentation of the
+ change files), is a mix of what authors of the engines wove into the source,
+ and most is of course from Don Knuths original. In \LUAMETATEX\ we try to
+ stay as close as possible to the original so that the documentation of the
fundamentals behind \TEX\ by Don Knuth still applies. However, because we use
\CCODE, some documentation is a bit off. Also, most global variables are now
- collected in structures, but the original names were kept. There are lots of
- so called macros too.
+ collected in structures, but the original names and level of abstraction were
+ mostly kept. On the other hand, opening up had its impact on the code, so
+ that makes some documentation a bit off too. Adapting that all will take time.
\stopitem
\startitem
@@ -74,14 +86,21 @@ most still comes from original Knuthian \TEX. But we divert a bit.
wherever we like. There are various options to control discretionary
injection and related penalties are now integrated in these nodes. Language
information is now bound to glyphs. The number of languages in \LUAMETATEX\
- is smaller than in \LUATEX.
+ is smaller than in \LUATEX. Control over discretionaries is more granular and
+ now managed by less variables.
\stopitem
\startitem
There is no pool file, all strings are embedded during compilation. This also
removed some memory constraints. We kept token and node memory management
because it is convenient and efficient but parts were reimplemented in order
- to remove some constraints. Token memory management is largely the same.
+ to remove some constraints. Token memory management is largely the same. All
+ the other large memory structures, like those related to nesting, the save
+ stack, input levels, the hash table and table of equivalents, etc. now all
+ start out small and are enlarged when needed, where maxima are controlled in
+ the usual way. In principle the initial memory footprint is smaller while at
+ the same time we can go real large. Because we have wide memory words some
+ data (arrays) used for housekeeping could be reorganized a bit.
\stopitem
\startitem
@@ -126,11 +145,9 @@ most still comes from original Knuthian \TEX. But we divert a bit.
\stopitem
\startitem
- When detailed logging is enabled more detail is output with respect to what
- nodes are involved. This is a side effect of the core nodes having more
- detailed subtype information. The benefit of more detail wins from any wish
- to be byte compatible in the logging. One can always write additional logging
- in \LUA.
+ The math style related primitives can use numbers as well as symbolic names.
+ There is some more (control over) math anyway, which is a side effect of
+ supporting \OPENTYPE\ math.
\stopitem
\stopitemize
@@ -172,10 +189,29 @@ features, but with a few small adaptations.
\stopitem
\startitem
+ Because we have more nodes, conditionals, etc.\ the \ETEX\ status related
+ variables are adapted to \LUAMETATEX: we use different \quote {constants},
+ but that should be no problem because any sane macro package uses
+ abstraction.
+\stopitem
+
+\startitem
+ The \type {\scantokens} primitive is now using the same mechanism as \LUA\
+ print|-|to|-|\TEX\ uses, which simplifies the code. There is a little
+ performance hit but it will not be noticed in \CONTEXT, because we never use
+ this primitive.
+\stopitem
+
+\startitem
Because we don't use change files on top of original \TEX, the integration of
\ETEX\ functionality is bit more natural, code wise.
\stopitem
+\startitem
+ The \prm {readline} primitive has to be implemented in \LUA. This is a side
+ effect of delegating all file \IO.
+\stopitem
+
\stopitemize
\stopsubsection
@@ -230,12 +266,7 @@ don't expect \LUAMETATEX\ to be compatible.
\startitem
When \lpr {adjustspacing} has value~2, hz optimization will be applied to
glyphs and kerns. When the value is~3, only glyphs will be treated. A value
- smaller than~2 disables this feature. With value of~1, font expansion is
- applied after \TEX's normal paragraph breaking routines have broken the
- paragraph into lines. In this case, line breaks are identical to standard
- \TEX\ behavior (as with \PDFTEX). But \unknown\ this is a left|-|over from
- the early days of \PDFTEX\ when this feature was part of a research topic. At
- some point level~1 might be dropped from \LUAMETATEX.
+ smaller than~2 disables this feature.
\stopitem
\startitem
@@ -292,7 +323,8 @@ Here is a summary of inherited functionality:
\startitem
Glues {\it immediately after} direction change commands are not legal
- breakpoints. There is a bit more sanity testing for the direction state.
+ breakpoints. There is a bit more sanity testing for the direction state. This
+ can be configured.
\stopitem
\startitem
@@ -303,7 +335,7 @@ Here is a summary of inherited functionality:
\startitem
There are no direction related primitives for page and body directions. The
paragraph, text and math directions are specified using primitives that
- take a number.
+ take a number. The three letter codes are dropped.
\stopitem
\stopitemize
@@ -334,7 +366,10 @@ The single internal memory heap that traditional \TEX\ used for tokens and nodes
is split into two separate arrays. Each of these will grow dynamically when
needed. Internally a token or node is an index into these arrays. This permits
for an efficient implementation and is also responsible for the performance of
-the core. The original documentation in \TEX\ The Program mostly applies!
+the core. All other data structures are mostly the same but managed dynamically
+too. Because we operate in a 64 bit world, the parallel table of equivalents
+needed for managing levels, is gone. Anyhow, the original documentation in \TEX\
+The Program mostly applies!
\stopsubsection
@@ -352,10 +387,6 @@ assignments don't show up when using the \ETEX\ tracing routines \prm
{tracingassigns} and \prm {tracingrestores} but we don't see that as a real
limitation. It also saves a lot of clutter.
-A side|-|effect of the current implementation is that \prm {global} is now more
-expensive in terms of processing than non|-|global assignments but not many users
-will notice that.
-
The glyph ids within a font are also managed by means of a sparse array as glyph
ids can go up to index $2^{21}-1$ but these are never accessed directly so again
users will not notice this.
@@ -367,26 +398,33 @@ users will not notice this.
\topicindex {csnames}
Single|-|character commands are no longer treated specially in the internals,
-they are stored in the hash just like the multiletter csnames.
+they are stored in the hash just like the multiletter control sequences. This is
+a side effect of going \UNICODE\ and \UTF. Where using 256 slots in an array add
+no burden supporting the whole \UNICODE\ range is a waste of space. Therefore,
+also active characters are internally implemented as a special type of
+multi|-|letter control sequences that uses a prefix that is otherwise impossible
+to obtain.
The code that displays control sequences explicitly checks if the length is one
when it has to decide whether or not to add a trailing space.
-Active characters are internally implemented as a special type of multi|-|letter
-control sequences that uses a prefix that is otherwise impossible to obtain.
-
\stopsubsection
\startsubsection[title=Binary file reading]
\topicindex {files+binary}
-All of the internal code is changed in such a way that if one of the \type
-{read_xxx_file} callbacks is not set, then the file is read by a \CCODE\ function
-using basically the same convention as the callback: a single read into a buffer
-big enough to hold the entire file contents. While this uses more memory than the
-previous code (that mostly used \type {getc} calls), it can be quite a bit faster
-(depending on your \IO\ subsystem). So far we never had issues with this approach.
+All input now goes via \LUA: files loaded with \type {\input} as well as files
+that are opened with \type {\openin}. Actually the later has to be implemented
+in terms of macros and \LUA\ calls. This also means that compared to \LUATEX\
+the internal handling of input has been changed but users won't notice that.
+
+Setting a callback is expected now. Although reading input natively using \type
+{getc} calls is more efficient, we now fetch lines from \LUA, put them in a
+buffer and then pick successive bytes (keep in mind that we read \UTF) from that.
+The performance is quite ok, also because \LUA\ is fast, todays operating systems
+cache, and storage media have become very fast. Also, \TEX\ is spending more time
+messing around with what it has input than actually reading input.
\stopsubsection
@@ -416,12 +454,17 @@ more details anyway.
\startsubsection[title=Logging]
+When detailed logging is enabled more detail is output with respect to what nodes
+are involved. This is a side effect of the core nodes having more detailed
+subtype information. The benefit of more detail wins from any wish to be byte
+compatible in the logging. One can always write additional logging in \LUA.
+
The information that goes into the log file can be different from \LUATEX, and
might even differ a bit more in the future. The main reason is that inside the
engine we have more granularity, which for instance means that we output subtype
-related information when nodes are printed. Of course we could have offered a
-compatibility mode but it serves no purpose. Over time there have been many
-subtle changes to control logs in the \TEX\ ecosystems so another one is
+and attribute related information when nodes are printed. Of course we could have
+offered a compatibility mode but it serves no purpose. Over time there have been
+many subtle changes to control logs in the \TEX\ ecosystems so another one is
bearable.
In a similar fashion, there is a bit different behaviour when \TEX\ expects
@@ -429,7 +472,38 @@ input, which in turn is a side effect of removing the interception of \type {*}
and \type {&} which made for cleaner code (quite a bit had accumulated as side
effect of continuous adaptations in the \TEX\ ecosystems). There was already code
that was never executed, simply as side effect of the way \LUATEX\ initializes
-itself (one needs to enable classes of primitives for instance).
+itself (one needs to enable classes of primitives for instance). Keep in mind
+that over time system dependencies have been handles with \TEX\ change files, the
+\WEBC\ infrastructure, \KPSE\ features, compilation variables and flags, etc. In
+\LUAMETATEX\ we try to minimize all that.
+
+\stopsubsection
+
+\startsubsection[title=Parsing]
+
+Token parsers have been upgraded for the sake of \LUA, \type {\csname} handling
+has been extended, macro definitions can be more flexible so there code was
+adapted, more conditionals also brought some changes. But we build upon the
+(reorganized) \TEX\ foundation so the basics can definitely be recognized.
+
+Because of interfacing in \LUA\ the internal token and node organization has
+been normalized (read: we cannot cheat because all is kind of visible). On
+the one hand this can come with a performance penalty but that is more than
+compensated by extensions, optimized parsers and such. Still the fact that we
+are \UTF\ based (32 bit) makes the machinery slower than the 8~bit original.
+The reworked \LUAMETATEX\ engine is substantially faster than the \LUATEX\
+predecessor.
+
+The handling of conditionals has been adapted so that we can have flatter
+branches (\type {\orelse} cum suis). This again has some consequences for
+parsing. Because parsing alignments is rather interwoven in general parsing and
+expansion the handling of related primitives has been slightly adapted (also for
+the sake of \LUA\ interfacing) and dealing with \type {\noalign} situations is a
+bit more convenient.
+
+This are just a few of the adaptations and most of this happened stepwise with
+testing in the \CONTEXT\ code base. It will be clear that \LUAMETATEX\ is a quite
+different extension to the original. You're warned.
\stopsubsection
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
index 50356da4211..6773d346d78 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
@@ -174,12 +174,12 @@ discussed here (yet).
\stopsubsection
-\startsubsection[title={\nod {ins} nodes}]
+\startsubsection[title={\nod {insert} nodes}]
\topicindex {nodes+insertions}
\topicindex {insertions}
-This node relates to the \prm {insert} primitive and support the fields: \showfields{ins}.
+This node relates to the \prm {insert} primitive and support the fields: \showfields{insert}.
\starttabulate[|l|l|p|]
\DB field \BC type \BC explanation \NC \NR
@@ -455,7 +455,7 @@ fields: \showfields {glyph}.
\NC \type{attr} \NC node \NC list of attributes \NC \NR
\NC \type{char} \NC number \NC the character index in the font \NC \NR
\NC \type{font} \NC number \NC the font identifier \NC \NR
-\NC \type{lang} \NC number \NC the language identifier \NC \NR
+\NC \type{language} \NC number \NC the language identifier \NC \NR
\NC \type{left} \NC number \NC the frozen \type {\lefthyphenmnin} value \NC \NR
\NC \type{right} \NC number \NC the frozen \type {\righthyphenmnin} value \NC \NR
\NC \type{uchyph} \NC boolean \NC the frozen \prm {uchyph} value \NC \NR
@@ -521,19 +521,19 @@ nodes: \showfields {boundary} are the only fields.
\TB
\NC \type{subtype} \NC number \NC \showsubtypes{boundary} \NC \NR
\NC \type{attr} \NC node \NC list of attributes \NC \NR
-\NC \type{value} \NC number \NC values 0--255 are reserved \NC \NR
+\NC \type{data} \NC number \NC values 0--255 are reserved \NC \NR
\LL
\stoptabulate
\stopsubsection
-\startsubsection[title={\nod {local_par} nodes}]
+\startsubsection[title={\nod {par} nodes}]
\topicindex {nodes+paragraphs}
\topicindex {paragraphs}
This node is inserted at the start of a paragraph. You should not mess
-too much with this one. Valid fields are: \showfields {local_par}.
+too much with this one. Valid fields are: \showfields {par}.
\starttabulate[|l|l|p|]
\DB field \BC type \BC explanation \NC \NR
@@ -613,7 +613,7 @@ into a single node type with separate subtypes for differentiation: \showfields
Many object fields in math mode are either simple characters in a specific family
or math lists or node lists: \type {math_char}, \type {math_text_char}, {sub_box}
-and \type {sub_mlist} and \type {delim}. These are endpoints and therefore the
+and \type {sub_mlist} and \type {delimiter}. These are endpoints and therefore the
\type {next} and \type {prev} fields of these these subnodes are unused.
Some of the more elaborate noads have an option field. The values in this bitset
@@ -681,11 +681,11 @@ its internal link structure is correct, otherwise an error is triggered.
\stopsubsubsection
-\startsubsubsection[title={\nod {delim} subnodes}]
+\startsubsubsection[title={\nod {delimiter} subnodes}]
There is a fifth subnode type that is used exclusively for delimiter fields. As
before, the \type {next} and \type {prev} fields are unused, but we do have:
-\showfields {delim}.
+\showfields {delimiter}.
\starttabulate[|l|l|p|]
\DB field \BC type \BC explanation \NC \NR
@@ -854,14 +854,14 @@ renderer and might get adapted in the process.
\starttabulate[|l|l|p|]
\DB field \BC type \BC explanation \NC \NR
\TB
-\NC \type{subtype} \NC number \NC \showsubtypes{fence} \NC \NR
-\NC \type{attr} \NC node \NC list of attributes \NC \NR
-\NC \type{delim} \NC delimiter node \NC delimiter specification \NC \NR
-\NC \type{italic} \NC number \NC italic correction \NC \NR
-\NC \type{height} \NC number \NC required height \NC \NR
-\NC \type{depth} \NC number \NC required depth \NC \NR
-\NC \type{options} \NC number \NC bitset of rendering options \NC \NR
-\NC \type{class} \NC number \NC spacing related class \NC \NR
+\NC \type{subtype} \NC number \NC \showsubtypes{fence} \NC \NR
+\NC \type{attr} \NC node \NC list of attributes \NC \NR
+\NC \type{delimiter} \NC delimiter node \NC delimiter specification \NC \NR
+\NC \type{italic} \NC number \NC italic correction \NC \NR
+\NC \type{height} \NC number \NC required height \NC \NR
+\NC \type{depth} \NC number \NC required depth \NC \NR
+\NC \type{options} \NC number \NC bitset of rendering options \NC \NR
+\NC \type{class} \NC number \NC spacing related class \NC \NR
\LL
\stoptabulate
@@ -1182,12 +1182,9 @@ tail node.
<node> tail = node.slide(<node> n)
\stopfunctioncall
-After some callbacks automatic sliding takes place. This feature can be turned
-off with \type {node.fix_node_lists(false)} but you better make sure then that
-you don't mess up lists. In most cases \TEX\ itself only uses \type {next}
-pointers but your other callbacks might expect proper \type {prev} pointers too.
-Future versions of \LUATEX\ can add more checking but this will not influence
-usage.
+In most cases \TEX\ itself only uses \type {next} pointers but your other
+callbacks might expect proper \type {prev} pointers too. So, when you run into
+issues or are in doubt, apply the slide function before you return the list.
\stopsubsubsection
@@ -1386,16 +1383,18 @@ The \type{traverse_char} iterator loops over the \nod {glyph} nodes in a list.
Only nodes with a subtype less than 256 are seen.
\startfunctioncall
-<node> n, font, char = node.traverse_char(<node> n)
+<direct> n, font, char = node.direct.traverse_char(<direct> n)
\stopfunctioncall
The \type{traverse_glyph} iterator loops over a list and returns the list and
filters all glyphs:
\startfunctioncall
-<node> n, font, char = node.traverse_glyph(<node> n)
+<direct> n, font, char = node.traverse_glyph(<direct> n)
\stopfunctioncall
+These functions are only available for direct nodes.
+
\stopsubsubsection
\startsubsubsection[title={\type {traverse_list}}]
@@ -1405,15 +1404,100 @@ filters all glyphs:
This iterator loops over the \nod {hlist} and \nod {vlist} nodes in a list.
\startfunctioncall
-<node> n, id, subtype, list = node.traverse_list(<node> n)
+<direct> n, id, subtype, list = node.traverse_list(<direct> n)
+\stopfunctioncall
+
+The four return values can save some time compared to fetching these fields but
+in practice you seldom need them all. This function is only available for direct
+nodes.
+
+\stopsubsubsection
+
+\startsubsubsection[title={\type {traverse_content}}]
+
+\libindex {traverse_content}
+
+This iterator loops over nodes that have content: \nod {hlist}, \nod {vlist}, \nod {glue}
+with leaders, \nod {glyphs}, \nod {disc} and \nod {rules} nodes.
+
+\startfunctioncall
+<direct> n, id, subtype[, list|leader] = node.traverse_list(<direct> n)
\stopfunctioncall
The four return values can save some time compared to fetching these fields but
-in practice you seldom need them all. So consider it a (side effect of
-experimental) convenience.
+in practice you seldom need them all. This function is only available for direct
+nodes.
\stopsubsubsection
+\startsubsubsection[title={Reverse traversing}]
+
+The traversers also support backward traversal. An optional extra boolean triggers
+this. Yet another optional boolean will automatically start at the end of the
+given list.
+
+\starttyping
+\setbox0\hbox{1 2 3 4 5}
+
+local l = tex.box[0].list
+for n in node.traverse(l) do
+ print("1>",n)
+end
+for n in node.traverse(l,true) do
+ print("2>",n)
+end
+for n in node.traverse(l,true,true) do
+ print("3>",n)
+end
+for n in node.traverse_id(nodes.nodecodes.glyph,l) do
+ print("4>",n)
+end
+for n in node.traverse_id(nodes.nodecodes.glyph,l,true) do
+ print("5>",n)
+end
+for n in node.traverse_id(nodes.nodecodes.glyph,l,true,true) do
+ print("6>",n)
+end
+\stoptyping
+
+This produces something similar to this (the glyph subtype indicates that it has
+been processed by the font handlers):
+
+\starttyping
+1> <node : nil <= 1112 => 590 : glyph 32768>
+1> <node : 1112 <= 590 => 1120 : glue spaceskip>
+1> <node : 590 <= 1120 => 849 : glyph 32768>
+1> <node : 1120 <= 849 => 1128 : glue spaceskip>
+1> <node : 849 <= 1128 => 880 : glyph 32768>
+1> <node : 1128 <= 880 => 1136 : glue spaceskip>
+1> <node : 880 <= 1136 => 1020 : glyph 32768>
+1> <node : 1136 <= 1020 => 1144 : glue spaceskip>
+1> <node : 1020 <= 1144 => nil : glyph 32768>
+2> <node : nil <= 1112 => 590 : glyph 32768>
+3> <node : 1020 <= 1144 => nil : glyph 32768>
+3> <node : 1136 <= 1020 => 1144 : glue spaceskip>
+3> <node : 880 <= 1136 => 1020 : glyph 32768>
+3> <node : 1128 <= 880 => 1136 : glue spaceskip>
+3> <node : 849 <= 1128 => 880 : glyph 32768>
+3> <node : 1120 <= 849 => 1128 : glue spaceskip>
+3> <node : 590 <= 1120 => 849 : glyph 32768>
+3> <node : 1112 <= 590 => 1120 : glue spaceskip>
+3> <node : nil <= 1112 => 590 : glyph 32768>
+4> <node : nil <= 1112 => 590 : glyph 32768>
+4> <node : 590 <= 1120 => 849 : glyph 32768>
+4> <node : 849 <= 1128 => 880 : glyph 32768>
+4> <node : 880 <= 1136 => 1020 : glyph 32768>
+4> <node : 1020 <= 1144 => nil : glyph 32768>
+5> <node : nil <= 1112 => 590 : glyph 32768>
+6> <node : 1020 <= 1144 => nil : glyph 32768>
+6> <node : 880 <= 1136 => 1020 : glyph 32768>
+6> <node : 849 <= 1128 => 880 : glyph 32768>
+6> <node : 590 <= 1120 => 849 : glyph 32768>
+6> <node : nil <= 1112 => 590 : glyph 32768>
+\stoptyping
+
+\stopsubsection
+
\startsubsubsection[title={\type {find_node}}]
\libindex {find_node}
@@ -1436,8 +1520,8 @@ This helper returns the location of the first match at or after node \type {n}:
\libindex {setglue}
-You can set the properties of a glue in one go. If you pass no values, the glue
-will become a zero glue.
+You can set the five properties of a glue in one go. If a non|-|numeric value is
+passed the property becomes zero.
\startfunctioncall
node.setglue(<node> n)
@@ -2095,6 +2179,16 @@ consistency. You can of course always define additional accessors using \type
{getfield} and \type {setfield} with little overhead. When the second argument of
\type {setattributelist} is \type {true} the current attribute list is assumed.
+The \type {reverse} function reverses a given list. The \type {exchange} function
+swaps two nodes; it takes upto three arguments: a head node, and one or two to be
+swapped nodes. When there is no third argument, it will assume that the node
+following node is to be used. So we have:
+
+\starttyping
+head = node.direct.reverse(head)
+head = node.direct.exchange(head,first,[second])
+\stoptyping
+
In \CONTEXT\ some of the not performance|-|critical userdata variants are
emulated in \LUA\ and not in the engine, so we retain downward compatibility.
@@ -2112,22 +2206,23 @@ emulated in \LUA\ and not in the engine, so we retain downward compatibility.
\TB
\supported {check_discretionaries} \nop \yes \yes
\supported {check_discretionary} \nop \yes \yes
-\supported {copy} \yes \yes \relax
\supported {copy_list} \yes \yes \relax
%supported {copy_only} \nop \yes \relax
+\supported {copy} \yes \yes \relax
\supported {count} \nop \yes \yes
-\supported {current_attr} \yes \yes \relax
+\supported {current_attributes} \yes \yes \relax
\supported {dimensions} \nop \yes \yes
\supported {effective_glue} \nop \yes \yes
\supported {end_of_math} \nop \yes \yes
+\supported {find_attribute_range} \nop \yes \relax
\supported {find_attribute} \nop \yes \yes
-%supported {find_attribute_range} \nop \yes \relax
-%supported {find_node} \nop \yes \relax
+\supported {find_node} \nop \yes \relax
\supported {first_glyph} \nop \yes \yes
\supported {flatten_discretionaries} \nop \yes \yes
\supported {flush_list} \yes \yes \relax
\supported {flush_node} \yes \yes \relax
\supported {free} \yes \yes \relax
+\supported {get_attributes} \nop \yes \relax
\supported {get_attribute} \yes \yes \relax
\supported {get_properties_table} \yes \yes \relax
\supported {get_synctex_fields} \nop \yes \relax
@@ -2135,7 +2230,6 @@ emulated in \LUA\ and not in the engine, so we retain downward compatibility.
\supported {getboth} \nop \yes \relax
\supported {getbox} \nop \yes \relax
\supported {getchar} \nop \yes \relax
-\supported {getstate} \nop \yes \relax
\supported {getdata} \nop \yes \relax
\supported {getdepth} \nop \yes \relax
\supported {getdirection} \nop \yes \relax
@@ -2145,42 +2239,65 @@ emulated in \LUA\ and not in the engine, so we retain downward compatibility.
\supported {getfield} \yes \yes \relax
\supported {getfont} \nop \yes \relax
\supported {getglue} \nop \yes \yes
-%supported {getglyphdata} \nop \yes \relax % experiment
+\supported {getglyphdata} \nop \yes \relax % old experiment
+\supported {getglyphdimensions} \nop \yes \yes
+\supported {getglyphscript} \nop \yes \relax % new experiment
+\supported {getglyphstate} \nop \yes \relax % new experiment
\supported {getheight} \nop \yes \relax
\supported {getid} \nop \yes \relax
+\supported {getindex} \nop \yes \relax
+\supported {getkerndimension} \nop \yes \yes
\supported {getkern} \nop \yes \relax
-\supported {getlang} \nop \yes \relax
+\supported {getlanguage} \nop \yes \relax
\supported {getleader} \nop \yes \relax
\supported {getlist} \nop \yes \relax
\supported {getnext} \nop \yes \relax
\supported {getnormalizedline} \nop \yes \relax
\supported {getnucleus} \nop \yes \relax
\supported {getoffsets} \nop \yes \relax
+\supported {getoptions} \nop \yes \relax
\supported {getorientation} \nop \yes \relax
+\supported {getparstate} \nop \yes \relax
\supported {getpenalty} \nop \yes \relax
\supported {getpost} \nop \yes \relax
-\supported {getpre} \nop \yes \relax
\supported {getprev} \nop \yes \relax
+\supported {getpre} \nop \yes \relax
\supported {getproperty} \yes \yes \relax
\supported {getreplace} \nop \yes \relax
+\supported {getscales} \nop \yes \relax
+\supported {getscript} \nop \yes \relax
\supported {getshift} \nop \yes \relax
-\supported {getsub} \nop \yes \relax
+\supported {getstate} \nop \yes \relax
+\supported {getsubpre} \nop \yes \relax
\supported {getsubtype} \nop \yes \relax
+\supported {getsub} \nop \yes \relax
+\supported {getsuppre} \nop \yes \relax
\supported {getsup} \nop \yes \relax
+\supported {gettotal} \yes \yes \relax
+%supported {getwhatever} \nop \yes \relax % experiment for myself
\supported {getwhd} \nop \yes \relax
\supported {getwidth} \nop \yes \relax
+\supported {getxscale} \nop \yes \relax
+\supported {getxyscale} \nop \yes \relax
+\supported {getyscale} \nop \yes \relax
\supported {has_attribute} \yes \yes \relax
\supported {has_dimensions} \nop \yes \relax
\supported {has_field} \yes \yes \relax
+\supported {has_glyph_option} \nop \yes \yes
\supported {has_glyph} \nop \yes \yes
\supported {hpack} \nop \yes \yes
-%supported {ignore_math_skip} \nop \yes \relax
+\supported {hyphenating} \nop \yes \yes
+\supported {ignore_math_skip} \nop \yes \relax
\supported {insert_after} \yes \yes \relax
\supported {insert_before} \yes \yes \relax
\supported {is_char} \nop \yes \relax
\supported {is_direct} \nop \yes \relax
\supported {is_glyph} \nop \yes \relax
+\supported {is_next_char} \nop \yes \relax
+\supported {is_next_glyph} \nop \yes \relax
\supported {is_node} \yes \yes \relax
+\supported {is_prev_char} \nop \yes \relax
+\supported {is_prev_glyph} \nop \yes \relax
\supported {is_valid} \nop \yes \relax
\supported {is_zero_glue} \nop \yes \yes
\supported {kerning} \nop \yes \yes
@@ -2188,22 +2305,23 @@ emulated in \LUA\ and not in the engine, so we retain downward compatibility.
\supported {length} \nop \yes \yes
\supported {ligaturing} \nop \yes \yes
\supported {make_extensible} \nop \yes \yes
+\supported {migrate} \nop \yes \relax
\supported {mlist_to_hlist} \nop \yes \yes
\supported {naturalwidth} \nop \yes \yes
\supported {new} \yes \yes \relax
-\supported {prepend_prevdepth} \nop \yes \yes
+%supported {prepend_prevdepth} \nop \yes \yes
\supported {protect_glyphs} \nop \yes \yes
\supported {protect_glyph} \nop \yes \yes
\supported {protrusion_skippable} \nop \yes \yes
\supported {rangedimensions} \nop \yes \yes
\supported {remove} \yes \yes \relax
+\supported {set_attributes} \nop \yes \relax
\supported {set_attribute} \yes \yes \relax
\supported {set_synctex_fields} \nop \yes \relax
\supported {setattributelist} \nop \yes \relax
\supported {setboth} \nop \yes \relax
\supported {setbox} \nop \yes \relax
\supported {setchar} \nop \yes \relax
-\supported {setstate} \nop \yes \relax
\supported {setdata} \nop \yes \relax
\supported {setdepth} \nop \yes \relax
\supported {setdirection} \nop \yes \relax
@@ -2213,24 +2331,36 @@ emulated in \LUA\ and not in the engine, so we retain downward compatibility.
\supported {setfield} \yes \yes \relax
\supported {setfont} \nop \yes \relax
\supported {setglue} \yes \yes \relax
-%supported {setglyphdata} \nop \yes \relax % experiment
+\supported {setglyphdata} \nop \yes \relax % old experiment
+\supported {setglyphscript} \nop \yes \relax % new experiment
+\supported {setglyphstate} \nop \yes \relax % new experiment
\supported {setheight} \nop \yes \relax
+\supported {setindex} \nop \yes \relax
\supported {setkern} \nop \yes \relax
-\supported {setlang} \nop \yes \relax
+\supported {setlanguage} \nop \yes \relax
\supported {setleader} \nop \yes \relax
\supported {setlink} \nop \yes \relax
\supported {setlist} \nop \yes \relax
\supported {setnext} \nop \yes \relax
\supported {setnucleus} \nop \yes \relax
\supported {setoffsets} \nop \yes \relax
+\supported {setoptions} \nop \yes \relax
\supported {setorientation} \nop \yes \relax
\supported {setpenalty} \nop \yes \relax
+\supported {setpost} \nop \yes \relax
\supported {setprev} \nop \yes \relax
+\supported {setpre} \nop \yes \relax
\supported {setproperty} \yes \yes \relax
+\supported {setreplace} \nop \yes \relax
+\supported {setscales} \nop \yes \relax
+\supported {setscript} \nop \yes \relax
\supported {setshift} \nop \yes \relax
\supported {setsplit} \nop \yes \relax
-\supported {setsub} \nop \yes \relax
+\supported {setstate} \nop \yes \relax
+\supported {setsubpre} \nop \yes \relax
\supported {setsubtype} \nop \yes \relax
+\supported {setsub} \nop \yes \relax
+\supported {setsuppre} \nop \yes \relax
\supported {setsup} \nop \yes \relax
\supported {setwhd} \nop \yes \relax
\supported {setwidth} \nop \yes \relax
@@ -2241,17 +2371,22 @@ emulated in \LUA\ and not in the engine, so we retain downward compatibility.
\supported {todirect} \nop \yes \relax
\supported {tonode} \nop \yes \relax
\supported {tostring} \yes \nop \relax
-\supported {traverse} \yes \yes \relax
+\supported {total} \nop \yes \relax
+\supported {tovaliddirect} \nop \yes \relax
\supported {traverse_char} \yes \yes \relax
+\supported {traverse_content} \yes \yes \relax
\supported {traverse_glyph} \yes \yes \relax
\supported {traverse_id} \yes \yes \relax
\supported {traverse_list} \yes \yes \relax
+\supported {traverse} \yes \yes \relax
\supported {type} \yes \nop \relax
\supported {unprotect_glyphs} \nop \yes \yes
\supported {unprotect_glyph} \nop \yes \yes
+\supported {unset_attributes} \nop \yes \relax
\supported {unset_attribute} \yes \yes \relax
\supported {usedlist} \nop \yes \yes
\supported {uses_font} \nop \yes \yes
+\supported {vertical_break} \nop \yes \relax
\supported {vpack} \nop \yes \yes
\supported {write} \yes \yes \relax
\LL
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-pdf.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-pdf.tex
index 79653fdf8a2..3412d0820fd 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-pdf.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-pdf.tex
@@ -183,6 +183,16 @@ b = getboolean(<pdfe array|dictionary>,index|key)
n = getname (<pdfe array|dictionary>,index|key)
\stoptyping
+The \type {getstring} function has two extra variants:
+
+\starttyping
+s, h = getstring (<pdfe array|dictionary>,index|key,false)
+s = getstring (<pdfe array|dictionary>,index|key,true)
+\stoptyping
+
+The first call returns the original string plus a boolean indicating if the
+string is hex encoded. The second call returns the unencoded string.
+
\stopsubsection
\startsubsection[title={\type {get[dictionary|array|stream]}}]
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-preamble.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-preamble.tex
index 8f1400c9f9c..5a2dae818b1 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-preamble.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-preamble.tex
@@ -10,12 +10,12 @@
\topicindex{boxes}
\topicindex{\LUA}
-This is a reference manual, not a tutorial. This means that we discuss changes
-relative to traditional \TEX\ and also present new functionality. As a
-consequence we will refer to concepts that we assume to be known or that might be
-explained later. Because the \LUATEX\ and \LUAMETATEX\ engines open up \TEX\
-there's suddenly quite some more to explain, especially about the way a (to be)
-typeset stream moves through the machinery. However, discussing all that in
+This is a reference manual and not a tutorial. This means that we discuss changes
+relative to traditional \TEX\ and also present new (or extended) functionality.
+As a consequence we will refer to concepts that we assume to be known or that
+might be explained later. Because the \LUATEX\ and \LUAMETATEX\ engines open up
+\TEX\ there's suddenly quite some more to explain, especially about the way a (to
+be) typeset stream moves through the machinery. However, discussing all that in
detail makes not much sense, because deep knowledge is only relevant for those
who write code not possible with regular \TEX\ and who are already familiar with
these internals (or willing to spend time on figuring it out).
@@ -47,19 +47,21 @@ If you still decide to read on, then it's good to know what nodes are, so we do
quick introduction here. If you input this text:
\starttyping
-Hi There
+Hi There ...
\stoptyping
eventually we will get a linked lists of nodes, which in \ASCII\ art looks like:
\starttyping
-H <=> i <=> [glue] <=> T <=> h <=> e <=> r <=> e
+H <=> i <=> [glue] <=> T <=> h <=> e <=> r <=> e ...
\stoptyping
-When we have a paragraph, we actually get something:
+When we have a paragraph, we actually get something like this, where a \type
+{par} node stores some metadata and is followed by a \type {hlist} flagged as
+indent box:
\starttyping
-[localpar] <=> H <=> i <=> [glue] <=> T <=> h <=> e <=> r <=> e <=> [glue]
+[par] <=> [hlist] <=> H <=> i <=> [glue] <=> T <=> h <=> e <=> r <=> e ...
\stoptyping
Each character becomes a so called glyph node, a record with properties like the
@@ -69,16 +71,27 @@ back to a previous node or next node, given that these exist. Sometimes
multiple characters are represented by one glyphs, so one can also get:
\starttyping
-[localpar] <=> H <=> i <=> [glue] <=> Th <=> e <=> r <=> e <=> [glue]
+[par] <=> [hlist] <=> H <=> i <=> [glue] <=> Th <=> e <=> r <=> e ...
\stoptyping
And maybe some characters get positioned relative to each other, so we might
see:
\starttyping
-[localpar] <=> H <=> [kern] <=> i <=> [glue] <=> Th <=> e <=> r <=> e <=> [glue]
+[par] <=> [hlist] <=> H <=> [kern] <=> i <=> [glue] <=> Th <=> e <=> r <=> e ...
\stoptyping
+Actually, the above representation is one view, because in \LUAMETATEX\ we can
+choose for this:
+
+\starttyping
+[par] <=> [glue] <=> H <=> [kern] <=> i <=> [glue] <=> Th <=> e <=> r <=> e ...
+\stoptyping
+
+where glue (currently fixed) is used instead of an empty hlist (think of a \type
+{\hbox}). Options like this are available because want a certain view on these
+lists from the \LUA\ end and the result being predicable is part of that.
+
It's also good to know beforehand that \TEX\ is basically centered around
creating paragraphs and pages. The par builder takes a list and breaks it into
lines. At some point horizontal blobs are wrapped into vertical ones. Lines are
@@ -96,6 +109,10 @@ means these hooks. The \TEX\ engine itself is pretty well optimized but when you
kick in much \LUA\ code, you will notices that performance drops. Don't blame and
bother the authors with performance issues. In \CONTEXT\ over 50\% of the time
can be spent in \LUA, but so far we didn't get many complaints about efficiency.
+Adding more callbacks makes no sense, also because at some point the performance
+hit gets too large. There are plenty ways to achieve one goals. For that reason:
+take remarks about \LUATEX, features, potential, performance etc.\ with a natural
+grain of salt.
Where plain \TEX\ is basically a basic framework for writing a specific style,
macro packages like \CONTEXT\ and \LATEX\ provide the user a whole lot of
@@ -108,7 +125,7 @@ mess around to much, the engine eventually might bark and quit. It can even
crash, because testing everywhere for what users can do wrong is no real option.
When you read about nodes in the following chapters it's good to keep in mind
-their commands that relate to then. Here are a few:
+what commands relate to them. Here are a few:
\starttabulate[|l|l|p|]
\DB command \BC node \BC explanation \NC \NR
@@ -127,6 +144,12 @@ their commands that relate to then. Here are a few:
\LL
\stoptabulate
+Whatever we feed into \TEX\ at some point becomes a token which is either
+interpreted directly or stored in a linked list. A token is just a number that
+encodes a specific command (operator) and some value (operand) that further
+specifies what that command is supposed to do. In addition to an interface to
+nodes, there is an interface to tokens, as later chapters will demonstrate.
+
Text (interspersed with macros) comes from an input medium. This can be a file,
token list, macro body cq.\ arguments, \ some internal quantity (like a number),
\LUA, etc. Macros get expanded. In the process \TEX\ can enter a group. Inside
@@ -137,23 +160,23 @@ all terms used in the next chapters. Don't worry, they loose their magic once yo
use \TEX\ a lot. You have access to most of the internals and when not, at least
it is possible to query some state we're in or level we're at.
-When we talk about packing it can mean two things. When \TEX\ has consumed some
-tokens that represent text the next can happen. When the text is put into a so
-called \type {\hbox} it (normally) first gets hyphenated, next ligatures are
-build, and finally kerns are added. Each of that stages can be overloaded using
-\LUA\ code. When these three stages are finished, the dimension of the content is
-calculated and the box gets its width, height and depth. What happens with the
-box depends on what macros do with it.
+When we talk about pack(ag)ing it can mean two things. When \TEX\ has consumed
+some tokens that represent text. When the text is put into a so called \type
+{\hbox} it (normally) first gets hyphenated (even in an horizontal list), next
+ligatures are build, and finally kerns are added. Each of these stages can be
+overloaded using \LUA\ code. When these three stages are finished, the dimension
+of the content is calculated and the box gets its width, height and depth. What
+happens with the box depends on what macros do with it.
The other thing that can happen is that the text starts a new paragraph. In that
-case some (directional) information is put in front, indentation is prepended and
-some skip appended at the end. Again the three stages are applied but this time,
-afterwards, the long line is broken into lines and the result is either added to
-the content of a box or to the main vertical list (the running text so to say).
-This is called par building. At some point \TEX\ decides that enough is enough
-and it will trigger the page builder. So, building is another concept we will
-encounter. Another example of a builder is the one that turns an intermediate
-math list into something typeset.
+case some information is stored in a leading \type {par} node. Then indentation
+is appended and the paragraph ends with some glue. Again the three stages are
+applied but this time, afterwards, the long line is broken into lines and the
+result is either added to the content of a box or to the main vertical list (the
+running text so to say). This is called par building. At some point \TEX\ decides
+that enough is enough and it will trigger the page builder. So, building is
+another concept we will encounter. Another example of a builder is the one that
+turns an intermediate math list into something typeset.
Wrapping something in a box is called packing. Adding something to a list is
described in terms of contributing. The more complicated processes are wrapped
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-primitives.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-primitives.tex
index 187bbd85acd..edbf7d32c19 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-primitives.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-primitives.tex
@@ -1,395 +1 @@
-% language=uk
-
-\environment luametatex-style
-
-\startcomponent luametatex-primitives
-
-\startchapter[reference=primitives,title={Primitives aka commands}]
-
-\startsection[title=Introduction]
-
-The starting point of \LUATEX\ is \PDFTEX, which itself contains regular \TEX\
-and \ETEX. Because directional support was needed we also took some code from
-\ALEPH\ (\OMEGA). In a later stage the backend specific commands were isolated in
-its own namespace which resulted in a cleaner code base where the backend code no
-longer was interwoven with the normal frontend primitives. We also promoted some
-generic constructs (like box resources and directions) to core functionality.
-
-Some of the \PDFTEX\ support primitives have been around from the start but when
-\LUA\ integration became better and when a token scanner library was added, not
-all of those made sense as primitives. In previous chapters we already mentioned
-what is gone from the core. Deep down some more has changed but not all is
-reflected at the primitive level. Because there is still a considerable amount of
-new primitives, a summary is given below.
-
-\stopsection
-
-\startsection[title=Languages]
-
-\starttabulate[|||p|]
-\NC \type {automatichyphenmode} \NC integer \NC \NC \NR
-\NC \type {automatichyphenpenalty} \NC integer \NC \NC \NR
-\NC \type {hyphenpenaltymode} \NC integer \NC \NC \NR
-\NC \type {compoundhyphenmode} \NC integer \NC \NC \NR
-\NC \type {exceptionpenalty} \NC integer \NC \NC \NR
-\NC \type {explicithyphenpenalty} \NC integer \NC \NC \NR
-\NC \type {hyphenationbounds} \NC integer \NC \NC \NR
-\NC \type {hjcode} \NC charactercode \NC \NC \NR
-\NC \type {hyphenationmin} \NC charactercode \NC \NC \NR
-\NC \type {postexhyphenchar} \NC charactercode \NC \NC \NR
-\NC \type {posthyphenchar} \NC charactercode \NC \NC \NR
-\NC \type {preexhyphenchar} \NC charactercode \NC \NC \NR
-\NC \type {prehyphenchar} \NC charactercode \NC \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=Fonts]
-
-\starttabulate[|||p|]
-\NC \type {tracingfonts} \NC integer \NC \NC \NR
-\NC \type {suppressfontnotfounderror} \NC integer \NC \NC \NR
-\NC \type {setfontid} \NC integer \NC \NC \NR
-\NC \type {fontid} \NC font \NC \NC \NR
-\NC \type {efcode} \NC font charactercode \NC \NC \NR
-\NC \type {lpcode} \NC font charactercode \NC \NC \NR
-\NC \type {rpcode} \NC font charactercode \NC \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=Math]
-
-\starttabulate[|||p|]
-\NC \type {matholdmode} \NC integer \NC \NC \NR
-\NC \type {mathstyle} \NC integer \NC \NC \NR
-\NC \type {matheqnogapstep} \NC integer \NC \NC \NR
-\NC \type {Uskewed} \NC \NC \NC \NR
-\NC \type {Uskewedwithdelims} \NC \NC \NC \NR
-\NC \type {Ustartdisplaymath} \NC \NC \NC \NR
-\NC \type {Ustartmath} \NC \NC \NC \NR
-\NC \type {Ustopdisplaymath} \NC \NC \NC \NR
-\NC \type {Ustopmath} \NC \NC \NC \NR
-\NC \type {crampeddisplaystyle} \NC \NC \NC \NR
-\NC \type {crampedtextstyle} \NC \NC \NC \NR
-\NC \type {crampedscriptstyle} \NC \NC \NC \NR
-\NC \type {crampedscriptscriptstyle} \NC \NC \NC \NR
-\NC \type {Umathchardef} \NC \NC \NC \NR
-\NC \type {Umathcharnumdef} \NC \NC \NC \NR
-\NC \type {mathdisplayskipmode} \NC integer \NC \NC \NR
-\NC \type {mathscriptsmode} \NC integer \NC \NC \NR
-\NC \type {mathnolimitsmode} \NC integer \NC \NC \NR
-\NC \type {mathitalicsmode} \NC integer \NC \NC \NR
-\NC \type {mathrulesmode} \NC integer \NC \NC \NR
-\NC \type {mathrulesfam} \NC integer \NC \NC \NR
-\NC \type {mathdelimitersmode} \NC integer \NC \NC \NR
-\NC \type {mathflattenmode} \NC integer \NC \NC \NR
-\NC \type {mathpenaltiesmode} \NC integer \NC \NC \NR
-\NC \type {mathrulethicknessmode} \NC integer \NC \NC \NR
-\NC \type {mathscriptboxmode} \NC integer \NC \NC \NR
-\NC \type {mathscriptcharmode} \NC integer \NC \NC \NR
-\NC \type {mathsurroundmode} \NC integer \NC \NC \NR
-\NC \type {nokerns} \NC integer \NC \NC \NR
-\NC \type {noligs} \NC integer \NC \NC \NR
-\NC \type {prebinoppenalty} \NC integer \NC \NC \NR
-\NC \type {predisplaygapfactor} \NC integer \NC \NC \NR
-\NC \type {prerelpenalty} \NC integer \NC \NC \NR
-\NC \type {Usuperscript} \NC command \NC \NC \NR
-\NC \type {Usubscript} \NC command \NC \NC \NR
-\NC \type {Unosuperscript} \NC command \NC \NC \NR
-\NC \type {Unosubscript} \NC command \NC \NC \NR
-\NC \type {Umathcode} \NC \NC \NC \NR
-\NC \type {Umathcodenum} \NC \NC \NC \NR
-\NC \type {Udelcode} \NC \NC \NC \NR
-\NC \type {Udelcodenum} \NC \NC \NC \NR
-\NC \type {Umathaxis} \NC family dimension \NC \NC \NR
-\NC \type {Umathbinbinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathbinclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathbininnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathbinopenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathbinopspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathbinordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathbinpunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathbinrelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathclosebinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathcloseclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathcloseinnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathcloseopenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathcloseopspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathcloseordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathclosepunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathcloserelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathconnectoroverlapmin} \NC family dimension \NC \NC \NR
-\NC \type {Umathfractiondelsize} \NC family dimension \NC \NC \NR
-\NC \type {Umathfractiondenomdown} \NC family dimension \NC \NC \NR
-\NC \type {Umathfractiondenomvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathfractionnumup} \NC family dimension \NC \NC \NR
-\NC \type {Umathfractionnumvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathfractionrule} \NC family dimension \NC \NC \NR
-\NC \type {Umathinnerbinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathinnerclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathinnerinnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathinneropenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathinneropspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathinnerordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathinnerpunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathinnerrelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathlimitabovebgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathlimitabovekern} \NC family dimension \NC \NC \NR
-\NC \type {Umathlimitabovevgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathlimitbelowbgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathlimitbelowkern} \NC family dimension \NC \NC \NR
-\NC \type {Umathlimitbelowvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathnolimitsubfactor} \NC family dimension \NC \NC \NR
-\NC \type {Umathnolimitsupfactor} \NC family dimension \NC \NC \NR
-\NC \type {Umathopbinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopenbinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopenclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopeninnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopenopenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopenopspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopenordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopenpunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopenrelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathoperatorsize} \NC family dimension \NC \NC \NR
-\NC \type {Umathopinnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopopenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopopspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathopordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathoppunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathoprelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordbinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordinnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordopenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordopspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordpunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathordrelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathoverbarkern} \NC family dimension \NC \NC \NR
-\NC \type {Umathoverbarrule} \NC family dimension \NC \NC \NR
-\NC \type {Umathoverbarvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathoverdelimiterbgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathoverdelimitervgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctbinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctinnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctopenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctopspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctpunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathpunctrelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathquad} \NC family dimension \NC \NC \NR
-\NC \type {Umathradicaldegreeafter} \NC family dimension \NC \NC \NR
-\NC \type {Umathradicaldegreebefore} \NC family dimension \NC \NC \NR
-\NC \type {Umathradicaldegreeraise} \NC family dimension \NC \NC \NR
-\NC \type {Umathradicalkern} \NC family dimension \NC \NC \NR
-\NC \type {Umathradicalrule} \NC family dimension \NC \NC \NR
-\NC \type {Umathradicalvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelbinspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelclosespacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelinnerspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelopenspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelopspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelordspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelpunctspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathrelrelspacing} \NC family dimension \NC \NC \NR
-\NC \type {Umathskewedfractionhgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathskewedfractionvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathspaceafterscript} \NC family dimension \NC \NC \NR
-\NC \type {Umathstackdenomdown} \NC family dimension \NC \NC \NR
-\NC \type {Umathstacknumup} \NC family dimension \NC \NC \NR
-\NC \type {Umathstackvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathsubshiftdown} \NC family dimension \NC \NC \NR
-\NC \type {Umathsubshiftdrop} \NC family dimension \NC \NC \NR
-\NC \type {Umathsubsupshiftdown} \NC family dimension \NC \NC \NR
-\NC \type {Umathsubsupvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathsubtopmax} \NC family dimension \NC \NC \NR
-\NC \type {Umathsupbottommin} \NC family dimension \NC \NC \NR
-\NC \type {Umathsupshiftdrop} \NC family dimension \NC \NC \NR
-\NC \type {Umathsupshiftup} \NC family dimension \NC \NC \NR
-\NC \type {Umathsupsubbottommax} \NC family dimension \NC \NC \NR
-\NC \type {Umathunderbarkern} \NC family dimension \NC \NC \NR
-\NC \type {Umathunderbarrule} \NC family dimension \NC \NC \NR
-\NC \type {Umathunderbarvgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathunderdelimiterbgap} \NC family dimension \NC \NC \NR
-\NC \type {Umathunderdelimitervgap} \NC family dimension \NC \NC \NR
-\NC \type {Udelimiter} \NC \NC \NC \NR
-\NC \type {Umathaccent} \NC \NC \NC \NR
-\NC \type {Umathchar} \NC \NC \NC \NR
-\NC \type {Umathcharnum} \NC \NC \NC \NR
-\NC \type {Ustack} \NC \NC \NC \NR
-\NC \type {Uradical} \NC \NC \NC \NR
-\NC \type {Uroot} \NC \NC \NC \NR
-\NC \type {Uunderdelimiter} \NC \NC \NC \NR
-\NC \type {Uoverdelimiter} \NC \NC \NC \NR
-\NC \type {Udelimiterunder} \NC \NC \NC \NR
-\NC \type {Udelimiterover} \NC \NC \NC \NR
-\NC \type {Uhextensible} \NC \NC \NC \NR
-\NC \type {Uchar} \NC \NC \NC \NR
-\NC \type {Umathcharclass} \NC \NC \NC \NR
-\NC \type {Umathcharfam} \NC \NC \NC \NR
-\NC \type {Umathcharslot} \NC \NC \NC \NR
-\NC \type {Uleft} \NC \NC \NC \NR
-\NC \type {Umiddle} \NC \NC \NC \NR
-\NC \type {Uright} \NC \NC \NC \NR
-\NC \type {Uvextensible} \NC \NC \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=Boxes and directions]
-
-\starttabulate[|||p|]
-\NC \type {pardirection} \NC direction \NC \NC \NR
-\NC \type {textdirection} \NC direction \NC \NC \NR
-\NC \type {mathdirection} \NC direction \NC \NC \NR
-\NC \type {linedirection} \NC direction \NC \NC \NR
-\NC \type {breakafterdirmode} \NC integer \NC \NC \NR
-\NC \type {shapemode} \NC integer \NC \NC \NR
-\NC \type {fixupboxesmode} \NC integer \NC \NC \NR
-\NC \type {boxdirection} \NC box direction \NC \NC \NR
-\NC \type {boxorientation} \NC box orientation \NC rotation over 90, 180, 270 degrees \NC \NR
-\NC \type {boxxoffset} \NC box xoffset \NC leaves dimensions untounched \NC \NR
-\NC \type {boxyoffset} \NC box yoffset \NC leaves dimensions untounched \NC \NR
-\NC \type {boxxmove} \NC box xmove \NC offsets that reflect on dimensions \NC \NR
-\NC \type {boxymove} \NC box ymove \NC offsets that reflect on dimensions \NC \NR
-\NC \type {boxtotal} \NC box ht+dp \NC height plus depth (and when assigned halfs) \NC \NR
-\NC \type {boxattr} \NC box attr value \NC (sets) a specific attribute to a value \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=Scanning]
-
-\starttabulate[|||p|]
-\NC \type {aftergrouped} \NC text \NC like aftergroup but for given list \NC \NR
-\NC \type {alignmark} \NC \NC equivalent to hash token \NC \NR
-\NC \type {aligntab} \NC \NC equivalent to tab token \NC \NR
-\NC \type {begincsname} \NC command \NC variant of \type {\csname} that ignores undefined commands \NC \NR
-\NC \type {catcodetable} \NC integer \NC switch to catcode table \NC \NR
-\NC \type {csstring} \NC command \NC the command without preceding escape character \NC \NR
-\NC \type {endlocalcontrol} \NC command \NC switches back to the main control loop \NC \NR
-\NC \type {etoksapp} \NC tokenregister text \NC append expanded text to given tokenregister \NC \NR
-\NC \type {etokspre} \NC tokenregister text \NC prepend expanded text to given tokenregister \NC \NR
-\NC \type {expanded} \NC text \NC expands the given text \NC \NR
-\NC \type {frozen} \NN prefix \NC \NC \NR
-\NC \type {futureexpand} \NN token command command \NC expands second ot third token depending on first match \NC \NR
-\NC \type {futureexpandis} \NN token command command \NC as \type {futureexpand} but also skips pars \NC \NR
-\NC \type {futureexpandisap} \NN token command command \NC same as idem but doesn't push back skipped spaces \NC \NR
-\NC \type {gtoksapp} \NC tokenregister text \NC globally append text to given tokenregister \NC \NR
-\NC \type {gtokspre} \NC tokenregister text \NC globally prepend text to given tokenregister \NC \NR
-\NC \type {ifabsdim} \NC dimension <=> dimension \NC test the absolute value of the given dimension \NC \NR
-\NC \type {ifabsnum} \NC integer <=> integer \NC test the absolute value of the given integer \NC \NR
-\NC \type {ifcondition} \NC command \NC assume the next token is a test (so skip as if) \NC \NR
-\NC \type {ifdimen} \NC possibly a dimension \NC acts like an \type {\ifcase} with 1 for valid and 2 for invalid \NC \NR
-\NC \type {ifincsname} \NC command \NC check if we're inside a csname expansion \NC \NR
-\NC \type {ifnumval} \NC \NC \NC \NR
-\NC \type {ifdimval} \NC \NC \NC \NR
-\NC \type {ifchknum} \NC \NC \NC \NR
-\NC \type {ifchkdim} \NC \NC \NC \NR
-\NC \type {ifcmpnum} \NC \NC \NC \NR
-\NC \type {ifcmpdim} \NC \NC \NC \NR
-\NC \type {ifusercmd} \NC command \NC \NC \NR
-\NC \type {ifprotected} \NC command \NC \NC \NR
-\NC \type {iffrozen} \NC command \NC \NC \NR
-\NC \type {iftok} \NC \NC \NC \NR
-\NC \type {ifcstok} \NC \NC \NC \NR
-\NC \type {internalcodesmode} \NC integer \NC \NC \NR
-%NC \type {ifprimitive} \NC command \NC check if the given command is a primitive \NC \NR
-\NC \type {immediateassigned} \NC command \NC (todo) expand the following assignment now\NC \NR
-\NC \type {immediateassignment} \NC command \NC (todo) expand the following assignment now\NC \NR
-\NC \type {initcatcodetable} \NC integer \NC initialize catcode table \NC \NR
-\NC \type {lastnamedcs} \NC command \NC last found command of \type {\ifcsname} construction \NC \NR
-\NC \type {nospaces} \NC integer \NC don't inject spaces \NC \NR
-%NC \type {primitive} \NC command \NC expands the next primitive equivalent \NC \NR
-\NC \type {orelse} \NC condition \NC \NC \NR
-\NC \type {pxdimen} \NC dimension \NC multiplier for the \type {px} unit \NC \NR
-\NC \type {savecatcodetable} \NC integer \NC save catcode table \NC \NR
-\NC \type {scantextokens} \NC text \NC \type {\scantokens} without file side effects \NC \NR
-\NC \type {suppressifcsnameerror} \NC integer \NC recover from issues in csname testing \NC \NR
-\NC \type {suppresslongerror} \NC integer \NC make \type {\long} a nop \NC \NR
-\NC \type {suppressmathparerror} \NC integer \NC accepts \type {\par} and empty lines in math \NC \NR
-\NC \type {suppressoutererror} \NC integer \NC make \type {\outer} a nop \NC \NR
-\NC \type {suppressprimitiveerror} \NC integer \NC don't report an invalid \type {\primitive} \NC \NR
-\NC \type {toksapp} \NC tokenregister text \NC append text to given tokenregister \NC \NR
-\NC \type {tokspre} \NC tokenregister text \NC prepend text to given tokenregister \NC \NR
-\NC \type {xtoksapp} \NC tokenregister text \NC globally append expanded text to given tokenregister \NC \NR
-\NC \type {xtokspre} \NC tokenregister text \NC globally prepend expanded text to given tokenregister \NC \NR
-% new (some end-of-the-year experiment)
-\NC \type {letfrozen} \NC macro \NC sets the frozen property of a macro \NC \NR
-\NC \type {letprotected} \NC macro \NC sets the protected property of a macro \NC \NR
-\NC \type {unletfrozen} \NC macro \NC unsets the frozen property of a macro \NC \NR
-\NC \type {unletprotected} \NC macro \NC unsets the protected property of a macro \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=Typesetting]
-
-\starttabulate[|||p|]
-\NC \type {protrudechars} \NC integer \NC \NC \NR
-\NC \type {localbrokenpenalty} \NC integer \NC \NC \NR
-\NC \type {localinterlinepenalty} \NC integer \NC \NC \NR
-\NC \type {adjustspacing} \NC integer \NC \NC \NR
-\NC \type {boundary} \NC command \NC \NC \NR
-\NC \type {noboundary} \NC command \NC \NC \NR
-\NC \type {protrusionboundary} \NC command \NC \NC \NR
-\NC \type {wordboundary} \NC command \NC \NC \NR
-\NC \type {nohrule} \NC command \NC \NC \NR
-\NC \type {novrule} \NC command \NC \NC \NR
-\NC \type {insertht} \NC number \NC \NC \NR
-\NC \type {quitvmode} \NC command \NC \NC \NR
-\NC \type {leftmarginkern} \NC dimension \NC \NC \NR
-\NC \type {rightmarginkern} \NC dimension \NC \NC \NR
-\NC \type {localleftbox} \NC box \NC \NC \NR
-\NC \type {localrightbox} \NC box \NC \NC \NR
-\NC \type {gleaders} \NC command \NC \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=\LUA]
-
-\starttabulate[|||p|]
-\NC \type {luacopyinputnodes} \NC integer \NC \NC \NR
-\NC \type {luadef} \NC \NC \NC \NR
-\NC \type {luabytecodecall} \NC \NC \NC \NR
-\NC \type {luafunctioncall} \NC \NC \NC \NR
-\NC \type {latelua} \NC \NC \NC \NR
-\NC \type {lateluafunction} \NC \NC \NC \NR
-\NC \type {luabytecode} \NC \NC \NC \NR
-\NC \type {luaescapestring} \NC \NC \NC \NR
-\NC \type {luafunction} \NC \NC \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=Management]
-
-\starttabulate[|||p|]
-\NC \type {outputbox} \NC integer \NC \NC \NR
-\NC \type {clearmarks} \NC \NC \NC \NR
-\NC \type {attribute} \NC \NC \NC \NR
-\NC \type {glet} \NC \NC \NC \NR
-\NC \type {letcharcode} \NC \NC \NC \NR
-\NC \type {attributedef} \NC \NC \NC \NR
-\stoptabulate
-
-\stopsection
-
-\startsection[title=Miscellaneous]
-
-\starttabulate[|||p|]
-\NC \type {luatexversion} \NC \NC \NC \NR
-\NC \type {formatname} \NC \NC \NC \NR
-\NC \type {luatexbanner} \NC \NC \NC \NR
-\NC \type {luatexrevision} \NC \NC \NC \NR
-\stoptabulate
-
-\stopsection
-
-\stopchapter
-
-\stopcomponent
+\starttext \ctxlua {table.save(tex.jobname .. '.lua',tex.primitives())} \stoptext \ No newline at end of file
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-style.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-style.tex
index 3f4b0ad2183..1b4a8d5a1c7 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-style.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-style.tex
@@ -115,11 +115,11 @@
\stopluacode
-\unexpanded\def\showfields #1{\ctxlua{document.functions.showfields("#1")}}
-\unexpanded\def\showid #1{\ctxlua{document.functions.showid("#1")}}
-\unexpanded\def\showsubtypes#1{\ctxlua{document.functions.showsubtypes("#1")}}
-\unexpanded\def\showvalues #1{\ctxlua{document.functions.showvalues(node.values("#1"))}}
-\unexpanded\def\showtypes {\ctxlua{document.functions.showvalues(node.types())}}
+\protected\def\showfields #1{\ctxlua{document.functions.showfields("#1")}}
+\protected\def\showid #1{\ctxlua{document.functions.showid("#1")}}
+\protected\def\showsubtypes#1{\ctxlua{document.functions.showsubtypes("#1")}}
+\protected\def\showvalues #1{\ctxlua{document.functions.showvalues(node.values("#1"))}}
+\protected\def\showtypes {\ctxlua{document.functions.showvalues(node.types())}}
\definecolor[blue] [b=.5]
@@ -130,11 +130,13 @@
%definecolor[othercolor][r=.5,g=.5]
\definecolor[maincolor] [b=.5]
+\definecolor[mooncolor] [b=.5]
\definecolor[keptcolor] [b=.5]
-\definecolor[othercolor][s=.5]
+%\efinecolor[othercolor][s=.5]
+\definecolor[othercolor][r=.5,g=.5]
\writestatus{luametatex manual}{}
-\writestatus{luametatex manual}{defining lucodaot} \usebodyfont [lucidaot]
+\writestatus{luametatex manual}{defining lucidaot} \usebodyfont [lucidaot]
\writestatus{luametatex manual}{defining pagella} \usebodyfont [pagella]
\writestatus{luametatex manual}{defining cambria} \usebodyfont [cambria]
\writestatus{luametatex manual}{defining modern} \usebodyfont [modern]
@@ -195,10 +197,11 @@
\startMPdefinitions
- color luaplanetcolor ; luaplanetcolor := \MPcolor{maincolor} ;
- color luaholecolor ; luaholecolor := white ;
+ string luaplanetcolor ; luaplanetcolor := "maincolor" ;
+ string luamooncolor ; luaplanetcolor := "mooncolor" ;
+ string luaholecolor ; luaholecolor := "white" ;
+ string luaorbitcolor ; luaorbitcolor := "darkgray" ;
numeric luaextraangle ; luaextraangle := 0 ;
- numeric luaorbitfactor ; luaorbitfactor := .25 ;
vardef lualogo = image (
@@ -213,10 +216,10 @@
draw fullcircle rotated 40.5 scaled (1+r)
dashed evenly scaled p
withpen pencircle scaled (p/2)
- withcolor (luaorbitfactor * luaholecolor) ;
+ withcolor luaorbitcolor ;
fill fullcircle scaled r shifted (d+1/8,d+1/8)
rotated - luaextraangle
- withcolor luaplanetcolor ;
+ withcolor luamooncolor ;
fill fullcircle scaled r shifted (d-1/8,d-1/8)
withcolor luaholecolor ;
luaorbitfactor := .25 ;
@@ -229,7 +232,8 @@
fill Page withcolor \MPcolor{othercolor} ;
- luaorbitfactor := 1 ;
+ luaorbitcolor := "white" ;
+ luamooncolor := "maincolor" ;
picture p ; p := lualogo ysized (5*\measure{paperheight}/10) ;
draw p
@@ -251,9 +255,11 @@
% \stoptexdefinition
\startuseMPgraphic{luanumber}
- % luaextraangle := \luaextraangle;
- luaextraangle := if (LastPageNumber < 10) : 10 else : (RealPageNumber / LastPageNumber) * 360 fi;
- luaorbitfactor := 0.25 ;
+ % luaextraangle := \luaextraangle;
+ luaextraangle := if (LastPageNumber < 10) : 10 else : (RealPageNumber / LastPageNumber) * 360 fi;
+ luaorbitcolor := "darkgray" ;
+ luamooncolor := "othercolor" ;
+ % luamooncolor := "maincolor" ;
picture p ; p := lualogo ;
setbounds p to boundingbox fullcircle ;
draw p ysized 1cm ;
@@ -314,13 +320,13 @@
{\strut\useMPgraphic{luanumber}}
\stopsetups
-\unexpanded\def\nonterminal#1>{\mathematics{\langle\hbox{\rm #1}\rangle}}
+\protected\def\nonterminal#1>{\mathematics{\langle\hbox{\rm #1}\rangle}}
% taco's brainwave -) .. todo: create a typing variant so that we can avoid the !crlf
\newcatcodetable\syntaxcodetable
-\unexpanded\def\makesyntaxcodetable
+\protected\def\makesyntaxcodetable
{\begingroup
\catcode`\<=13 \catcode`\|=12
\catcode`\!= 0 \catcode`\\=12
@@ -329,11 +335,11 @@
\makesyntaxcodetable
-\unexpanded\def\startsyntax {\begingroup\catcodetable\syntaxcodetable \dostartsyntax}
-\unexpanded\def\syntax {\begingroup\catcodetable\syntaxcodetable \dosyntax}
+\protected\def\startsyntax {\begingroup\catcodetable\syntaxcodetable \dostartsyntax}
+\protected\def\syntax {\begingroup\catcodetable\syntaxcodetable \dosyntax}
\let\stopsyntax \relax
-\unexpanded\def\syntaxenvbody#1%
+\protected\def\syntaxenvbody#1%
{\par
\tt
\startnarrower
@@ -342,7 +348,7 @@
\stopnarrower
\par}
-\unexpanded\def\syntaxbody#1%
+\protected\def\syntaxbody#1%
{\begingroup
% \maincolor
\tt #1%
@@ -427,13 +433,13 @@
\defineregister[nodeindex]
\defineregister[libraryindex]
-\unexpanded\def\lpr#1{\doifmode{*bodypart}{\primitiveindex[#1]{\bf\tex {#1}}}\tex {#1}}
-\unexpanded\def\prm#1{\doifmode{*bodypart}{\primitiveindex[#1]{\tex {#1}}}\tex {#1}}
-\unexpanded\def\orm#1{\doifmode{*bodypart}{\primitiveindex[#1]{\tex {#1}}}\tex {#1}}
-\unexpanded\def\cbk#1{\doifmode{*bodypart}{\callbackindex [#1]{\type {#1}}}\type{#1}}
-\unexpanded\def\nod#1{\doifmode{*bodypart}{\nodeindex [#1]{\bf\type{#1}}}\type{#1}}
-\unexpanded\def\whs#1{\doifmode{*bodypart}{\nodeindex [#1]{\type {#1}}}\type{#1}}
-\unexpanded\def\noa#1{\doifmode{*bodypart}{\nodeindex [#1]{\type {#1}}}\type{#1}}
+\protected\def\lpr#1{\doifmode{*bodypart}{\primitiveindex[#1]{\bf\tex {#1}}}\tex {#1}}
+\protected\def\prm#1{\doifmode{*bodypart}{\primitiveindex[#1]{\tex {#1}}}\tex {#1}}
+\protected\def\orm#1{\doifmode{*bodypart}{\primitiveindex[#1]{\tex {#1}}}\tex {#1}}
+\protected\def\cbk#1{\doifmode{*bodypart}{\callbackindex [#1]{\type {#1}}}\type{#1}}
+\protected\def\nod#1{\doifmode{*bodypart}{\nodeindex [#1]{\bf\type{#1}}}\type{#1}}
+\protected\def\whs#1{\doifmode{*bodypart}{\nodeindex [#1]{\type {#1}}}\type{#1}}
+\protected\def\noa#1{\doifmode{*bodypart}{\nodeindex [#1]{\type {#1}}}\type{#1}}
\hyphenation{sub-nodes}
@@ -457,14 +463,14 @@
[2]
[textstyle=\tttf]
-\unexpanded\def\lib #1{\doifmode{*bodypart}{\expanded{\libraryindex{\currentlibraryindex+#1}}}\type{\currentlibraryindex.#1}}
-\unexpanded\def\libindex#1{\doifmode{*bodypart}{\expanded{\libraryindex{\currentlibraryindex+#1}}}}
-\unexpanded\def\libidx#1#2{\doifmode{*bodypart}{\expanded{\libraryindex{#1+#2}}\type{#1.#2}}}
-\unexpanded\def\lix #1#2{\doifmode{*bodypart}{\expanded{\libraryindex{#1+#2}}}}
+\protected\def\lib #1{\doifmode{*bodypart}{\expanded{\libraryindex{\currentlibraryindex+#1}}}\type{\currentlibraryindex.#1}}
+\protected\def\libindex#1{\doifmode{*bodypart}{\expanded{\libraryindex{\currentlibraryindex+#1}}}}
+\protected\def\libidx#1#2{\doifmode{*bodypart}{\expanded{\libraryindex{#1+#2}}\type{#1.#2}}}
+\protected\def\lix #1#2{\doifmode{*bodypart}{\expanded{\libraryindex{#1+#2}}}}
% \setstructurepageregister[][keys:1=,entries:1=]
-\unexpanded\def\inlineluavalue#1%
+\protected\def\inlineluavalue#1%
{{\maincolor \ctxlua {
local t = #1
if type(t) == "table" then
@@ -475,4 +481,74 @@
context.typ(t)
}}}
+% Common:
+
+% Added December 9 2020 after being energized by Becca Stevens's Slow Burn music
+% video: interesting what comes out of top musicians working remote.
+
+\startluacode
+local list = token.getprimitives()
+
+-- redo this:
+
+function document.showprimitives(tag)
+ local t = tex.extraprimitives(tag)
+ table.sort(t)
+ for i=1,#t do
+ local v = t[i]
+ if v ~= ' ' and v ~= "/" and v ~= "-" then
+ context.type(v)
+ context.space()
+ end
+ end
+end
+
+-- inspect(tokens.commands)
+
+function document.filteredprimitives(cmd)
+ local t = { }
+ local c = tokens.commands[cmd]
+ for i=1,#list do
+ local l = list[i]
+ if l[1] == c then
+ t[#t+1] = l[3]
+ end
+ end
+ table.sort(t)
+ for i=1,#t do
+ if i > 1 then
+ context(", ")
+ elseif i == #t then
+ context(" and ")
+ end
+ context.typ(t[i])
+ end
+end
+
+function document.allprimitives()
+ local c = tokens.commands
+ local o = tex.getprimitiveorigins()
+ table.sort(list, function(a,b)
+ return a[3] < b[3]
+ end)
+ context.starttabulate { "|T|T|Tc|T|" }
+ context.DB() context("primitive")
+ context.BC() context("command code")
+ context.BC() context("char code")
+ context.BC() context("origin")
+ context.NC() context.NR()
+ context.TB()
+ for i=1,#list do
+ local l = list[i]
+ context.NC() context.tex(l[3])
+ context.NC() context(c[l[1]])
+ context.NC() context(l[2])
+ context.NC() context(o[l[4]])
+ context.NC() context.NR()
+ end
+ context.LL()
+ context.stoptabulate()
+end
+\stopluacode
+
\stopenvironment
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
index 737f3b96081..503bbfd8293 100644
--- a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
@@ -4,6 +4,16 @@
\startcomponent luametatex-tex
+% { "getbytecode", lualib_get_bytecode },
+% { "setbytecode", lualib_set_bytecode },
+% { "callbytecode", lualib_call_bytecode },
+% { "getfunctionstable", lualib_get_functions_table },
+% { "getstartupfile", lualib_get_startupfile },
+% { "getversion", lualib_get_version },
+% { "setexitcode", lualib_set_exitcode },
+% { NULL, NULL },
+
+
\startchapter[reference=tex,title={The \TEX\ related libraries}]
\startsection[title={The \type {lua} library}][library=lua]
@@ -11,21 +21,28 @@
\startsubsection[title={Version information}]
\topicindex{libraries+\type{lua}}
-\topicindex{version}
-\topicindex{startupfile}
+\topicindex{getversion}
+\topicindex{getstartupfile}
+
+\libindex{getversion}
+\libindex{getstartupfile}
+
+This version of the used \LUA\ interpreter (currently {\tttf \cldcontext
+{lua.getversion()}}) can be queried with:
-\libindex{version}
-\libindex{startupfile}
+\starttyping
+<string> v = lua.getversion()
+\stoptyping
-This library contains two read|-|only items:
+The name of used startup file, if at all, is returned by:
\starttyping
-<string> v = lua.version
-<string> s = lua.startupfile
+<string> s = lua.getstartupfile()
\stoptyping
-This returns the \LUA\ version identifier string. The value currently is
-\directlua {tex.print(lua.version)}.
+For this document the reported value is:
+
+\blank {\ttx \cldcontext {lua.getstartupfile()}} \blank
\stopsubsection
@@ -60,6 +77,7 @@ This preallocates 100 hash entries and 5000 index entries. The \type
\libindex{bytecode}
\libindex{setbytecode}
\libindex{getbytecode}
+\libindex{callbytecode}
\LUA\ registers can be used to store \LUA\ code chunks. The accepted values for
assignments are functions and \type {nil}. Likewise, the retrieved value is
@@ -67,7 +85,7 @@ either a function or \type {nil}.
\starttyping
lua.bytecode[<number> n] = <function> f
-lua.bytecode[<number> n]()
+<function> f = lua.bytecode[<number> n] % -- f()
\stoptyping
The contents of the \type {lua.bytecode} array is stored inside the format file
@@ -75,8 +93,8 @@ as actual \LUA\ bytecode, so it can also be used to preload \LUA\ code. The
function must not contain any upvalues. The associated function calls are:
\startfunctioncall
-<function> f = lua.getbytecode(<number> n)
lua.setbytecode(<number> n, <function> f)
+<function> f = lua.getbytecode(<number> n)
\stopfunctioncall
Note: Since a \LUA\ file loaded using \type {loadfile(filename)} is essentially
@@ -84,14 +102,14 @@ an anonymous function, a complete file can be stored in a bytecode register like
this:
\startfunctioncall
-lua.bytecode[n] = loadfile(filename)
+lua.setbytecode(n,loadfile(filename))
\stopfunctioncall
Now all definitions (functions, variables) contained in the file can be
created by executing this bytecode register:
\startfunctioncall
-lua.bytecode[n]()
+lua.callbytecode(n)
\stopfunctioncall
Note that the path of the file is stored in the \LUA\ bytecode to be used in
@@ -102,30 +120,6 @@ mind when preloading files into a bytecode register in \INITEX.
\stopsubsection
-\startsubsection[title={Chunk name registers}]
-
-\libindex{name}
-\libindex{setluaname}
-\libindex{getluaname}
-
-There is an array of 65536 (0--65535) potential chunk names for use with the
-\prm {directlua} and \lpr {latelua} primitives.
-
-\startfunctioncall
-lua.name[<number> n] = <string> s
-<string> s = lua.name[<number> n]
-\stopfunctioncall
-
-If you want to unset a \LUA\ name, you can assign \type {nil} to it. The function
-accessors are:
-
-\startfunctioncall
-lua.setluaname(<string> s,<number> n)
-<string> s = lua.getluaname(<number> n)
-\stopfunctioncall
-
-\stopsubsection
-
\startsubsection[title={Introspection}]
\libindex{getstacktop}
@@ -166,96 +160,100 @@ and values as a table.
<table> info = status.list()
\stopfunctioncall
-The keys in the table are the known items, the value is the current value. Almost
-all of the values in \type {status} are fetched through a metatable at run|-|time
-whenever they are accessed, so you cannot use \type {pairs} on \type {status},
-but you {\it can\/} use \type {pairs} on \type {info}, of course. If you do not
-need the full list, you can also ask for a single item by using its name as an
-index into \type {status}. The current list is:
+The keys in the table are the known items, the value is the current value. There are
+toplevel items and items that are tables with subentries. The current list is:
-\starttabulate[|l|p|]
-\DB key \BC explanation \NC \NR
+\startluacode
+ local list = status.list()
+
+ context.starttabulate { "|Tw(10em)|Tp|" }
+ context.DB()
+ context("toplevel statistics")
+ context.BC()
+ context.NC()
+ context.NR()
+ context.TB()
+ for k, v in table.sortedhash(list) do
+ if type(v) ~= "table" then
+ context.NC()
+ context(k)
+ context.NC()
+ context(tostring(v))
+ context.NC()
+ context.NR()
+ end
+ end
+ context.LL()
+ context.stoptabulate()
+
+ for k, v in table.sortedhash(list) do
+ if type(v) == "table" then
+ context.starttabulate { "|Tw(10em)|Tp|" }
+ context.DB()
+ context(k ..".*")
+ context.BC()
+ context.NC()
+ context.NR()
+ context.TB()
+ for k, v in table.sortedhash(v) do
+ context.NC()
+ context(k)
+ context.NC()
+ context(v == "" and "unset" or tostring(v))
+ context.NC()
+ context.NR()
+ end
+ context.LL()
+ context.stoptabulate()
+ end
+ end
+\stopluacode
+
+There are also getters for the subtables. The whole repertoire of functions in
+the \type {status} table is: {\tttf \cldcontext {table . concat ( table .
+sortedkeys (status), ", ")}}. The error and warning messages can be wiped with
+the \type {resetmessages} function. The states in subtables relate to memory
+management and are mostly there for development purposes.
+
+The \type {getconstants} query gives back a table with all kind of internal
+quantities and again these are only relevant for diagnostic and development
+purposes. Many are good old \TEX\ constants that are describes in the original
+documentation of the source but some are definitely \LUAMETATEX\ specific.
+
+\startluacode
+ context.starttabulate { "|Tw(15em)|Tp|" }
+ context.DB()
+ context("constants.*")
+ context.BC()
+ context.NC()
+ context.NR()
+ context.TB()
+ for k, v in table.sortedhash(status.getconstants()) do
+ if type(v) ~= "table" then
+ context.NC()
+ context(k)
+ context.NC()
+ context(tostring(v))
+ context.NC()
+ context.NR()
+ end
+ end
+ context.LL()
+ context.stoptabulate()
+\stopluacode
+
+Most variables speak for themselves, some are more obscure. For instance the
+\type {run_state} variable indicates what the engine is doing:
+
+\starttabulate[|l|l|p|]
+\DB n \NC meaning \NC explanation \NC \NR
\TB
-\NC \type{banner} \NC terminal display banner \NC \NR
-\NC \type{best_page_break} \NC the current best break (a node) \NC \NR
-\NC \type{buf_size} \NC current allocated size of the line buffer \NC \NR
-\NC \type{callbacks} \NC total number of executed callbacks so far \NC \NR
-\NC \type{cs_count} \NC number of control sequences \NC \NR
-\NC \type{dest_names_size} \NC \PDF\ destination table size \NC \NR
-\NC \type{dvi_gone} \NC written \DVI\ bytes \NC \NR
-\NC \type{dvi_ptr} \NC not yet written \DVI\ bytes \NC \NR
-\NC \type{dyn_used} \NC token (multi|-|word) memory in use \NC \NR
-\NC \type{filename} \NC name of the current input file \NC \NR
-\NC \type{fix_mem_end} \NC maximum number of used tokens \NC \NR
-\NC \type{fix_mem_min} \NC minimum number of allocated words for tokens \NC \NR
-\NC \type{fix_mem_max} \NC maximum number of allocated words for tokens \NC \NR
-\NC \type{font_ptr} \NC number of active fonts \NC \NR
-\NC \type{hash_extra} \NC extra allowed hash \NC \NR
-\NC \type{hash_size} \NC size of hash \NC \NR
-\NC \type{indirect_callbacks} \NC number of those that were themselves a result of other callbacks (e.g. file readers) \NC \NR
-\NC \type{ini_version} \NC \type {true} if this is an \INITEX\ run \NC \NR
-\NC \type{init_pool_ptr} \NC \INITEX\ string pool index \NC \NR
-\NC \type{init_str_ptr} \NC number of \INITEX\ strings \NC \NR
-\NC \type{input_ptr} \NC the level of input we're at \NC \NR
-\NC \type{inputid} \NC numeric id of the current input \NC \NR
-\NC \type{largest_used_mark} \NC max referenced marks class \NC \NR
-\NC \type{lasterrorcontext} \NC last error context string (with newlines) \NC \NR
-\NC \type{lasterrorstring} \NC last \TEX\ error string \NC \NR
-\NC \type{lastluaerrorstring} \NC last \LUA\ error string \NC \NR
-\NC \type{lastwarningstring} \NC last warning tag, normally an indication of in what part\NC \NR
-\NC \type{lastwarningtag} \NC last warning string\NC \NR
-\NC \type{linenumber} \NC location in the current input file \NC \NR
-\NC \type{log_name} \NC name of the log file \NC \NR
-\NC \type{luabytecode_bytes} \NC number of bytes in \LUA\ bytecode registers \NC \NR
-\NC \type{luabytecodes} \NC number of active \LUA\ bytecode registers \NC \NR
-\NC \type{luastate_bytes} \NC number of bytes in use by \LUA\ interpreters \NC \NR
-\NC \type{luatex_engine} \NC the \LUATEX\ engine identifier \NC \NR
-\NC \type{luatex_hashchars} \NC length to which \LUA\ hashes strings ($2^n$) \NC \NR
-\NC \type{luatex_hashtype} \NC the hash method used (in \LUAJITTEX) \NC \NR
-\NC \type{luatex_version} \NC the \LUATEX\ version number \NC \NR
-\NC \type{luatex_revision} \NC the \LUATEX\ revision string \NC \NR
-\NC \type{max_buf_stack} \NC max used buffer position \NC \NR
-\NC \type{max_in_stack} \NC max used input stack entries \NC \NR
-\NC \type{max_nest_stack} \NC max used nesting stack entries \NC \NR
-\NC \type{max_param_stack} \NC max used parameter stack entries \NC \NR
-\NC \type{max_save_stack} \NC max used save stack entries \NC \NR
-\NC \type{max_strings} \NC maximum allowed strings \NC \NR
-\NC \type{nest_size} \NC nesting stack size \NC \NR
-\NC \type{node_mem_usage} \NC a string giving insight into currently used nodes \NC \NR
-\NC \type{obj_ptr} \NC max \PDF\ object pointer \NC \NR
-\NC \type{obj_tab_size} \NC \PDF\ object table size \NC \NR
-\NC \type{output_active} \NC \type {true} if the \prm {output} routine is active \NC \NR
-\NC \type{output_file_name} \NC name of the \PDF\ or \DVI\ file \NC \NR
-\NC \type{param_size} \NC parameter stack size \NC \NR
-\NC \type{pdf_dest_names_ptr} \NC max \PDF\ destination pointer \NC \NR
-\NC \type{pdf_gone} \NC written \PDF\ bytes \NC \NR
-\NC \type{pdf_mem_ptr} \NC max \PDF\ memory used \NC \NR
-\NC \type{pdf_mem_size} \NC \PDF\ memory size \NC \NR
-\NC \type{pdf_os_cntr} \NC max \PDF\ object stream pointer \NC \NR
-\NC \type{pdf_os_objidx} \NC \PDF\ object stream index \NC \NR
-\NC \type{pdf_ptr} \NC not yet written \PDF\ bytes \NC \NR
-\NC \type{pool_ptr} \NC string pool index \NC \NR
-\NC \type{pool_size} \NC current size allocated for string characters \NC \NR
-\NC \type{save_size} \NC save stack size \NC \NR
-\NC \type{shell_escape} \NC \type {0} means disabled, \type {1} means anything is permitted, and \type {2} is restricted \NC \NR
-\NC \type{safer_option} \NC \type {1} means safer is enforced \NC \NR
-\NC \type{kpse_used} \NC \type {1} means that kpse is used \NC \NR
-\NC \type{stack_size} \NC input stack size \NC \NR
-\NC \type{str_ptr} \NC number of strings \NC \NR
-\NC \type{total_pages} \NC number of written pages \NC \NR
-\NC \type{var_mem_max} \NC number of allocated words for nodes \NC \NR
-\NC \type{var_used} \NC variable (one|-|word) memory in use \NC \NR
-\NC \type{lc_collate} \NC the value of \type {LC_COLLATE} at startup time (becomes \type {C} at startup) \NC \NR
-\NC \type{lc_ctype} \NC the value of \type {LC_CTYPE} at startup time (becomes \type {C} at startup) \NC \NR
-%NC \type{lc_monetary} \NC the value of \type {LC_MONETARY} at startup time \NC \NR
-\NC \type{lc_numeric} \NC the value of \type {LC_NUMERIC} at startup time \NC \NR
-%NC \type{lc_time} \NC the value of \type {LC_TIME} at startup time (becomes \type {C} at startup) \NC \NR
+\NC 0 \NC initializing \NC \type {--ini} mnode \NC \NR
+\NC 1 \NC updating \NC relates to \lpr {overloadmode} \NC \NR
+\NC 2 \NC production \NC a regular (format driven) run \NC \NR
\LL
\stoptabulate
-The error and warning messages can be wiped with the \type {resetmessages}
-function. A return value can be set with \type {setexitcode}.
-
\stopsection
\startsection[title={The \type {tex} library}][library=tex]
@@ -284,7 +282,8 @@ readable (like \prm {tracingcommands} and \prm {parindent}).
\topicindex{parameters+internal}
-\libindex{set} \libindex{get}
+\libindex{set}
+\libindex{get}
For all the parameters in this section, it is possible to access them directly
using their names as index in the \type {tex} table, or by using one of the
@@ -307,152 +306,8 @@ get all five values. Otherwise you get a node which is a copy of the internal
value so you are responsible for its freeing at the \LUA\ end. When you set a
glue quantity you can either pass a \nod {glue_spec} or upto five numbers.
-\subsubsection{Integer parameters}
-
-The integer parameters accept and return \LUA\ numbers. These are read|-|write:
-
-\starttwocolumns
-\starttyping
-tex.adjdemerits
-tex.binoppenalty
-tex.brokenpenalty
-tex.catcodetable
-tex.clubpenalty
-tex.day
-tex.defaulthyphenchar
-tex.defaultskewchar
-tex.delimiterfactor
-tex.displaywidowpenalty
-tex.doublehyphendemerits
-tex.endlinechar
-tex.errorcontextlines
-tex.escapechar
-tex.exhyphenpenalty
-tex.fam
-tex.finalhyphendemerits
-tex.floatingpenalty
-tex.globaldefs
-tex.hangafter
-tex.hbadness
-tex.holdinginserts
-tex.hyphenpenalty
-tex.interlinepenalty
-tex.language
-tex.lastlinefit
-tex.lefthyphenmin
-tex.linepenalty
-tex.localbrokenpenalty
-tex.localinterlinepenalty
-tex.looseness
-tex.mag
-tex.maxdeadcycles
-tex.month
-tex.newlinechar
-tex.outputpenalty
-tex.pausing
-tex.postdisplaypenalty
-tex.predisplaydirection
-tex.predisplaypenalty
-tex.pretolerance
-tex.relpenalty
-tex.righthyphenmin
-tex.savinghyphcodes
-tex.savingvdiscards
-tex.showboxbreadth
-tex.showboxdepth
-tex.time
-tex.tolerance
-tex.tracingassigns
-tex.tracingcommands
-tex.tracinggroups
-tex.tracingifs
-tex.tracinglostchars
-tex.tracingmacros
-tex.tracingnesting
-tex.tracingonline
-tex.tracingoutput
-tex.tracingpages
-tex.tracingparagraphs
-tex.tracingrestores
-tex.tracingscantokens
-tex.tracingstats
-tex.uchyph
-tex.vbadness
-tex.widowpenalty
-tex.year
-\stoptyping
-\stoptwocolumns
-
-These are read|-|only:
-
-\startthreecolumns
-\starttyping
-tex.deadcycles
-tex.insertpenalties
-tex.parshape
-tex.prevgraf
-tex.spacefactor
-\stoptyping
-\stopthreecolumns
-
-\subsubsection{Dimension parameters}
-
-The dimension parameters accept \LUA\ numbers (signifying scaled points) or
-strings (with included dimension). The result is always a number in scaled
-points. These are read|-|write:
-
-\startthreecolumns
-\starttyping
-tex.boxmaxdepth
-tex.delimitershortfall
-tex.displayindent
-tex.displaywidth
-tex.emergencystretch
-tex.hangindent
-tex.hfuzz
-tex.hoffset
-tex.hsize
-tex.lineskiplimit
-tex.mathsurround
-tex.maxdepth
-tex.nulldelimiterspace
-tex.overfullrule
-tex.pagebottomoffset
-tex.pageheight
-tex.pageleftoffset
-tex.pagerightoffset
-tex.pagetopoffset
-tex.pagewidth
-tex.parindent
-tex.predisplaysize
-tex.scriptspace
-tex.splitmaxdepth
-tex.vfuzz
-tex.voffset
-tex.vsize
-tex.prevdepth
-tex.prevgraf
-tex.spacefactor
-\stoptyping
-\stopthreecolumns
-
-These are read|-|only:
-
-\startthreecolumns
-\starttyping
-tex.pagedepth
-tex.pagefilllstretch
-tex.pagefillstretch
-tex.pagefilstretch
-tex.pagegoal
-tex.pageshrink
-tex.pagestretch
-tex.pagetotal
-\stoptyping
-\stopthreecolumns
-
-Beware: as with all \LUA\ tables you can add values to them. So, the following is
-valid:
+Beware: as with regular \LUA\ tables you can add values to the \type {tex} table.
+So, the following is valid:
\starttyping
tex.foo = 123
@@ -484,70 +339,52 @@ as there is no way \LUATEX\ can guess your intentions. By using the accessor in
the \type {tex} tables, you get and set the values at the top of the nesting
stack.
-\subsubsection{Direction parameters}
+\subsubsection{Integer parameters}
-The direction states can be queried and set with:
+The integer parameters accept and return \LUA\ integers. In some cases the values
+are checked, trigger other settings or result in some immediate change of
+behaviour: \ctxlua {document.filteredprimitives ("internal_int")}.
-\startthreecolumns
-\starttyping
-tex.gettextdir()
-tex.getlinedir()
-tex.getmathdir()
-tex.getpardir()
-tex.settextdir(<number>)
-tex.setlinedir(<number>)
-tex.setmathdir(<number>)
-tex.setpardir(<number>)
-\stoptyping
-\stopthreecolumns
+Some integer parameters are read only, because they are actually referring not to
+some internal integer register but to an engine property: \typ {deadcycles},
+\typ {insertpenalties}, \typ {parshape}, \typ {interlinepenalties}, \typ
+{clubpenalties}, \typ {widowpenalties}, \typ {displaywidowpenalties}, \typ
+{prevgraf} and \typ {spacefactor}.
-and also with:
+\subsubsection{Dimension parameters}
-\startthreecolumns
-\starttyping
-tex.textdirection
-tex.linedirection
-tex.mathdirection
-tex.pardirection
-\stoptyping
-\stopthreecolumns
+The dimension parameters accept \LUA\ numbers (signifying scaled points) or
+strings (with included dimension). The result is always a number in scaled
+points. These are read|-|write: \ctxlua {document.filteredprimitives
+("internal_dimen")}.
+These are read|-|only: \typ {pagedepth}, \typ {pagefilllstretch}, \typ
+{pagefillstretch}, \typ {pagefilstretch}, \typ {pagegoal}, \typ {pageshrink},
+\typ {pagestretch} and \typ {pagetotal}.
-\subsubsection{Glue parameters}
+\subsubsection{Direction parameters}
-The glue parameters accept and return a userdata object that represents a \nod {glue_spec} node.
+The direction states can be queried with: \typ {gettextdir}, \typ {getlinedir},
+\typ {getmathdir} and \typ {getpardir}. You can set them with \typ
+{settextdir}, \typ {setlinedir}, \typ {setmathdir} and \typ {setpardir},
+commands that accept a number. You can also set these parameters as table
+key|/|values: \typ {textdirection}, \typ {linedirection}, \typ {mathdirection}
+and \typ {pardirection}, so the next code sets the text direction to \typ
+{r2l}:
-\startthreecolumns
\starttyping
-tex.abovedisplayshortskip
-tex.abovedisplayskip
-tex.baselineskip
-tex.belowdisplayshortskip
-tex.belowdisplayskip
-tex.leftskip
-tex.lineskip
-tex.parfillskip
-tex.parskip
-tex.rightskip
-tex.spaceskip
-tex.splittopskip
-tex.tabskip
-tex.topskip
-tex.xspaceskip
+tex.textdirection = 1
\stoptyping
-\stopthreecolumns
-\subsubsection{Muglue parameters}
+\subsubsection{Glue parameters}
-All muglue parameters are to be used read|-|only and return a \LUA\ string.
+The internal glue parameters accept and return a userdata object that represents
+a \nod {glue_spec} node: \ctxlua {document.filteredprimitives ("internal_glue")}.
-\startthreecolumns
-\starttyping
-tex.medmuskip
-tex.thickmuskip
-tex.thinmuskip
-\stoptyping
-\stopthreecolumns
+\subsubsection{Muglue parameters}
+
+All muglue parameters are to be used read|-|only and return a \LUA\ string
+\ctxlua {document.filteredprimitives ("internal_mu_glue")}.
\subsubsection{Tokenlist parameters}
@@ -556,22 +393,8 @@ converted to and from token lists using \prm {the} \prm {toks} style expansion:
all category codes are either space (10) or other (12). It follows that assigning
to some of these, like \quote {tex.output}, is actually useless, but it feels bad
to make exceptions in view of a coming extension that will accept full|-|blown
-token strings.
-
-\startthreecolumns
-\starttyping
-tex.errhelp
-tex.everycr
-tex.everydisplay
-tex.everyeof
-tex.everyhbox
-tex.everyjob
-tex.everymath
-tex.everypar
-tex.everyvbox
-tex.output
-\stoptyping
-\stopthreecolumns
+token strings. Here is the lot: \ctxlua {document.filteredprimitives
+("internal_toks")}.
\stopsubsection
@@ -580,55 +403,23 @@ tex.output
\topicindex{convert commands}
All \quote {convert} commands are read|-|only and return a \LUA\ string. The
-supported commands at this moment are:
-
-\starttwocolumns
-\starttyping
-tex.formatname
-tex.jobname
-tex.luatexbanner
-tex.luatexrevision
-tex.fontname(number)
-tex.uniformdeviate(number)
-tex.number(number)
-tex.romannumeral(number)
-tex.fontidentifier(number)
-\stoptyping
-\stoptwocolumns
-
-If you are wondering why this list looks haphazard; these are all the cases of
-the \quote {convert} internal command that do not require an argument, as well as
-the ones that require only a simple numeric value. The special (\LUA|-|only) case
-of \type {tex.fontidentifier} returns the \type {csname} string that matches a
-font id number (if there is one).
+supported commands at this moment are: \ctxlua {document.filteredprimitives
+("convert")}. You will get an error message if an operation is not (yet)
+permitted. Some take an string or number argument, just like at the \TEX\ end
+some extra input is expected.
\stopsubsection
-\startsubsection[title={Last item commands}]
+\startsubsection[title={Item commands}]
\topicindex{last items}
-All \quote {last item} commands are read|-|only and return a number. The
-supported commands at this moment are:
-
-\startthreecolumns
-\starttyping
-tex.lastpenalty
-tex.lastkern
-tex.lastskip
-tex.lastnodetype
-tex.inputlineno
-tex.lastxpos
-tex.lastypos
-tex.randomseed
-tex.luatexversion
-tex.currentgrouplevel
-tex.currentgrouptype
-tex.currentiflevel
-tex.currentiftype
-tex.currentifbranch
-\stoptyping
-\stopthreecolumns
+All so called \quote {item} commands are read|-|only and return a number. The
+complete list of these commands is: \ctxlua {document.filteredprimitives
+("some_item")}. No all are currently supported but eventually that might be the
+case. Like the lists in previous sections, there are differences between \LUATEX\
+and \LUAMETATEX, where some commands are organized differently in order to
+provide a consistent \LUA\ interface.
\stopsubsection
@@ -649,6 +440,8 @@ tex.currentifbranch
\libindex{scantoks}
+\libindex{getmark}
+
\TEX's attributes (\lpr {attribute}), counters (\prm {count}), dimensions (\prm
{dimen}), skips (\prm {skip}, \prm {muskip}) and token (\prm {toks}) registers
can be accessed and written to using two times five virtual sub|-|tables of the
@@ -734,17 +527,17 @@ We have similar setters for \type {count}, \type {dimen}, \type {muskip}, and
nodes, and toks by strings.
Again the glue variants are not using the \nod {glue-spec} userdata nodes. The
-\type {setglue} function accepts upto 5 arguments: width, stretch, shrink,
-stretch order and shrink order and the \type {getglue} function reports them,
-unless the second argument is \type {false} in which case only the width is
-returned.
+\type {setglue} function accepts upto five arguments: width, stretch, shrink,
+stretch order and shrink order. Non|-|numeric values set the property to zero.
+The \type {getglue} function reports all five properties, unless the second
+argument is \type {false} in which case only the width is returned.
Here is an example using a threesome:
\startfunctioncall
local d = tex.getdimen("foo")
-if tex.isdimen("bar") then
- tex.setdimen("bar",d)
+if tex.isdimen("oof") then
+ tex.setdimen("oof",d)
end
\stopfunctioncall
@@ -784,6 +577,15 @@ tex.scantoks("global",0,3,"$\int\limits^1_2$")
In the function-based interface, it is possible to define values globally by
using the string \type {global} as the first function argument.
+There is a dedicated getter for marks: \type {getmark} that takes two arguments.
+The first argument is one of \type {top}, \type {bottom}, \type {first}, \type
+{splitbottom} or \type {splitfirst}, and the second argument is a marks class
+number. When no arguments are given the current maximum number of classes is
+returned.
+
+When \type {tex.gettoks} gets an extra argument \type {true} it will return a
+table with userdata tokens.
+
\stopsubsection
\startsubsection[title={Character code registers: \type {[get|set]*code[s]}}]
@@ -1061,7 +863,7 @@ keep track of building page lists.
\DB field \BC explanation \NC \NR
\TB
\NC \type{page_ins_head} \NC circular list of pending insertions \NC \NR
-\NC \type{contrib_head} \NC the recent contributions \NC \NR
+\NC \type{contribute_head} \NC the recent contributions \NC \NR
\NC \type{page_head} \NC the current page content \NC \NR
%NC \type{temp_head} \NC \NC \NR
\NC \type{hold_head} \NC used for held-over items for next page \NC \NR
@@ -1120,7 +922,7 @@ The known fields are:
\NC \type{prevgraf} \NC number \NC vmode \NC number of lines in the previous paragraph \NC \NR
\NC \type{prevdepth} \NC number \NC vmode \NC depth of the previous paragraph \NC \NR
\NC \type{spacefactor} \NC number \NC hmode \NC the current space factor \NC \NR
-\NC \type{dirs} \NC node \NC hmode \NC used for temporary storage by the line break algorithm\NC \NR
+\NC \type{direction} \NC node \NC hmode \NC stack used for temporary storage by the line break algorithm \NC \NR
\NC \type{noad} \NC node \NC mmode \NC used for temporary storage of a pending fraction numerator,
for \prm {over} etc. \NC \NR
\NC \type{delimptr} \NC node \NC mmode \NC used for temporary storage of the previous math delimiter,
@@ -1524,14 +1326,15 @@ an error:
This function has two optional arguments in case a token register is passed:
\startfunctioncall
-tex.runtoks(<token register>,force,grouped)
+tex.runtoks(<token register>,force,grouped,obeymode)
\stopfunctioncall
Inside for instance an \type {\edef} the \type {runtoks} function behaves (at
least tries to) like it were an \type {\the}. This prevents unwanted side
effects: normally in such an definition tokens remain tokens and (for instance)
characters don't become nodes. With the second argument you can force the local
-main loop, no matter what. The third argument adds a level of grouping.
+main loop, no matter what. The third argument adds a level of grouping. The last
+argument tells the scanner to stay in the current mode.
You can quit the local loop with \type {\endlocalcontrol} or from the \LUA\ end
with \type {tex.quittoks}. In that case you end one level up! Of course in the
@@ -1657,20 +1460,6 @@ given by the requested string value(s). The possible values and their (current)
return values are given in the following table. In addition the somewhat special
primitives \quote{\tex{ }}, \quote{\tex {/}} and \quote{\type {-}} are defined.
-\startluacode
-function document.showprimitives(tag)
- local t = tex.extraprimitives(tag)
- table.sort(t)
- for i=1,#t do
- local v = t[i]
- if v ~= ' ' and v ~= "/" and v ~= "-" then
- context.type(v)
- context.space()
- end
- end
-end
-\stopluacode
-
\starttabulate[|l|pl|]
\DB name \BC values \NC \NR
\TB
@@ -1750,7 +1539,7 @@ The understood parameters are as follows:
\NC \type{exhyphenpenalty} \NC number \NC \NC \NR
\NC \type{pdfadjustspacing} \NC number \NC \NC \NR
\NC \type{adjdemerits} \NC number \NC \NC \NR
-\NC \type{pdfprotrudechars} \NC number \NC \NC \NR
+\NC \type{protrudechars} \NC number \NC \NC \NR
\NC \type{linepenalty} \NC number \NC \NC \NR
\NC \type{lastlinefit} \NC number \NC \NC \NR
\NC \type{doublehyphendemerits} \NC number \NC \NC \NR
@@ -1781,7 +1570,7 @@ horizontal mode):
\startitemize
\startitem
- add an \quote {indent box} and perhaps a \nod {local_par} node at the start
+ add an \quote {indent box} and perhaps a \nod {par} node at the start
(only if you need them)
\stopitem
\startitem
@@ -1969,8 +1758,8 @@ texio.write(<string> s, ...)
Without the \type {target} argument, writes all given strings to the same
location(s) \TEX\ writes messages to at this moment. If \prm {batchmode} is in
effect, it writes only to the log, otherwise it writes to the log and the
-terminal. The optional \type {target} can be one of three possibilities: \type
-{term}, \type {log} or \type {term and log}.
+terminal. The optional \type {target} can be one of \type {terminal},
+\type {logfile} or \type {terminal_and_logfile}.
Note: If several strings are given, and if the first of these strings is or might
be one of the targets above, the \type {target} must be specified explicitly to
@@ -2071,6 +1860,9 @@ most intriguing.
\LL
\stoptabulate
+The integer, dimension and glue scanners take an extra optional argument that
+signals that en optional equal is permitted.
+
The scanners can be considered stable apart from the one scanning for a token.
The \type {scan_code} function takes an optional number, the \type {scan_keyword}
function a normal \LUA\ string. The \type {infinity} boolean signals that we also
@@ -2080,14 +1872,19 @@ macro, in which case the result will also provide information about what
arguments are expected and in the result this is separated from the meaning by a
separator token. The \type {expand} flag determines if the list will be expanded.
+The \type {scan_argument} function expands the given argument. When a braced
+argument is scanned, expansion can be prohibited by passing \type {false}
+(default is \type {true}). In case of a control sequence passing \type {false}
+will result in a one|-|level expansion (the meaning of the macro).
+
The string scanner scans for something between curly braces and expands on the
way, or when it sees a control sequence it will return its meaning. Otherwise it
will scan characters with catcode \type {letter} or \type {other}. So, given the
following definition:
\startbuffer
-\def\bar{bar}
-\def\foo{foo-\bar}
+\def\oof{oof}
+\def\foo{foo-\oof}
\stopbuffer
\typebuffer \getbuffer
@@ -2108,7 +1905,9 @@ expanded definition (\prm {edef}'d). In the case of the braced variant one can o
course use the \prm {detokenize} and \prm {unexpanded} primitives since there we
do expand.
-The \type {scan_word} scanner can be used to implement for instance a number scanner:
+The \type {scan_word} scanner can be used to implement for instance a number
+scanner. An optional boolean argument can signal that a trailing space or \type
+{\relax} should be gobbled:
\starttyping
function token.scan_number(base)
@@ -2176,22 +1975,43 @@ a bit but for passing strings conversion to and from tokens has to be done anywa
\startsubsection[title={Picking up one token}]
-\libindex {get_next}
+\libindex {scan_next}
+\libindex {scan_next_expanded}
+\libindex {skip_next}
+\libindex {skip_next_expanded}
+\libindex {peek_next}
+\libindex {peek_next_expanded}
\libindex {scan_token}
\libindex {expand}
The scanners look for a sequence. When you want to pick up one token from the
-input you use \type {get_next}. This creates a token with the (low level)
+input you use \type {scan_next}. This creates a token with the (low level)
properties as discussed next. This token is just the next one. If you want to
-enforce expansion first you can use \type {scan_token}. Internally tokens are
-characterized by a number that packs a lot of information. In order to access
-the bits of information a token is wrapped in a userdata object.
+enforce expansion first you can use \type {scan_token} or the \type {_expanded}
+variants. Internally tokens are characterized by a number that packs a lot of
+information. In order to access the bits of information a token is wrapped in a
+userdata object.
The \type {expand} function will trigger expansion of the next token in the
input. This can be quite unpredictable but when you call it you probably know
enough about \TEX\ not to be too worried about that. It basically is a call to
the internal expand related function.
+\starttabulate[|lT|p|]
+\DB name \BC explanation \NC \NR
+\TB
+\NC scan_next \NC get the next token \NC \NR
+\NC scan_next_expanded \NC get the next expanded token \NC \NR
+\NC skip_next \NC skip the next token \NC \NR
+\NC skip_next_expanded \NC skip the next expanded token \NC \NR
+\NC peek_next \NC get the next token and put it back in the input \NC \NR
+\NC peek_next_expanded \NC get the next expanded token and put it back in the input \NC \NR
+\LL
+\stoptabulate
+
+The peek function accept a boolean argument that triggers skipping spaces and
+alike.
+
\stopsubsection
\startsubsection[title={Creating tokens}]
@@ -2216,8 +2036,10 @@ the internal expand related function.
\libindex{get_mode}
\libindex{get_index}
\libindex{get_tok}
+\libindex{get_frozen}
+\libindex{get_user}
-\libindex{get_next}
+\libindex{scan_next}
The creator function can be used as follows:
@@ -2241,8 +2063,9 @@ primitive. The possible properties of tokens are:
\NC \type {active} \NC a boolean indicating the active state of the token \NC \NR
\NC \type {expandable} \NC a boolean indicating if the token (macro) is expandable \NC \NR
\NC \type {protected} \NC a boolean indicating if the token (macro) is protected \NC \NR
-\NC \type {mode} \NC a number either representing a character or another entity \NC \NR
-\NC \type {index} \NC a number running from 0x0000 upto 0xFFFF indicating a \TEX\ register index \NC \NR
+\NC \type {frozen} \NC a boolean indicating if the token is a frozen command \NC \NR
+\NC \type {user} \NC a boolean indicating if the token is a user defined command \NC \NR
+\NC \type {index} \NC a number that indicated the subcommand; differs per command \NC \NR
\LL
\stoptabulate
@@ -2307,6 +2130,8 @@ need to know that boundary condition.
\libindex{set_char}
\libindex{set_lua}
\libindex{get_functions_table}
+\libindex{push_macro}
+\libindex{pop_macro}
The \type {set_macro} function can get upto 4 arguments:
@@ -2356,20 +2181,25 @@ that you can access with \type {lua.get_functions_table}. It is the companion
to \lpr {luadef}. When the first (and only) argument is true the size will preset
to the value of \type {texconfig.function_size}.
+The \type {push_macro} and \type {pop_macro} function are very experimental and
+can be used to get and set an existing macro. The push call returns a user data
+object and the pop takes such a userdata object. These object have no accessors
+and are to be seen as abstractions.
+
\stopsubsection
\startsubsection[title={Pushing back}]
-\libindex{get_next}
+\libindex{scan_next}
\libindex{put_next}
There is a (for now) experimental putter:
\starttyping
-local t1 = token.get_next()
-local t2 = token.get_next()
-local t3 = token.get_next()
-local t4 = token.get_next()
+local t1 = token.scan_next()
+local t2 = token.scan_next()
+local t3 = token.scan_next()
+local t4 = token.scan_next()
-- watch out, we flush in sequence
token.put_next { t1, t2 }
-- but this one gets pushed in front
@@ -2410,28 +2240,28 @@ When scanning for the next token you need to keep in mind that we're not scannin
like \TEX\ does: expanding, changing modes and doing things as it goes. When we
scan with \LUA\ we just pick up tokens. Say that we have:
-\pushmacro\bar \let\bar\undefined
+\pushmacro\oof \let\oof\undefined
\starttyping
-\bar
+\oof
\stoptyping
-but \type {\bar} is undefined. Normally \TEX\ will then issue an error message.
+but \type {\oof} is undefined. Normally \TEX\ will then issue an error message.
However, when we have:
\starttyping
-\def\foo{\bar}
+\def\foo{\oof}
\stoptyping
-We get no error, unless we expand \type {\foo} while \type {\bar} is still
+We get no error, unless we expand \type {\foo} while \type {\oof} is still
undefined. What happens is that as soon as \TEX\ sees an undefined macro it will
create a hash entry and when later it gets defined that entry will be reused. So,
-\type {\bar} really exists but can be in an undefined state.
+\type {\oof} really exists but can be in an undefined state.
\startbuffer[demo]
-bar : \directlua{tex.print(token.scan_csname())}\bar
+oof : \directlua{tex.print(token.scan_csname())}\oof
foo : \directlua{tex.print(token.scan_csname())}\foo
-myfirstbar : \directlua{tex.print(token.scan_csname())}\myfirstbar
+myfirstoof : \directlua{tex.print(token.scan_csname())}\myfirstoof
\stopbuffer
\startlines
@@ -2442,8 +2272,8 @@ This was entered as:
\typebuffer[demo]
-The reason that you see \type {bar} reported and not \type {myfirstbar} is that
-\type {\bar} was already used in a previous paragraph.
+The reason that you see \type {oof} reported and not \type {myfirstoof} is that
+\type {\oof} was already used in a previous paragraph.
If we now say:
@@ -2462,7 +2292,7 @@ we get:
And if we say
\startbuffer
-\def\foo{\bar}
+\def\foo{\oof}
\stopbuffer
\typebuffer \getbuffer
@@ -2477,9 +2307,9 @@ When scanning from \LUA\ we are not in a mode that defines (undefined) macros at
all. There we just get the real primitive undefined macro token.
\startbuffer
-\directlua{local t = token.get_next() tex.print(t.id.." "..t.tok)}\myfirstbar
-\directlua{local t = token.get_next() tex.print(t.id.." "..t.tok)}\mysecondbar
-\directlua{local t = token.get_next() tex.print(t.id.." "..t.tok)}\mythirdbar
+\directlua{local t = token.scan_next() tex.print(t.id.." "..t.tok)}\myfirstoof
+\directlua{local t = token.scan_next() tex.print(t.id.." "..t.tok)}\mysecondoof
+\directlua{local t = token.scan_next() tex.print(t.id.." "..t.tok)}\mythirdoof
\stopbuffer
\startlines
@@ -2494,7 +2324,7 @@ So, we do get a unique token because after all we need some kind of \LUA\ object
that can be used and garbage collected, but it is basically the same one,
representing an undefined control sequence.
-\popmacro\bar
+\popmacro\oof
\stopsubsection
diff --git a/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex.tex b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex.tex
new file mode 100644
index 00000000000..1327ea3a14b
--- /dev/null
+++ b/Master/texmf-dist/doc/context/sources/general/manuals/luametatex/luametatex.tex
@@ -0,0 +1,132 @@
+% ------------------------ ------------------------ ------------------------
+% 2019-12-17 32bit 64bit 2020-01-10 32bit 64bit 2020-11-30 32bit 64bit
+% ------------------------ ------------------------ ------------------------
+% freebsd 2270k 2662k freebsd 2186k 2558k freebsd 2108k 2436k
+% openbsd6.6 2569k 2824k openbsd6.6 2472k 2722k openbsd6.8 2411k 2782k
+% linux-armhf 2134k linux-armhf 2063k linux-armhf 2138k 2860k
+% linux 2927k 2728k linux 2804k 2613k linux (?) 3314k 2762k
+% linux-musl 2532k 2686k
+% osx 2821k osx 2732k osx 2711k
+% ms mingw 2562k 2555k ms mingw 2481k 2471k ms mingw 2754k 2760k
+% ms intel 2448k
+% ms arm 3894k
+% ms clang 2159k
+% ------------------------ ------------------------ ------------------------
+%
+% performance of mingw and native are getting close (small win for mingw) but
+% clang bins are still slower .. quite inconsistent differences between 32 and
+% 64 bit (not all compilers are the same version)
+
+% \loggingall
+% \tracingonline \zerocount
+% \tracingmacros \plusone
+
+% \enableautoglyphscaling % saves only a few instances ... no gain .. a few pages more
+
+% \enabletexdirective{vspacing.experimental}
+
+% \nopdfcompression
+
+% 20200509 : 258 pages
+%
+% my 2013 i7 laptop with windows : 11.8 sec mingw64
+% raspberry pi 64 bit with ssd : 39.5 sec gcc 9.2
+% idem with native bin : 38.5 sec
+% idem overclocked f=2000/v=6 : 31.5 sec
+%
+% 20200526 : 258 pages
+% mingw all in: 10.9 / rpi 32 bit: 33.1
+
+% 20200610 : 258 pages
+% mingw all in: 10.6
+
+% 20201204 : 260 pages
+% mingw all in: 10.0
+
+% 20201204 : 262 pages
+% mingw all in: 9.9
+
+% 20200720 : 258 pages / all in
+%
+% mingw : 10.6 (sometimes less)
+% rpi 32 : 32.3
+% rpi 64 : 26.0 (overclocked)
+% amd 10 fitlet : 28.2
+
+% But speed can differ a bit per compilation (upto .5 seconds maybe due to how
+% compiled code is organized which might influence caching. Who knows ... (or
+% cares). For instance at 20200407 I measured 10.9 seconds after some new low level
+% metapost magic was added but who knows if that was the reason, because mp
+% processing is already fast. A week later, at 20200415, a by then 254 page file
+% took 10.5 seconds, that is, we were at exactly 24 pages per second but after
+% switching to gcc9 it dropped again. In december 2020, with IPO enabled I crossed
+% the 26 pps barrier and went below 10 seconds but that was also after some further
+% cleanup in \LMTX.
+
+% msvc 1899k (2% slower than mingw)
+% msvc /GL 2297k (similar to mingw)
+% msvc /GL /Ob3 2847k (not faster than /GL)
+% msvc /Ob3 2052k (slower than /Ob2)
+% msvc /Ob1 1763k (slower than /Ob2)
+% clang 2460k (15% slower than mingw)
+
+% Thanks to sebastian.miele@gmail.com for close reading the manual and sending
+% fixes.
+
+% \listcallbackmode0
+
+% 290 pages, 10.8 sec, 292M lua, 99M tex, 158 instances
+% 290 pages, 9.5 sec, 149M lua, 35M tex, 30 instances
+
+\enableexperiments[fonts.compact]
+% \enabledirectives[fonts.injections.method=advance] % tricky ... not all xoffsets are advance robust
+
+\pushoverloadmode \unprotect
+ % test code
+\protect \popoverloadmode
+
+\enabletrackers[system.usage=summary]
+
+\environment luametatex-style
+\environment luametatex-private
+
+\startdocument
+ [manual=LuaMeta\TeX,
+ %status=experimental,
+ version=\cldcontext{status.luatex_verbose}]
+
+\component luametatex-titlepage
+\component luametatex-firstpage
+
+\startfrontmatter
+ \component luametatex-contents
+ \component luametatex-introduction
+\stopfrontmatter
+
+\startbodymatter
+ \component luametatex-preamble
+ \component luametatex-differences
+ \component luametatex-modifications
+ \component luametatex-lua
+ \component luametatex-enhancements
+ \component luametatex-fonts
+ \component luametatex-languages
+ \component luametatex-math
+ \component luametatex-nodes
+ \component luametatex-callbacks
+ \component luametatex-tex
+ \component luametatex-metapost
+ \component luametatex-pdf
+ \component luametatex-libraries
+ \component luametatex-primitives
+\stopbodymatter
+
+\startbackmatter
+ % \component luametatex-rejected % local file
+ \component luametatex-codes
+ \component luametatex-registers
+ \component luametatex-statistics
+\stopbackmatter
+
+\stopdocument
+