summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2020-07-17 21:45:53 +0000
committerKarl Berry <karl@freefriends.org>2020-07-17 21:45:53 +0000
commit522964b40afca63660485cdcfc577b25bdf02e35 (patch)
treea49bd7b714409b6b06338b793d7c9921f0398f8f /Master/texmf-dist/source/latex/l3kernel/l3token.dtx
parent7f75afca3aa3d645c67ee84efeb01c08482d71ac (diff)
l3 (17jul20)
git-svn-id: svn://tug.org/texlive/trunk@55860 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3token.dtx42
1 files changed, 28 insertions, 14 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
index d12c5deeed7..cffbea2c975 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-06-18}
+% \date{Released 2020-07-17}
%
% \maketitle
%
@@ -992,11 +992,7 @@
% other engines) and category code~$13$.
% \item A character token, characterized by its character code and
% category code (one of $1$, $2$, $3$, $4$, $6$, $7$, $8$, $10$,
-% $11$ or~$12$ whose meaning is described below).\footnote{In
-% \LuaTeX{}, there is also the case of \enquote{bytes}, which behave as
-% character tokens of category code $12$~(other) and character code
-% between $1114112$ and~$1114366$. They are used to output
-% individual bytes to files, rather than UTF-8.}
+% $11$ or~$12$ whose meaning is described below).
% \end{itemize}
% There are also a few internal tokens. The following list may be
% incomplete in some engines.
@@ -1017,6 +1013,19 @@
% \item Tricky programming might access a frozen |\endwrite|.
% \item Some frozen tokens can only be accessed in interactive
% sessions: |\cr|, |\right|, |\endgroup|, |\fi|, |\inaccessible|.
+% \item In \LuaTeX{}, there is also the strange case of \enquote{bytes}
+% |^^^^^^1100|$x$$y$ where $x,y$ are any two lowercase hexadecimal
+% digits, so that the hexadecimal number ranges from
+% $"\text{110000}=1114112$ to~$"\text{1100ff}=1114367$. These are
+% used to output individual bytes to files, rather than UTF-8. For
+% the purposes of token comparisons they behave like non-expandable
+% primitive control sequences (\emph{not characters}) whose
+% \tn{meaning} is \verb*|the character | followed by the given byte.
+% If this byte is in the range |80|--|ff| this gives an ``invalid
+% utf-8 sequence'' error: applying \cs{token_to_str:N} or
+% \cs{token_to_meaning:N} to these tokens is unsafe. Unfortunately,
+% they don't seem to be detectable safely by any means except perhaps
+% Lua code.
% \end{itemize}
%
% The meaning of a (non-active) character token is fixed by its category
@@ -1028,7 +1037,7 @@
% \begin{itemize}
% \item[1] begin-group character (|group_begin|, often |{|),
% \item[2] end-group character (|group_end|, often |}|),
-% \item[3] math shift character (|math_toggle|, often |$|),
+% \item[3] math shift character (|math_toggle|, often |$|), %^^A $
% \item[4] alignment tab character (|alignment|, often |&|),
% \item[6] macro parameter character (|parameter|, often |#|),
% \item[7] superscript character (|math_superscript|, often |^|),
@@ -1058,18 +1067,16 @@
% \item a font selection command,
% \item undefined.
% \end{itemize}
-% Macros be \tn{protected} or not, \tn{long} or not (the opposite of
+% Macros can be \tn{protected} or not, \tn{long} or not (the opposite of
% what \LaTeX3 calls |nopar|), and \tn{outer} or not (unused in
% \LaTeX3). Their \tn{meaning} takes the form
% \begin{quote}
-% \meta{properties} |macro:|\meta{parameters}|->|\meta{replacement}
+% \meta{prefix} |macro:|\meta{argument}|->|\meta{replacement}
% \end{quote}
-% where \meta{properties} is among \tn{protected}\tn{long}\tn{outer},
-% \meta{parameters} describes parameters that the macro expects, such as
+% where \meta{prefix} is among \tn{protected}\tn{long}\tn{outer},
+% \meta{argument} describes parameters that the macro expects, such as
% |#1#2#3|, and \meta{replacement} describes how the parameters are
-% manipulated, such as~|#2/#1/#3|.
-%
-% ^^A todo Bruno: discuss here some other subtleties of space tokens? when looking for numbers, when looking for equal signs in let, in expressions, etc.
+% manipulated, such as~|\int_eval:n{#2+#1*#3}|.
%
% Now is perhaps a good time to mention some subtleties relating to
% tokens with category code $10$ (space). Any input character with this
@@ -1087,6 +1094,13 @@
% single tokens \enquote{N-type}, as they are suitable to be used as an
% argument for a function with the signature~\texttt{:N}.
%
+% When a macro takes a delimited argument \TeX{} scans ahead until
+% finding the delimiter (outside any pairs of begin-group/end-group
+% explicit characters), and the resulting list of tokens (with outer
+% braces removed) becomes the argument. Note that explicit space
+% characters at the start of the argument are \emph{not} ignored in this
+% case (and they prevent brace-stripping).
+%
% \end{documentation}
%
% \begin{implementation}