diff options
Diffstat (limited to 'macros/latex/contrib')
28 files changed, 5740 insertions, 998 deletions
diff --git a/macros/latex/contrib/pythonimmediate/DEPENDS.txt b/macros/latex/contrib/pythonimmediate/DEPENDS.txt new file mode 100644 index 0000000000..74052c8330 --- /dev/null +++ b/macros/latex/contrib/pythonimmediate/DEPENDS.txt @@ -0,0 +1,4 @@ +saveenv +currfile +l3keys2e +precattl diff --git a/macros/latex/contrib/pythonimmediate/README b/macros/latex/contrib/pythonimmediate/README new file mode 100644 index 0000000000..c164e2ca9b --- /dev/null +++ b/macros/latex/contrib/pythonimmediate/README @@ -0,0 +1,24 @@ +pythonimmediate -- Library to run Python code + \fileinfo. + +Released under the LaTeX Project Public License v1.3c or later +See http://www.latex-project.org/lppl.txt + +Report bugs at https://github.com/user202729/TeXlib + +======== + +Copyright 2022 user202729 + +This work may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this license or +(at your option) any later version. The latest version of this license is in +the file: + + http://www.latex-project.org/lppl.txt + +This work has the LPPL maintenance status `maintained'. + +The Current Maintainer of this work is user202729. + +This work consists of the files pythonimmediate.sty, pythonimmediate_script_pytotex.py, pythonimmediate_script_textopy.py. diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate.pdf b/macros/latex/contrib/pythonimmediate/pythonimmediate.pdf Binary files differnew file mode 100644 index 0000000000..67bb6f1259 --- /dev/null +++ b/macros/latex/contrib/pythonimmediate/pythonimmediate.pdf diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate.sty b/macros/latex/contrib/pythonimmediate/pythonimmediate.sty new file mode 100644 index 0000000000..6ed72780bf --- /dev/null +++ b/macros/latex/contrib/pythonimmediate/pythonimmediate.sty @@ -0,0 +1,183 @@ +% File: pythonimmediate.sty +% Copyright 2022 user202729 +% +% This work may be distributed and/or modified under the conditions of the +% LaTeX Project Public License (LPPL), either version 1.3c of this license or +% (at your option) any later version. The latest version of this license is in +% the file: +% +% http://www.latex-project.org/lppl.txt +% +% This work has the LPPL maintenance status `maintained'. +% +% The Current Maintainer of this work is user202729. + +\ProvidesExplPackage{pythonimmediate}{2022/12/23}{0.0.0}{Library to run Python code} + +\RequirePackage{saveenv} +\RequirePackage{currfile} +\RequirePackage{l3keys2e} +\RequirePackage{precattl} + + +\cs_generate_variant:Nn \str_set:Nn {NV} +\cs_generate_variant:Nn \str_if_eq:nnT {VnT} +\cs_generate_variant:Nn \msg_error:nnn {nnV} +\cs_generate_variant:Nn \str_if_eq:nnF {xnF} +\cs_generate_variant:Nn \str_range:nnn {Vnn} +\cs_generate_variant:Nn \str_if_eq:nnF {VnF} +\cs_generate_variant:Nn \str_if_in:nnF {VnF} +\cs_generate_variant:Nn \tl_build_put_right:Nn {NV} + +%\bench before~rescan. +%\bench after~rescan. + +%\GenerateVariantsFile:n{\rescansynclastfilename} + +\str_set:Nn \_pythonimmediate_outputdir{.} +\tl_set:Nn \_pythonimmediate_textopy_script_path{} +\str_set:Nn \_pythonimmediate_mode{multiprocessing-network} +\str_set:Nn \_pythonimmediate_python_executable{python3} +\keys_define:nn{pythonimmediate}{ + outputdir.tl_set_x:N=\_pythonimmediate_outputdir, + %outputdir.default:n={.}, % huh does not work? + + mode.tl_set:N=\_pythonimmediate_mode, + scriptpath.tl_set_x:N=\_pythonimmediate_textopy_script_path, + + python-executable.tl_set:N=\_pythonimmediate_python_executable, +} +\ProcessKeysOptions{pythonimmediate} + +\str_set:NV \_pythonimmediate_outputdir \_pythonimmediate_outputdir +\str_set:NV \_pythonimmediate_mode \_pythonimmediate_mode +\str_set:NV \_pythonimmediate_textopy_script_path \_pythonimmediate_textopy_script_path + +\msg_new:nnn {pythonimmediate} {shell-fail} {Shell~command~execution~failed!} +\msg_new:nnn {pythonimmediate} {cannot-determine-script-path} {Cannot~determine~script~path!} +\msg_new:nnn {pythonimmediate} {cannot-read-line} {Cannot~read~line!} +\msg_new:nnn {pythonimmediate} {internal-error} {Internal~error!} +\msg_new:nnn {pythonimmediate} {invalid-mode} {Invalid~mode:~'#1'.} + + +\bool_new:N \_pythonimmediate_mode_multiprocessing_network +\bool_new:N \_pythonimmediate_mode_unnamed_pipe +\str_if_eq:VnT \_pythonimmediate_mode {multiprocessing-network} {\bool_set_true:N \_pythonimmediate_mode_multiprocessing_network} +\str_if_eq:VnT \_pythonimmediate_mode {unnamed-pipe} {\bool_set_true:N \_pythonimmediate_mode_unnamed_pipe} +\bool_if:nF {\_pythonimmediate_mode_multiprocessing_network || \_pythonimmediate_mode_unnamed_pipe} { + \msg_error:nnV {pythonimmediate} {invalid-mode} \_pythonimmediate_mode +} + +% we need to persistently open the file anyway, so using LaTeX3 stream reference counting doesn't help +\newread \_pythonimmediate_read_file + + + + +%\bench before~setup. + +\begingroup + \endlinechar=-1~ + \tl_if_empty:NT \_pythonimmediate_textopy_script_path { + % ======== first use kpsewhich to get the _pythonimmediate_script_path here ======== + % (abuse \_pythonimmediate_read_file variable for this purpose) + \openin \_pythonimmediate_read_file=|"kpsewhich~ pythonimmediate_script_textopy.py"~ + \readline \_pythonimmediate_read_file to \_pythonimmediate_textopy_script_path + %\bench after~get~textopy~path. + \ifeof \_pythonimmediate_read_file + \msg_error:nn {pythonimmediate} {cannot-determine-script-path} + \fi + } + + \str_if_eq:xnF {\str_range:Vnn \_pythonimmediate_textopy_script_path {-10} {-1}} {textopy.py} { + \msg_error:nn {pythonimmediate} {cannot-determine-script-path} + } + + \newwrite \_pythonimmediate_write_file + + %\bench before~openin. + + % ======== open persistent pipes to the child processes + \bool_if:NTF \_pythonimmediate_mode_unnamed_pipe { % in this case make sure the pipe remains open... + \openin \_pythonimmediate_read_file=|"sleep~ 0.5s|\_pythonimmediate_python_executable \space \str_range:Vnn \_pythonimmediate_textopy_script_path {1} {-11} pytotex.py \space \_pythonimmediate_mode"~ % we must use the primitive here to use the pipe file path + % TODO sleep infinity causes some resource leak (the process will not exit after TeX exits). Need to fix + % but sleep too little might be problematic that it exits before the setup is done + % we can just assume machines are not that slow + } { + \openin \_pythonimmediate_read_file=|"\_pythonimmediate_python_executable \space \str_range:Vnn \_pythonimmediate_textopy_script_path {1} {-11} pytotex.py \space \_pythonimmediate_mode"~ % we must use the primitive here to use the pipe file path + } + + %\bench after~openin. + + %\bench before~openout. + \immediate\openout \_pythonimmediate_write_file=|"\_pythonimmediate_python_executable \space \_pythonimmediate_textopy_script_path \space \_pythonimmediate_mode"~ + % both processes must be before the \readline above so that the 2 Python processes are started "in parallel" + %\bench after~openout. + + \readline \_pythonimmediate_read_file to \_pythonimmediate_dummy_line % endlinechar still -1 + %\bench after~read~line. + \bool_if:NT \_pythonimmediate_mode_multiprocessing_network { + \str_if_eq:VnF \_pythonimmediate_dummy_line {listener-setup-done} { + \msg_error:nn {pythonimmediate} {cannot-read-line} + } + } + \bool_if:NT \_pythonimmediate_mode_unnamed_pipe { + \str_if_in:VnF \_pythonimmediate_dummy_line {pytotex_pid=} { + \msg_error:nn {pythonimmediate} {cannot-read-line} + } + } + \bool_if:NT \_pythonimmediate_mode_unnamed_pipe { + \immediate\write\_pythonimmediate_write_file {\_pythonimmediate_dummy_line} % which contains pytotex's pid + } +\endgroup + +%\bench after~setup. + + +% read one block of \TeX\ code from Python, store into the specified variable +% the block is delimited using |surround_delimiter()| in Python i.e. the first and last line are identical +% new lines are represented with ^^J +\cs_new_protected:Npn \_pythonimmediate_gread_block:N #1 { + \begingroup + \endlinechar=10~ % affect \readline + \readline \_pythonimmediate_read_file to \_pythonimmediate_delimiter + + \tl_build_gbegin:N #1 + \readline \_pythonimmediate_read_file to \_pythonimmediate_line + + %\bench read~first~line. + + \bool_do_until:nn {\tl_if_eq_p:NN \_pythonimmediate_delimiter \_pythonimmediate_line} { + \tl_build_gput_right:NV #1 \_pythonimmediate_line + \ifeof \_pythonimmediate_read_file + \msg_error:nn {pythonimmediate} {internal-error} + \fi + \readline \_pythonimmediate_read_file to \_pythonimmediate_line + } + \tl_build_gend:N #1 + \endgroup +} +\cs_generate_variant:Nn \tl_build_gput_right:Nn {NV} + +\cs_new_protected:Npn \_pythonimmediate_read_block:N #1 { + \_pythonimmediate_gread_block:N \_pythonimmediate_block + \tl_set_eq:NN #1 \_pythonimmediate_block +} + + +% read one block of \TeX\ code from Python and |\scantokens|-run it +% the content inside is the actual TeX code to be executed +\cs_new_protected:Npn \_pythonimmediate_run_block: { + \_pythonimmediate_gread_block:N \_pythonimmediate_code + \begingroup + \newlinechar=10~ + \expandafter + \endgroup + \scantokens \expandafter{\_pythonimmediate_code} +} % trick described in https://tex.stackexchange.com/q/640274 to scantokens the code with \newlinechar=10 + +% bootstrap code +%\bench before~bootstrap. +\_pythonimmediate_run_block: +%\bench after~bootstrap. + diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate.tex b/macros/latex/contrib/pythonimmediate/pythonimmediate.tex new file mode 100644 index 0000000000..877bc3a05e --- /dev/null +++ b/macros/latex/contrib/pythonimmediate/pythonimmediate.tex @@ -0,0 +1,535 @@ +\ProvidesFile{pythonimmediate.tex}[2022/12/23 0.0.0 Library to run Python code] +\RequirePackage{fvextra} +\documentclass{l3doc} +\usepackage{tikz} +\usetikzlibrary{calc} +\usetikzlibrary{arrows.meta} +\EnableCrossrefs +\CodelineIndex +\fvset{breaklines=true,gobble=0,tabsize=4,frame=single,numbers=left,numbersep=3pt} + +\AtBeginDocument{\DeleteShortVerb\"} % https://tex.stackexchange.com/a/650966/250119 +\MakeOuterQuote{"} + +\newcommand\reg{régime} +\newcommand\DescribePython{\DescribeMacro} % hack + + +\begin{document} + + +% hack to make quotes inside |...| straight +\edef\temp{\def\noexpand|{\noexpand\Verb\string|}}\temp +% basically execute \def| { \Verb ⟨catcode other |⟩ } + +\GetFileInfo{\jobname.tex} + +\title{\pkg{\jobname} --- \fileinfo +\thanks{This file describes version \fileversion, last revised \filedate.} +} +\author{user202729} +\date{Released \filedate} + +\maketitle + +\begin{abstract} + \fileinfo. +\end{abstract} + +\section{Motivation} + +Just like \pkg{Perl\TeX} or \pkg{PyLua\TeX} (and unlike \pkg{Python\TeX} or \pkg{lt3luabridge}), +this only requires a single run, and variables are persistent throughout the run. + +Unlike \pkg{Perl\TeX} or \pkg{PyLua\TeX}, there's no restriction on compiler or script required to run the code. + +There's also debugging functionalities -- \TeX\ errors results in Python traceback, and Python error results in \TeX\ traceback. +Errors in code executed with the |pycode| environment gives the correct traceback point to the Python line of code in the \TeX\ file. + +For advanced users, this package allows the user to manipulate the \TeX\ state directly from within Python, +so you don't need to write a single line of \TeX\ code. + +\section{Installation} + +%You need to install Python \pkg{daemoniker} and \pkg{psutil} package. + +The package should work out of the box with no additional Python package needed. + +\subsection{Installation on Overleaf} + +At the point of writing, this package can be used on Overleaf. + +Instruction: + +\begin{itemize} + \item Download the following files and place it in the root folder of Overleaf: + \begin{itemize} + \item \file{saveenv.sty} + \item \file{precattl.sty} + \item \file{pythonimmediate.sty} + \item \file{pythonimmediate_script_textopy.py} + \item \file{pythonimmediate_script_pytotex.py} + \end{itemize} + \item Write the following in the preamble: +\begin{verbatim} +\usepackage[abspath]{currfile} +\usepackage[mode=unnamed-pipe]{pythonimmediate} +\end{verbatim} +Refer to \ref{troubleshoot-source-file-not-found} for explanation of the |abspath| option. +\end{itemize} + +For some unknown reason in the default mode on Overleaf (|\nonstopmode|), when there's an error +the log file might be truncated, +so in that case consider writing |\errorstopmode|. + +Refer to \ref{troubleshoot-Python-error} to read the error traceback in case of Python error. + +\section{Usage} + +\subsection{Package options} + +\DescribeOption{outputdir=} +Specify the output directory if it's not the default value. + +\begin{texnote} + The value will be |x|-expanded. +\end{texnote} + +Using the \pkg{outputdir} package, it's possible to determine the output directory automatically, subject to restrictions. + +An usage example, if you invoked |pdflatex| with the command +\begin{verbatim} +pdflatex --output-directory=/tmp/ a.tex +\end{verbatim} +then in the file \file{a.tex} you should have +\begin{verbatim} +\usepackage[outputdir=/tmp/]{pythonimmediate} +\end{verbatim} + +\DescribeOption{mode=} +Method to communicate between \TeX\ and Python. Usually the method will be automatically detected. + +Possible values include |multiprocessing-network| and |unnamed-pipe|. + +\subsection{\TeX\ interface} + +The interface mimics those in popular packages such as \pkg{Python\TeX} or \pkg{PyLua\TeX}. + +\subsubsection{Inline commands} + +\DescribeMacro{\py}\label{py-command} +Evaluate some Python expression, consider the result as a string, then execute the result as \TeX\ command. + +\begin{texnote} + The command is not expandable, and the argument will be fully expanded with the active |~| set to |\relax|, + |\set@display@protect| executed and |\escapechar=-1|, then the result + passed to Python as a string. +\end{texnote} + +Which, for the users who are not familiar with \TeX\ terminology, roughly means the following: +\begin{itemize} + \item the value can only be used to typeset text, + it must not be used to pass "values" to other \LaTeX\ commands. + + The following is legal: +\begin{verbatim} +The value of $1+1$ is $\py{1+1}$. +\end{verbatim} +The following is illegal, as the result (2) can only be used to typeset text, not passed to another command that expect a "value": +\begin{verbatim} +\setcounter{abc}{\py{1+1}} +\end{verbatim} +A possible workaround is: +\begin{verbatim} +\py{ r'\\setcounter{abc}{' + str(1+1) + '}' } +\end{verbatim} +In this example it works without escaping the |{}| characters, but if the Python code has those unbalanced then you can escape them as mentioned below. + + \item Special characters can be "escaped" simply by prefixing the character with backslash. + + For example +\begin{verbatim} +\pyc{assert len('\ \ ')==2} +\pyc{assert ord('\\\\')==0x5c} +\pyc{assert ord('\%') ==0x25} +\end{verbatim} +In the examples above, Python "sees" (i.e. the Python code being executed is) +\begin{verbatim} +assert len(' ')==2 +assert ord('\\')==0x5c +assert ord('%') ==0x25 +\end{verbatim} +respectively. + + \item Macros will be expanded. +\begin{verbatim} +\def\mycode{1+1} +The value of $1+1$ is $\py{\mycode}$. +\end{verbatim} +\end{itemize} + + +\DescribeMacro{\pyc} +Execute some Python code provided as an argument (the argument will be interpreted as described above). + +The command is not expandable -- roughly speaking, you can only use this at "top level". + +Any output (as described in \ref{print-to-TeX}) will be typesetted. + +The difference between |\py| and |\pyc| is that the argument of |\py| should be a Python expression +(suitable for passing into |eval()| Python function) while the argument of |\pyc| should be a Python +statement (suitable for passing into |exec()| Python function). + +Therefore, +\begin{itemize} + \item |\py{1+1}| will typeset 2. + \item |\pyc{1+1}| is valid, but will do nothing just like |exec("1+1")|. + \item |\py{x=1}| is invalid. + \item |\pyc{x=1}| is valid and assigns the variable |x| to be 1. +\end{itemize} + +\DescribeMacro{\pycq} +Same as above, but output (\ref{print-to-TeX}) will not be typesetted. + +\DescribeMacro{\pyfile} +Given an argument being the file name, execute that file. + +\DescribeMacro{\pys} +Performs "string interpolation", the same way as \pkg{Python\TeX}. +(not yet implemented) + +\subsubsection{Environments} + +\DescribeEnv{pycode} +Verbatim-like environment that executes the code inside as Python. + +Example usage: The following will typeset |123| +\begin{verbatim} +\begin{pycode} +pythonimmediate.print("123") +\end{pycode} +\end{verbatim} + +Special note: white spaces at the end of lines are preserved. + +Any output (as described in \ref{print-to-TeX}) will be typesetted. + +\DescribeEnv{pycodeq} +Same as above, but output will not be typesetted. + +\DescribeEnv{pysub} +Not yet implemented. + +\subsection{Python interface} + +The \TeX\ interface is only used to call Python. Apart from that, all the work can be done on the Python side. + +All functions in this section should be imported from |pythonimmediate| package, unless specified otherwise. + +\subsubsection{Print to \TeX}\label{print-to-TeX} + +\DescribePython{.print()} +\DescribePython{.file} +These functions are used in |\pyc| command or |pycode| environment. + +Unlike most other packages, using |print()| function in Python will print to the console (\TeX\ standard output). +In order to print \TeX\ code to be executed, you can do one of +\begin{verbatim} +pythonimmediate.print(...) +print(..., file=pythonimmediate.file) +with contextlib.redirect_stdout(pythonimmediate.file): + print(...) +\end{verbatim} +Note that in quiet environments, |pythonimmediate.file| is None, the second variant using |print()| will print to stdout +instead of suppress output. The third variant works as expected. + +All output will be buffered until the whole Python code finishes executing. +In order to typeset the text immediately use one of the advanced commands. + + + +\DescribePython{.newcommand()} +\DescribePython{.renewcommand()} +Same as \LaTeX's |\newcommand| and |\renewcommand|. Can be used as follows: + +\begin{verbatim} +from pythonimmediate import newcommand, renewcommand + +@newcommand +def function(): + ... +# define |\function| in TeX + +@newcommand("controlsequencename") +def function(): + ... +# define |\controlsequencename| in TeX + +def function(): + ... +newcommand("controlsequencename", function) +\end{verbatim} + +\DescribePython{.get_arg_str()} +\DescribePython{.get_optional_arg_str()} +\DescribePython{.get_verb_arg()} +\DescribePython{.get_multiline_verb_arg()} +\DescribePython{.peek_next_char()} +\DescribePython{.get_next_char()} +There are those functions that is mostly understandable to an inexperienced \LaTeX\ user, +and should be sufficient for a lot of programming works. + +This is an example of how the functions could be used. The name should be mostly self-explanatory. + +\begin{verbatim} +\documentclass{article} +\usepackage{pythonimmediate} +\begin{document} +\begin{pycode} +from pythonimmediate import newcommand, peek_next_char, get_next_char, get_arg_str, print +@newcommand +def innerproduct(): + s = get_arg_str() # in the example below this will have the value '\mathbf{a},\mathbf{b}' + x, y = s.split(",") # it's just a Python string, manipulate normally (but be careful of comma inside braces, parse the string yourself) + print(r"\left\langle" + x + r"\middle|" + y + r"\right\rangle") + +@newcommand +def fx(): + if peek_next_char() == "_": + get_next_char() + subscript = get_arg_str() + print("f_{" + subscript + "}(x)") + else: + print("f(x)") + +@newcommand +def sumManyArgs(): + s = 0 + while peek_next_char() == "{": + i = get_arg_str() + s += int(i) + print(str(s)) +\end{pycode} +$1+2+3 = \sumManyArgs{1}{2}{3}$ + +$\innerproduct{\mathbf{a},\mathbf{b}}=1$ + +$\fx = 1$, $\fx_i = 2$, $\fx_{ij} = 3$ +\end{document} +\end{verbatim} + +It will typeset: + +\begin{quote} +$1+2+3=6$ + +$\left\langle\mathbf{a}\middle\vert\mathbf{b}\right\rangle=1$ + +$f(x)=1$, $f_i(x)=2$, $f_{ij}(x)=3$ +\end{quote} + +\DescribePython{.get_arg_estr()} +\DescribePython{.get_optional_arg_estr()} +Similar to some functions above, except that the argument is fully expanded and "escapes" of common characters are handled correctly, +similar to how |\py| command (\ref{py-command}) reads its arguments. + +\DescribePython{.execute()} +Takes a string and execute it immediately. (so that any |.execute()| will be executed before any |.print()|) + +Assuming \TeX\ is in errorstopmode (i.e. errors halt \TeX\ execution), +any error in \TeX\ will create an error in Python and the traceback should point to the correct line of code. + +For example, in the following code + +\begin{verbatim} +\documentclass{article} +\usepackage{tikz} +\usepackage{pythonimmediate} +\begin{document} + +\begin{tikzpicture} +\begin{pycode} +from pythonimmediate import execute +execute(r'\draw (0, 0) to (1, 1);') +execute(r'\draw (2, 2) to (p);') +execute(r'\draw (3, 3) to (4, 4);') +\end{pycode} +\end{tikzpicture} + +\end{document} +\end{verbatim} +each |\draw| command will be executed immediately when the Python |.execute()| function is executed, +and as the second line throws an error, the Python traceback will point to that line. + +\section{Troubleshooting} + +\subsection{"Source file not found!" error message}\label{troubleshoot-source-file-not-found} + +In order to obtain the exact code with trailing spaces and produce error traceback +point to the correct \TeX\ file, the Python code need to know the full path to the current +\TeX\ file for the |pycode| environment. + +Nevertheless, this is difficult and does not always work +(refer to the documentation of \pkg{currfile} for details), so this message is issued +when the file cannot be found. + +In that case try the following fixes: + +\begin{itemize} + \item Include |\usepackage[abspath]{currfile}| at the start of the document, after the |\documentclass| line. + (this option is not included by default because it's easy to get package clash, and usually \pkg{currfile} without + the |abspath| option works fine -- unless custom |jobname| is used) + \item Explicitly override |currfilename| or |currfileabspath| -- for example +\begin{verbatim} +\def\currfilename{main.tex} +\end{verbatim} + Technically this is an abuse of the \pkg{currfile} package API, but it usually works regardless. +\end{itemize} + +\subsection{"Python error" error message}\label{troubleshoot-Python-error} + +In case of Python error, the Python traceback is included in the terminal and \TeX\ log file. + +Search for "Python error traceback" before the error line in the log file. + +On Overleaf, you can either view the log file ("Raw logs" section) +or the traceback on stderr (download \file{output.stderr} file) + +\subsection{"\TeX\ error" error message}\label{troubleshoot-TeX-error} + +If an error occur in \TeX, traceback cannot be included in the log file. + +Besides, this can only be detected in |\errorstopmode|. Such an error will always halt \TeX, +and Python will be force-exited after printing the error traceback. + +On Overleaf, download \file{output.stderr} file to read the traceback. + +\section{Implementation note} + +Communication between \TeX\ and Python are done by opening two pseudo-files from the output of a Python process |textopy| +(similar to |\ior_shell_open:Nn|) +and to the input of another Python process |pytotex| (this would be |\iow_shell_open:Nn|, if \LaTeX3 have such a function). + +There are various methods for the 2 Python child processes to communicate with each other. +After some initial bootstrapping to setup the communication, we can consider only the |textopy| script, the other +merely serves as the bridge to send input to \TeX. + +The communication protocol is a little complicated, since it must support nesting bidirectional execution of \TeX\ and Python. + +Besides, I believe it's not possible to make a "background listener" on the \TeX\ side, so it must keep track of whether a command should be read from Python and executed. + +Currently, exception handling (throwing a Python exception in a nested Python function, catch it in the outer Python function) is not supported. + +These are some examples of what could happen in the communication protocol. + +\ExplSyntaxOn +\let\fpEval\fp_eval:n +\ExplSyntaxOff + +\begingroup + +\tikzset{myarrow/.style={-{Latex[length=2mm]}}} + +\def\blockWidth{1} +\def\blockHeight{7} +\def\separation{10.5} +\def\step{0.7} + +\def\left #1{ + \draw [myarrow] (p) ++(\separation, 0) -- node [above] {\small #1} ++(-\separation, 0); + \path (p) ++ (0, -\step) coordinate (p); +} +\def\right #1{ + \draw [myarrow] (p) -- node [above] {\small #1} ++(\separation, 0); + \path (p) ++ (0, -\step) coordinate (p); +} + +\def\initDrawing{ + \path (\blockWidth, \fpEval{-\step}) coordinate (p); +} + +\newdimen \tempx +\newdimen \tempy + +\ExplSyntaxOn +\let \dimToDecimalInUnit \dim_to_decimal_in_unit:nn +\ExplSyntaxOff + +\def\drawBlocks{ + \path [transform canvas] (p); + \pgfgetlastxy{\tempx}{\tempy} + \edef\blockHeight{ \fpEval{ - \dimToDecimalInUnit{\tempy}{1cm} } } + + \draw (0, \fpEval{-\blockHeight}) rectangle node [rotate=90] {\TeX} (\blockWidth, 0); + \draw (\fpEval{\blockWidth+\separation}, \fpEval{-\blockHeight}) rectangle node [rotate=270] {Python} (\fpEval{\separation+2*\blockWidth}, 0); +} + +\vspace{5pt} +\begin{tikzpicture} +\edef\blockHeight{\fpEval{\step*3}} +\initDrawing +\right{execute Python code: print(1)} +\left{execute \TeX\ code: 1} +\drawBlocks +\end{tikzpicture} + +Nevertheless, there may be more complicated cases where the Python code itself may call \TeX\ code before actually returns: + +\vspace{5pt} +\begin{tikzpicture} +\edef\blockHeight{\fpEval{\step*5}} +\initDrawing +\right{execute Python code: print(var(a)*2)} +\left{execute \TeX\ code: sendtopy(a); execute another command} +\right{123} +\left{execute \TeX\ code: 123123} +\drawBlocks +\end{tikzpicture} + +Or: + +\vspace{5pt} +\begin{tikzpicture} +\edef\blockHeight{\fpEval{\step*7}} +\initDrawing +\right{execute Python code: tex.exec(a=456); print(var(a)*2)} +\left{execute \TeX\ code: a=456; sendtopy(done); execute another command} +\right{done} +\left{execute \TeX\ code: sendtopy(a); execute another command} +\right{456} +\left{456456} +\drawBlocks +\end{tikzpicture} + +\endgroup + +The Python side must not just listen for "done" command back, but must potentially call a nested loop. + +The exact protocol is: +\begin{itemize} + \item "execute Python code" sends from \TeX\ to Python has a single line "|i|\meta{handler name}", + followed by any number of arguments (depends on the handler). + + Refer to the |define_TeX_call_Python| internal function for details. + + \item "done" sends from \TeX\ to Python has the format "|r|\meta{optional return value as a string in a single line}". + + This is sent by executing \TeX\ command |\pythonimmediatecontinue|, which takes a single argument to be e-expanded using |\write| + as the "return value". + + \item "execute \TeX\ code" sends from Python to \TeX\ must only be sent when the \TeX\ side listens for a command. + It consist of a single line specify the "command name", which \TeX\ will + execute the command named + |\__run_|\meta{command name}|:| + which must already be defined on the \TeX\ side. + + The command itself might contain additional code to execute more code, e.g. by reading more lines from Python. + + Refer to the |define_Python_call_TeX| internal function for details. +\end{itemize} + + + +\PrintIndex + +\end{document} diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate_script_pytotex.py b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_pytotex.py new file mode 100755 index 0000000000..abb71a2631 --- /dev/null +++ b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_pytotex.py @@ -0,0 +1,57 @@ +#!/bin/python3 +""" +======== Py-to-TeX half ======== + +receive things that should be passed to TeX from TeX-to-Py half, +then pass to TeX. + +the things that are sent should already be newline-terminated if necessary. + +user code are not executed here. +""" + +import sys +import signal +signal.signal(signal.SIGINT, signal.SIG_IGN) # when the other half terminates this one will terminates "gracefully" + +#debug_file=open(Path(tempfile.gettempdir())/"pythonimmediate_debug_pytotex.txt", "w", encoding='u8', buffering=2) +#debug=functools.partial(print, file=debug_file, flush=True) +debug=lambda *args, **kwargs: None + +import argparse +parser=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("mode", choices=["multiprocessing-network", "unnamed-pipe"]) +args=parser.parse_args() + + +# ======== setup communication method. Just an infinite loop print whatever being sent. + +if args.mode=="multiprocessing-network": + from multiprocessing.connection import Listener + + address=("localhost", 7348) + #address="./pythonimmediate.socket" + with Listener(address) as listener: + print("listener-setup-done", flush=True) + with listener.accept() as connection: + debug("accepted a connection") + while True: + try: + data=connection.recv_bytes() + debug(" data=", data) + sys.__stdout__.buffer.write(data) # will go to TeX + sys.__stdout__.buffer.flush() + except EOFError: break + +elif args.mode=="unnamed-pipe": + import os + sys.stdout.write("pytotex_pid=" + str(os.getpid()) + "\n") + sys.stdout.flush() + for line in sys.stdin: + sys.stdout.write(line) + sys.stdout.flush() + +else: + assert False + +# ======== diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate_script_textopy.py b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_textopy.py new file mode 100755 index 0000000000..7e4f4a7de9 --- /dev/null +++ b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_textopy.py @@ -0,0 +1,2295 @@ +#!/bin/python3 +""" +======== TeX-to-Py half ======== + +receive commands from TeX, then execute it here +""" + + +#from __future__ import annotations +import sys +import os +import inspect +import contextlib +import io +import functools +from typing import Optional, Union, Callable, Any, Iterator, Protocol, Iterable, Sequence, Type, Tuple, List, Dict +import typing +from abc import ABC, abstractmethod +from pathlib import Path +from dataclasses import dataclass +import tempfile +import signal +import traceback +import re +import collections +import enum + + +def user_documentation(x: Union[Callable, str])->Any: + return x + + + +#debug=functools.partial(print, file=sys.stderr, flush=True) # unfortunately this is async ... or so it seems...? +#debug_file=open(Path(tempfile.gettempdir())/"pythonimmediate_debug_textopy.txt", "w", encoding='u8', buffering=2) +#debug=functools.partial(print, file=debug_file, flush=True) +debug=lambda *args, **kwargs: None + + +import argparse +parser=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("mode", choices=["multiprocessing-network", "unnamed-pipe"]) +args=parser.parse_args() + +expansion_only_can_call_Python=False # normally. May be different in LuaTeX etc. + +# ======== setup communication method. Requires raw_readline() and send_raw() methods. + +if True: + sys.stdin=None # type: ignore + # avoid user mistakenly read + + raw_readline=sys.__stdin__.readline # raw_readline() should return "⟨line⟩\n" or "" (if EOF) on each call + +if args.mode=="multiprocessing-network": + address=("localhost", 7348) # this must be identical to that of the other half-script + #address="./pythonimmediate.socket" + + from multiprocessing.connection import Client + connection=Client(address) + debug("connected") + + def send_raw(s: str)->None: # send_raw() should get pass the s = "⟨line⟩\n" + global connection + connection.send_bytes(s.encode('u8')) + +elif args.mode=="unnamed-pipe": + pytotex_pid_line=raw_readline() + match_=re.fullmatch("pytotex_pid=(\d+)\n", pytotex_pid_line) + assert match_ + pytotex_pid=int(match_[1]) + + connection_=open("/proc/" + str(pytotex_pid) + "/fd/0", "w", encoding='u8', + buffering=1 # line buffering + ) + + def send_raw(s: str)->None: + global connection_ + connection_.write(s) + connection_.flush() # just in case + +else: + assert False + +# ======== done. + +# https://stackoverflow.com/questions/5122465/can-i-fake-a-package-or-at-least-a-module-in-python-for-testing-purposes +from types import ModuleType +pythonimmediate: Any=ModuleType("pythonimmediate") +pythonimmediate.__file__="pythonimmediate.py" +sys.modules["pythonimmediate"]=pythonimmediate + +pythonimmediate.debugging=True + +def export_function_to_module(f: Callable)->Callable: + """ + the functions decorated with this decorator are accessible from user code with + + import pythonimmediate + pythonimmediate.⟨function name⟩(...) + """ + setattr(pythonimmediate, f.__name__, f) + return f + +action_done=False + + +def check_not_finished()->None: + global action_done + if action_done: + raise RuntimeError("can only do one action per block!") + +def send_finish(s: str)->None: + check_not_finished() + global action_done + action_done=True + send_raw(s) + + +import random +def surround_delimiter(block: str)->str: + while True: + delimiter=str(random.randint(0, 10**12)) + if delimiter not in block: break + return delimiter + "\n" + block + "\n" + delimiter + "\n" + +bootstrap_code: Optional[str]="" +def mark_bootstrap(code: str)->None: + global bootstrap_code + assert bootstrap_code is not None + bootstrap_code+=code + +def substitute_private(code: str)->str: + return (code + #.replace("\n", ' ') # because there are comments in code, cannot + .replace("__", "_" + "pythonimmediate" + "_") + ) + +def send_bootstrap_code()->None: + global bootstrap_code + assert bootstrap_code is not None + send_raw(surround_delimiter(substitute_private(bootstrap_code))) + bootstrap_code = None + +# ======== + + +# as the name implies, this reads one "command" from Python side and execute it. +# the command might do additional tasks e.g. read more \TeX\ code. +# +# e.g. if `block' is read from the communication channel, run |\__run_block:|. + +mark_bootstrap( +r""" +\cs_new_protected:Npn \__read_do_one_command: { + \begingroup + \endlinechar=-1~ + \readline \__read_file to \__line + \expandafter + \endgroup % also this will give an error instead of silently do nothing when command is invalid + \csname __run_ \__line :\endcsname +} + +% read documentation of |_peek| commands for details what this command does. +\cs_new_protected:Npn \pythonimmediatecontinue #1 { + \immediate\write \__write_file {r #1} + \__read_do_one_command: +} + +\cs_new_protected:Npn \pythonimmediatecontinuenoarg { + \pythonimmediatecontinue {} +} + +% internal function. Just send an arbitrary block of data to Python. +\cs_new_protected:Npn \__send_block:e #1 { + \immediate\write \__write_file { + #1 ^^J + pythonimm?""" + '"""' + r"""?'''? % following character will be newline + } +} + +\cs_new_protected:Npn \__send_block:n #1 { + \__send_block:e {\unexpanded{#1}} +} + +\AtEndDocument{ + \immediate\write \__write_file {r} +} +""") + + +# ======== + +# when 'i⟨string⟩' is sent from TeX to Python, the function with index ⟨string⟩ in this dict is called +TeX_handlers: Dict[str, Callable[[], None]]={} + +TeXToPyObjectType=Optional[str] + +def run_main_loop()->TeXToPyObjectType: + while True: + line=readline() + if not line: return None + + if line[0]=="i": + TeX_handlers[line[1:]]() + elif line[0]=="r": + return line[1:] + else: + raise RuntimeError("Internal error: unexpected line "+line) + +def run_main_loop_get_return_one()->str: + line=readline() + assert line[0]=="r" + return line[1:] + + + +user_documentation( +""" +All exported functions can be accessed through the module as |import pythonimmediate|. + +The |_finish| functions are internal functions, which must be called \emph{at most} once in each +|\pythonimmediate:n| call from \TeX\ to tell \TeX\ what to do. + +The |_local| functions simply execute the code. These functions will only return when +the \TeX\ code finishes executing; nevertheless, the \TeX\ code might recursively execute some Python code +inside it. + +A simple example is |pythonimmediate.run_block_local('123')| which simply typesets |123|. + +The |_peek| functions is the same as above; however, the \TeX\ code must contain an explicit command +|\pythonimmediatecontinue{...}|. + +The argument of |\pythonimmediatecontinue| will be |e|-expanded +by |\write| (note that the written content must not contain any newline character, +otherwise the behavior is undefined), then returned as a string by the Python code. +The Python function will only return when |\pythonimmediatecontinue| is called. + +In other words, |run_*_local(code)| is almost identical to |run_*_peek(code + "\pythonimmediatecontinue {}")|. +""") + +@export_function_to_module +def run_block_finish(block: str)->None: + send_finish("block\n" + surround_delimiter(block)) + + +@user_documentation +@export_function_to_module +def execute(block: str)->None: + """ + Run a block of \TeX\ code (might consist of multiple lines). + Catcode-changing commands are allowed inside. + + A simple example is |pythonimmediate.run_block_local('123')| which simply typesets |123|. + + A more complicated example is |pythonimmediate.run_block_local(r'\verb+%+')|. + """ + run_block_local(block) + +def check_line(line: str, *, braces: bool, newline: bool, continue_: Optional[bool])->None: + """ + check user-provided line before sending to TeX for execution + """ + if braces: + assert line.count("{") == line.count("}") + if newline: + assert '\n' not in line + assert '\r' not in line # this is not the line separator but just in case + if continue_==True: assert "pythonimmediatecontinue" in line + elif continue_==False: assert "pythonimmediatecontinue" not in line + + + + +do_run_error_finish=True + + + + + +user_scope: Dict[str, Any]={} # consist of user's local variables etc. + +def readline()->str: + line=raw_readline() + if not line: + sys.stderr.write("\n\nTraceback (most recent call last):\n") + traceback.print_stack(file=sys.stderr) + sys.stderr.write("RuntimeError: Fatal irrecoverable TeX error\n\n") + os._exit(1) + + + assert line[-1]=='\n' + line=line[:-1] + debug("======== saw line", line) + return line + +block_delimiter: str="pythonimm?\"\"\"?'''?" + +def read_block()->str: + """ + Internal function to read one block sent from \TeX\ (including the final delimiter line, + but the delimiter line is not returned) + """ + lines: List[str]=[] + while True: + line=readline() + if line==block_delimiter: + return '\n'.join(lines) + else: + lines.append(line) + + +@export_function_to_module +class NToken(ABC): + """ + Represent a possibly-notexpanded token. + For convenience, a notexpanded token is called a blue token. + It's not always possible to determine the notexpanded status of a following token in the input stream. + Remark: Token objects must be frozen. + """ + + @abstractmethod + def __str__(self)->str: ... + + @abstractmethod + def repr1(self)->str: ... + + @property + @abstractmethod + def assignable(self)->bool: ... + + def assign(self, other: "NToken")->None: + assert self.assignable + NTokenList([T.let, self, C.other("="), C.space(' '), other]).execute() + + def assign_future(self)->None: + assert self.assignable + futurelet_(PTTBalancedTokenList(BalancedTokenList([self.no_blue]))) + + def assign_futurenext(self)->None: + assert self.assignable + futureletnext_(PTTBalancedTokenList(BalancedTokenList([self.no_blue]))) + + def meaning_str(self)->str: + """ + get the meaning of this token as a string. + """ + return NTokenList([T.meaning, self]).expand_x().str() + + @property + @abstractmethod + def blue(self)->"BlueToken": ... + + @property + @abstractmethod + def no_blue(self)->"Token": ... + + def meaning_equal(self, other: "Token")->bool: + return NTokenList([T.ifx, self, other, Catcode.other("1"), T["else"], Catcode.other("0"), T.fi]).expand_x().bool() + + def str(self)->str: + """ + self must represent a character of a TeX string. (i.e. equal to itself when detokenized) + return the string content. + + default implementation below. Not necessarily correct. + """ + raise ValueError("Token does not represent a string!") + + def degree(self)->int: + """ + return the imbalance degree for this token ({ -> 1, } -> -1, everything else -> 0) + + default implementation below. Not necessarily correct. + """ + return 0 + + +@export_function_to_module +class Token(NToken): + """ + Represent a TeX token, excluding the notexpanded possibility. + See also documentation of NToken. + """ + + @abstractmethod + def serialize(self)->str: ... + + @property + def blue(self)->"BlueToken": return BlueToken(self) + + @property + def no_blue(self)->"Token": return self + + def __repr__(self)->str: + return f"<Token: {self.repr1()}>" + + @staticmethod + def deserialize(s: str)->"Token": + t=TokenList.deserialize(s) + assert len(t)==1 + return t[0] + + @staticmethod + def get_next()->"Token": + """ + Get the following token. + + Note: in LaTeX3 versions without the commit |https://github.com/latex3/latex3/commit/24f7188904d6| + sometimes this may error out. + + Note: because of the internal implementation of |\peek_analysis_map_inline:n|, this may + tokenize up to 2 tokens ahead (including the returned token), + as well as occasionally return the wrong token in unavoidable cases. + """ + return Token.deserialize(str(get_next_()[0])) + + @staticmethod + def peek_next()->"Token": + """ + Get the following token without removing it from the input stream. + + Equivalent to get_next() then put_next() immediately. See documentation of get_next() for some notes. + """ + return Token.deserialize( + typing.cast(Callable[[], TTPLine], Python_call_TeX_local( + r""" + \cs_new_protected:Npn \__peek_next_callback: #1 { + \immediate\write \__write_file { r^^J #1 } + \expandafter % expand the ##1 in (*) + \__read_do_one_command: + } + + \cs_new_protected:Npn %name% { + \peek_analysis_map_inline:n { + \peek_analysis_map_break:n { + \__tlserialize_char_unchecked:nnNN {##1}{##2}##3 \__peek_next_callback: ##1 % (*) + } + } + } + """, recursive=False))() + ) + + def put_next(self)->None: + d=self.degree() + if d==0: + BalancedTokenList([self]).put_next() + else: + assert isinstance(self, CharacterToken) + if d==1: + put_next_bgroup(PTTInt(self.index)) + else: + assert d==-1 + put_next_egroup(PTTInt(self.index)) + + + + + + +""" +TeX code for serializing and deserializing a token list. +Convert a token list from/to a string. +""" + + +mark_bootstrap( +r""" +\precattl_exec:n { + +% here #1 is the target token list to store the result to, #2 is a string with the final '.'. +\cs_new_protected:Npn \__tldeserialize_dot:Nn #1 #2 { + \begingroup + \tl_set:Nn \__tmp {#2} + \tl_replace_all:Nnn \__tmp {~} {\cO\ } + + \def \start ##1 { \csname ##1 \endcsname } + + \def \> ##1 ##2 \cO\ { \csname ##1 \endcsname ##2 \cU\ } + \def \\ ##1 \cO\ ##2 { \expandafter \noexpand \csname ##1 \endcsname \csname ##2 \endcsname } + \def \1 ##1 ##2 { \char_generate:nn {`##1} {1} \csname ##2 \endcsname } + \def \2 ##1 ##2 { \char_generate:nn {`##1} {2} \csname ##2 \endcsname } + \def \3 ##1 ##2 { \char_generate:nn {`##1} {3} \csname ##2 \endcsname } + \def \4 ##1 ##2 { \char_generate:nn {`##1} {4} \csname ##2 \endcsname } + \def \6 ##1 ##2 { #### \char_generate:nn {`##1} {6} \csname ##2 \endcsname } + \def \7 ##1 ##2 { \char_generate:nn {`##1} {7} \csname ##2 \endcsname } + \def \8 ##1 ##2 { \char_generate:nn {`##1} {8} \csname ##2 \endcsname } + \def \A ##1 ##2 { \char_generate:nn {`##1} {10} \csname ##2 \endcsname } + \def \B ##1 ##2 { \char_generate:nn {`##1} {11} \csname ##2 \endcsname } + \def \C ##1 ##2 { \char_generate:nn {`##1} {12} \csname ##2 \endcsname } + \def \D ##1 ##2 { \expandafter \expandafter \expandafter \noexpand \char_generate:nn {`##1} {13} \csname ##2 \endcsname } + \def \R ##1 { \cFrozenRelax \csname ##1 \endcsname } + + \let \. \empty + + \exp_args:NNNx + \endgroup \tl_set:Nn #1 {\expandafter \start \__tmp} +} + +\cs_new_protected:Npn \__tlserialize_char_unchecked:nnNN #1 #2 #3 #4 { + % #1=token, #2=char code, #3=catcode, #4: callback (will be called exactly once and with nothing following the input stream) + \int_compare:nNnTF {#2} = {-1} { + % token is control sequence + \tl_if_eq:onTF {#1} {\cFrozenRelax} { + #4 {\cStr{ R }} + } { + \tl_if_eq:onTF {#1} { \cC{} } { + #4 {\cStr{ \\\ }} + } { + \tl_set:Nx \__name { \expandafter \cs_to_str:N #1 } + \exp_args:Nx #4 { \prg_replicate:nn {\str_count_spaces:N \__name} {>} \cStr\\ \__name \cStr\ } + } + } + } { + % token is not control sequence + % (hex catcode) (character) (or escape sequence with that character) + \exp_args:Nx #4 { #3 \expandafter \string #1 } + } +} + +} + +% deserialize as above but #2 does not end with '.'. +\cs_new_protected:Npn \__tldeserialize_nodot:Nn #1 #2 { + \__tldeserialize_dot:Nn #1 {#2 .} +} + +% serialize token list in #2 store to #1. +\cs_new_protected:Npn \__tlserialize_nodot_unchecked:Nn #1 #2 { + \tl_build_begin:N #1 + \tl_set:Nn \__tlserialize_callback { \tl_build_put_right:Nn #1 } + \tl_analysis_map_inline:nn {#2} { + \__tlserialize_char_unchecked:nnNN {##1}{##2}##3 \__tlserialize_callback + } + \tl_build_end:N #1 +} + +% serialize token list in #2 store to #1. Call T or F branch depends on whether serialize is successful. +% #1 must be different from \__tlserialize_tmp. +\cs_new_protected:Npn \__tlserialize_nodot:NnTF #1 #2 { + \__tlserialize_nodot_unchecked:Nn #1 {#2} + \__tldeserialize_nodot:NV \__tlserialize_nodot_tmp #1 + + \tl_if_eq:NnTF \__tlserialize_nodot_tmp {#2} % dangling +} + +\cs_new_protected:Npn \__tlserialize_nodot:NnF #1 #2 { + \__tlserialize_nodot:NnTF #1 {#2} {} % dangling +} + +\cs_new_protected:Npn \__tlserialize_nodot:NnT #1 #2 #3 { \__tlserialize_nodot:NnTF #1 {#2} {#3} {} } + +\msg_new:nnn {pythonimmediate} {cannot-serialize} {Token~list~cannot~be~serialized} + +\cs_new_protected:Npn \__tlserialize_nodot:Nn #1 #2{ + \__tlserialize_nodot:NnF #1 {#2} { + \msg_error:nn {pythonimmediate} {cannot-serialize} + } +} + +\cs_generate_variant:Nn \__tldeserialize_dot:Nn {NV} +\cs_generate_variant:Nn \__tldeserialize_nodot:Nn {NV} +\cs_generate_variant:Nn \__tlserialize_nodot:Nn {NV} +""") + + +class ControlSequenceTokenMaker: + """ + shorthand to create control sequence objects in Python easier. + """ + def __init__(self, prefix: str)->None: + self.prefix=prefix + def __getattribute__(self, a: str)->"ControlSequenceToken": + return ControlSequenceToken(object.__getattribute__(self, "prefix")+a) + def __getitem__(self, a: str)->"ControlSequenceToken": + return ControlSequenceToken(object.__getattribute__(self, "prefix")+a) + + +@export_function_to_module +@dataclass(repr=False, frozen=True) +class ControlSequenceToken(Token): + make=typing.cast(ControlSequenceTokenMaker, None) # some interference makes this incorrect. Manually assign below + csname: str + @property + def assignable(self)->bool: + return True + def __str__(self)->str: + if self.csname=="": return r"\csname\endcsname" + return "\\"+self.csname + def serialize(self)->str: + return ">"*self.csname.count(" ") + "\\" + self.csname + " " + def repr1(self)->str: + return f"\\{self.csname}" + + +ControlSequenceToken.make=ControlSequenceTokenMaker("") + +T=ControlSequenceToken.make +P=ControlSequenceTokenMaker("_pythonimmediate_") # create private tokens + +@export_function_to_module +class Catcode(enum.Enum): + begin_group=bgroup=1 + end_group=egroup=2 + math_toggle=math=3 + alignment=4 + parameter=param=6 + math_superscript=superscript=7 + math_subscript=subscript=8 + space=10 + letter=11 + other=12 + active=13 + + escape=0 + end_of_line=paragraph=line=5 + ignored=9 + comment=14 + invalid=15 + + @property + def for_token(self)->bool: + """ + Return whether a token may have this catcode. + """ + return self not in (Catcode.escape, Catcode.line, Catcode.ignored, Catcode.comment, Catcode.invalid) + + def __call__(self, ch: Union[str, int])->"CharacterToken": + """ + Shorthand: + Catcode.letter("a") = Catcode.letter(97) = CharacterToken(index=97, catcode=Catcode.letter) + """ + if isinstance(ch, str): ch=ord(ch) + return CharacterToken(ch, self) + +C=Catcode + +@export_function_to_module +@dataclass(repr=False, frozen=True) # must be frozen because bgroup and egroup below are reused +class CharacterToken(Token): + index: int + catcode: Catcode + @property + def chr(self)->str: + return chr(self.index) + def __post_init__(self)->None: + assert self.catcode.for_token + def __str__(self)->str: + return self.chr + def serialize(self)->str: + return f"{self.catcode.value:X}{self.chr}" + def repr1(self)->str: + cat=str(self.catcode.value).translate(str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉")) + return f"{self.chr}{cat}" + @property + def assignable(self)->bool: + return self.catcode==Catcode.active + def degree(self)->int: + if self.catcode==Catcode.bgroup: + return 1 + elif self.catcode==Catcode.egroup: + return -1 + else: + return 0 + def str(self)->str: + catcode=Catcode.space if self.index==32 else Catcode.other + if catcode!=self.catcode: + raise ValueError("this CharacterToken does not represent a string!") + return self.chr + +class FrozenRelaxToken(Token): + def __str__(self)->str: + return r"\relax" + def serialize(self)->str: + return "R" + def repr1(self)->str: + return r"[frozen]\relax" + @property + def assignable(self)->bool: + return False + +frozen_relax_token=FrozenRelaxToken() +pythonimmediate.frozen_relax_token=frozen_relax_token + +# other special tokens later... + +bgroup=Catcode.bgroup("{") +egroup=Catcode.egroup("}") +space=Catcode.space(" ") + + + +@export_function_to_module +@dataclass(frozen=True) +class BlueToken(NToken): + token: Token + + @property + def blue(self)->"BlueToken": return self + + @property + def no_blue(self)->"Token": return self.token + + def __str__(self)->str: return str(self.token) + + def repr1(self)->str: return "notexpanded:"+self.token.repr1() + + @property + def assignable(self)->bool: return self.token.assignable + + def put_next(self)->None: + put_next_blue(PTTBalancedTokenList(BalancedTokenList([self.token]))) + + +doc_catcode_table: Dict[int, Catcode]={} +doc_catcode_table[ord("{")]=Catcode.begin_group +doc_catcode_table[ord("}")]=Catcode.end_group +doc_catcode_table[ord("$")]=Catcode.math_toggle +doc_catcode_table[ord("&")]=Catcode.alignment +doc_catcode_table[ord("#")]=Catcode.parameter +doc_catcode_table[ord("^")]=Catcode.math_superscript +doc_catcode_table[ord("_")]=Catcode.math_subscript +doc_catcode_table[ord(" ")]=Catcode.space +doc_catcode_table[ord("~")]=Catcode.active +for ch in range(ord('a'), ord('z')+1): doc_catcode_table[ch]=Catcode.letter +for ch in range(ord('A'), ord('Z')+1): doc_catcode_table[ch]=Catcode.letter +doc_catcode_table[ord("\\")]=Catcode.escape +doc_catcode_table[ord("%")]=Catcode.comment + +e3_catcode_table=dict(doc_catcode_table) +e3_catcode_table[ord("_")]=Catcode.letter +e3_catcode_table[ord(":")]=Catcode.letter +e3_catcode_table[ord(" ")]=Catcode.ignored +e3_catcode_table[ord("~")]=Catcode.space + + +TokenListType = typing.TypeVar("TokenListType", bound="TokenList") + +if typing.TYPE_CHECKING: + TokenListBaseClass = collections.UserList[Token] +else: # Python 3.8 compatibility + TokenListBaseClass = collections.UserList + +@export_function_to_module +class TokenList(TokenListBaseClass): + @staticmethod + def force_token_list(a: Iterable)->Iterable[Token]: + for x in a: + if isinstance(x, Token): + yield x + elif isinstance(x, Sequence): + yield bgroup + child=BalancedTokenList(x) + assert child.is_balanced() + yield from child + yield egroup + else: + raise RuntimeError(f"Cannot make TokenList from object {x} of type {type(x)}") + + def is_balanced(self)->bool: + """ + check if this is balanced. + """ + degree=0 + for x in self: + degree+=x.degree() + if degree<0: return False + return degree==0 + + def check_balanced(self)->None: + """ + ensure that this is balanced. + """ + if not self.is_balanced(): + raise ValueError("Token list is not balanced") + + def balanced_parts(self)->"List[Union[BalancedTokenList, Token]]": + """ + split this TokenList into a list of balanced parts and unbalanced {/}tokens + """ + degree=0 + min_degree=0, 0 + for i, token in enumerate(self): + degree+=token.degree() + min_degree=min(min_degree, (degree, i+1)) + min_degree_pos=min_degree[1] + + left_half: List[Union[BalancedTokenList, Token]]=[] + degree=0 + last_pos=0 + for i in range(min_degree_pos): + d=self[i].degree() + degree+=d + if degree<0: + degree=0 + if last_pos!=i: + left_half.append(BalancedTokenList(self[last_pos:i])) + left_half.append(self[i]) + last_pos=i+1 + if min_degree_pos!=last_pos: + left_half.append(BalancedTokenList(self[last_pos:min_degree_pos])) + + right_half: List[Union[BalancedTokenList, Token]]=[] + degree=0 + last_pos=len(self) + for i in range(len(self)-1, min_degree_pos-1, -1): + d=self[i].degree() + degree-=d + if degree<0: + degree=0 + if i+1!=last_pos: + right_half.append(BalancedTokenList(self[i+1:last_pos])) + right_half.append(self[i]) + last_pos=i + if min_degree_pos!=last_pos: + right_half.append(BalancedTokenList(self[min_degree_pos:last_pos])) + + return left_half+right_half[::-1] + + def put_next(self)->None: + for part in reversed(self.balanced_parts()): part.put_next() + + @property + def balanced(self)->"BalancedTokenList": + """ + return a BalancedTokenList containing the content of this object. + it must be balanced. + """ + return BalancedTokenList(self) + + def __init__(self, a: Iterable=())->None: + super().__init__(TokenList.force_token_list(a)) + + @staticmethod + def iterable_from_string(s: str, get_catcode: Callable[[int], Catcode])->Iterable[Token]: + """ + refer to documentation of from_string() for details. + """ + i=0 + while i<len(s): + ch=s[i] + i+=1 + cat=get_catcode(ord(ch)) + if cat==Catcode.space: + yield space + # special case: collapse multiple spaces into one but only if character code is space + if get_catcode(32) in (Catcode.space, Catcode.ignored): + while i<len(s) and s[i]==' ': + i+=1 + elif cat.for_token: + yield cat(ch) + elif cat==Catcode.ignored: + continue + else: + assert cat==Catcode.escape, f"cannot create TokenList from string containing catcode {cat}" + cat=get_catcode(ord(s[i])) + if cat!=Catcode.letter: + yield ControlSequenceToken(s[i]) + i+=1 + else: + csname=s[i] + i+=1 + while i<len(s) and get_catcode(ord(s[i]))==Catcode.letter: + csname+=s[i] + i+=1 + yield ControlSequenceToken(csname) + # special case: remove spaces after control sequence but only if character code is space + if get_catcode(32) in (Catcode.space, Catcode.ignored): + while i<len(s) and s[i]==' ': + i+=1 + + @classmethod + def from_string(cls: Type[TokenListType], s: str, get_catcode: Callable[[int], Catcode])->TokenListType: + """ + convert a string to a TokenList approximately. + The tokenization algorithm is slightly different from TeX's in the following respect: + + * multiple spaces are collapsed to one space, but only if it has character code space (32). + * spaces with character code different from space (32) after a control sequence is not ignored. + * ^^ syntax are not supported. Use Python's escape syntax as usual. + """ + return cls(TokenList.iterable_from_string(s, get_catcode)) + + @classmethod + def e3(cls: Type[TokenListType], s: str)->TokenListType: + """ + approximate tokenizer in expl3 catcode, implemented in Python. + refer to documentation of from_string() for details. + """ + return cls.from_string(s, lambda x: e3_catcode_table.get(x, Catcode.other)) + + @classmethod + def doc(cls: Type[TokenListType], s: str)->TokenListType: + """ + approximate tokenizer in document catcode, implemented in Python. + refer to documentation of from_string() for details. + """ + return cls.from_string(s, lambda x: doc_catcode_table.get(x, Catcode.other)) + + def serialize(self)->str: + return "".join(t.serialize() for t in self) + + @classmethod + def deserialize(cls: Type[TokenListType], data: str)->TokenListType: + result: List[Token]=[] + i=0 + cs_skip_space_count=0 + while i<len(data): + if data[i]==">": + cs_skip_space_count+=1 + i+=1 + elif data[i]=="\\": + j=data.index(' ', i+1) + for __ in range(cs_skip_space_count): + j=data.index(' ', j+1) + cs_skip_space_count=0 + result.append(ControlSequenceToken(data[i+1:j])) + i=j+1 + elif data[i]=="R": + result.append(frozen_relax_token) + i+=1 + else: + result.append(CharacterToken(index=ord(data[i+1]), catcode=Catcode(int(data[i], 16)))) + i+=2 + return cls(result) + + def __repr__(self)->str: + return '<' + type(self).__name__ + ': ' + ' '.join(t.repr1() for t in self) + '>' + + def execute(self)->None: + NTokenList(self).execute() + + def expand_x(self)->"BalancedTokenList": + return NTokenList(self).expand_x() + + def bool(self)->bool: + return NTokenList(self).bool() + + def str(self)->str: + return NTokenList(self).str() + + + +@export_function_to_module +class BalancedTokenList(TokenList): + """ + Represents a balanced token list. + Note that runtime checking is not strictly enforced, + use `is_balanced()` method explicitly if you need to check. + """ + + def __init__(self, a: Iterable=())->None: + """ + constructor. This must check for balanced-ness as balanced() method depends on this. + """ + super().__init__(a) + self.check_balanced() + + def expand_o(self)->"BalancedTokenList": + return BalancedTokenList(expand_o_(PTTBalancedTokenList(self))[0]) # type: ignore + def expand_x(self)->"BalancedTokenList": + return BalancedTokenList(expand_x_(PTTBalancedTokenList(self))[0]) # type: ignore + def execute(self)->None: + execute_(PTTBalancedTokenList(self)) + + def put_next(self)->None: + put_next_tokenlist(PTTBalancedTokenList(self)) + + @staticmethod + def get_next()->"BalancedTokenList": + """ + get an (undelimited) argument from the TeX input stream. + """ + return BalancedTokenList(get_argument_tokenlist_()[0]) # type: ignore + + + +if typing.TYPE_CHECKING: + NTokenListBaseClass = collections.UserList[NToken] +else: # Python 3.8 compatibility + NTokenListBaseClass = collections.UserList + +@export_function_to_module +class NTokenList(NTokenListBaseClass): + @staticmethod + def force_token_list(a: Iterable)->Iterable[NToken]: + for x in a: + if isinstance(x, NToken): + yield x + elif isinstance(x, Sequence): + yield bgroup + child=NTokenList(x) + assert child.is_balanced() + yield from child + yield egroup + else: + raise RuntimeError(f"Cannot make NTokenList from object {x} of type {type(x)}") + + def __init__(self, a: Iterable=())->None: + super().__init__(NTokenList.force_token_list(a)) + + def is_balanced(self)->bool: + return TokenList(self).is_balanced() # a bit inefficient (need to construct a TokenList) but good enough + + def simple_parts(self)->List[Union[BalancedTokenList, Token, BlueToken]]: + """ + Split this NTokenList into a list of balanced non-blue parts, unbalanced {/} tokens, and blue tokens. + """ + parts: List[Union[TokenList, BlueToken]]=[TokenList()] + for i in self: + if isinstance(i, BlueToken): + parts+=i, TokenList() + else: + assert isinstance(i, Token) + last_part=parts[-1] + assert isinstance(last_part, TokenList) + last_part.append(i) + result: List[Union[BalancedTokenList, Token, BlueToken]]=[] + for large_part in parts: + if isinstance(large_part, BlueToken): + result.append(large_part) + else: + result+=large_part.balanced_parts() + return result + + def put_next(self)->None: + for part in reversed(self.simple_parts()): part.put_next() + + def execute(self)->None: + """ + Execute self. + """ + parts=self.simple_parts() + if len(parts)==1: + x=parts[0] + if isinstance(x, BalancedTokenList): + x.execute() + return + NTokenList([*self, T.pythonimmediatecontinue, []]).put_next() + continue_until_passed_back() + + def expand_x(self)->BalancedTokenList: + """ + x-expand self. The result must be balanced. + """ + NTokenList([T.edef, P.tmp, bgroup, *self, egroup]).execute() + return BalancedTokenList([P.tmp]).expand_o() + + def str(self)->str: + """ + self must represent a TeX string. (i.e. equal to itself when detokenized) + return the string content. + """ + return "".join(t.str() for t in self) + + def bool(self)->bool: + s=self.str() + return {"0": False, "1": True}[s] + + +class TeXToPyData(ABC): + @staticmethod + @abstractmethod + def read()->"TeXToPyData": + ... + @staticmethod + @abstractmethod + def send_code(arg: str)->str: + pass + @staticmethod + @abstractmethod + def send_code_var(var: str)->str: + pass + +# tried and failed +#@typing.runtime_checkable +#class TeXToPyData(Protocol): +# @staticmethod +# def read()->"TeXToPyData": +# ... +# +# #send_code: str +# +# #@staticmethod +# #@property +# #def send_code()->str: +# # ... + + +class TTPLine(TeXToPyData, str): + send_code=r"\immediate \write \__write_file {{\unexpanded{{ {} }}}}".format + send_code_var=r"\immediate \write \__write_file {{\unexpanded{{ {} }}}}".format + @staticmethod + def read()->"TTPLine": + return TTPLine(readline()) + +# some old commands e.g. \$, \^, \_, \~ require \set@display@protect to be robust. +# ~ needs to be redefined directly. +mark_bootstrap( +r""" +\precattl_exec:n { + \cs_new_protected:Npn \__begingroup_setup_estr: { + \begingroup + \escapechar=-1~ + \cC{set@display@protect} + \let \cA\~ \relax + } +} +""") + +class TTPELine(TeXToPyData, str): + """ + Same as TTPEBlock, but for a single line only. + """ + send_code=r"\__begingroup_setup_estr: \immediate \write \__write_file {{ {} }} \endgroup".format + send_code_var=r"\__begingroup_setup_estr: \immediate \write \__write_file {{ {} }} \endgroup".format + @staticmethod + def read()->"TTPELine": + return TTPELine(readline()) + +class TTPEmbeddedLine(TeXToPyData, str): + @staticmethod + def send_code(self)->str: + raise RuntimeError("Must be manually handled") + @staticmethod + def send_code_var(self)->str: + raise RuntimeError("Must be manually handled") + @staticmethod + def read()->"TTPEmbeddedLine": + raise RuntimeError("Must be manually handled") + +class TTPBlock(TeXToPyData, str): + send_code=r"\__send_block:n {{ {} }}".format + send_code_var=r"\__send_block:V {}".format + @staticmethod + def read()->"TTPBlock": + return TTPBlock(read_block()) + +class TTPEBlock(TeXToPyData, str): + """ + A kind of argument that interprets "escaped string" and fully expand anything inside. + For example, {\\} sends a single backslash to Python, {\{} sends a single '{' to Python. + Done by fully expand the argument in \escapechar=-1 and convert it to a string. + Additional precaution is needed, see the note above. + """ + send_code=r"\__begingroup_setup_estr: \__send_block:e {{ {} }} \endgroup".format + send_code_var=r"\__begingroup_setup_estr: \__send_block:e {} \endgroup".format + @staticmethod + def read()->"TTPEBlock": + return TTPEBlock(read_block()) + +class TTPBalancedTokenList(TeXToPyData, BalancedTokenList): + send_code=r"\__tlserialize_nodot:Nn \__tmp {{ {} }} \immediate \write \__write_file {{\unexpanded\expandafter{{ \__tmp }}}}".format + send_code_var=r"\__tlserialize_nodot:NV \__tmp {} \immediate \write \__write_file {{\unexpanded\expandafter{{ \__tmp }}}}".format + @staticmethod + def read()->"TTPBalancedTokenList": + return TTPBalancedTokenList(BalancedTokenList.deserialize(readline())) + + +class PyToTeXData(ABC): + @staticmethod + @abstractmethod + def read_code(var: str)->str: + ... + @abstractmethod + def write(self)->None: + ... + +@dataclass +class PTTVerbatimLine(PyToTeXData): + """ + Represents a line to be tokenized verbatim. Internally the |\readline| primitive is used, as such, any trailing spaces are stripped. + The trailing newline is not included, i.e. it's read under |\endlinechar=-1|. + """ + data: str + read_code=r"\ior_str_get:NN \__read_file {} ".format + def write(self)->None: + assert "\n" not in self.data + assert self.data.rstrip()==self.data, "Cannot send verbatim line with trailing spaces!" + send_raw(self.data+"\n") + +@dataclass +class PTTInt(PyToTeXData): + data: int + read_code=PTTVerbatimLine.read_code + def write(self)->None: + PTTVerbatimLine(str(self.data)).write() + +@dataclass +class PTTTeXLine(PyToTeXData): + """ + Represents a line to be tokenized in \TeX's current catcode regime. + The trailing newline is not included, i.e. it's tokenized under |\endlinechar=-1|. + """ + data: str + read_code=r"\ior_get:NN \__read_file {} ".format + def write(self)->None: + assert "\n" not in self.data + send_raw(self.data+"\n") + +@dataclass +class PTTBlock(PyToTeXData): + data: str + read_code=r"\__read_block:N {}".format + def write(self)->None: + send_raw(surround_delimiter(self.data)) + +@dataclass +class PTTBalancedTokenList(PyToTeXData): + data: BalancedTokenList + read_code=r"\ior_str_get:NN \__read_file {0} \__tldeserialize_dot:NV {0} {0}".format + def write(self)->None: + PTTVerbatimLine(self.data.serialize()+".").write() + + +# ======== define TeX functions that execute Python code ======== +# ======== implementation of |\py| etc. Doesn't support verbatim argument yet. ======== + +import itertools +import string + +def random_identifiers()->Iterator[str]: # do this to avoid TeX hash collision while keeping the length short + for len_ in itertools.count(0): + for value in range(1<<len_): + for initial in string.ascii_letters: + yield initial + f"{value:0{len_}b}".translate({ord("0"): "a", ord("1"): "b"}) + +random_identifier_iterable=random_identifiers() + +def get_random_identifier()->str: + return next(random_identifier_iterable) + + +def define_TeX_call_Python(f: Callable[..., None], name: Optional[str]=None, argtypes: Optional[List[Type[TeXToPyData]]]=None, identifier: Optional[str]=None)->str: + """ + This function setups some internal data structure, and + returns the \TeX\ code to be executed on the \TeX\ side to define the macro. + + f: the Python function to be executed. + It should take some arguments and eventually (optionally) call one of the |_finish| functions. + + name: the macro name on the \TeX\ side. This should only consist of letter characters in |expl3| catcode regime. + + argtypes: list of argument types. If it's None it will be automatically deduced from the function |f|'s signature. + + Returns: some code (to be executed in |expl3| catcode regime) as explained above. + """ + if argtypes is None: argtypes=[p.annotation for p in inspect.signature(f).parameters.values()] + if name is None: name=f.__name__ + + if identifier is None: identifier=get_random_identifier() + assert identifier not in TeX_handlers + + @functools.wraps(f) + def g()->None: + assert argtypes is not None + args=[argtype.read() for argtype in argtypes] + + + global action_done + old_action_done=action_done + + action_done=False + try: + f(*args) + except: + if action_done: + # error occurred after 'finish' is called, cannot signal the error to TeX, will just ignore (after printing out the traceback)... + pass + else: + # TODO what should be done here? What if the error raised below is caught + action_done=True + raise + finally: + if not action_done: + run_none_finish() + + action_done=old_action_done + + + TeX_handlers[identifier]=g + + TeX_argspec = "" + TeX_send_input_commands = "" + for i, argtype in enumerate(argtypes): + if isinstance(argtype, str): + raise RuntimeError("string annotation or `from __future__ import annotations' not yet supported") + if not issubclass(argtype, TeXToPyData): + raise RuntimeError(f"Argument type {argtype} is incorrect, should be a subclass of TeXToPyData") + arg = f"#{i+1}" + TeX_send_input_commands += argtype.send_code(arg) + TeX_argspec += arg + + return """ + \\cs_new_protected:Npn \\""" + name + TeX_argspec + """ { + \immediate \write \__write_file { i """ + identifier + """ } + """ + TeX_send_input_commands + """ + \__read_do_one_command: + } + """ + + +def define_internal_handler(f: Callable)->Callable: + mark_bootstrap(define_TeX_call_Python(f)) + return f + + +import linecache + +# https://stackoverflow.com/questions/47183305/file-string-traceback-with-line-preview +def exec_or_eval_with_linecache(code: str, globals: dict, mode: str)->Any: + sourcename: str="<usercode>" + i=0 + while sourcename in linecache.cache: + sourcename="<usercode" + str(i) + ">" + i+=1 + + lines=code.splitlines(keepends=True) + linecache.cache[sourcename] = len(code), None, lines, sourcename + + compiled_code=compile(code, sourcename, mode) + return (exec if mode=="exec" else eval)(compiled_code, globals) + + #del linecache.cache[sourcename] + # we never delete the cache, in case some function is defined here then later are called... + +def exec_with_linecache(code: str, globals: Dict[str, Any])->None: + exec_or_eval_with_linecache(code, globals, "exec") + +def eval_with_linecache(code: str, globals: Dict[str, Any])->Any: + return exec_or_eval_with_linecache(code, globals, "eval") + + +@define_internal_handler +def py(code: TTPEBlock)->None: + pythonimmediate.run_block_finish(str(eval_with_linecache(code, user_scope))+"%") + +@define_internal_handler +def pyfile(filename: TTPELine)->None: + with open(filename, "r") as f: + source=f.read() + exec(compile(source, filename, "exec"), user_scope) + +def print_TeX(*args, **kwargs)->None: + if not hasattr(pythonimmediate, "file"): + raise RuntimeError("Internal error: attempt to print to TeX outside any environment!") + if pythonimmediate.file is not None: + functools.partial(print, file=pythonimmediate.file)(*args, **kwargs) # allow user to override `file` kwarg +pythonimmediate.print=print_TeX + +class RedirectPrintTeX: + def __init__(self, t)->None: + self.t=t + + def __enter__(self)->None: + if hasattr(pythonimmediate, "file"): + self.old=pythonimmediate.file + pythonimmediate.file=self.t + + def __exit__(self, exc_type, exc_value, tb)->None: + if hasattr(self, "old"): + pythonimmediate.file=self.old + else: + del pythonimmediate.file + +def run_code_redirect_print_TeX(f: Callable[[], Any])->None: + with io.StringIO() as t: + with RedirectPrintTeX(t): + result=f() + if result is not None: + t.write(str(result)+"%") + content=t.getvalue() + if content.endswith("\n"): + content=content[:-1] + else: + #content+=r"\empty" # this works too + content+="%" + pythonimmediate.run_block_finish(content) + +@define_internal_handler +def pyc(code: TTPEBlock)->None: + run_code_redirect_print_TeX(lambda: exec_with_linecache(code, user_scope)) + +@define_internal_handler +def pycq(code: TTPEBlock)->None: + with RedirectPrintTeX(None): + exec_with_linecache(code, user_scope) + run_none_finish() + +mark_bootstrap( +r""" +\NewDocumentCommand\pyv{v}{\py{#1}} +\NewDocumentCommand\pycv{v}{\pyc{#1}} +""") + +# ======== implementation of |pycode| environment +mark_bootstrap( +r""" +\NewDocumentEnvironment{pycode}{}{ + \saveenvreinsert \__code { + \exp_last_unbraced:Nx \__pycodex {{\__code} {\the\inputlineno} { + \ifdefined\currfilename \currfilename \fi + } { + \ifdefined\currfileabspath \currfileabspath \fi + }} + } +}{ + \endsaveenvreinsert +} +""") + +def normalize_lines(lines: List[str])->List[str]: + return [line.rstrip() for line in lines] + +@define_internal_handler +def __pycodex(code: TTPBlock, lineno_: TTPLine, filename: TTPLine, fileabspath: TTPLine)->None: + if not code: return + + lineno=int(lineno_) + # find where the code comes from... (for easy meaningful traceback) + target_filename: Optional[str] = None + + code_lines_normalized=normalize_lines(code.splitlines(keepends=True)) + + for f in (fileabspath, filename): + if not f: continue + p=Path(f) + if not p.is_file(): continue + file_lines=p.read_text().splitlines(keepends=True)[lineno-len(code_lines_normalized)-1:lineno-1] + if normalize_lines(file_lines)==code_lines_normalized: + target_filename=f + break + + if not target_filename: + raise RuntimeError("Source file not found! (attempted {})".format((fileabspath, filename))) + + with io.StringIO() as t: + with RedirectPrintTeX(t): + if target_filename: + code_=''.join(file_lines) # restore missing trailing spaces + code_="\n"*(lineno-len(code_lines_normalized)-1)+code_ + if target_filename: + compiled_code=compile(code_, target_filename, "exec") + exec(compiled_code, user_scope) + else: + exec(code_, user_scope) + pythonimmediate.run_block_finish(t.getvalue()) + +# ======== Python-call-TeX functions +# ======== additional functions... + +user_documentation( +r""" +These functions get an argument in the input stream and returns it detokenized. + +Which means, for example, |#| are doubled, multiple spaces might be collapsed into one, spaces might be introduced +after a control sequence. + +It's undefined behavior if the message's "string representation" contains a "newline character". +""") + +def template_substitute(template: str, pattern: str, substitute: Union[str, Callable[[re.Match], str]], optional: bool=False)->str: + """ + pattern is a regex + """ + if not optional: + #assert template.count(pattern)==1 + assert len(re.findall(pattern, template))==1 + return re.sub(pattern, substitute, template) + +#typing.TypeVarTuple(PyToTeXData) + +#PythonCallTeXFunctionType=Callable[[PyToTeXData], Optional[Tuple[TeXToPyData, ...]]] + +class PythonCallTeXFunctionType(Protocol): # https://stackoverflow.com/questions/57658879/python-type-hint-for-callable-with-variable-number-of-str-same-type-arguments + def __call__(self, *args: PyToTeXData)->Optional[Tuple[TeXToPyData, ...]]: ... + +class PythonCallTeXSyncFunctionType(PythonCallTeXFunctionType, Protocol): # https://stackoverflow.com/questions/57658879/python-type-hint-for-callable-with-variable-number-of-str-same-type-arguments + def __call__(self, *args: PyToTeXData)->Tuple[TeXToPyData, ...]: ... + + +@dataclass(frozen=True) +class Python_call_TeX_data: + TeX_code: str + recursive: bool + finish: bool + sync: Optional[bool] + +@dataclass(frozen=True) +class Python_call_TeX_extra: + ptt_argtypes: Tuple[Type[PyToTeXData], ...] + ttp_argtypes: Union[Type[TeXToPyData], Tuple[Type[TeXToPyData], ...]] + +Python_call_TeX_defined: Dict[Python_call_TeX_data, Tuple[Python_call_TeX_extra, Callable]]={} + +def Python_call_TeX_local(TeX_code: str, *, recursive: bool=True, sync: Optional[bool]=None, finish: bool=False)->Callable: + data=Python_call_TeX_data( + TeX_code=TeX_code, recursive=recursive, sync=sync, finish=finish + ) + return Python_call_TeX_defined[data][1] + +def build_Python_call_TeX(T: Type, TeX_code: str, *, recursive: bool=True, sync: Optional[bool]=None, finish: bool=False)->None: + assert T.__origin__ == typing.Callable[[], None].__origin__ # type: ignore + # might be typing.Callable or collections.abc.Callable depends on Python version + data=Python_call_TeX_data( + TeX_code=TeX_code, recursive=recursive, sync=sync, finish=finish + ) + + tmp: Any = T.__args__[-1] + ttp_argtypes: Union[Type[TeXToPyData], Tuple[Type[TeXToPyData], ...]] + if tmp is type(None): + ttp_argtypes = () + elif isinstance(tmp, type) and issubclass(tmp, TeXToPyData): + # special case, return a single object instead of a tuple of length 1 + ttp_argtypes = tmp + else: + ttp_argtypes = tmp.__args__ # type: ignore + + extra=Python_call_TeX_extra( + ptt_argtypes=T.__args__[:-1], + ttp_argtypes=ttp_argtypes + ) # type: ignore + if data in Python_call_TeX_defined: + assert Python_call_TeX_defined[data][0]==extra + else: + if isinstance(ttp_argtypes, type) and issubclass(ttp_argtypes, TeXToPyData): + # special case, return a single object instead of a tuple of length 1 + code, result1=define_Python_call_TeX(TeX_code=TeX_code, ptt_argtypes=[*extra.ptt_argtypes], ttp_argtypes=[ttp_argtypes], + recursive=recursive, sync=sync, finish=finish, + ) + def result(*args): + [tmp]=result1(*args) + return tmp + else: + code, result=define_Python_call_TeX(TeX_code=TeX_code, ptt_argtypes=[*extra.ptt_argtypes], ttp_argtypes=[*ttp_argtypes], + recursive=recursive, sync=sync, finish=finish, + ) + mark_bootstrap(code) + Python_call_TeX_defined[data]=extra, result + +def scan_Python_call_TeX(filename: str)->None: + """ + scan the file in filename for occurrences of typing.cast(T, Python_call_TeX_local(...)), then call build_Python_call_TeX(T, ...) for each occurrence. + + Don't use on untrusted code. + """ + import ast + from copy import deepcopy + for node in ast.walk(ast.parse(Path(filename).read_text(), mode="exec")): + try: + if isinstance(node, ast.Call): + if ( + isinstance(node.func, ast.Attribute) and + isinstance(node.func.value, ast.Name) and + node.func.value.id == "typing" and + node.func.attr == "cast" + ): + T = node.args[0] + if isinstance(node.args[1], ast.Call): + f_call = node.args[1] + if isinstance(f_call.func, ast.Name): + if f_call.func.id == "Python_call_TeX_local": + f_call=deepcopy(f_call) + assert isinstance(f_call.func, ast.Name) + f_call.func.id="build_Python_call_TeX" + f_call.args=[T]+f_call.args + eval(compile(ast.Expression(body=f_call), "<string>", "eval")) + except: + print("======== error on line", node.lineno, "========", file=sys.stderr) + raise + +def define_Python_call_TeX(TeX_code: str, ptt_argtypes: List[Type[PyToTeXData]], ttp_argtypes: List[Type[TeXToPyData]], + *, + recursive: bool=True, + sync: Optional[bool]=None, + finish: bool=False, + )->Tuple[str, PythonCallTeXFunctionType]: + r""" + |TeX_code| should be some expl3 code that defines a function with name |%name%| that when called should: + * run some \TeX\ code (which includes reading the arguments, if any) + * do the following if |sync|: + * send |r| to Python (equivalently write %sync%) + * send whatever needed for the output (as in |ttp_argtypes|) + * call |\__read_do_one_command:| iff not |finish|. + + This is allowed to contain the following: + * %name%: the name of the function to be defined as explained above. + * %read_arg0(\var_name)%, %read_arg1(...)%: will be expanded to code that reads the input. + * %send_arg0(...)%, %send_arg1(...)%: will be expanded to code that sends the content. + * %send_arg0_var(\var_name)%, %send_arg1_var(...)%: will be expanded to code that sends the content in the variable. + * %optional_sync%: expanded to code that writes |r| (to sync), if |sync| is True. + + ptt_argtypes: list of argument types to be sent from Python to TeX (i.e. input of the TeX function) + + ttp_argtypes: list of argument types to be sent from TeX to Python (i.e. output of the TeX function) + + recursive: whether the TeX_code might call another Python function. Default to True. + It does not hurt to always specify True, but performance would be a bit slower. + + sync: whether the Python function need to wait for the TeX function to finish. + Required if |ttp_argtypes| is not empty. + This should be left to be the default None most of the time. (which will make it always sync if |debugging|, + otherwise only sync if needed i.e. there's some output) + + finish: Include this if and only if |\__read_do_one_command:| is omitted. + Normally this is not needed, but it can be used as a slight optimization; and it's needed internally to implement + |run_none_finish| among others. + For each TeX-call-Python layer, \emph{exactly one} |finish| call can be made. If the function itself doesn't call + any |finish| call (which happens most of the time), then the wrapper will call |run_none_finish|. + + Return some TeX code to be executed, and a Python function object that when called will call the TeX function + and return the result. + + Possible optimizations: + * the |r| is not needed if not recursive and |ttp_argtypes| is nonempty + (the output itself tells Python when the \TeX\ code finished) + * the first line of the output may be on the same line as the |r| itself (done, use TTPEmbeddedLine type, although a bit hacky) + """ + if ttp_argtypes!=[]: + assert sync!=False + sync=True + + if sync is None: + sync=pythonimmediate.debugging + + TeX_code=template_substitute(TeX_code, "%optional_sync%", + lambda _: r'\immediate\write\__write_file { r }' if sync else '',) + + TeX_code=template_substitute(TeX_code, "%sync%", + lambda _: r'\immediate\write\__write_file { r }' if sync else '', optional=True) + + assert sync is not None + if ttp_argtypes: assert sync + assert ttp_argtypes.count(TTPEmbeddedLine)<=1 + identifier=get_random_identifier() # TODO to be fair it isn't necessary to make the identifier both ways distinct, can reuse + + TeX_code=template_substitute(TeX_code, "%name%", lambda _: r"\__run_" + identifier + ":") + + for i, argtype_ in enumerate(ptt_argtypes): + TeX_code=template_substitute(TeX_code, r"%read_arg" + str(i) + r"\(([^)]*)\)%", + lambda match: argtype_.read_code(match[1]), + optional=True) + + for i, argtype in enumerate(ttp_argtypes): + TeX_code=template_substitute(TeX_code, f"%send_arg{i}" + r"\(([^)]*)\)%", + lambda match: argtype.send_code(match[1]), + optional=True) + TeX_code=template_substitute(TeX_code, f"%send_arg{i}_var" + r"\(([^)]*)\)%", + lambda match: argtype.send_code_var(match[1]), + optional=True) + + def f(*args)->Optional[Tuple[TeXToPyData, ...]]: + assert len(args)==len(ptt_argtypes) + + # send function header + check_not_finished() + if finish: + global action_done + action_done=True + send_raw(identifier+"\n") + + # send function args + for arg, argtype in zip(args, ptt_argtypes): + assert isinstance(arg, argtype) + arg.write() + + if not sync: return None + + # wait for the result + if recursive: + result_=run_main_loop() + else: + result_=run_main_loop_get_return_one() + + result: List[TeXToPyData]=[] + if TTPEmbeddedLine not in ttp_argtypes: + assert not result_ + for argtype_ in ttp_argtypes: + if argtype_==TTPEmbeddedLine: + result.append(TTPEmbeddedLine(result_)) + else: + result.append(argtype_.read()) + return tuple(result) + + return TeX_code, f + +scan_Python_call_TeX(__file__) + +def define_Python_call_TeX_local(*args, **kwargs)->PythonCallTeXFunctionType: + """ + used to define "local" handlers i.e. used by this library. + The code will be included in mark_bootstrap(). + """ + code, result=define_Python_call_TeX(*args, **kwargs) + mark_bootstrap(code) + return result + +# essentially this is the same as the above, but just that the return type is guaranteed to be not None to satisfy type checkers +def define_Python_call_TeX_local_sync(*args, **kwargs)->PythonCallTeXSyncFunctionType: + return define_Python_call_TeX_local(*args, **kwargs, sync=True) # type: ignore + +run_none_finish=define_Python_call_TeX_local( +r""" +\cs_new_eq:NN %name% \relax +""", [], [], finish=True, sync=False) + + +""" +|run_error_finish| is fatal to TeX, so we only run it when it's fatal to Python. + +We want to make sure the Python traceback is printed strictly before run_error_finish() is called, +so that the Python traceback is not interleaved with TeX error messages. +""" +run_error_finish=define_Python_call_TeX_local( +r""" +\msg_new:nnn {pythonimmediate} {python-error} {Python~error.} +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \wlog{^^JPython~error~traceback:^^J\__data^^J} + \msg_error:nn {pythonimmediate} {python-error} +} +""", [PTTBlock], [], finish=True, sync=False) + + +put_next_blue=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn \__put_next_blue_tmp { + %optional_sync% + \expandafter \__read_do_one_command: \noexpand +} +\cs_new_protected:Npn %name% { + %read_arg0(\__target)% + \expandafter \__put_next_blue_tmp \__target +} +""" + , [PTTBalancedTokenList], [], recursive=False) + + +put_next_tokenlist=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn \__put_next_tmp { + %optional_sync% + \__read_do_one_command: +} +\cs_new_protected:Npn %name% { + %read_arg0(\__target)% + \expandafter \__put_next_tmp \__target +} +""" + , [PTTBalancedTokenList], [], recursive=False) + +get_next_=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% { + \peek_analysis_map_inline:n { + \peek_analysis_map_break:n { + \__tlserialize_char_unchecked:nnNN {##1}{##2}##3 \pythonimmediatecontinue + } + } +} +""", [], [TTPEmbeddedLine], recursive=False) + +put_next_bgroup=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__index)% + \expandafter \expandafter \expandafter \pythonimmediatecontinuenoarg + \char_generate:nn {\__index} {1} +} +""", [PTTInt], [], recursive=False) + +put_next_egroup=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__index)% + \expandafter \expandafter \expandafter \pythonimmediatecontinuenoarg + \char_generate:nn {\__index} {2} +} +""", [PTTInt], [], recursive=False) + + +get_argument_tokenlist_=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% #1 { + %sync% + %send_arg0(#1)% + \__read_do_one_command: +} +""", [], [TTPBalancedTokenList], recursive=False) + + +run_tokenized_line_local_=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \__data + %optional_sync% + \__read_do_one_command: +} +""", [PTTTeXLine], []) + +@export_function_to_module +def run_tokenized_line_local(line: str, *, check_braces: bool=True, check_newline: bool=True, check_continue: bool=True)->None: + check_line(line, braces=check_braces, newline=check_newline, continue_=(False if check_continue else None)) + run_tokenized_line_local_(PTTTeXLine(line)) + + + +@export_function_to_module +def run_tokenized_line_peek(line: str, *, check_braces: bool=True, check_newline: bool=True, check_continue: bool=True)->str: + check_line(line, braces=check_braces, newline=check_newline, continue_=(True if check_continue else None)) + return typing.cast( + Callable[[PTTTeXLine], Tuple[TTPEmbeddedLine]], + Python_call_TeX_local( + r""" + \cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \__data + } + """) + )(PTTTeXLine(line))[0] + + +run_block_local_=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \begingroup \newlinechar=10~ \expandafter \endgroup + \scantokens \expandafter{\__data} + % trick described in https://tex.stackexchange.com/q/640274 to scantokens the code with \newlinechar=10 + + %optional_sync% + \__read_do_one_command: +} +""", [PTTBlock], []) + +@export_function_to_module +def run_block_local(block: str)->None: + run_block_local_(PTTBlock(block)) + +expand_o_=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \exp_args:NNV \tl_set:No \__data \__data + %sync% + %send_arg0_var(\__data)% + \__read_do_one_command: +} +""", [PTTBalancedTokenList], [TTPBalancedTokenList], recursive=expansion_only_can_call_Python) + +expand_x_=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \tl_set:Nx \__data {\__data} + %sync% + %send_arg0_var(\__data)% + \__read_do_one_command: +} +""", [PTTBalancedTokenList], [TTPBalancedTokenList], recursive=expansion_only_can_call_Python) + +execute_=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \__data + %optional_sync% + \__read_do_one_command: +} +""", [PTTBalancedTokenList], []) + +futurelet_=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \expandafter \futurelet \__data \pythonimmediatecontinuenoarg +} +""", [PTTBalancedTokenList], []) + +futureletnext_=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% { + %read_arg0(\__data)% + \afterassignment \pythonimmediatecontinuenoarg \expandafter \futurelet \__data +} +""", [PTTBalancedTokenList], []) + +continue_until_passed_back_=define_Python_call_TeX_local_sync( +r""" +\cs_new_eq:NN %name% \relax +""", [], [TTPEmbeddedLine]) + +@export_function_to_module +def continue_until_passed_back_str()->str: + """ + Usage: + + First put some tokens in the input stream that includes |\pythonimmediatecontinue{...}| + (or |%sync% \__read_do_one_command:|), then call |continue_until_passed_back()|. + + The function will only return when the |\pythonimmediatecontinue| is called. + """ + return str(continue_until_passed_back_()[0]) + +@export_function_to_module +def continue_until_passed_back()->None: + """ + Same as |continue_until_passed_back_str()| but nothing can be returned from TeX to Python. + """ + result=continue_until_passed_back_str() + assert not result + + +@export_function_to_module +def expand_once()->None: + typing.cast(Callable[[], None], Python_call_TeX_local( + r""" + \cs_new_protected:Npn %name% { \expandafter \pythonimmediatecontinuenoarg } + """, recursive=False, sync=True))() + + +@export_function_to_module +@user_documentation +def get_arg_str()->str: + """ + Get a mandatory argument. + """ + return typing.cast(Callable[[], TTPEmbeddedLine], Python_call_TeX_local( + r""" + \cs_new_protected:Npn %name% #1 { + \immediate\write\__write_file { \unexpanded { + r #1 + }} + \__read_do_one_command: + } + """, recursive=False))() + +get_arg_estr_=define_Python_call_TeX_local_sync( +r""" +\cs_new_protected:Npn %name% #1 { + %sync% + %send_arg0(#1)% + \__read_do_one_command: +} +""", [], [TTPEBlock], recursive=False) +@export_function_to_module +@user_documentation +def get_arg_estr()->str: + return str(get_arg_estr_()[0]) + + +get_optional_argument_detokenized_=define_Python_call_TeX_local_sync( +r""" +\NewDocumentCommand %name% {o} { + \immediate\write \__write_file { + r ^^J + \IfNoValueTF {#1} { + 0 + } { + \unexpanded{1 #1} + } + } + \__read_do_one_command: +} +""", [], [TTPLine], recursive=False) +@export_function_to_module +@user_documentation +def get_optional_arg_str()->Optional[str]: + """ + Get an optional argument. + """ + [result]=get_optional_argument_detokenized_() + result_=str(result) + if result_=="0": return None + assert result_[0]=="1", result_ + return result_[1:] + + +get_optional_arg_estr_=define_Python_call_TeX_local_sync( +r""" +\NewDocumentCommand %name% {o} { + %sync% + \IfNoValueTF {#1} { + %send_arg0(0)% + } { + %send_arg0(1 #1)% + } + \__read_do_one_command: +} +""", [], [TTPEBlock], recursive=False) + +@export_function_to_module +@user_documentation +def get_optional_arg_estr()->Optional[str]: + [result]=get_optional_arg_estr_() + result_=str(result) + if result_=="0": return None + assert result_[0]=="1", result_ + return result_[1:] + + +get_verbatim_argument_=define_Python_call_TeX_local_sync( +r""" +\NewDocumentCommand %name% {v} { + \immediate\write\__write_file { \unexpanded { + r ^^J + #1 + }} + \__read_do_one_command: +} +""", [], [TTPLine], recursive=False) +@export_function_to_module +@user_documentation +def get_verb_arg()->str: + """ + Get a verbatim argument. Since it's verbatim, there's no worry of |#| being doubled, + but it can only be used at top level. + """ + return str(get_verbatim_argument_()[0]) + +get_multiline_verbatim_argument_=define_Python_call_TeX_local_sync( +r""" +\NewDocumentCommand %name% {+v} { + \immediate\write\__write_file { r } + \begingroup + \newlinechar=13~ % this is what +v argument type in xparse uses + \__send_block:n { #1 } + \endgroup + \__read_do_one_command: +} +""", [], [TTPBlock], recursive=False) +@export_function_to_module +@user_documentation +def get_multiline_verb_arg()->str: + """ + Get a multi-line verbatim argument. + """ + return str(get_multiline_verbatim_argument_()[0]) + +newcommand2=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn %name% { + \begingroup + \endlinechar=-1~ + %read_arg0(\__line)% + %read_arg1(\__identifier)% + \cs_new_protected:cpx {\__line} { + \unexpanded{\immediate\write \__write_file} { i \__identifier } + \unexpanded{\__read_do_one_command:} + } + \endgroup + %optional_sync% + \__read_do_one_command: +} +""", [PTTVerbatimLine, PTTVerbatimLine], [], recursive=False) + +renewcommand2=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn %name% { + \begingroup + \endlinechar=-1~ + \readline \__read_file to \__line + \readline \__read_file to \__identifier + \exp_args:Ncx \renewcommand {\__line} { + \unexpanded{\immediate\write \__write_file} { i \__identifier } + \unexpanded{\__read_do_one_command:} + } + \exp_args:Nc \MakeRobust {\__line} % also make the command global + \endgroup + %optional_sync% + \__read_do_one_command: +} +""", [PTTVerbatimLine, PTTVerbatimLine], [], recursive=False) + +def check_function_name(name: str)->None: + if not re.fullmatch("[A-Za-z]+", name) or (len(name)==1 and ord(name)<=0x7f): + raise RuntimeError("Invalid function name: "+name) + +def newcommand_(name: str, f: Callable)->Callable: + identifier=get_random_identifier() + + newcommand2(PTTVerbatimLine(name), PTTVerbatimLine(identifier)) + + _code=define_TeX_call_Python( + lambda: run_code_redirect_print_TeX(f), + name, argtypes=[], identifier=identifier) + # ignore _code, already executed something equivalent in the TeX command + return f + +def renewcommand_(name: str, f: Callable)->Callable: + identifier=get_random_identifier() + + renewcommand2(PTTVerbatimLine(name), PTTVerbatimLine(identifier)) + # TODO remove the redundant entry from TeX_handlers (although technically is not very necessary, just cause slight memory leak) + #try: del TeX_handlers["u"+name] + #except KeyError: pass + + _code=define_TeX_call_Python( + lambda: run_code_redirect_print_TeX(f), + name, argtypes=[], identifier=identifier) + # ignore _code, already executed something equivalent in the TeX command + return f + + + +@export_function_to_module +def newcommand(x: Union[str, Callable, None]=None, f: Optional[Callable]=None)->Callable: + """ + Define a new \TeX\ command. + If name is not provided, it's automatically deduced from the function. + """ + if f is not None: return newcommand(x)(f) + if x is None: return newcommand # weird design but okay (allow |@newcommand()| as well as |@newcommand|) + if isinstance(x, str): return functools.partial(newcommand_, x) + return newcommand_(x.__name__, x) + +@export_function_to_module +def renewcommand(x: Union[str, Callable, None]=None, f: Optional[Callable]=None)->Callable: + """ + Redefine a \TeX\ command. + If name is not provided, it's automatically deduced from the function. + """ + if f is not None: return newcommand(x)(f) + if x is None: return newcommand # weird design but okay (allow |@newcommand()| as well as |@newcommand|) + if isinstance(x, str): return functools.partial(renewcommand_, x) + return renewcommand_(x.__name__, x) + + +# ======== + +put_next_TeX_line=define_Python_call_TeX_local( +r""" +\cs_new_protected:Npn \__put_next_tmpa { + %optional_sync% + \__read_do_one_command: +} +\cs_new_protected:Npn %name% { + %read_arg0(\__target)% + \expandafter \__put_next_tmpa \__target +} +""" + , [PTTTeXLine], [], recursive=False) + +@export_function_to_module +@user_documentation +def put_next(arg: Union[str, Token, BalancedTokenList])->None: + """ + Put some content forward in the input stream. + + arg: has type |str| (will be tokenized in the current catcode regime, must be a single line), + or |BalancedTokenList|. + """ + if isinstance(arg, str): put_next_TeX_line(PTTTeXLine(arg)) + else: arg.put_next() + + + +# TODO I wonder which one is faster. Need to benchmark... +@export_function_to_module +@user_documentation +def peek_next_meaning()->str: + """ + Get the meaning of the following token, as a string, using the current |\escapechar|. + + This is recommended over |peek_next_token()| as it will not tokenize an extra token. + + It's undefined behavior if there's a newline (|\newlinechar| or |^^J|, the latter is OS-specific) + in the meaning string. + """ + return typing.cast(Callable[[], TTPEmbeddedLine], Python_call_TeX_local( + r""" + \cs_new_protected:Npn \__peek_next_meaning_callback: { + + \edef \__tmp {\meaning \__tmp} % just in case |\__tmp| is outer, |\write| will not be able to handle it + %\immediate\write \__write_file { r \unexpanded\expandafter{\__tmp} } + \immediate\write \__write_file { r \__tmp } + + \__read_do_one_command: + } + \cs_new_protected:Npn %name% { + \futurelet \__tmp \__peek_next_meaning_callback: + } + """, recursive=False))() + + +if 0: + peek_next_char_=define_Python_call_TeX_local_sync( + + # first attempt. Slower than peek_next_meaning. + r""" + \cs_new_protected:Npn \__peek_next_char_callback: { + \edef \__tmpb { \expandafter\str_item:nn\expandafter{\meaning \__tmp} {-1} } % \expandafter just in case \__tmp is \outer + \if \noexpand\__tmp \__tmpb % is a character + \immediate\write \__write_file { r^^J \__tmpb . } + \else % is not? + \immediate\write \__write_file { r^^J } + \fi + \__read_do_one_command: + } + \cs_new_protected:Npn %name% { + \futurelet \__tmp \__peek_next_char_callback: + } + """ + + # second attempt. Faster than before but still slower than peek_next_meaning. + #r""" + #\cs_new_protected:Npn %name% { + # \futurelet \__tmp \__peek_next_char_callback: + #} + # + #\cs_new_protected:Npn \__peek_next_char_callback: { + # %\if \noexpand\__tmp \c_space_token % there's also this case and that \__tmp is some TeX primitive conditional... + # \expandafter \__peek_next_char_callback_b: \meaning \__tmp \relax + #} + # + #\cs_new_protected:Npn \__peek_next_char_callback_b: #1 #2 { + # \ifx #2 \relax + # \if \noexpand\__tmp #1 % is a character + # \immediate\write \__write_file { r^^J #1 } + # \else % is not? + # \immediate\write \__write_file { r^^J } + # \fi + # \expandafter \__read_do_one_command: + # \else + # \expandafter \__peek_next_char_callback_b: \expandafter #2 + # \fi + #} + # + #""" + + , [], [TTPLine], recursive=False) + + + +meaning_str_to_catcode: Dict[str, Catcode]={ + "begin-group character ": Catcode.bgroup, + "end-group character ": Catcode.egroup, + "math shift character ": Catcode.math, + "alignment tab character ": Catcode.alignment, + "macro parameter character ": Catcode.parameter, + "superscript character ": Catcode.superscript, + "subscript character ": Catcode.subscript, + "blank space ": Catcode.space, + "the letter ": Catcode.letter, + "the character ": Catcode.other, + } + +def parse_meaning_str(s: str)->Optional[Tuple[Catcode, str]]: + if s and s[:-1] in meaning_str_to_catcode: + return meaning_str_to_catcode[s[:-1]], s[-1] + return None + +@export_function_to_module +@user_documentation +def peek_next_char()->str: + """ + Get the character of the following token, or empty string if it's not a character. + Will also return nonempty if the next token is an implicit character token. + + Uses peek_next_meaning() under the hood to get the meaning of the following token. See peek_next_meaning() for a warning on undefined behavior. + """ + + #return str(peek_next_char_()[0]) + # too slow (marginally slower than peek_next_meaning) + + r=parse_meaning_str(peek_next_meaning()) + if r is None: + return "" + return r[1] + +@export_function_to_module +def get_next_char()->str: + result=Token.get_next() + assert isinstance(result, CharacterToken), "Next token is not a character!" + return result.chr + +# ======== + +try: + send_bootstrap_code() + run_main_loop() # if this returns cleanly TeX has no error. Otherwise some readline() will reach eof and print out a stack trace + assert not raw_readline(), "Internal error: TeX sends extra line" + +except: + # see also documentation of run_error_finish. + sys.stderr.write("\n") + traceback.print_exc(file=sys.stderr) + + if do_run_error_finish: + action_done=False # force run it + run_error_finish(PTTBlock("".join(traceback.format_exc()))) + + os._exit(0) + diff --git a/macros/latex/contrib/saveenv/saveenv.pdf b/macros/latex/contrib/saveenv/saveenv.pdf Binary files differindex 20bf11c9fd..f93c230efa 100644 --- a/macros/latex/contrib/saveenv/saveenv.pdf +++ b/macros/latex/contrib/saveenv/saveenv.pdf diff --git a/macros/latex/contrib/saveenv/saveenv.sty b/macros/latex/contrib/saveenv/saveenv.sty index 454275c878..9a694a5f4d 100644 --- a/macros/latex/contrib/saveenv/saveenv.sty +++ b/macros/latex/contrib/saveenv/saveenv.sty @@ -12,7 +12,7 @@ % % The Current Maintainer of this work is user202729. -\ProvidesExplPackage{saveenv}{2022/07/08}{0.0.0}{Save environment content verbatim} +\ProvidesExplPackage{saveenv}{2022/12/23}{0.0.1}{Save environment content verbatim} \RequirePackage{precattl} \msg_new:nnn {saveenv} {trailing-content-or-pretokenized} {Trailing~content~found~on~line,~or~content~pretokenized!} @@ -51,14 +51,11 @@ \tl_build_gbegin:N #1 - \prg_replicate:nn {\the\inputlineno-\_senv_firstline-2} { + \prg_replicate:nn {\the\inputlineno-\_senv_firstline-1} { \ior_str_get:NN \_senv_file \_senv_line - \exp_args:NNo \tl_build_gput_right:Nn #1 {\_senv_line \cO\^^J} + \exp_args:NNo \tl_build_gput_right:Nn #1 {\_senv_line ^^J} } - \ior_str_get:NN \_senv_file \_senv_line - \exp_args:NNo \tl_build_gput_right:Nn #1 {\_senv_line} - \ior_close:N \_senv_file \tl_build_gend:N #1 @@ -67,16 +64,19 @@ % #1: target macro -\NewDocumentEnvironment {saveenvkeeplast} {m} { +% #2: content to be inserted after the \end +\NewDocumentEnvironment {saveenvkeeplastreinsert} {mm} { \begingroup \edef \_senv_old_endlinechar {\the\endlinechar} + %\bench before cctab. \cctab_select:N \c_other_cctab % note that this changes the value of \endlinechar as well + %\bench after cctab. \int_compare:nNnTF {\_senv_old_endlinechar} < {0} { - \_senv_start_get_body:N #1 + \_senv_start_get_body:Nn #1 {#2} } { \exp_last_unbraced:Nf \peek_meaning_remove:NTF { \char_generate:nn{\_senv_old_endlinechar}{12} } %12 is other { - \_senv_start_get_body:N #1 + \_senv_start_get_body:Nn #1 {#2} } { \msg_error:nn {saveenv} {trailing-content-or-pretokenized} @@ -85,24 +85,33 @@ } { } + % #1: target macro -\cs_new_protected:Npn \_senv_start_get_body:N #1 { +% #2: content to be inserted after the \end +\cs_new_protected:Npn \_senv_start_get_body:Nn #1 #2 { \endlinechar=10~ \str_set:NV \_senv_env \@currenvir \tl_replace_all:Nnn \_senv_env {~} {\cO\ } - \exp_args:NNVV \_senv_helper #1 \_senv_env \@currenvir + \_senv_helper:NVVn #1 \_senv_env \@currenvir {#2} } % #1: target macro % #2: value of \@currenvir but with all tokens catcode 12 (other) % #3: value of \@currenvir -\def \_senv_helper #1 #2 #3 { +% #4 content to be inserted after the \end +\cs_new_protected:Npn \_senv_helper:Nnnn #1 #2 #3 #4 { \cs_set_protected:cpn {[saveenv]~verbatim~body~scanner~for~#2} ##1 \cO{ \\end\{ } #2 \cO\} { + %\bench X3. % ##1: the body - \peek_meaning_remove:NTF \cO\^^J { + \peek_meaning_remove:NTF ^^J { + %\bench inside peek. \endgroup + %\bench after endgroup. \str_gset:Nn #1 {##1} + %\bench X5. \end{#3} + %\bench X6. + #4 } { \msg_error:nn {saveenv} {trailing-content-end-line} } @@ -110,14 +119,23 @@ \use:c {[saveenv]~verbatim~body~scanner~for~#2} } +\cs_generate_variant:Nn \_senv_helper:Nnnn {NVVn} + +\NewDocumentEnvironment {saveenvkeeplast} {m} { + \saveenvkeeplastreinsert #1 {} +} { + \endsaveenvkeeplastreinsert +} % set variable #1 to have content of #2, but with last line dropped. % lines are separated by \^^J. +\cs_new:Npn \_senv_append_newline:n #1 { #1 ^^J } + \cs_new_protected:Npn \saveenv_set_drop_last:Nn #1 #2 { \tl_set:Nn \_senv_body {#2} \tl_replace_all:Nnn \_senv_body {~} {\cO\ } % keep spaces in seq_set_split (support older expl3 versions) - \seq_set_split:NnV \_senv_lines {\cO\^^J} \_senv_body + \seq_set_split:NnV \_senv_lines {^^J} \_senv_body \seq_pop_right:NN \_senv_lines \_senv_lastline \tl_map_inline:Nn \_senv_lastline { % debug check, ensure last line is empty \int_case:nnF {`##1} { @@ -128,17 +146,24 @@ \msg_error:nnoVV {saveenv} {leading-content-last-line} {\number`##1} \_senv_lastline \@currenvir } } - \str_gset:Nx #1 {\seq_use:Nn \_senv_lines {\cO\^^J}} + %\str_gset:Nx #1 {\seq_use:Nn \_senv_lines {^^J}} % this is extremely slow because \seq_use:Nn is ⋆-expandable + \str_gset:Nx #1 {\seq_map_function:NN \_senv_lines \_senv_append_newline:n } } \cs_generate_variant:Nn \saveenv_set_drop_last:Nn {NV} +\NewDocumentEnvironment {saveenvreinsert} {mm} { + \saveenvkeeplastreinsert #1 {#2} +} { + \endsaveenvkeeplastreinsert + \saveenv_set_drop_last:NV #1 #1 +} \NewDocumentEnvironment {saveenv} {m} { - \saveenvkeeplast \_senv_content + \saveenvkeeplast #1 } { \endsaveenvkeeplast - \saveenv_set_drop_last:NV #1 \_senv_content + \saveenv_set_drop_last:NV #1 #1 } diff --git a/macros/latex/contrib/saveenv/saveenv.tex b/macros/latex/contrib/saveenv/saveenv.tex index c6d4ba87bb..5a5f741a25 100644 --- a/macros/latex/contrib/saveenv/saveenv.tex +++ b/macros/latex/contrib/saveenv/saveenv.tex @@ -1,5 +1,5 @@ %%! TEX program = lualatex -\ProvidesFile{saveenv.tex} [2022/07/08 v0.0.0 ] +\ProvidesFile{saveenv.tex} [2022/12/23 v0.0.1 ] \documentclass{l3doc} \EnableCrossrefs \CodelineIndex @@ -41,8 +41,8 @@ This package provides tools to create your own verbatim environments, and works \begin{itemize} \item \pkg{scontents} package is similar; however it does not allow programmer to programmatically retrieve the macro content, only do a limited number of actions with them (execute, typeset as verbatim code, etc.). - \item \pkg{fancyvrb} and \pkg{verbatimcopy} provides some internal macro for defining similar environments, however they're internal. - \item \pkg{filecontentsdef} and \pkg{newverbs} provides very similar facilities (\env{filecontentsdefmacro} environment and \tn{Collectverbenv} command respectively); however it requires \verb+\endlinechar=13+ and does not support \cs{ExplSyntaxOn} environment. + \item \pkg{fancyvrb}, \pkg{xsimverb} and \pkg{verbatimcopy} provides some internal macro for defining similar environments, however they're internal and/or too specialized (writes to file etc.). + \item \pkg{filecontentsdef} and \pkg{newverbs} provides very similar facilities (\env{filecontentsdefmacro} environment and |\Collectverbenv| command respectively); however it requires |\endlinechar=13| and does not support |\ExplSyntaxOn| environment. \item \pkg{verbatim} provides facilities to define custom verbatim environment that processes the content line-by-line, however the interface is more complicated. For comparison, the following code @@ -80,7 +80,7 @@ This package provides tools to create your own verbatim environments, and works \end{verbatim} defines an environment that saves the data similar to \env{saveenv} environment described below (with the overhead of |\tl_build_*| functions), -but inside \cs{ExplSyntaxOn} environment it generates a spurious space at the beginning of the string. +but inside |\ExplSyntaxOn| environment it generates a spurious space at the beginning of the string. \end{itemize} @@ -91,17 +91,28 @@ but inside \cs{ExplSyntaxOn} environment it generates a spurious space at the be Environment that saves its body. For example, the following code - \begin{verbatim} \begin{saveenv}{\data} 123 456 \end{saveenv} \end{verbatim} +will save the string \texttt{123\meta{newline}456\meta{newline}} globally into |\data|. + +Remark: it is consistent to keep the trailing newline, as +\begin{verbatim} +\begin{saveenv}{\data} +\end{saveenv} +\end{verbatim} +will make |\data| empty, and +\begin{verbatim} +\begin{saveenv}{\data} -will save the string \texttt{123\meta{newline}456} globally into \cs{data}. +\end{saveenv} +\end{verbatim} +will make |\data| consist of a single \meta{newline}. -The braces around \verb+{\data}+ is optional; however, in the unlikely case if \cs{endlinechar} has the \enquote{letter} catcode, it might be absorbed and gives unexpected result. +The braces around |{\data}| is optional; however, in the unlikely case if \cs{endlinechar} has the \enquote{letter} catcode, it might be absorbed and gives unexpected result. A newline is represented as a token with charcode 10 (|^^J|) and catcode other. @@ -116,7 +127,7 @@ The assignment is global, and done before the macro \cs{endsaveenv} is executed. The data is represented as an \pkg{expl3}-string, that is, a sequence of tokens with catcode 12 (for non-space characters) or 10 (for space character). -In other words, the token list is equal to its own \cs{detokenize}. +In other words, the token list is equal to its own |\detokenize|. \DescribeEnv{saveenvghost} Similar to above; however the content inside is still typesetted/executed, and the Sync\TeX\ information is preserved. @@ -127,7 +138,43 @@ Note that this environment is implemented by reading the whole file from the beg Similar to above; however the final newline and the space characters before \texttt{\cs{end}\{\meta{environment name}\}} are preserved. For example, the example above with \texttt{saveenv} replaced with \texttt{saveenvkeeplast} -will save the string \texttt{123\meta{newline}456\meta{newline}} into \cs{data} instead. +will save the string \texttt{123\meta{newline}456\meta{newline}} into |\data| instead. + +\section{Reinsert environments} + +Sometimes it's desirable to execute something (e.g. do some local assignments) after the group ends. + +There are a few options: +\begin{itemize} + \item use |\aftergroup|, + \item close the group, execute the code and open a new one (remember to preserve the value of |\@currenvir|), + \item use one of the environments below. +\end{itemize} + +\DescribeEnv{saveenvreinsert} +Environment that takes two arguments, the \meta{str var} to be set and the code to be put after the group end. + +Usage example: If the following code is executed +\begin{verbatim} +\begin{saveenvreinsert}{\data}{\myfunction {args etc.}} +123 +456 +\end{saveenvreinsert} +some other content +\end{verbatim} +the effect would be identical to as if |\data| is set to \texttt{123\meta{newline}456\meta{newline}}, +then the following code is executed +\begin{verbatim} +\myfunction {args etc.}some other content +\end{verbatim} +If the second argument is empty, this environment is identical to the \env{saveenv} environment. + +Note that in the example above |\myfunction| is executed after the group ends, +while any code in the second block of the |\NewDocumentEnvironment| definition +would be executed before the group ends (such as in the example below), thus any local assignment will not persist. + +\DescribeEnv{saveenvkeeplastreinsert} +Same as above, but the trailing whitespaces (after the last newline) are preserved. \section{Extending the environments} @@ -148,6 +195,8 @@ For example, the following definition \end{verbatim} will define an environment \env{custom} that prints out the content inside the environment using |\tl_show:N|. +Note that in this case |\tl_show:N \data| is executed before the group ends. + \section{Limitation} \begin{itemize} @@ -157,13 +206,15 @@ will define an environment \env{custom} that prints out the content inside the e Nevertheless, in modern computers, the overhead is negligible. \item In case it's desired to typeset the content afterwards, it's difficult to preserve the Sync\TeX\ data. + + (although with Lua\LaTeX\ it's possible, see \pkg{rescansync} package) \end{itemize} \item Currently nested environments with the same name are not supported (unlike \pkg{scontents} package). - \item Note that there must be nothing after the \texttt{\cs{begin}\{\meta{environment name}\}} or the \texttt{\cs{end}\{\meta{environment name}\}} line. + \item Note that there must be nothing after the |\begin|\texttt{\{\meta{environment name}\}} or the |\end|\texttt{\{\meta{environment name}\}} line. - \item Note that \texttt{\cs{end}\{\meta{environment name}\}} must not be in the middle of any line. + \item Note that |\end|\texttt{\{\meta{environment name}\}} must not be in the middle of any line. \item The \cs{endlinechar} must not be tokenized in advance (or if it is, its catcode must be 12/other). This might happen when for example your environment looks ahead for optional argument. @@ -172,10 +223,9 @@ will define an environment \env{custom} that prints out the content inside the e Alternatively, you can \begin{itemize} \item manually remove the end line token, - \item set \verb+\endlinechar=-1\relax+ before calling \cs{saveenv} - \item (optional) reset the value of \cs{endlinechar} afterwards. - + \item set |\endlinechar=-1\relax| inside the environment before calling \cs{saveenv}. \end{itemize} + The value of |\endlinechar| will automatically be reset after the group of the environment is closed. \end{itemize} diff --git a/macros/latex/contrib/tagpdf/README.md b/macros/latex/contrib/tagpdf/README.md index 265428377f..dc8ac843f8 100644 --- a/macros/latex/contrib/tagpdf/README.md +++ b/macros/latex/contrib/tagpdf/README.md @@ -1,6 +1,6 @@ #tagpdf — A package to create tagged pdf -Packageversion: 0.97 -Packagedate: 2022/08/24 +Packageversion: 0.98 +Packagedate: 2022/12/22 Author: Ulrike Fischer ## License diff --git a/macros/latex/contrib/tagpdf/tagpdf-backend.dtx b/macros/latex/contrib/tagpdf/tagpdf-backend.dtx index 800a6bfa46..47369b519b 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-backend.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-backend.dtx @@ -47,13 +47,13 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{implementation} % \begin{macrocode} %<@@=tag> %<*luatex> -\ProvidesExplFile {tagpdf-luatex.def} {2022-08-24} {0.97} +\ProvidesExplFile {tagpdf-luatex.def} {2022-12-22} {0.98} {tagpdf~driver~for~luatex} % \end{macrocode} % \section{Loading the lua} @@ -146,6 +146,7 @@ } % \end{macrocode} % \end{macro} +% % \begin{macrocode} %</luatex> % \end{macrocode} @@ -157,8 +158,8 @@ local ProvidesLuaModule = { name = "tagpdf", - version = "0.97", --TAGVERSION - date = "2022-08-24", --TAGDATE + version = "0.98", --TAGVERSION + date = "2022-12-22", --TAGDATE description = "tagpdf lua code", license = "The LATEX Project Public License 1.3c" } @@ -282,6 +283,7 @@ ltx.@@.struct = ltx.@@.struct or { } -- struct data ltx.@@.tables = ltx.@@.tables or { } -- tables created with new prop and new seq. -- wasn't a so great idea ... -- g_@@_role_tags_seq used by tag<-> is in this tables! + -- used for pure lua tables too now! ltx.@@.page = ltx.@@.page or { } -- page data, currently only i->{0->mcnum,1->mcnum,...} ltx.@@.trace = ltx.@@.trace or { } -- show commands ltx.@@.func = ltx.@@.func or { } -- functions @@ -437,7 +439,28 @@ local function @@_get_mathsubtype (mathnode) end % \end{macrocode} % \end{macro} - +% +% \begin{variable}{ltx.@@.tables.role_tag_attribute,ltx.@@.tables.role_tag_attribute} +% The first is a table with key a tag and value a number (the attribute) +% The second is an array with the attribute value as key. +% \begin{macrocode} +ltx.@@.tables.role_tag_attribute = {} +ltx.@@.tables.role_attribute_tag = {} +% \end{macrocode} +% \end{variable} +% \begin{macro}{ltx.@@.func.alloctag} +% \begin{macrocode} +local @@_alloctag = + function (tag) + if not ltx.@@.tables.role_tag_attribute[tag] then + table.insert(ltx.@@.tables.role_attribute_tag,tag) + ltx.@@.tables.role_tag_attribute[tag]=#ltx.@@.tables.role_attribute_tag + @@_log ("Add "..tag.." "..ltx.@@.tables.role_tag_attribute[tag],3) + end + end +ltx.@@.func.alloctag = @@_alloctag +% \end{macrocode} +% \end{macro} % \begin{macro} % { % @@_get_num_from, @@ -451,8 +474,8 @@ end % \begin{macrocode} local @@_get_num_from = function (tag) - if ltx.@@.tables["g_@@_role_tags_prop"][tag] then - a= ltx.@@.tables["g_@@_role_tags_prop"][tag] + if ltx.@@.tables.role_tag_attribute[tag] then + a= ltx.@@.tables.role_tag_attribute[tag] else a= -1 end @@ -479,13 +502,13 @@ end % } % These functions are the opposites to the previous function: % they take as argument a number (the attribute value) and return the string |tag|. -% The first function outputs the number for lua, while the |output| function +% The first function outputs the string for lua, while the |output| function % outputs to tex. % \begin{macrocode} local @@_get_tag_from = function (num) - if ltx.@@.tables["g_@@_role_tags_seq"][num] then - a = ltx.@@.tables["g_@@_role_tags_seq"][num] + if ltx.@@.tables.role_attribute_tag[num] then + a = ltx.@@.tables.role_attribute_tag[num] else a= "UNKNOWN" end @@ -924,7 +947,7 @@ function ltx.@@.func.mark_page_elements (box,mcpagecnt,mccntprev,mcopen,name,mct elseif n.id == VLIST then -- enter the vlist mcopen,mcpagecnt,mccntprev,mctypeprev= ltx.@@.func.mark_page_elements (n,mcpagecnt,mccntprev,mcopen,"INTERNAL VLIST",mctypeprev) - elseif n.id == GLUE then -- at glue real space chars are inserted, but this has + elseif n.id == GLUE and not n.leader then -- at glue real space chars are inserted, but this has -- been done if the previous shipout wandering, so here it is ignored elseif n.id == LOCAL_PAR then -- local_par is ignored elseif n.id == PENALTY then -- penalty is ignored diff --git a/macros/latex/contrib/tagpdf/tagpdf-checks.dtx b/macros/latex/contrib/tagpdf/tagpdf-checks.dtx index e5ba53195f..52113f348f 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-checks.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-checks.dtx @@ -48,7 +48,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{documentation} % \section{Commands} @@ -295,7 +295,7 @@ % \begin{macrocode} %<@@=tag> %<*header> -\ProvidesExplPackage {tagpdf-checks-code} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-checks-code} {2022-12-22} {0.98} {part of tagpdf - code related to checks, conditionals, debugging and messages} %</header> % \end{macrocode} @@ -402,9 +402,22 @@ % Informational message shown if log-mode is high enough % \begin{macrocode} \msg_new:nnn { tag } {struct-show-closing} - { closing~structure~#1~tagged~\prop_item:cn{g_@@_struct_#1_prop}{S} } + { closing~structure~#1~tagged~\use:e{\prop_item:cn{g_@@_struct_#1_prop}{S}} } % \end{macrocode} % \end{macro} +% \begin{macro}{tree-struct-still-open} +% Message issued at the end if there are beside Root other +% open structures on the stack. +% \begin{macrocode} +\msg_new:nnn { tag } {tree-struct-still-open} + { + There~are~still~open~structures~on~the~stack!\\ + The~stack~contains~\seq_use:Nn\g_@@_struct_tag_stack_seq{,}.\\ + The~structures~are~automatically~closed,\\ + but~their~nesting~can~be~wrong. + } +% \end{macrocode} +% \end{macro} % % \subsection{Attributes} % Not much yet, as attributes aren't used so much. @@ -423,11 +436,30 @@ \msg_new:nnn { tag } {role-unknown-tag} { tag~#1~is~not~known } % \end{macrocode} % \end{macro} +% \begin{macro}{role-parent-child} +% This is info and warning message about the containment rules between child and +% parent tags. +% \begin{macrocode} +\msg_new:nnn { tag } {role-parent-child} + { The~rule~between~parent~'#1'~\\and~child~'#2'~is~#3} +% \end{macrocode} +% \end{macro} +% \begin{macro}{role-parent-child} +% This is info and warning message about the containment rules between child and +% parent tags. +% \begin{macrocode} +\msg_new:nnn { tag } {role-remapping} + { remapping~tag~to~#1 } +% \end{macrocode} +% \end{macro} % \begin{macro}{role-tag,new-tag} % Info messages. % \begin{macrocode} \msg_new:nnn { tag } {role-tag} { mapping~tag~#1~to~role~#2 } \msg_new:nnn { tag } {new-tag} { adding~new~tag~#1 } +\msg_new:nnn { tag } {read-namespace} { reading~namespace~definitions~tagpdf-ns-#1.def } +\msg_new:nnn { tag } {namespace-missing}{ namespace~definitions~tagpdf-ns-#1.def~not~found } +\msg_new:nnn { tag } {namespace-unknown}{ namespace~#1~is~not~declared } % \end{macrocode} % \end{macro} % @@ -562,7 +594,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_check_structure_tag:N #1 { - \prop_if_in:NoF \g_@@_role_tags_prop {#1} + \prop_if_in:NoF \g_@@_role_tags_NS_prop {#1} { \msg_warning:nnx { tag } {role-unknown-tag} {#1} } @@ -618,10 +650,10 @@ { \tl_if_empty:nTF {#2} { - \msg_warning:nnn { tag } {role-missing} {#1} + \msg_error:nnn { tag } {role-missing} {#1} } { - \prop_get:NnNTF \g_@@_role_tags_prop {#2} \l_tmpa_tl + \prop_get:NnNTF \g_@@_role_tags_NS_prop {#2} \l_tmpa_tl { \int_compare:nNnT {\l_@@_loglevel_int} > { 0 } { @@ -629,13 +661,35 @@ } } { - \msg_warning:nnn { tag } {role-unknown} {#2} + \msg_error:nnn { tag } {role-unknown} {#2} + } + } + } +% \end{macrocode} +% Similar with a namespace +% \begin{macrocode} +\cs_new_protected:Npn \@@_check_add_tag_role:nnn #1 #2 #3 %#1 tag/NS, #2 role #3 namespace + { + \tl_if_empty:nTF {#2} + { + \msg_error:nnn { tag } {role-missing} {#1} + } + { + \prop_get:cnNTF { g_@@_role_NS_#3_prop } {#2} \l_tmpa_tl + { + \int_compare:nNnT {\l_@@_loglevel_int} > { 0 } + { + \msg_info:nnnn { tag } {role-tag} {#1} {#2/#3} + } + } + { + \msg_error:nnn { tag } {role-unknown} {#2/#3} } } } % \end{macrocode} % \end{macro} -% + % \subsection{Check related to mc-chunks} % % \begin{macro}{\@@_check_mc_if_nested:,\@@_check_mc_if_open:} diff --git a/macros/latex/contrib/tagpdf/tagpdf-code.pdf b/macros/latex/contrib/tagpdf/tagpdf-code.pdf Binary files differindex 551a700624..6dfd5b74a4 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-code.pdf +++ b/macros/latex/contrib/tagpdf/tagpdf-code.pdf diff --git a/macros/latex/contrib/tagpdf/tagpdf-data.dtx b/macros/latex/contrib/tagpdf/tagpdf-data.dtx new file mode 100644 index 0000000000..560856bf28 --- /dev/null +++ b/macros/latex/contrib/tagpdf/tagpdf-data.dtx @@ -0,0 +1,581 @@ +% \iffalse meta-comment +% +%% File: tagpdf-data.dtx +% +% Copyright (C) 2022 Ulrike Fischer +% +% It may be distributed and/or modified under the conditions of the +% LaTeX Project Public License (LPPL), either version 1.3c of this +% license or (at your option) any later version. The latest version +% of this license is in the file +% +% https://www.latex-project.org/lppl.txt +% +% This file is part of the "tagpdf bundle" (The Work in LPPL) +% and all files in that bundle must be distributed together. +% +% ----------------------------------------------------------------------- +% +% The development version of the bundle can be found at +% +% https://github.com/u-fischer/tagpdf +% +% for those people who are interested. +%<*driver> +\DocumentMetadata{} +\documentclass{l3doc} +\usepackage{array,booktabs,caption} +\hypersetup{pdfauthor=Ulrike Fischer, + pdftitle=tagpdf-data module (tagpdf)} +\begin{document} + \DocInput{\jobname.dtx} +\end{document} +%</driver> +% \fi +% \title{^^A +% The \pkg{tagpdf-data} module\\ +% Data files for tag namespaces and containment rules^^A +% \\ Part of the tagpdf package +% } +% +% \author{^^A +% Ulrike Fischer\thanks +% {^^A +% E-mail: +% \href{mailto:fischer@troubleshooting-tex.de} +% {fischer@troubleshooting-tex.de}^^A +% }^^A +% } +% +% \date{Version 0.98, released 2022-12-22} +% \maketitle +% \begin{documentation} +% This files contains a various data files which are read in +% by \pkg{tagpdf} to setup name spaces and rules for tag relationships. +% +% \end{documentation} +% \begin{implementation} +% \section{The LaTeX namespace} +% This is the main new namespace. It is bound to change a lot! +% It lists the new tag, the rolemap and the namespace of the rolemap. +% \begin{macrocode} +%<*ns-latex> +%% \ProvidesExplFile {tagpdf-ns-latex.def} {2022-12-22} {0.98} +%% {latex} {https://www.latex-project.org/ns/dflt/2022}{} +title, Title, pdf2, +part, Title, pdf2, +section, H1, pdf2, +subsection, H2, pdf2, +subsubsection,H3, pdf2, +paragraph, H4, pdf2, +subparagraph, H5, pdf2, +itemize, L, pdf2, +enumerate, L, pdf2, +description, L, pdf2, +item, LI, pdf2, +itemlabel, Lbl, pdf2, +itembody, LBody, pdf2, +footnote, FENote, pdf2, +footnotemark, Lbl, pdf2, +footnotelabel,Lbl, pdf2, +%</ns-latex> +% \end{macrocode} +% \section{The LaTeX-book namespace} +% This changes the rolemapping of some tags for heading with +% to values more suitable in book classes. +% It is bound to change +% \begin{macrocode} +%<*ns-latex-book> +%% \ProvidesExplFile {tagpdf-ns-latex-book.def} {2022-12-22} {0.98} +%% {latex-book} {https://www.latex-project.org/ns/book/2022}{} +chapter, H1,pdf2, +section, H2,pdf2, +subsection, H3,pdf2, +subsubsection, H4,pdf2, +paragraph, H5,pdf2, +subparagraph, H6,pdf2, +%</ns-latex-book> +% \end{macrocode} +% \section{The LaTeX-inline namespace} +% This degrades the rolemapping of some tags. It is an experimental name space. +% It can disappear again if it doesn't look usefull. +% It is bound to change a lot! +% \begin{macrocode} +%<*ns-latex-inline> +%% \ProvidesExplFile {tagpdf-ns-latex-inline.def} {2022-12-22} {0.98} +%% {latex-inline} {https://www.latex-project.org/ns/inline/2022}{} +chapter, Span, pdf2, +section, Span, pdf2, +subsection, Span, pdf2, +subsubsection, Span, pdf2, +paragraph, Span, pdf2, +subparagraph, Span, pdf2, +P, Span, pdf2, +%</ns-latex-inline> +% \end{macrocode} +% \section{The pdf namespace data} +% \begin{macrocode} +%<*ns-pdf> +%% \ProvidesExplFile {tagpdf-ns-pdf.def} {2022-12-22} {0.98} +%% {pdf} {http://iso.org/pdf/ssn}{} +StructTreeRoot,,,D, +Document,,,D, +Part,,,G, +Sect,,,G, +Div,,,G, +Caption,,,GB, +NonStruct,,,G, +H,,,B, +H1,,,B, +H2,,,B, +H3,,,B, +H4,,,B, +H5,,,B, +H6,,,B, +P,,,B, +L,,,BI, +LI,,,L, +Lbl,,,I, +LBody,,,L, +Table,,,B, +TR,,,T, +TH,,,T, +TD,,,T, +THead,,,T, +TBody,,,T, +TFoot,,,T, +Span,,,I, +Link,,,GBI, +Annot,,,GBI, +Figure,,,GBI, +Formula,,,GBI, +Form,,,GBI, +Ruby,,,I, +RB,,,I, +RT,,,I, +Warichu,,,I, +WT,,,I, +WP,,,I, +Artifact,,,GBI, +Art,,,,G, % only pdf, types unclear, needs correction later +BlockQuote,,,G, %or GB? +TOC,,,G, +TOCI,,,G, +Index,,,G, +Private,,,G, +Quote,,,I, +Note,,,GBI, % I in reference, but +Reference,,,B, % +BibEntry,,,B, % +Code,,,I, % +%</ns-pdf> +% \end{macrocode} +% \section{The pdf 2.0 namespace data} +% \begin{macrocode} +%<*ns-pdf2> +%% \ProvidesExplFile {tagpdf-ns-pdf2.def} {2022-12-22} {0.98} +%% {pdf2} {http://iso.org/pdf2/ssn}{} +Document,,,D, +Part,,,G, +Sect,,,G, +Div,,,G, +Caption,,,GB, +NonStruct,,,G, +H,,,B, +H1,,,B, +H2,,,B, +H3,,,B, +H4,,,B, +H5,,,B, +H6,,,B, +P,,,B, +L,,,BI, +LI,,,L, +Lbl,,,I, +LBody,,,L, +Table,,,B, +TR,,,T, +TH,,,T, +TD,,,T, +THead,,,T, +TBody,,,T, +TFoot,,,T, +Span,,,I, +Link,,,GBI, +Annot,,,GBI, +Figure,,,GBI, +Formula,,,GBI, +Form,,,GBI, +Ruby,,,I, +RB,,,I, +RT,,,I, +Warichu,,,I, +WT,,,I, +WP,,,I, +Artifact,,,GBI, +DocumentFragment,Art, pdf,D, %rest only pdf2 +Aside, Note, pdf,G, +H7, H6, pdf,B, +H8, H6, pdf,B, +H9, H6, pdf,B, +H10, H6, pdf,B, +Title, P, pdf,GB, +FENote, Note, pdf,GBI, +Sub, Span, pdf,I, +Em, Span, pdf,I, +Strong, Span, pdf,I, +%</ns-pdf2> +% \end{macrocode} +% \section{The mathml namespace data} +% \begin{macrocode} +%<*ns-mathml> +%% \ProvidesExplFile {tagpdf-ns-mathml.def} {2022-12-22} {0.98} +% {mathml}{http://www.w3.org/1998/Math/MathML}{} +abs,,, +and,,, +annotation,,, +apply,,, +approx,,, +arccos,,, +arccosh,,, +arccot,,, +arccoth,,, +arccsc,,, +arccsch,,, +arcsec,,, +arcsech,,, +arcsin,,, +arcsinh,,, +arctan,,, +arctanh,,, +arg,,, +bind,,, +bvar,,, +card,,, +cartesianproduct,,, +cbytes,,, +ceiling,,, +cerror,,, +ci,,, +cn,,, +codomain,,, +complexes,,, +compose,,, +condition,,, +conjugate,,, +cos,,, +cosh,,, +cot,,, +coth,,, +cs,,, +csc,,, +csch,,, +csymbol,,, +curl,,, +declare,,, +degree,,, +determinant,,, +diff,,, +divergence,,, +divide,,, +domain,,, +domainofapplication,,, +emptyset,,, +eq,,, +equivalent,,, +eulergamma,,, +exists,,, +exp,,, +exponentiale,,, +factorial,,, +factorof,,, +false,,, +floor,,, +fn,,, +forall,,, +gcd,,, +geq,,, +grad,,, +gt,,, +ident,,, +image,,, +imaginary,,, +imaginaryi,,, +implies,,, +in,,, +infinity,,, +int,,, +integers,,, +intersect,,, +interval,,, +inverse,,, +lambda,,, +laplacian,,, +lcm,,, +leq,,, +limit,,, +ln,,, +log,,, +logbase,,, +lowlimit,,, +lt,,, +maction,,, +maligngroup,,, +malignmark,,, +math,,, +matrix,,, +matrixrow,,, +max,,, +mean,,, +median,,, +menclose,,, +merror,,, +mfenced,,, +mfrac,,, +mglyph,,, +mi,,, +min,,, +minus,,, +mlabeledtr,,, +mlongdiv,,, +mmultiscripts,,, +mn,,, +mo,,, +mode,,, +moment,,, +momentabout,,, +mover,,, +mpadded,,, +mphantom,,, +mprescripts,,, +mroot,,, +mrow,,, +ms,,, +mscarries,,, +mscarry,,, +msgroup,,, +msline,,, +mspace,,, +msqrt,,, +msrow,,, +mstack,,, +mstyle,,, +msub,,, +msubsup,,, +msup,,, +mtable,,, +mtd,,, +mtext,,, +mtr,,, +munder,,, +munderover,,, +naturalnumbers,,, +neq,,, +none,,, +not,,, +notanumber,,, +notin,,, +notprsubset,,, +notsubset,,, +or,,, +otherwise,,, +outerproduct,,, +partialdiff,,, +pi,,, +piece,,, +piecewise,,, +plus,,, +power,,, +primes,,, +product,,, +prsubset,,, +quotient,,, +rationals,,, +real,,, +reals,,, +reln,,, +rem,,, +root,,, +scalarproduct,,, +sdev,,, +sec,,, +sech,,, +selector,,, +semantics,,, +sep,,, +set,,, +setdiff,,, +share,,, +sin,,, +sinh,,, +subset,,, +sum,,, +tan,,, +tanh,,, +tendsto,,, +times,,, +transpose,,, +true,,, +union,,, +uplimit,,, +variance,,, +vector,,, +vectorproduct,,, +xor,,, +%</ns-mathml> +% \end{macrocode} +% \section{Rule naming} +% \begin{macro}{\c_@@_role_rules_prop} +% These two properties map the rule strings to numbers and back. +% There are in tagpdf-data.dtx near the csv files for easier maintainance. +% \begin{macrocode} +%<*package> +\prop_const_from_keyval:Nn \c_@@_role_rules_prop + { + 0..n = 1, + 0..1 = 2, + 1 = 3, %StructTreeRoot, not really needed + [a] = 4, %ruby + [b] = 5, %warichu + c = 6, % WP ?? + ‡ = 7, % Part,Div,NonStruct -> "check parent" + ∅* = 8, % or negative by default? + ∅ = -1, + } +%in debug?? +\prop_const_from_keyval:Nn \c_@@_role_rules_num_prop + { + 1 = 0..n, + 2 = 0..1, + 3 = 1 , %StructTreeRoot, not really needed + 4 = [a] , %ruby + 5 = [b] , %warichu + 6 = c , % WP ?? + 7 = ‡ , % Part,Div,NonStruct -> "check parent" + 8 = ∅* , % or negative by default? + -1 = ∅ , + } +%</package> +% \end{macrocode} +% \end{macro} +% \section{The parent-child rules for pdf <2.0} +% This csv contains the parent-child matrix for pdf <2.0. +% The values are currently the strings used in the PDF reference. +% This will perhaps change in future. +% \begin{macrocode} +%<*parent-child> +%% \ProvidesExplFile {tagpdf-parent-child.csv} {2022-12-22} {0.98} +,,,StructTreeRoot,Document,Art,Part,Div,Sect,BlockQuote,NonStruct,TOC,TOCI,Index,Private,Quote,Note,Reference,BibEntry,P,Hn,H,Lbl,Code,Span,Link,Annot,Form,Ruby,RB,RT,RP,Warichu,WT,WP,L,LI,LBody,Table,TR,TH,TD,THead,TBody,TFoot,Caption,Figure,Formula,MC +Document,both,document level,1,0..n,∅,‡,‡,∅,0..n,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +Art,1.7,grouping,∅,0..n,∅,‡,‡,0..n,0..n,‡,∅,∅,∅,0..n,∅,0..n,∅,∅,∅,0..1,0..1,∅,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,∅,∅ +Part,both,grouping,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,∅,0..n,∅,0..n,∅,∅,∅,∅,0..n,∅,∅*,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,0..n,∅ +Div,both,grouping,∅,0..n,0..n,‡,‡,0..n,0..n,‡,0..n,0..n,0..n,0..n,∅,0..n,∅,0..n,∅,∅,∅,∅,0..n,∅,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Sect,both,grouping,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,∅,0..n,∅,∅,∅,0..1,0..1,∅,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,∅,∅ +BlockQuote,1.7,grouping?,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,∅,0..n,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,0..n,∅ +NonStruct,both,grouping,∅,0..n,0..n,‡,‡,0..n,0..n,‡,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅ +TOC,1.7,grouping,∅,0..n,0..n,‡,‡,0..n,0..n,‡,0..n,0..n,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +TOCI,1.7,TOC,∅,∅,∅,‡,‡,∅,∅,‡,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +Index,1.7,grouping?,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,∅,0..n,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Private,1.7,grouping,∅,0..n,0..n,‡,‡,0..n,0..n,‡,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅ +P,both,block,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,0..n,0..n,0..n,∅,0..n,∅,0..n,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Hn,both,block,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +H,both,block,∅,0..1,0..1,‡,‡,0..1,0..1,‡,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..1,∅,∅,0..1,0..1,∅,∅,∅,0..1,0..1,0..1,∅ +Quote,1.7,inline,∅,∅,∅,‡,‡,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Note,1.7,inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Reference,1.7,inline,∅,∅,∅,‡,‡,∅,0..n,‡,∅,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅*,∅*,∅,0..n,0..n,0..n,∅,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +BibEntry,1.7,inline,∅,∅,∅,‡,‡,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Lbl,both,inline,∅,∅,0..n,‡,‡,0..n,0..n,‡,∅,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Code,1.7,inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Span,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Link,both,grouping!block!inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Annot,both,grouping!block!inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Form,both,grouping!block!inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,∅*,∅*,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Ruby,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +RB,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[a],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +RT,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[a],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +RP,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[a],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +Warichu,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +WT,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[b],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +WP,both,inline,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,[b],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +L,both,block!inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,∅,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅*,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,0..n,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +LI,both,in L,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +LBody,both,in LI,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +Table,both,block,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +TR,both,Table,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,∅,∅ +TH,both,Table,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅ +TD,both,Table,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅ +THead,both,Table,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..1,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +TBody,both,Table,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +TFoot,both,Table,∅,∅,∅,∅,‡,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..1,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅ +Caption,both,grouping!block,∅,∅,0..n,‡,‡,0..n,0..1,‡,0..1,0..1,0..n,0..n,∅,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,0..1,∅,∅,∅,∅,∅,∅,∅,0..1,∅,0..1,0..1,∅,∅,∅,∅,∅,∅,∅,0..1,0..1,∅ +Figure,both,grouping!block!inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅*,∅,∅,∅,∅,∅,∅,c,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +Formula,both,grouping!block!inline,∅,0..n,0..n,‡,‡,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +MC,both,---,∅,∅,∅,∅,∅,∅,0..n,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,‡,‡,‡,0..n,‡,‡,∅,0..n,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅ +%</parent-child> +% \end{macrocode} +% \section{The parent-child rules for pdf <2.0} +% This csv contains the parent-child matrix for pdf <2.0. +% The values are currently the strings used in the PDF reference. +% This will perhaps change in future. +% \begin{macrocode} +%<*parent-child-2> +%% \ProvidesExplFile {tagpdf-parent-child-2.csv} {2022-12-22} {0.98} +,,,StructTreeRoot,Document,DocumentFragment,Art,Part,Div,Sect,Aside,BlockQuote,NonStruct,TOC,TOCI,Index,Private,Title,Sub,Quote,Note,Reference,BibEntry,P,Hn,H,Lbl,Code,Em,Strong,Span,Link,Annot,Form,Ruby,RB,RT,RP,Warichu,WT,WP,FENote,L,LI,LBody,Table,TR,TH,TD,THead,TBody,TFoot,Caption,Figure,Formula,math,mathml,Artifact,MC +Document,both,document level,1,0..n,0..n,∅,‡,‡,∅,0..n,0..n,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +DocumentFragment,2.0,document level,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,∅,0..n,∅,∅,∅,∅*,∅,∅,∅,∅,∅,∅,∅*,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅,∅,∅,∅,0..n,∅ +Art,1.7,grouping,∅,0..n,0..n,∅,‡,‡,0..n,0..n,0..n,‡,∅,∅,∅,0..n,∅,∅,∅,0..n,∅,∅,∅,0..1,0..1,∅,∅,∅,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,∅,∅,∅,0..n,∅ +Part,both,grouping,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅,0..n,∅,∅,∅,∅,0..n,∅,∅,∅,∅*,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Div,both,grouping,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,∅,0..n,∅,∅,∅,∅,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,∅,0..n,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Sect,both,grouping,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,∅,∅,∅,0..n,∅,∅,∅,0..1,0..1,∅,∅,∅,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,∅,∅,∅,0..n,∅ +Aside,2.0,grouping,∅,0..n,0..n,0..n,‡,‡,0..n,∅,∅,‡,∅,∅,∅,0..n,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +BlockQuote,1.7,grouping?,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,∅,0..n,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +NonStruct,both,grouping,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,∅ +TOC,1.7,grouping,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,0..n,0..n,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +TOCI,1.7,TOC,∅,∅,∅,∅,‡,‡,∅,∅,∅,‡,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +Index,1.7,grouping?,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,∅,0..n,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Private,1.7,grouping,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,∅ +P,both,block,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,0..n,0..n,0..n,0..n,∅,∅,0..n,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Hn,both,block,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +H,both,block,∅,0..1,0..1,0..1,‡,‡,0..1,0..1,0..1,‡,∅,∅,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..1,∅,∅,0..1,0..1,∅,∅,∅,0..1,0..1,0..1,∅,∅,0..1,∅ +Title,2.0,grouping!block,∅,0..n,0..n,0..n,‡,‡,0..n,∅,0..n,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +Sub,2.0,inline,∅,∅,∅,∅,‡,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,∅*,0..n,∅,∅,0..n,∅ +Quote,1.7,inline,∅,∅,∅,∅,‡,‡,∅,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Note,1.7,inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Reference,1.7,inline,∅,∅,∅,∅,‡,‡,∅,0..n,0..n,‡,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅*,∅*,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +BibEntry,1.7,inline,∅,∅,∅,∅,‡,‡,∅,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Lbl,both,inline,∅,∅,∅,0..n,‡,‡,0..n,0..n,0..n,‡,∅,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,∅,0..n,∅,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Code,1.7,inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,∅*,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Em,2.0,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Strong,2.0,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Span,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Link,both,grouping!block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Annot,both,grouping!block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Form,both,grouping!block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅*,∅*,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Ruby,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +RB,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[a],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +RT,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[a],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +RP,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[a],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +Warichu,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +WT,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[b],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +WP,both,inline,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,[b],∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +FENote,2.0,grouping!block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +L,both,block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +LI,both,in L,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +LBody,both,in LI,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +Table,both,block,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,∅*,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +TR,both,Table,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,∅,∅,∅,0..n,∅ +TH,both,Table,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +TD,both,Table,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +THead,both,Table,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..1,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +TBody,both,Table,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +TFoot,both,Table,∅,∅,∅,∅,∅,‡,∅,∅,∅,‡,∅,∅,∅,0..n,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..1,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅ +Caption,both,grouping!block,∅,∅,∅,0..n,‡,‡,0..n,0..1,0..1,‡,0..1,0..1,0..n,0..n,0..1,∅,∅,∅*,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅*,∅*,0..1,∅,∅,∅,∅,∅,∅,∅,∅*,0..1,∅,0..1,0..1,∅,∅,∅,∅,∅,∅,∅,0..1,0..1,∅,∅,0..n,∅ +Figure,both,grouping!block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,∅*,∅,∅,∅,∅,∅,∅,c,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Formula,both,grouping!block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,∅*,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,∅,0..n,∅ +Artifact,2.0,grouping!block!inline,∅,0..n,0..n,0..n,‡,‡,0..n,0..n,0..n,‡,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,0..n,0..n,0..n,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,∅,∅,0..n,∅ +math,2.0,?,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,∅,∅,0..n,∅ +mathml,2.0,?,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,0..n,∅ +MC,both,---,∅,∅,∅,∅,∅,∅,∅,0..n,0..n,‡,∅,∅,∅,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,0..n,‡,‡,‡,0..n,‡,‡,0..n,∅,0..n,0..n,∅,∅,0..n,0..n,∅,∅,∅,0..n,0..n,0..n,∅,0..n,0..n,∅ +%</parent-child-2> +% \end{implementation} +% \PrintIndex diff --git a/macros/latex/contrib/tagpdf/tagpdf-mc-generic.dtx b/macros/latex/contrib/tagpdf/tagpdf-mc-generic.dtx index 42017f4836..ea460e0104 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-mc-generic.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-mc-generic.dtx @@ -46,7 +46,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{documentation} % \end{documentation} @@ -55,11 +55,11 @@ % \begin{macrocode} %<@@=tag> %<*generic> -\ProvidesExplPackage {tagpdf-mc-code-generic} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-mc-code-generic} {2022-12-22} {0.98} {part of tagpdf - code related to marking chunks - generic mode} %</generic> %<*debug> -\ProvidesExplPackage {tagpdf-debug-generic} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-debug-generic} {2022-12-22} {0.98} {part of tagpdf - debugging code related to marking chunks - generic mode} %</debug> % \end{macrocode} @@ -683,6 +683,11 @@ \group_begin: %hm \@@_check_mc_if_nested: \bool_gset_true:N \g_@@_in_mc_bool +% \end{macrocode} +% set default MC tags to structure: +% \begin{macrocode} + \tl_set_eq:NN \l_@@_mc_key_tag_tl \g_@@_struct_tag_tl + \tl_gset_eq:NN\g_@@_mc_key_tag_tl \g_@@_struct_tag_tl \keys_set:nn { @@ / mc } {#1} \bool_if:NTF \l_@@_mc_artifact_bool { %handle artifact @@ -703,6 +708,23 @@ } \bool_if:NF \l_@@_mc_key_stash_bool { + \exp_args:NV\@@_struct_get_tag_info:nNN + \g_@@_struct_stack_current_tl + \l_@@_tmpa_tl + \l_@@_tmpb_tl + \@@_check_parent_child:VVnnN + \l_@@_tmpa_tl \l_@@_tmpb_tl + {MC}{} + \l_@@_parent_child_check_tl + \int_compare:nNnT {\l_@@_parent_child_check_tl}<{0} + { + \msg_warning:nnxxx + { tag } + {role-parent-child} + { \g_@@_struct_tag_tl/\g_@@_struct_tag_NS_tl } + { MC~(=~real content) } + { 'not~allowed'. } + } \@@_mc_handle_stash:x { \int_use:N \c@g_@@_MCID_abs_int } } } diff --git a/macros/latex/contrib/tagpdf/tagpdf-mc-luacode.dtx b/macros/latex/contrib/tagpdf/tagpdf-mc-luacode.dtx index fd4362227c..ca8645e76c 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-mc-luacode.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-mc-luacode.dtx @@ -46,7 +46,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{implementation} % The code is splitted into three parts: code shared by all engines, @@ -86,7 +86,7 @@ % \begin{macrocode} %<@@=tag> %<*luamode> -\ProvidesExplPackage {tagpdf-mc-code-lua} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-mc-code-lua} {2022-12-22} {0.98} {tagpdf - mc code only for the luamode } %</luamode> % \end{macrocode} @@ -182,8 +182,7 @@ % ,\@@_mc_lua_set_mc_type_attr:o % ,\@@_mc_lua_unset_mc_type_attr: % } -% This takes a tag name, and sets the attributes to the related number. -% It is not decided yet if this will be global or local, see the global-mc option. +% This takes a tag name, and sets the attributes globally to the related number. % \begin{macrocode} \cs_new:Nn \@@_mc_lua_set_mc_type_attr:n % #1 is a tag name { @@ -300,6 +299,15 @@ \bool_set_false:N\l_@@_mc_artifact_bool \tl_clear:N \l_@@_mc_key_properties_tl \int_gincr:N \c@g_@@_MCID_abs_int +% \end{macrocode} +% set the default tag to the structure: +% \begin{macrocode} + \tl_set_eq:NN \l_@@_mc_key_tag_tl \g_@@_struct_tag_tl + \tl_gset_eq:NN\g_@@_mc_key_tag_tl \g_@@_struct_tag_tl + \lua_now:e + { + ltx.@@.func.store_mc_data(\@@_get_mc_abs_cnt:,"tag","\g_@@_struct_tag_tl") + } \keys_set:nn { @@ / mc }{ label={}, #1 } %check that a tag or artifact has been used \@@_check_mc_tag:N \l_@@_mc_key_tag_tl @@ -315,6 +323,23 @@ % if not stashed record the absolute number \bool_if:NF \l_@@_mc_key_stash_bool { + \exp_args:NV\@@_struct_get_tag_info:nNN + \g_@@_struct_stack_current_tl + \l_@@_tmpa_tl + \l_@@_tmpb_tl + \@@_check_parent_child:VVnnN + \l_@@_tmpa_tl \l_@@_tmpb_tl + {MC}{} + \l_@@_parent_child_check_tl + \int_compare:nNnT {\l_@@_parent_child_check_tl}<{0} + { + \msg_warning:nnxxx + { tag } + {role-parent-child} + { \g_@@_struct_tag_tl/\g_@@_struct_tag_NS_tl } + { MC~(=~real content) } + { 'not~allowed'. } + } \@@_mc_handle_stash:x { \@@_get_mc_abs_cnt: } } } diff --git a/macros/latex/contrib/tagpdf/tagpdf-mc-shared.dtx b/macros/latex/contrib/tagpdf/tagpdf-mc-shared.dtx index 6ea1b35544..2e49c84725 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-mc-shared.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-mc-shared.dtx @@ -46,7 +46,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{documentation} % \section{Public Commands} @@ -174,7 +174,7 @@ % \begin{macrocode} %<@@=tag> %<*header> -\ProvidesExplPackage {tagpdf-mc-code-shared} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-mc-code-shared} {2022-12-22} {0.98} {part of tagpdf - code related to marking chunks - code shared by generic and luamode } %</header> diff --git a/macros/latex/contrib/tagpdf/tagpdf-roles.dtx b/macros/latex/contrib/tagpdf/tagpdf-roles.dtx index 0b1899bf71..b17515882e 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-roles.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-roles.dtx @@ -46,7 +46,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{documentation} % \begin{function} @@ -55,106 +55,222 @@ % tag (rolemap-key), % namespace (rolemap-key), % role (rolemap-key), -% role-namespace (rolemap-key) +% role-namespace (rolemap-key), % } -% This key can be used in \cs{tagpdfsetup} to declare and rolemap new tags. +% The \texttt{add-new-tag} key can be used in \cs{tagpdfsetup} to declare and rolemap new tags. % It takes as value a key-value list or a simple |new-tag/old-tag|. % % The key-value list knows the following keys: % \begin{description} -% \item[\texttt{tag}] This is the name of the new type as it should then be used in \cs{tagstructbegin}. -% \item[\texttt{namespace}] This is the namespace of the new type. +% \item[\texttt{tag}] This is the name of the new tag as it should +% then be used in \cs{tagstructbegin}. +% \item[\texttt{namespace}] This is the namespace of the new tag. % The value should be a shorthand of a namespace. -% The allowed values are currently |pdf|, |pdf2|, |mathml| and |user|. +% The allowed values are currently |pdf|, |pdf2|, |mathml|,|latex|, |latex-book| and |user|. % The default value (and recommended value for a new tag) is |user|. % The public name of the user namespace is |tag/NS/user|. This can be used to reference % the namespace e.g. in attributes. -% \item[\texttt{role}] This is the type the tag should be mapped too. -% In a PDF 1.7 or earlier this is normally a type from the |pdf| set, +% \item[\texttt{role}] This is the tag the tag should be mapped too. +% In a PDF 1.7 or earlier this is normally a tag from the |pdf| set, % in PDF 2.0 from the |pdf|, |pdf2| and |mathml| set. -% It can also be a user type, or a still unknown type. +% It can also be a user tag. The tag must be declared before, as the code retrieves +% the class of the new tag from it. % The PDF format allows mapping to be done transitively. % But tagpdf can't/won't check such unusual role mapping. -% \item[\texttt{role-namespace}] If the role is a known type -% the default value is the default namespace of this type. -% If the role is unknown, |user| is used and -% the code hopes that the type will be defined later. With this key a specific -% namespace can be forced. +% \item[\texttt{role-namespace}] If the role is a known tag +% the default value is the default namespace of this tag. +% With this key a specific namespace can be forced. % \end{description} +% +% Namespaces are mostly a PDF 2.0 property, but it doesn't harm to +% set them also in a PDF 1.7 or earlier. % \end{function} % \end{documentation} % \begin{implementation} % \begin{macrocode} %<@@=tag> %<*header> -\ProvidesExplPackage {tagpdf-roles-code} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-roles-code} {2022-12-22} {0.98} {part of tagpdf - code related to roles and structure names} %</header> % \end{macrocode} % \section{Code related to roles and structure names} -% \subsection{Variables} -% Tags have both a name (a string) and a number (for the lua attribute). -% Testing a name is easier with a prop, while accessing with a number is -% better done with a seq. So both are used and must be kept in sync if a new -% tag is added. The number is only relevant for the MC type, tags with the same -% name from different names spaces can have the same number. -% \begin{variable}{\g_@@_role_tags_seq,\g_@@_role_tags_prop} % \begin{macrocode} %<*package> -\@@_seq_new:N \g_@@_role_tags_seq %to get names (type/NS) from numbers -\@@_prop_new:N \g_@@_role_tags_prop %to get numbers from names (type/NS) % \end{macrocode} -% \end{variable} -% \begin{variable}{\g_@@_role_tags_NS_prop} -% in pdf 2.0 tags belong to a name space. For every tag we store a default +% +% +% \subsection{Variables} +% Tags are used in structures (\cs{tagstructbegin}) and mc-chunks (\cs{tagmcbegin}). +% +% They have a name (a string), in lua a number (for the lua attribute), and +% in PDF 2.0 belong to one or more name spaces, with one being the default % name space. -% The keys are the tags, the value shorthands like pdf2, or mathml. +% +% Tags of structures are classified, e.g. as grouping, +% inline or block level structure (and a few special classes like lists and tables), +% and must follow containments rules depending on their classification +% (for example a inline structure can not contain +% a block level structure). New tags inherit their +% classification from their rolemapping to the standard namespaces (\texttt{pdf} +% and/or \texttt{pdf2}). +% We store this classification as it will probably +% be needed for tests but currently the data is not much used. +% The classification for math (and the containment rules) +% is unclear currently and so not set. +% +% The attribute number is only relevant in lua and only for the MC chunks +% (so tags with the same name from different names spaces can have the same number), +% and so only stored if luatex is detected. +% +% Due to the namespaces the storing and processing of tags and there data +% are different in various places for PDF~2.0 and PDF~<2.0, which makes +% things a bit difficult and leads to some duplications. Perhaps at some time +% there should be a clear split. +% +% This are the main variables used by the code: +% \begin{description} +% \item[\cs{g_@@_role_tags_NS_prop}] +% This is the core list of tag names. It uses tags as keys +% and the shorthand (e.g. pdf2, or mathml) of the default name space as value. +% +% In pdf 2.0 the value is needed in the structure dictionaries. +% +% \item[\cs{g_@@_role_tags_class_prop}] +% This contains for each tag a classification type. It is used in pdf <2.0. +% +% \item[\cs{g_@@_role_NS_prop}] This contains the names spaces. The +% values are the object references. They are used in pdf 2.0. +% +% \item[\cs{g_@@_role_rolemap_prop}] +% This contains for each tag the role to a standard tag. +% It is used in pdf<2.0 for tag checking and to fill at the end the +% RoleMap dictionary. +% +% \item[\texttt{g\_@@\_role/RoleMap\_dict}] This dictionary contains +% the standard rolemaps. It is relevant only for pdf <2.0. +% +% \item[\cs{g_@@_role_NS_<ns>_prop}] This prop contains the tags of +% a name space and their role. The props are also use for +% remapping. As value they contain two brace groups: tag and namespace. +% In pdf <2.0 the namespace is empty. +% +% \item[\cs{g_@@_role_NS_<ns>_class_prop}] +% This prop contains the tags of +% a name space and their type. The value is only needed for pdf 2.0. +% +% \item[\cs{g_@@_role_index_prop}] +% This prop contains the standard tags (pdf in pdf<2.0, +% pdf,pdf2 + mathml in pdf 2.0) as keys, the values are a two-digit +% number. These numbers are used to get the containment rule of two tags +% from the intarray. +% +%\item[\cs{l_@@_role_debug_prop}] This property is used to pass some info +% around for info messages or debugging. +% \end{description} +% +% +% \begin{variable}{\g_@@_role_tags_NS_prop} +% This is the core list of tag names. It uses tags as keys +% and the shorthand (e.g. pdf2, or mathml) of the default name space as value. +% We store the default name space also in pdf <2.0, even if not needed: +% it doesn't harm and simplifies the code. % There is no need to access this from lua, so we use the standard prop commands. % \begin{macrocode} -\prop_new:N \g_@@_role_tags_NS_prop %to namespace info +\prop_new:N \g_@@_role_tags_NS_prop % \end{macrocode} % \end{variable} +% +% \begin{variable}{\g_@@_role_tags_class_prop} +% With pdf 2.0 we store the class in the NS dependant props. +% With pdf <2.0 we store for now the type(s) of a tag in a common +% prop. +% Tags that are rolemapped should get the type from +% the target. +% \begin{macrocode} +\prop_new:N \g_@@_role_tags_class_prop +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\g_@@_role_NS_prop} -% The standard names spaces are the following. The keys are the name -% tagpdf will use, the urls are the identifier in the namespace object. +% This holds the list of supported name spaces. +% The keys are the name tagpdf will use, the values the object reference. +% The urls identifier are stored in related dict object. % \begin{description} % \item[mathml] http://www.w3.org/1998/Math/MathML % \item[pdf2] http://iso.org/pdf2/ssn % \item[pdf] http://iso.org/pdf/ssn (default) % \item[user] \cs{c_@@_role_userNS_id_str} (random id, for user tags) +% \item[latex] https://www.latex-project.org/ns/dflt/2022 +% \item[latex-book] https://www.latex-project.org/ns/book/2022 +% \item[latex-inline] https://www.latex-project.org/ns/inline/2022 % \end{description} % More namespaces are possible and -% their objects references and the ones of the namespaces must be collected +% their objects references and their rolemaps must be collected % so that an array can be written to the StructTreeRoot at the end (see tagpdf-tree). -% We use a prop to store also the object reference as it will be needed rather +% We use a prop to store the object reference as it will be needed rather % often. % \begin{macrocode} -\prop_new:N \g_@@_role_NS_prop % collect namespaces +\prop_new:N \g_@@_role_NS_prop % \end{macrocode} % \end{variable} -% -% We need also a bunch of temporary variables: +% +% \begin{variable}{\g_@@_role_index_prop} +% This prop contains the standard tags (pdf in pdf<2.0, +% pdf,pdf2 + mathml in pdf 2.0) as keys, the values are a two-digit +% number. These numbers are used to get the containment rule of two tags +% from the intarray. +% \begin{macrocode} +\prop_new:N \g_@@_role_index_prop +% \end{macrocode} +% \end{variable} +% \begin{variable}{\l_@@_role_debug_prop} +% This variable is used to pass more infos to debug messages. +% \begin{macrocode} +\prop_new:N \l_@@_role_debug_prop +% \end{macrocode} +% \end{variable} +% We need also a bunch of temporary variables. % \begin{variable} % { -% \l_@@_role_tag_tmpa_tl +% ,\l_@@_role_tag_tmpa_tl % ,\l_@@_role_tag_namespace_tmpa_tl % ,\l_@@_role_role_tmpa_tl -% ,\l_@@_role_role_namespace_tmpa_tl +% ,\l_@@_role_role_namespace_tmpa_tl +% ,\l_@@_role_tmpa_seq % } % \begin{macrocode} \tl_new:N \l_@@_role_tag_tmpa_tl \tl_new:N \l_@@_role_tag_namespace_tmpa_tl \tl_new:N \l_@@_role_role_tmpa_tl -\tl_new:N \l_@@_role_role_namespace_tmpa_tl +\tl_new:N \l_@@_role_role_namespace_tmpa_tl +\seq_new:N\l_@@_role_tmpa_seq % \end{macrocode} % \end{variable} -% \subsection{Namesspaces} -% The following commands setups a names space. Namespace dictionaries can +% +% \subsection{Namespaces} +% The following commands setups a name space. +% With pdf version $<$2.0 this is only a prop with the +% rolemap. With pdf 2.0 a dictionary must be set up. +% Such a name space dictionaries can % contain an optional |/Schema| and |/RoleMapNS| entry. We only reserve the % objects but delay the writing to the finish code, where we can test if the -% keys and the name spaces are actually needed +% keys and the name spaces are actually needed. % This commands setups objects for the name space and its rolemap. It also -% initialize a prop to collect the rolemaps if needed. +% initialize a dict to collect the rolemaps if needed, and a property +% with the tags of the name space and their rolemapping for loops. +% It is unclear if a reference to a schema file will be ever needed, +% but it doesn't harm \ldots. +% +% \begin{variable}{g_@@_role/RoleMap_dict,\g_@@_role_rolemap_prop} +% This is the object which contains the normal RoleMap. It is probably not +% needed in pdf 2.0 but currently kept. +% \begin{macrocode} +\pdfdict_new:n {g_@@_role/RoleMap_dict} +\prop_new:N \g_@@_role_rolemap_prop +% \end{macrocode} +% \end{variable} +% % \begin{function}{\@@_role_NS_new:nnn} % \begin{syntax} % \cs{@@_role_NS_new:nnn}\Arg{shorthand}\Arg{URI-ID}{Schema} @@ -162,38 +278,51 @@ % \end{function} % \begin{macro}{\@@_role_NS_new:nnn} % \begin{macrocode} -\cs_new_protected:Npn \@@_role_NS_new:nnn #1 #2 #3 - { - \pdf_object_new:n {tag/NS/#1} - \pdfdict_new:n {g_@@_role/Namespace_#1_dict} - \pdf_object_new:n {@@/RoleMapNS/#1} - \pdfdict_new:n {g_@@_role/RoleMapNS_#1_dict} - \pdfdict_gput:nnn - {g_@@_role/Namespace_#1_dict} - {Type} - {/Namespace} - \pdf_string_from_unicode:nnN{utf8/string}{#2}\l_tmpa_str - \tl_if_empty:NF \l_tmpa_str - { - \pdfdict_gput:nnx - {g_@@_role/Namespace_#1_dict} - {NS} - {\l_tmpa_str} - } - %RoleMapNS is added in tree - \tl_if_empty:nF {#3} - { - \pdfdict_gput:nnx{g_@@_role/Namespace_#1_dict} - {Schema}{#3} - } - \prop_gput:Nnx \g_@@_role_NS_prop {#1}{\pdf_object_ref:n{tag/NS/#1}~} - } +\pdf_version_compare:NnTF < {2.0} + { + \cs_new_protected:Npn \@@_role_NS_new:nnn #1 #2 #3 + { + \prop_new:c { g_@@_role_NS_#1_prop } + \prop_new:c { g_@@_role_NS_#1_class_prop } + \prop_gput:Nnx \g_@@_role_NS_prop {#1}{} + } + } + { + \cs_new_protected:Npn \@@_role_NS_new:nnn #1 #2 #3 + { + \prop_new:c { g_@@_role_NS_#1_prop } + \prop_new:c { g_@@_role_NS_#1_class_prop } + \pdf_object_new:n {tag/NS/#1} + \pdfdict_new:n {g_@@_role/Namespace_#1_dict} + \pdf_object_new:n {@@/RoleMapNS/#1} + \pdfdict_new:n {g_@@_role/RoleMapNS_#1_dict} + \pdfdict_gput:nnn + {g_@@_role/Namespace_#1_dict} + {Type} + {/Namespace} + \pdf_string_from_unicode:nnN{utf8/string}{#2}\l_@@_tmpa_str + \tl_if_empty:NF \l_@@_tmpa_str + { + \pdfdict_gput:nnx + {g_@@_role/Namespace_#1_dict} + {NS} + {\l_@@_tmpa_str} + } + %RoleMapNS is added in tree + \tl_if_empty:nF {#3} + { + \pdfdict_gput:nnx{g_@@_role/Namespace_#1_dict} + {Schema}{#3} + } + \prop_gput:Nnx \g_@@_role_NS_prop {#1}{\pdf_object_ref:n{tag/NS/#1}~} + } + } % \end{macrocode} % \end{macro} % We need an id for the user space. For the tests it should be possible % to set it to a fix value. So we use random numbers which can % be fixed by setting a seed. We fake a sort of -% GUID but not try to be really exact as it doesn't matter ... +% GUID but do not try to be really exact as it doesn't matter ... % % \begin{variable}{\c_@@_role_userNS_id_str} % \begin{macrocode} @@ -213,423 +342,126 @@ } % \end{macrocode} % \end{variable} -% Now we setup the standard names spaces. Currently only if we detect pdf2.0 but -% this will perhaps have to change if the structure code gets to messy. +% Now we setup the standard names spaces. +% The mathml space is currently only loaded for pdf 2.0. % \begin{macrocode} -\pdf_version_compare:NnT > {1.9} - { - \@@_role_NS_new:nnn {pdf} {http://iso.org/pdf/ssn}{} - \@@_role_NS_new:nnn {pdf2} {http://iso.org/pdf2/ssn}{} +\@@_role_NS_new:nnn {pdf} {http://iso.org/pdf/ssn}{} +\@@_role_NS_new:nnn {pdf2} {http://iso.org/pdf2/ssn}{} +\pdf_version_compare:NnF < {2.0} + { \@@_role_NS_new:nnn {mathml}{http://www.w3.org/1998/Math/MathML}{} - %\@@_role_NS_new:nnn {latex} {https://www.latex-project.org/ns/2022}{} - \exp_args:Nnx - \@@_role_NS_new:nnn {user}{\c_@@_role_userNS_id_str}{} - } + } +\@@_role_NS_new:nnn {latex} {https://www.latex-project.org/ns/dflt/2022}{} +\@@_role_NS_new:nnn {latex-book} {https://www.latex-project.org/ns/book/2022}{} +\@@_role_NS_new:nnn {latex-inline} {https://www.latex-project.org/ns/inline/2022}{} +\exp_args:Nnx + \@@_role_NS_new:nnn {user}{\c_@@_role_userNS_id_str}{} % \end{macrocode} % -% \subsection{Data} -% In this section we setup the standard data. -% At first the list of structure types. We split them in three lists, -% the tags with which are both in the pdf and pdf2 namespace, -% the one only in pdf and the one with the tags only in pdf2. -% We also define a rolemap for the pdfII only type to pdf so that -% they can always be used. -% \begin{variable} -% { -% \c_@@_role_sttags_pdf_pdfII_clist, -% \c_@@_role_sttags_only_pdf_clist, -% \c_@@_role_sttags_only_pdfII_clist, -% \c_@@_role_sttags_mathml_clist, -% \c_@@_role_sttags_pdfII_to_pdf_prop -% } +% \subsection{Adding a new tag} +% Both when reading the files and when setting up a tag manually +% we have to store data in various places. +% +% \begin{macro}{\@@_role_alloctag:nnn} +% This command allocates a new tag without role mapping. In the +% lua backend it will also record the attribute value. % \begin{macrocode} -% -\clist_const:Nn \c_@@_role_sttags_pdf_pdfII_clist - { - Document, %A complete document. This is the root element - %of any structure tree containing - %multiple parts or multiple articles. - Part, %A large-scale division of a document. - Sect, %A container for grouping related content elements. - Div, %A generic block-level element or group of elements - Caption, %A brief portion of text describing a table or figure. - Index, - NonStruct, %probably not needed - H, - H1, - H2, - H3, - H4, - H5, - H6, - P, - L, %list - LI, %list item (around label and list item body) - Lbl, %list label - LBody, %list item body - Table, - TR, %table row - TH, %table header cell - TD, %table data cell - THead, %table header (n rows) - TBody, %table rows - TFoot, %table footer - Span, %generic inline marker - Link, % - Annot, - Figure, - Formula, - Form, - % ruby warichu etc .. - Ruby, - RB, - RT, - Warichu, - WT, - WP, - Artifact % only MC-tag ?... - } - -\clist_const:Nn \c_@@_role_sttags_only_pdf_clist - { - Art, %A relatively self-contained body of text - %constituting a single narrative or exposition - BlockQuote, %A portion of text consisting of one or more paragraphs - %attributed to someone other than the author of the - %surrounding text. - TOC, %A list made up of table of contents item entries - %(structure tag TOCI; see below) and/or other - %nested table of contents entries - TOCI, %An individual member of a table of contents. - %This entry's children can be any of the following structure tags: - %Lbl,Reference,NonStruct,P,TOC - Index, - Private, - Quote, %inline quote - Note, %footnote, endnote. Lbl can be child - Reference, %A citation to content elsewhere in the document. - BibEntry, %bibentry - Code - } - -\clist_const:Nn \c_@@_role_sttags_only_pdfII_clist - { - DocumentFragment - ,Aside - ,H7 - ,H8 - ,H9 - ,H10 - ,Title - ,FENote - ,Sub - ,Em - ,Strong - ,Artifact - } - -\clist_const:Nn \c_@@_role_sttags_mathml_clist - { - abs - ,and - ,annotation - ,apply - ,approx - ,arccos - ,arccosh - ,arccot - ,arccoth - ,arccsc - ,arccsch - ,arcsec - ,arcsech - ,arcsin - ,arcsinh - ,arctan - ,arctanh - ,arg - ,bind - ,bvar - ,card - ,cartesianproduct - ,cbytes - ,ceiling - ,cerror - ,ci - ,cn - ,codomain - ,complexes - ,compose - ,condition - ,conjugate - ,cos - ,cosh - ,cot - ,coth - ,cs - ,csc - ,csch - ,csymbol - ,curl - ,declare - ,degree - ,determinant - ,diff - ,divergence - ,divide - ,domain - ,domainofapplication - ,emptyset - ,eq - ,equivalent - ,eulergamma - ,exists - ,exp - ,exponentiale - ,factorial - ,factorof - ,false - ,floor - ,fn - ,forall - ,gcd - ,geq - ,grad - ,gt - ,ident - ,image - ,imaginary - ,imaginaryi - ,implies - ,in - ,infinity - ,int - ,integers - ,intersect - ,interval - ,inverse - ,lambda - ,laplacian - ,lcm - ,leq - ,limit - ,ln - ,log - ,logbase - ,lowlimit - ,lt - ,maction - ,maligngroup - ,malignmark - ,math - ,matrix - ,matrixrow - ,max - ,mean - ,median - ,menclose - ,merror - ,mfenced - ,mfrac - ,mglyph - ,mi - ,min - ,minus - ,mlabeledtr - ,mlongdiv - ,mmultiscripts - ,mn - ,mo - ,mode - ,moment - ,momentabout - ,mover - ,mpadded - ,mphantom - ,mprescripts - ,mroot - ,mrow - ,ms - ,mscarries - ,mscarry - ,msgroup - ,msline - ,mspace - ,msqrt - ,msrow - ,mstack - ,mstyle - ,msub - ,msubsup - ,msup - ,mtable - ,mtd - ,mtext - ,mtr - ,munder - ,munderover - ,naturalnumbers - ,neq - ,none - ,not - ,notanumber - ,notin - ,notprsubset - ,notsubset - ,or - ,otherwise - ,outerproduct - ,partialdiff - ,pi - ,piece - ,piecewise - ,plus - ,power - ,primes - ,product - ,prsubset - ,quotient - ,rationals - ,real - ,reals - ,reln - ,rem - ,root - ,scalarproduct - ,sdev - ,sec - ,sech - ,selector - ,semantics - ,sep - ,set - ,setdiff - ,share - ,sin - ,sinh - ,subset - ,sum - ,tan - ,tanh - ,tendsto - ,times - ,transpose - ,true - ,union - ,uplimit - ,variance - ,vector - ,vectorproduct - ,xor - } - -\prop_const_from_keyval:Nn \c_@@_role_sttags_pdfII_to_pdf_prop +\pdf_version_compare:NnTF < {2.0} { - DocumentFragment = Art, - Aside = Note, - Title = H1, - Sub = Span, - H7 = H6 , - H8 = H6 , - H9 = H6 , - H10 = H6, - FENote= Note, - Em = Span, - Strong= Span, - } -% \end{macrocode} -% \end{variable} -% We fill the structure tags in to the seq. We allow all pdf1.7 and pdf2.0, -% and role map if needed the 2.0 tags. -% \begin{macrocode} -% get tag name from number: \seq_item:Nn \g_@@_role_tags_seq { n } -% get tag number from name: \prop_item:Nn \g_@@_role_tags_prop { name } - -\clist_map_inline:Nn \c_@@_role_sttags_pdf_pdfII_clist - { - \@@_seq_gput_right:Nn \g_@@_role_tags_seq { #1 } - \prop_gput:Nnn \g_@@_role_tags_NS_prop { #1 }{ pdf2 } - } -\clist_map_inline:Nn \c_@@_role_sttags_only_pdf_clist - { - \@@_seq_gput_right:Nn \g_@@_role_tags_seq { #1 } - \prop_gput:Nnn \g_@@_role_tags_NS_prop { #1 }{ pdf } - } -\clist_map_inline:Nn \c_@@_role_sttags_only_pdfII_clist - { - \@@_seq_gput_right:Nn \g_@@_role_tags_seq { #1 } - \prop_gput:Nnn \g_@@_role_tags_NS_prop { #1 }{ pdf2 } - } -\pdf_version_compare:NnT > {1.9} - { - \clist_map_inline:Nn \c_@@_role_sttags_mathml_clist + \sys_if_engine_luatex:TF + { + \cs_new_protected:Npn \@@_role_alloctag:nnn #1 #2 #3 %#1 tagname, ns, type { - \@@_seq_gput_right:Nn \g_@@_role_tags_seq { #1 } - \prop_gput:Nnn \g_@@_role_tags_NS_prop { #1 }{ mathml } - } + \lua_now:e { ltx.@@.func.alloctag ('#1') } + \prop_gput:Nnn \g_@@_role_tags_NS_prop {#1}{#2} + \prop_gput:cnn {g_@@_role_NS_#2_prop} {#1}{{}{}} + \prop_gput:Nnn \g_@@_role_tags_class_prop {#1}{#3} + \prop_gput:cnn {g_@@_role_NS_#2_class_prop} {#1}{--UNUSED--} + } + } + { + \cs_new_protected:Npn \@@_role_alloctag:nnn #1 #2 #3 + { + \prop_gput:Nnn \g_@@_role_tags_NS_prop {#1}{#2} + \prop_gput:cnn {g_@@_role_NS_#2_prop} {#1}{{}{}} + \prop_gput:Nnn \g_@@_role_tags_class_prop {#1}{#3} + \prop_gput:cnn {g_@@_role_NS_#2_class_prop} {#1}{--UNUSED--} + } + } } -% \end{macrocode} -% For luatex and the MC we need a name/number relation. -% The name space is not relevant. -% \begin{macrocode} -\int_step_inline:nnnn { 1 }{ 1 }{ \seq_count:N \g_@@_role_tags_seq } { - \@@_prop_gput:Nxn \g_@@_role_tags_prop - { - \seq_item:Nn \g_@@_role_tags_seq { #1 } - } - { #1 } - } + \sys_if_engine_luatex:TF + { + \cs_new_protected:Npn \@@_role_alloctag:nnn #1 #2 #3 %#1 tagname, ns, type + { + \lua_now:e { ltx.@@.func.alloctag ('#1') } + \prop_gput:Nnn \g_@@_role_tags_NS_prop {#1}{#2} + \prop_gput:cnn {g_@@_role_NS_#2_prop} {#1}{{}{}} + \prop_gput:Nnn \g_@@_role_tags_class_prop {#1}{--UNUSED--} + \prop_gput:cnn {g_@@_role_NS_#2_class_prop} {#1}{#3} + } + } + { + \cs_new_protected:Npn \@@_role_alloctag:nnn #1 #2 #3 + { + \prop_gput:Nnn \g_@@_role_tags_NS_prop {#1}{#2} + \prop_gput:cnn {g_@@_role_NS_#2_prop} {#1}{{}{}} + \prop_gput:Nnn \g_@@_role_tags_class_prop {#1}{--UNUSED--} + \prop_gput:cnn {g_@@_role_NS_#2_class_prop} {#1}{#3} + } + } + } +\cs_generate_variant:Nn \@@_role_alloctag:nnn {nnV} % \end{macrocode} -% -% \subsection{Adding new tags and rolemapping} +% \end{macro} % % \subsubsection{pdf 1.7 and earlier} -% With this versions only RoleMap is filled. -% At first the dictionary: -% \begin{variable}{g_@@_role/RoleMap_dict} -% \begin{macrocode} -\pdfdict_new:n {g_@@_role/RoleMap_dict} -% \end{macrocode} -% \end{variable} +% % \begin{macro}{\@@_role_add_tag:nn} % The pdf 1.7 version has only two arguments: new and rolemap name. -% To make pdf 2.0 types usable we directly define a rolemapping for them. -% +% The role must be an existing tag and should not be empty. +% We allow to change the role of an existing tag: as the rolemap is written +% at the end not confusion can happen. % \begin{macrocode} -\cs_new_protected:Nn \@@_role_add_tag:nn %(new) name, reference to old +\cs_new_protected:Nn \@@_role_add_tag:nn % (new) name, reference to old { - \prop_if_in:NnF \g_@@_role_tags_prop {#1} +% \end{macrocode} +% checks and messages +% \begin{macrocode} + \@@_check_add_tag_role:nn {#1}{#2} + \prop_if_in:NnF \g_@@_role_tags_NS_prop {#1} { \int_compare:nNnT {\l_@@_loglevel_int} > { 0 } { \msg_info:nnn { tag }{new-tag}{#1} } - \@@_seq_gput_right:Nn \g_@@_role_tags_seq { #1 } - \@@_prop_gput:Nnx \g_@@_role_tags_prop { #1 } - { - \seq_count:N \g_@@_role_tags_seq - } - \prop_gput:Nnn \g_@@_role_tags_NS_prop { #1 }{ user } } - \@@_check_add_tag_role:nn {#1}{#2} +% \end{macrocode} +% now the addition +% \begin{macrocode} + \prop_get:NnN \g_@@_role_tags_class_prop {#2}\l_@@_tmpa_tl + \quark_if_no_value:NT \l_@@_tmpa_tl + { + \tl_set:Nn\l_@@_tmpa_tl{--UNKNOWN--} + } + \@@_role_alloctag:nnV {#1}{user}\l_@@_tmpa_tl +% \end{macrocode} +% We resolve rolemapping recursively so that all targets are stored as standard +% tags. +% \begin{macrocode} \tl_if_empty:nF { #2 } { - \pdfdict_gput:nnx {g_@@_role/RoleMap_dict} - {#1} - {\pdf_name_from_unicode_e:n{#2}} + \prop_get:NnN \g_@@_role_rolemap_prop {#2}\l_@@_tmpa_tl + \quark_if_no_value:NTF \l_@@_tmpa_tl + { + \prop_gput:Nnx \g_@@_role_rolemap_prop {#1}{\tl_to_str:n{#2}} + } + { + \prop_gput:NnV \g_@@_role_rolemap_prop {#1}\l_@@_tmpa_tl + } } } -\cs_generate_variant:Nn \@@_role_add_tag:nn {VV} - -\pdf_version_compare:NnT < {2.0} - { - \prop_map_inline:Nn \c_@@_role_sttags_pdfII_to_pdf_prop - { - \@@_role_add_tag:nn {#1}{#2} - } - } - +\cs_generate_variant:Nn \@@_role_add_tag:nn {VV,ne} % \end{macrocode} % \end{macro} % \subsubsection{The pdf 2.0 version} @@ -639,17 +471,17 @@ % \begin{macrocode} \cs_new_protected:Nn \@@_role_add_tag:nnnn %tag/namespace/role/namespace { + \@@_check_add_tag_role:nnn {#1/#2}{#3}{#4} \int_compare:nNnT {\l_@@_loglevel_int} > { 0 } { \msg_info:nnn { tag }{new-tag}{#1} } - \@@_seq_gput_right:Nn \g_@@_role_tags_seq { #1 } - \@@_prop_gput:Nnx \g_@@_role_tags_prop { #1 } - { - \seq_count:N \g_@@_role_tags_seq - } - \prop_gput:Nnn \g_@@_role_tags_NS_prop { #1 }{ #2 } - \@@_check_add_tag_role:nn {#1}{#3} + \prop_get:cnN { g_@@_role_NS_#4_class_prop } {#3}\l_@@_tmpa_tl + \quark_if_no_value:NT \l_@@_tmpa_tl + { + \tl_set:Nn\l_@@_tmpa_tl{--UNKNOWN--} + } + \@@_role_alloctag:nnV {#1}{#2}\l_@@_tmpa_tl \pdfdict_gput:nnx {g_@@_role/RoleMapNS_#2_dict}{#1} { [ @@ -658,10 +490,697 @@ \pdf_object_ref:n {tag/NS/#4} ] } +% \end{macrocode} +% We resolve rolemapping recursively so that all targets are stored as standard +% tags for the tests. +% \begin{macrocode} + \tl_if_empty:nF { #2 } + { + \prop_get:cnN { g__tag_role_NS_#4_prop } {#3}\l_@@_tmpa_tl + \quark_if_no_value:NTF \l_@@_tmpa_tl + { + \prop_gput:cnx { g__tag_role_NS_#2_prop } {#1} + {{\tl_to_str:n{#3}}{\tl_to_str:n{#4}}} + } + { + \prop_gput:cno { g__tag_role_NS_#2_prop } {#1}{\l_@@_tmpa_tl} + } + } } \cs_generate_variant:Nn \@@_role_add_tag:nnnn {VVVV} % \end{macrocode} % \end{macro} +% +% \subsection{Helper command to read the data from files} +% In this section we setup the helper command to read namespace files. + +% \begin{macro}{\@@_role_read_namespace_line:nw} +% This command will process a line in the name space file. +% The first argument is the name of the name space. +% The definition differ for pdf 2.0. as we have proper name spaces there. +% With pdf<2.0 not special name spaces shouldn't update the default role or add to the rolemap +% again. We use a boolean here. +% \begin{macrocode} +\bool_new:N\l_@@_role_update_bool +\bool_set_true:N \l_@@_role_update_bool +% \end{macrocode} +% +% \begin{macrocode} +\pdf_version_compare:NnTF < {2.0} + { + \cs_new_protected:Npn \@@_role_read_namespace_line:nw #1#2,#3,#4,#5,#6\q_stop % + % #1 NS, #2 tag, #3 rolemap, #4 NS rolemap #5 type + { + \tl_if_empty:nF { #2 } + { + \bool_if:NTF \l_@@_role_update_bool + { + \tl_if_empty:nTF {#5} + { + \prop_get:NnN \g_@@_role_tags_class_prop {#3}\l_@@_tmpa_tl + \quark_if_no_value:NT \l_@@_tmpa_tl + { + \tl_set:Nn\l_@@_tmpa_tl{--UNKNOWN--} + } + } + { + \tl_set:Nn \l_@@_tmpa_tl {#5} + } + \@@_role_alloctag:nnV {#2}{#1}\l_@@_tmpa_tl + \tl_if_empty:nF {#3} + { + \@@_role_add_tag:nn {#2}{#3} + } + \prop_gput:cnn {g_@@_role_NS_#1_prop} {#2}{{#3}{}} + } + { + \prop_gput:cnn {g_@@_role_NS_#1_prop} {#2}{{#3}{}} + \prop_gput:cnn {g_@@_role_NS_#1_class_prop} {#2}{--UNUSED--} + } + } + } + } + { + \cs_new_protected:Npn \@@_role_read_namespace_line:nw #1#2,#3,#4,#5,#6\q_stop % + % #1 NS, #2 tag, #3 rolemap, #4 NS rolemap #5 type + { + \tl_if_empty:nF {#2} + { + \tl_if_empty:nTF {#5} + { + \prop_get:cnN { g_@@_role_NS_#4_class_prop } {#3}\l_@@_tmpa_tl + \quark_if_no_value:NT \l_@@_tmpa_tl + { + \tl_set:Nn\l_@@_tmpa_tl{--UNKNOWN--} + } + } + { + \tl_set:Nn \l_@@_tmpa_tl {#5} + } + \@@_role_alloctag:nnV {#2}{#1}\l_@@_tmpa_tl + \bool_lazy_and:nnT + { ! \tl_if_empty_p:n {#3} }{! \str_if_eq_p:nn {#1}{pdf2}} + { + \@@_role_add_tag:nnnn {#2}{#1}{#3}{#4} + \prop_gput:cnn {g_@@_role_NS_#1_prop} {#2}{{#3}{#4}} + } + } + } + } +% \end{macrocode} +% \end{macro} +% \begin{macro}{\@@_role_read_namespace:n} +% This command reads the namespace file. +% \begin{macrocode} +\cs_new_protected:Npn \@@_role_read_namespace:n #1 %name of namespace + { + \prop_if_exist:cF {g__tag_role_NS_#1_prop} + { \msg_warning:nnn {tag}{namespace-unknown}{#1} } + \file_if_exist:nTF { tagpdf-ns-#1.def} + { + \ior_open:Nn \g_tmpa_ior {tagpdf-ns-#1.def} + \msg_info:nnn {tag}{read-namespace}{#1} + \ior_map_inline:Nn \g_tmpa_ior + { + \@@_role_read_namespace_line:nw {#1} ##1,,,,\q_stop + } + \ior_close:N\g_tmpa_ior + } + { + \msg_warning:nnn{tag}{namespace-missing}{#1} + } + } + +% \end{macrocode} +% \end{macro} +% + +% +% \subsection{Reading the default data} +% The order is important as we want pdf2 and latex as default. +% \begin{macrocode} +\@@_role_read_namespace:n {pdf} +\@@_role_read_namespace:n {pdf2} +\pdf_version_compare:NnF < {2.0} + {\@@_role_read_namespace:n {mathml}} +\bool_set_false:N\l_@@_role_update_bool +\@@_role_read_namespace:n {latex-inline} +\@@_role_read_namespace:n {latex-book} +\bool_set_true:N\l_@@_role_update_bool +\@@_role_read_namespace:n {latex} +\@@_role_read_namespace:n {pdf} +\@@_role_read_namespace:n {pdf2} +% \end{macrocode} +% +% But is the class provides a \cs{chapter} command then we switch +% \begin{macrocode} +\pdf_version_compare:NnTF < {2.0} + { + \hook_gput_code:nnn {begindocument}{tagpdf} + { + \cs_if_exist:NT \chapter + { + \prop_map_inline:cn{g_@@_role_NS_latex-book_prop} + { + \@@_role_add_tag:ne {#1}{\use_i:nn #2\c_empty_tl\c_empty_tl} + } + } + } + } + { + \hook_gput_code:nnn {begindocument}{tagpdf} + { + \cs_if_exist:NT \chapter + { + \prop_map_inline:cn{g_@@_role_NS_latex-book_prop} + { + \prop_gput:Nnn \g_@@_role_tags_NS_prop { #1 }{ latex-book } + } + } + } + } +% \end{macrocode} +% \subsection{Parent-child rules} +% PDF define various rules about which tag can be a child of another tag. +% The following code implements the matrix to allow to use it in tests. +% \begin{variable}{\g_@@_role_parent_child_intarray} +% This intarray will store the rule as a number. For parent nm and child ij +% (n,m,i,j digits) the rule is at position nmij. As we have around 56 tags, +% we need roughly a size 6000. +% \begin{macrocode} +\intarray_new:Nn \g_@@_role_parent_child_intarray {6000} +% \end{macrocode} +% \end{variable} +% \begin{macro}{\c_@@_role_rules_prop,\c_@@_role_rules_num_prop} +% These two properties map the rule strings to numbers and back. +% There are in tagpdf-data.dtx near the csv files for easier maintenance. +% \end{macro} +% +% \begin{macro}{\@@_store_parent_child_rule:nnn} +% The helper command is used to store the rule. +% It assumes that parent and child are given as 2-digit number! +% \begin{macrocode} +\cs_new_protected:Npn \@@_store_parent_child_rule:nnn #1 #2 #3 % num parent, num child, #3 string + { + \intarray_gset:Nnn \g_@@_role_parent_child_intarray + { #1#2 }{0\prop_item:Nn\c_@@_role_rules_prop{#3}} + } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Reading in the csv-files} +% This counter will be used to identify the first (non-comment) line +% \begin{macrocode} +\int_zero:N \l_@@_tmpa_int +% \end{macrocode} +% Open the file depending on the PDF version +% \begin{macrocode} +\pdf_version_compare:NnTF < {2.0} + { + \ior_open:Nn \g_tmpa_ior {tagpdf-parent-child.csv} + } + { + \ior_open:Nn \g_tmpa_ior {tagpdf-parent-child-2.csv} + } +% \end{macrocode} + +% Now the main loop over the file +% \begin{macrocode} +\ior_map_inline:Nn \g_tmpa_ior + { +% \end{macrocode} +% ignore lines containing only comments +% \begin{macrocode} + \tl_if_empty:nF{#1} + { +% \end{macrocode} +% count the lines ... +% \begin{macrocode} + \int_incr:N\l_@@_tmpa_int +% \end{macrocode} +% put the line into a seq. Attention! empty cells are dropped. +% \begin{macrocode} + \seq_set_from_clist:Nn\l_@@_tmpa_seq { #1 } + \int_compare:nNnTF {\l_@@_tmpa_int}=1 +% \end{macrocode} +% This handles the header line. It gives the tags 2-digit numbers +% \begin{macrocode} + { + \seq_map_indexed_inline:Nn \l_@@_tmpa_seq + { + \prop_gput:Nnx\g_@@_role_index_prop + {##2} + {\int_compare:nNnT{##1}<{10}{0}##1} + } + } +% \end{macrocode} +% now the data lines. +% \begin{macrocode} + { + \seq_set_from_clist:Nn\l_@@_tmpa_seq { #1 } +% \end{macrocode} +% get the name of the child tag from the first column +% \begin{macrocode} + \seq_pop_left:NN\l_@@_tmpa_seq\l_@@_tmpa_tl +% \end{macrocode} +% get the number of the child, and store it in \cs{l_@@_tmpb_tl} +% \begin{macrocode} + \prop_get:NVN \g_@@_role_index_prop \l_@@_tmpa_tl \l_@@_tmpb_tl +% \end{macrocode} +% remove column 2+3 +% \begin{macrocode} + \seq_pop_left:NN\l_@@_tmpa_seq\l_@@_tmpa_tl + \seq_pop_left:NN\l_@@_tmpa_seq\l_@@_tmpa_tl +% \end{macrocode} +% Now map over the rest. The index \verb+##1+ gives us the +% number of the parent, \verb+##2+ is the data. +% \begin{macrocode} + \seq_map_indexed_inline:Nn \l_@@_tmpa_seq + { + \exp_args:Nnx + \@@_store_parent_child_rule:nnn {##1}{\l_@@_tmpb_tl}{ ##2 } + } + } + } + } +% \end{macrocode} +% close the read handle. +% \begin{macrocode} +\ior_close:N\g_tmpa_ior +% \end{macrocode} +% The Root, +% Hn and mathml tags are special and need to be added explicitly +% \begin{macrocode} +\prop_get:NnN\g_@@_role_index_prop{StructTreeRoot}\l_@@_tmpa_tl +\prop_gput:Nnx\g_@@_role_index_prop{Root}{\l_@@_tmpa_tl} +\prop_get:NnN\g_@@_role_index_prop{Hn}\l_@@_tmpa_tl +\pdf_version_compare:NnTF < {2.0} + { + \int_step_inline:nn{6} + { + \prop_gput:Nnx\g_@@_role_index_prop{H#1}{\l_@@_tmpa_tl} + } + } + { + \int_step_inline:nn{10} + { + \prop_gput:Nnx\g_@@_role_index_prop{H#1}{\l_@@_tmpa_tl} + } +% \end{macrocode} +% all mathml tags are currently handled identically +% \begin{macrocode} + \prop_get:NnN\g_@@_role_index_prop {mathml}\l_@@_tmpa_tl + \prop_get:NnN\g_@@_role_index_prop {math}\l_@@_tmpb_tl + \prop_map_inline:Nn \g_@@_role_NS_mathml_prop + { + \prop_gput:NnV\g_@@_role_index_prop{#1}\l_@@_tmpa_tl + } + \prop_gput:NnV\g_@@_role_index_prop{math}\l_@@_tmpb_tl + } +% \end{macrocode} +% +% \subsubsection{Retrieving the parent-child rule} +% +% \begin{macro}{\l_@@_role_real_parent_tl} +% Part, Div and NonStruct have no own rules, instead +% the parent(s) have to be inspected. To store this real parent +% we use this tlvar +% \begin{macrocode} +\tl_new:N \l_@@_role_real_parent_tl +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_role_get_parent_child_rule:nnN} +% This command retrieves the rule (as a number) and stores it in the tl-var. +% TODO check temporary variables. Check if the tl-var should be fix. +% The arguments should be standard tags for which a rule exist and +% role mapping should have already be done. +% \begin{macrocode} +\tl_new:N \l_@@_parent_child_check_tl +\cs_new_protected:Npn \@@_role_get_parent_child_rule:nnN #1 #2 #3 + % #1 parent (string) #2 child (string) #3 tl for state + { + \tl_set:Nn \l_@@_role_real_parent_tl {#1} +% \end{macrocode} +% +% \begin{macrocode} + \prop_get:NnN \g_@@_role_index_prop{#1}\l_@@_tmpa_tl + \prop_get:NnN \g_@@_role_index_prop{#2}\l_@@_tmpb_tl + \bool_lazy_and:nnTF + { ! \quark_if_no_value_p:N \l_@@_tmpa_tl } + { ! \quark_if_no_value_p:N \l_@@_tmpb_tl } + { +% \end{macrocode} +% Get the rule from the intarray +% \begin{macrocode} + \tl_set:Nx#3 + { + \intarray_item:Nn + \g_@@_role_parent_child_intarray + {\l_@@_tmpa_tl\l_@@_tmpb_tl} + } +% \end{macrocode} +% If the state is ‡ we have to check the parents from the stack and +% use the first which is not Part, Div or NonStruct +% \begin{macrocode} + \int_compare:nNnT + {#3} = {\prop_item:Nn\c_@@_role_rules_prop{‡}} + { + \seq_set_eq:NN \l_@@_role_tmpa_seq \g_@@_struct_tag_stack_seq +% \end{macrocode} +% we must take the current child from the stack if is already there, +% depending on location the check is called, this could also remove the +% parent, but that is ok too. +% \begin{macrocode} + \seq_pop_left:NN \l_@@_role_tmpa_seq\l_@@_get_tmpc_tl + \seq_map_inline:Nn\l_@@_role_tmpa_seq + { + \tl_if_in:nnF {-Part-Div-NonStruct-}{-##1-} + { + \tl_set:Nn\l_@@_role_real_parent_tl {##1} + \int_zero:N\l_@@_tmpa_int + \@@_role_get_parent_child_rule:nnN {##1}{#2}#3 + \int_set:Nn\l_@@_tmpa_int{1} + \seq_map_break: + } + } + } +% \end{macrocode} +% This is the message, this can perhaps go into debug mode. +% \begin{macrocode} + \group_begin: + \int_compare:nNnT {\l_@@_tmpa_int*\l_@@_loglevel_int} > { 0 } + { + \prop_get:NVNF\c_@@_role_rules_num_prop #3 \l_@@_tmpa_tl + { + \tl_set:Nn \l_@@_tmpa_tl {unknown} + } + \tl_set:Nn \l_@@_tmpb_tl {#1} + \msg_note:nnxxx + { tag } + { role-parent-child } + { #1 + \tl_if_eq:NNTF\l_@@_tmpb_tl\l_@@_role_real_parent_tl + { + \bool_lazy_and:nnT + { + \prop_if_in_p:Nn \l_@@_role_debug_prop {parent} + } + { + !\str_if_eq_p:ee {#1}{\prop_item:Nn\l_@@_role_debug_prop {parent}} + } + { + \c_space_tl (from~\prop_item:Nn\l_@@_role_debug_prop {parent}) + } + } + { + \c_space_tl(inherited~from~\l_@@_role_real_parent_tl) + } + } + { + #2 + \bool_lazy_and:nnT + { + \prop_if_in_p:Nn \l_@@_role_debug_prop {child} + } + { + !\str_if_eq_p:ee {#2}{\prop_item:Nn\l_@@_role_debug_prop {child}} + } + { + \c_space_tl (from~\prop_item:Nn\l_@@_role_debug_prop {child}) + } + } + { '#3~(\l_@@_tmpa_tl)' } + } + \group_end: + } + { + \tl_set:Nn#3 {0} + \msg_warning:nnxxx + { tag } + {role-parent-child} + { #1 } + { #2 } + { unknown! } + } + } +\cs_generate_variant:Nn\@@_role_get_parent_child_rule:nnN {VVN} +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{@@_check_parent_child:nnnnN} +% This is the main command. +% It has to retrieve the standard tags for a comparison. +% In pdf 2.0 the name spaces of the tags are relevant, so we +% have arguments for them, but in pdf <2.0 they are ignored and can +% be left empty. +% \begin{macrocode} +\pdf_version_compare:NnTF < {2.0} + { + \cs_new_protected:Npn \@@_check_parent_child:nnnnN #1 #2 #3 #4 #5 + { +% \end{macrocode} +% for debugging messages we store the arguments. +% \begin{macrocode} + \prop_put:Nnn \l_@@_role_debug_prop {parent} {#1} + \prop_put:Nnn \l_@@_role_debug_prop {child} {#3} +% \end{macrocode} +% get the standard tags through rolemapping if needed +% at first the parent +% \begin{macrocode} + \prop_get:NnNTF \g_@@_role_index_prop {#1}\l_@@_tmpa_tl + { + \tl_set:Nn \l_@@_tmpa_tl {#1} + } + { + \prop_get:NnNF \g_@@_role_rolemap_prop {#1}\l_@@_tmpa_tl + { + \tl_set:Nn \l_@@_tmpa_tl {\q_no_value} + } + } +% \end{macrocode} +% now the child +% \begin{macrocode} + \prop_get:NnNTF \g_@@_role_index_prop {#3}\l_@@_tmpb_tl + { + \tl_set:Nn \l_@@_tmpb_tl {#3} + } + { + \prop_get:NnNF \g_@@_role_rolemap_prop {#3}\l_@@_tmpb_tl + { + \tl_set:Nn \l_@@_tmpb_tl {\q_no_value} + } + } +% \end{macrocode} +% if we got tags for parent and child we call the checking command +% \begin{macrocode} + \bool_lazy_and:nnTF + { ! \quark_if_no_value_p:N \l_@@_tmpa_tl } + { ! \quark_if_no_value_p:N \l_@@_tmpb_tl } + { + \@@_role_get_parent_child_rule:VVN \l_@@_tmpa_tl \l_@@_tmpb_tl #5 + } + { + \tl_set:Nn #5 {0} + \msg_warning:nnxxx + { tag } + {role-parent-child} + { #1 } + { #3 } + { unknown! } + } + } + \cs_new_protected:Npn \@@_check_parent_child:nnN #1#2#3 + { + \@@_check_parent_child:nnnnN {#1}{}{#2}{}#3 + } + } +% \end{macrocode} +% and now the pdf 2.0 version +% The version with three arguments retrieves the default +% names space and then calls the full command. +% Not sure if this will ever be needed but we leave it for now. +% \begin{macrocode} + { + \cs_new_protected:Npn \@@_check_parent_child:nnN #1 #2 #3 + { + \prop_get:NnN\g_@@_role_tags_NS_prop {#1}\l_@@_role_tag_namespace_tmpa_tl + \prop_get:NnN\g_@@_role_tags_NS_prop {#2}\l_@@_role_tag_namespace_tmpb_tl + \str_if_eq:nnT{#2}{MC}{\tl_clear:N \l_@@_role_tag_namespace_tmpb_tl} + \bool_lazy_and:nnTF + { ! \quark_if_no_value_p:N \l_@@_role_tag_namespace_tmpa_tl } + { ! \quark_if_no_value_p:N \l_@@_role_tag_namespace_tmpb_tl } + { + \@@_check_parent_child:nVnVN + {#1}\l_@@_role_tag_namespace_tmpa_tl + {#2}\l_@@_role_tag_namespace_tmpb_tl + #3 + } + { + \tl_set:Nn #3 {0} + \msg_warning:nnxxx + { tag } + {role-parent-child} + { #1 } + { #2 } + { unknown! } + } + } +% \end{macrocode} +% and now the real command. +% \begin{macrocode} + \cs_new_protected:Npn \@@_check_parent_child:nnnnN #1 #2 #3 #4 #5 %tag,NS,tag,NS, tl var + { + \prop_put:Nnn \l_@@_role_debug_prop {parent} {#1/#2} + \prop_put:Nnn \l_@@_role_debug_prop {child} {#3/#4} +% \end{macrocode} +% If the namespace is empty, we assume a standard tag, +% otherwise we retrieve the rolemapping from the namespace +% \begin{macrocode} + \tl_if_empty:nTF {#2} + { + \tl_set:Nn \l_@@_tmpa_tl {#1} + } + { + \prop_get:cnNTF + { g_@@_role_NS_#2_prop } + {#1} + \l_@@_tmpa_tl + { + \tl_set:Nx \l_@@_tmpa_tl {\tl_head:N\l_@@_tmpa_tl} + \tl_if_empty:NT\l_@@_tmpa_tl + { + \tl_set:Nn \l_@@_tmpa_tl {#1} + } + } + { + \tl_set:Nn \l_@@_tmpa_tl {\q_no_value} + } + } +% \end{macrocode} +% and the same for the child +% If the namespace is empty, we assume a standard tag, +% otherwise we retrieve the rolemapping from the namespace +% \begin{macrocode} + \tl_if_empty:nTF {#4} + { + \tl_set:Nn \l_@@_tmpb_tl {#3} + } + { + \prop_get:cnNTF + { g_@@_role_NS_#4_prop } + {#3} + \l_@@_tmpb_tl + { + \tl_set:Nx \l_@@_tmpb_tl { \tl_head:N\l_@@_tmpb_tl } + \tl_if_empty:NT\l_@@_tmpb_tl + { + \tl_set:Nn \l_@@_tmpb_tl {#3} + } + } + { + \tl_set:Nn \l_@@_tmpb_tl {\q_no_value} + } + } +% \end{macrocode} +% and now get the relation +% \begin{macrocode} + \bool_lazy_and:nnTF + { ! \quark_if_no_value_p:N \l_@@_tmpa_tl } + { ! \quark_if_no_value_p:N \l_@@_tmpb_tl } + { + \@@_role_get_parent_child_rule:VVN \l_@@_tmpa_tl \l_@@_tmpb_tl #5 + } + { + \tl_set:Nn #5 {0} + \msg_warning:nnxxx + { tag } + {role-parent-child} + { #1 } + { #3 } + { unknown! } + } + } + } +\cs_generate_variant:Nn\@@_check_parent_child:nnN {VVN} +\cs_generate_variant:Nn\@@_check_parent_child:nnnnN {VVVVN,nVnVN,VVnnN} +% \end{macrocode} +% \end{macro} +% +% \subsection{Remapping of tags} +% In some context it can be necessary to remap or replace the tags. +% That means instead of tag=H1 or tag=section one wants the effect of tag=Span. +% Or instead of tag=P one wants tag=Code. +% +% The following command provide some general interface for this. +% The core idea is that before a tag is set it is fed through a function +% that can change it. We want to be able to chain such functions, +% so all of them manipulate the same variables. +% +% \begin{variable}{\l_@@_role_remap_tag_tl,\l_@@_role_remap_NS_tl} +% \begin{macrocode} +\tl_new:N \l_@@_role_remap_tag_tl +\tl_new:N \l_@@_role_remap_NS_tl +% \end{macrocode} +% \end{variable} +% \begin{macro}{\@@_role_remap:} +% This function is used in the structure and the mc code before using a tag. By default it +% does nothing with the tl vars. Perhaps this should be a hook? +% \begin{macrocode} +\cs_new_protected:Npn \@@_role_remap: { } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_role_remap_id: } +% This is copy in case we have to restore the main command. +% \begin{macrocode} +\cs_set_eq:NN \@@_role_remap_id: \@@_role_remap: +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_role_remap_inline:} +% The mapping is meant to \enquote{degrade} tags, e.g. if used +% inside some complex object. +% The pdf<2.0 code maps the tag to the new role, the pdf 2.0 code only +% switch the NS. +% \begin{macrocode} +\pdf_version_compare:NnTF < {2.0} + { + \cs_new_protected:Npn \@@_role_remap_inline: + { + \prop_get:cVNT { g__tag_role_NS_latex-inline_prop }\l_@@_role_remap_tag_tl\l_@@_tmpa_tl + { + \tl_set:Nx\l_@@_role_remap_tag_tl + { + \exp_last_unbraced:NV\use_i:nn \l_@@_tmpa_tl + } + \tl_set:Nx\l_@@_role_remap_NS_tl + { + \exp_last_unbraced:NV\use_ii:nn \l_@@_tmpa_tl + } + } + \int_compare:nNnT {\l_@@_loglevel_int} > { 0 } + { + \msg_note:nnx { tag } { role-remapping }{ \l_@@_role_remap_tag_tl } + } + } + } + { + \cs_new_protected:Npn \@@_role_remap_inline: + { + \prop_get:cVNT { g__tag_role_NS_latex-inline_prop }\l_@@_role_remap_tag_tl\l_@@_tmpa_tl + { + \tl_set:Nn\l_@@_role_remap_NS_tl {latex-inline} + } + \int_compare:nNnT {\l_@@_loglevel_int} > { 0 } + { + \msg_note:nnx { tag } { role-remapping }{ \l_@@_role_remap_tag_tl/latex-inline } + } + } + } +% \end{macrocode} +% \end{macro} +% % \subsection{Key-val user interface} % The user interface uses the key |add-new-tag|, which takes either a % keyval list as argument, or a tag/role. diff --git a/macros/latex/contrib/tagpdf/tagpdf-space.dtx b/macros/latex/contrib/tagpdf/tagpdf-space.dtx index 20f189f8a5..ad5e17ff5b 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-space.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-space.dtx @@ -46,7 +46,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{documentation} % \begin{function}{interwordspace (setup-key)} @@ -64,7 +64,7 @@ % \begin{macrocode} %<@@=tag> %<*header> -\ProvidesExplPackage {tagpdf-space-code} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-space-code} {2022-12-22} {0.98} {part of tagpdf - code related to real space chars} %</header> % \end{macrocode} diff --git a/macros/latex/contrib/tagpdf/tagpdf-struct.dtx b/macros/latex/contrib/tagpdf/tagpdf-struct.dtx index dd85e47277..6f6d60ba4b 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-struct.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-struct.dtx @@ -46,7 +46,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{documentation} % \section{Public Commands} @@ -242,7 +242,7 @@ % \begin{macrocode} %<@@=tag> %<*header> -\ProvidesExplPackage {tagpdf-struct-code} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-struct-code} {2022-12-22} {0.98} {part of tagpdf - code related to storing structure} %</header> % \end{macrocode} @@ -298,6 +298,14 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\g_@@_struct_tag_NS_prop} +% For the parent-child check, we need the tag and NS of every structure +% \begin{macrocode} +\prop_new:N\g_@@_struct_tag_NS_prop +% \end{macrocode} +% \end{variable} +% +% % \begin{variable}{\g_@@_struct_stack_current_tl,\l_@@_struct_stack_parent_tmpa_tl} % The global variable will hold the current structure number. It is already % defined in \texttt{tagpdf-base}. @@ -399,6 +407,28 @@ \bool_new:N \l_@@_struct_elem_stash_bool % \end{macrocode} % \end{variable} +% +% \subsection{Variables used by tagging code of basic elements} +% +% \begin{variable}{\g_@@_struct_dest_num_prop} +% This variable records for (some or all, not clear yet) +% destination names the related structure number to allow +% to reference them in a Ref. The key is the destination. +% It is currently used by the toc-tagging and sec-tagging code. +% \begin{macrocode} +\prop_new:N \g_@@_struct_dest_num_prop +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_@@_struct_ref_by_dest_prop} +% This variable contains structures whose Ref key should be updated +% at the end to point to structured related with this destination. +% As this is probably need in other places too, it is not only a toc-variable. +% \begin{macrocode} +\prop_new:N \g_@@_struct_ref_by_dest_prop +% \end{macrocode} +% \end{variable} +% % \section{Commands} % % The properties must be in some places handled expandably. @@ -442,7 +472,11 @@ % \begin{macrocode} \tl_gset:Nn \g_@@_struct_stack_current_tl {0} % \end{macrocode} - +% \begin{macro}{\@@_pdf_name_e:n} +% \begin{macrocode} +\cs_new:Npn \@@_pdf_name_e:n #1{\pdf_name_from_unicode_e:n{#1}} +% \end{macrocode} +% \end{macro} % % \begin{variable}{g_@@_struct_0_prop,g_@@_struct_kids_0_seq} % \begin{macrocode} @@ -450,13 +484,17 @@ \@@_new_output_prop_handler:n {0} \@@_seq_new:c { g_@@_struct_kids_0_seq } -\@@_prop_gput:cnn +\@@_prop_gput:cnx { g_@@_struct_0_prop } { Type } - { /StructTreeRoot } - + { \pdf_name_from_unicode_e:n {StructTreeRoot} } +\@@_prop_gput:cnx + { g_@@_struct_0_prop } + { S } + { \pdf_name_from_unicode_e:n {StructTreeRoot} } +\prop_gput:Nnn \g_@@_struct_tag_NS_prop {0}{{StructTreeRoot}{pdf}} % \end{macrocode} % Namespaces are pdf 2.0 but it doesn't harm % to have an empty entry. We could add a test, but if the code moves into @@ -468,6 +506,65 @@ { \pdf_object_ref:n { @@/tree/namespaces } } % \end{macrocode} % \end{variable} +% +% \subsection{Filling in the tag info} + +% \begin{macro}{\@@_struct_set_tag_info:nnn } +% This adds or updates the tag info to a structure given by a number. +% We need also the original data, so we store both. +% \begin{macrocode} +\pdf_version_compare:NnTF < {2.0} + { + \cs_new_protected:Npn \@@_struct_set_tag_info:nnn #1 #2 #3 + %#1 structure number, #2 tag, #3 NS + { + \@@_prop_gput:cnx + { g_@@_struct_#1_prop } + { S } + { \pdf_name_from_unicode_e:n {#2} } % + \prop_gput:Nnn \g_@@_struct_tag_NS_prop {#1}{{#2}{#3}} + } + } + { + \cs_new_protected:Npn \@@_struct_set_tag_info:nnn #1 #2 #3 + { + \@@_prop_gput:cnx + { g_@@_struct_#1_prop } + { S } + { \pdf_name_from_unicode_e:n {#2} } % + \prop_get:NnNT \g_@@_role_NS_prop {#3} \l_@@_get_tmpc_tl + { + \@@_prop_gput:cnx + { g_@@_struct_#1_prop } + { NS } + { \l_@@_get_tmpc_tl } % + } + \prop_gput:Nnn \g_@@_struct_tag_NS_prop {#1}{{#2}{#3}} + } + } +\cs_generate_variant:Nn \@@_struct_set_tag_info:nnn {eVV} +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\cc} +% We also need a way to get the tag info back from parent structures. +% \begin{macrocode} +\cs_new_protected:Npn \@@_struct_get_tag_info:nNN #1 #2 #3 + %#1 struct num, #2 tlvar for tag , #3 tlvar for NS + { + \prop_get:NnNTF \g_@@_struct_tag_NS_prop {#1}\l_@@_get_tmpc_tl + { + \tl_set:Nx #2{\exp_last_unbraced:NV\use_i:nn \l_@@_get_tmpc_tl} + \tl_set:Nx #3{\exp_last_unbraced:NV\use_ii:nn \l_@@_get_tmpc_tl} + } + { + \tl_clear:N#2 + \tl_clear:N#3 + } + } +\cs_generate_variant:Nn\@@_struct_get_tag_info:nNN {eNN} +% \end{macrocode} +% \end{macro} % \subsection{Handlings kids} % Commands to store the kids. Kids in a structure can be a reference to a mc-chunk, % an object reference to another structure element, or a object reference to an @@ -857,21 +954,10 @@ parent .default:n = {-1}, tag .code:n = % S property { - \seq_set_split:Nne \l_@@_tmpa_seq { / } {#1/\prop_item:No\g__tag_role_tags_NS_prop{#1}} - \tl_gset:Nx \g_@@_struct_tag_tl { \seq_item:Nn\l_@@_tmpa_seq {1} } - \tl_gset:Nx \g_@@_struct_tag_NS_tl { \seq_item:Nn\l_@@_tmpa_seq {2} } + \seq_set_split:Nne \l_@@_tmpa_seq { / } {#1/\prop_item:Ne\g__tag_role_tags_NS_prop{#1}} + \tl_gset:Nx \g_@@_struct_tag_tl { \seq_item:Nn\l_@@_tmpa_seq {1} } + \tl_gset:Nx \g_@@_struct_tag_NS_tl{ \seq_item:Nn\l_@@_tmpa_seq {2} } \@@_check_structure_tag:N \g_@@_struct_tag_tl - \@@_prop_gput:cnx - { g_@@_struct_\int_eval:n {\c@g_@@_struct_abs_int}_prop } - { S } - { \pdf_name_from_unicode_e:n{ \g_@@_struct_tag_tl} } % - \prop_get:NVNT \g_@@_role_NS_prop\g_@@_struct_tag_NS_tl\l_@@_tmpa_tl - { - \@@_prop_gput:cnx - { g_@@_struct_\int_eval:n {\c@g_@@_struct_abs_int}_prop } - { NS } - { \l_@@_tmpa_tl } % - } }, title .code:n = % T property { @@ -1114,7 +1200,13 @@ { /StructElem } \tl_set:Nn \l_@@_struct_stack_parent_tmpa_tl {-1} \keys_set:nn { @@ / struct} { #1 } - \@@_check_structure_has_tag:n { \int_eval:n {\c@g_@@_struct_abs_int} } +% \end{macrocode} +% \begin{macrocode} + \@@_struct_set_tag_info:eVV + { \int_eval:n {\c@g_@@_struct_abs_int} } + \g_@@_struct_tag_tl + \g_@@_struct_tag_NS_tl + \@@_check_structure_has_tag:n { \int_eval:n {\c@g_@@_struct_abs_int} } \tl_if_empty:NF \l_@@_struct_key_label_tl { @@ -1139,7 +1231,43 @@ %\seq_show:N \g_@@_struct_stack_seq \bool_if:NF \l_@@_struct_elem_stash_bool - {%set the parent + { +% \end{macrocode} +% check if the tag can be used inside the parent. It only makes sense, +% if the structure is actually used here, so it is guarded by the stash boolean. +% For now we ignore the namespace! +% \begin{macrocode} + \@@_struct_get_tag_info:eNN + {\l_@@_struct_stack_parent_tmpa_tl} + \l_@@_get_parent_tmpa_tl + \l_@@_get_parent_tmpb_tl + \@@_check_parent_child:VVVVN + \l_@@_get_parent_tmpa_tl + \l_@@_get_parent_tmpb_tl + \g_@@_struct_tag_tl + \g_@@_struct_tag_NS_tl + \l_@@_parent_child_check_tl + \int_compare:nNnT {\l_@@_parent_child_check_tl}<0 + { + \msg_warning:nnxxx + { tag } + {role-parent-child} + { \l_@@_get_parent_tmpa_tl/\l_@@_get_parent_tmpb_tl } + { \g_@@_struct_tag_tl/\g_@@_struct_tag_NS_tl } + { not~allowed~(struct~\g_@@_struct_stack_current_tl) } + \cs_set_eq:NN \l_@@_role_remap_tag_tl \g_@@_struct_tag_tl + \cs_set_eq:NN \l_@@_role_remap_NS_tl \g_@@_struct_tag_NS_tl + \@@_role_remap: + \cs_gset_eq:NN \g_@@_struct_tag_tl \l_@@_role_remap_tag_tl + \cs_gset_eq:NN \g_@@_struct_tag_NS_tl \l_@@_role_remap_NS_tl + \@@_struct_set_tag_info:eVV + { \int_eval:n {\c@g_@@_struct_abs_int} } + \g_@@_struct_tag_tl + \g_@@_struct_tag_NS_tl + } +% \end{macrocode} +% Set the Parent. +% \begin{macrocode} \@@_prop_gput:cnx { g_@@_struct_\int_eval:n {\c@g_@@_struct_abs_int}_prop } { P } @@ -1202,7 +1330,7 @@ %<base>\cs_new_protected:Npn \tag_struct_use:n #1 {} %<*package> \cs_set_protected:Npn \tag_struct_use:n #1 %#1 is the label - { + { \@@_check_if_active_struct:T { \prop_if_exist:cTF @@ -1220,6 +1348,33 @@ { \pdf_object_ref:e { @@/struct/\g_@@_struct_stack_current_tl } } +% \end{macrocode} +% check if the tag is allowed as child. Here we have to retrieve the +% tag info for the child, while the data for the parent is in +% the global tl-vars: +% \begin{macrocode} + \@@_struct_get_tag_info:eNN + {\__tag_ref_value:enn{tagpdfstruct-#1}{tagstruct}{0}} + \l_@@_tmpa_tl + \l_@@_tmpb_tl + \@@_check_parent_child:VVVVN + \g_@@_struct_tag_tl + \g_@@_struct_tag_NS_tl + \l_@@_tmpa_tl + \l_@@_tmpb_tl + \l_@@_parent_child_check_tl + \int_compare:nNnT {\l_@@_parent_child_check_tl}<0 + { + \cs_set_eq:NN \l_@@_role_remap_tag_tl \g_@@_struct_tag_tl + \cs_set_eq:NN \l_@@_role_remap_NS_tl \g_@@_struct_tag_NS_tl + \@@_role_remap: + \cs_gset_eq:NN \g_@@_struct_tag_tl \l_@@_role_remap_tag_tl + \cs_gset_eq:NN \g_@@_struct_tag_NS_tl \l_@@_role_remap_NS_tl + \@@_struct_set_tag_info:eVV + { \int_eval:n {\c@g_@@_struct_abs_int} } + \g_@@_struct_tag_tl + \g_@@_struct_tag_NS_tl + } } { \msg_warning:nnn{ tag }{struct-label-unknown}{#1} @@ -1271,11 +1426,11 @@ \prop_get:cnN { g_@@_struct_#1_prop } {Ref} - \l_@@_tmpb_tl + \l_@@_get_tmpc_tl \@@_prop_gput:cnx { g_@@_struct_#1_prop } { Ref } - { \quark_if_no_value:NF\l_@@_tmpb_tl { \l_@@_tmpb_tl\c_space_tl }#2 } + { \quark_if_no_value:NF\l_@@_get_tmpc_tl { \l_@@_get_tmpc_tl\c_space_tl }#2 } } \cs_generate_variant:Nn \@@_struct_gput_data_ref:nn {ee} % \end{macrocode} @@ -1319,7 +1474,7 @@ % \section{Attributes and attribute classes} % \begin{macrocode} %<*header> -\ProvidesExplPackage {tagpdf-attr-code} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-attr-code} {2022-12-22} {0.98} {part of tagpdf - code related to attributes and attribute classes} %</header> % \end{macrocode} diff --git a/macros/latex/contrib/tagpdf/tagpdf-tree.dtx b/macros/latex/contrib/tagpdf/tagpdf-tree.dtx index 6bb4aa9980..ecbd96f04e 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-tree.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-tree.dtx @@ -47,13 +47,13 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{implementation} % \begin{macrocode} %<@@=tag> %<*header> -\ProvidesExplPackage {tagpdf-tree-code} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-tree-code} {2022-12-22} {0.98} {part of tagpdf - code related to writing trees and dictionaries to the pdf} %</header> % \end{macrocode} @@ -81,6 +81,21 @@ } } % \end{macrocode} +% \subsection{Check structure} +% \begin{macro}{\@@_tree_final_checks:} +% \begin{macrocode} +\cs_new_protected:Npn \@@_tree_final_checks: + { + \int_compare:nNnF {\seq_count:N\g_@@_struct_stack_seq}={1} + { + \msg_warning:nn {tag}{tree-struct-still-open} + \int_step_inline:nnn{2}{\seq_count:N\g_@@_struct_stack_seq} + {\tag_struct_end:} + } + } +% \end{macrocode} +% \end{macro} +% % \subsection{Catalog: MarkInfo and StructTreeRoot} % The StructTreeRoot and the MarkInfo entry must be added to the catalog. % We do it late so that we can win, but before the pdfmanagement hook. @@ -109,18 +124,34 @@ % \begin{macro}{\@@_tree_write_structtreeroot:} % This writes out the root object. % \begin{macrocode} -\cs_new_protected:Npn \@@_tree_write_structtreeroot: - { - \@@_prop_gput:cnx - { g_@@_struct_0_prop } - { ParentTree } - { \pdf_object_ref:n { @@/tree/parenttree } } - \@@_prop_gput:cnx - { g_@@_struct_0_prop } - { RoleMap } - { \pdf_object_ref:n { @@/tree/rolemap } } - \@@_struct_write_obj:n { 0 } - } +\pdf_version_compare:NnTF < {2.0} + { + \cs_new_protected:Npn \@@_tree_write_structtreeroot: + { + \@@_prop_gput:cnx + { g_@@_struct_0_prop } + { ParentTree } + { \pdf_object_ref:n { @@/tree/parenttree } } + \@@_prop_gput:cnx + { g_@@_struct_0_prop } + { RoleMap } + { \pdf_object_ref:n { @@/tree/rolemap } } + \@@_struct_write_obj:n { 0 } + } + } +% \end{macrocode} +% no RoleMap in pdf 2.0 +% \begin{macrocode} + { + \cs_new_protected:Npn \@@_tree_write_structtreeroot: + { + \@@_prop_gput:cnx + { g_@@_struct_0_prop } + { ParentTree } + { \pdf_object_ref:n { @@/tree/parenttree } } + \@@_struct_write_obj:n { 0 } + } + } % \end{macrocode} % \end{macro} % @@ -300,7 +331,10 @@ % \begin{variable}{@@/tree/rolemap} % At first we reserve again an object. % \begin{macrocode} -\pdf_object_new:n { @@/tree/rolemap } +\pdf_version_compare:NnT < {2.0} + { + \pdf_object_new:n { @@/tree/rolemap } + } % \end{macrocode} % \end{variable} % @@ -308,13 +342,26 @@ % This writes out the rolemap, basically it simply pushes out % the dictionary which has been filled in the role module. % \begin{macrocode} -\cs_new_protected:Npn \@@_tree_write_rolemap: +\pdf_version_compare:NnTF < {2.0} { - \pdf_object_write:nnx { @@/tree/rolemap }{dict} - { + \cs_new_protected:Npn \@@_tree_write_rolemap: + { + \prop_map_inline:Nn\g_@@_role_rolemap_prop + { + \pdfdict_gput:nnx {g_@@_role/RoleMap_dict} + {##1} + {\pdf_name_from_unicode_e:n{##2}} + } + \pdf_object_write:nnx { @@/tree/rolemap }{dict} + { \pdfdict_use:n{g_@@_role/RoleMap_dict} - } + } + } + } + { + \cs_new_protected:Npn \@@_tree_write_rolemap:{} } + % \end{macrocode} % \end{macro} % @@ -360,36 +407,39 @@ % \end{macro} % \subsection{Namespaces} % Namespaces are handle in the role module, here is the code to write them out. -% Namespaces are only relevant for pdf2.0 but we don't care, it doesn't harm. +% Namespaces are only relevant for pdf2.0. % \begin{variable}{@@/tree/namespaces} % \begin{macrocode} -\pdf_object_new:nn{ @@/tree/namespaces }{array} +\pdf_object_new:n { @@/tree/namespaces } % \end{macrocode} % \end{variable} % \begin{macro}{\@@_tree_write_namespaces:} % \begin{macrocode} \cs_new_protected:Npn \@@_tree_write_namespaces: { - \prop_map_inline:Nn \g_@@_role_NS_prop - { - \pdfdict_if_empty:nF {g_@@_role/RoleMapNS_##1_dict} - { - \pdf_object_write:nnx {@@/RoleMapNS/##1}{dict} - { - \pdfdict_use:n {g_@@_role/RoleMapNS_##1_dict} - } - \pdfdict_gput:nnx{g_@@_role/Namespace_##1_dict} - {RoleMapNS}{\pdf_object_ref:n {@@/RoleMapNS/##1}} - } - \pdf_object_write:nnx{tag/NS/##1}{dict} - { - \pdfdict_use:n {g_@@_role/Namespace_##1_dict} - } - } - \pdf_object_write:nx {@@/tree/namespaces} %array - { - \prop_map_tokens:Nn \g_@@_role_NS_prop{\use_ii:nn} - } + \pdf_version_compare:NnF < {2.0} + { + \prop_map_inline:Nn \g_@@_role_NS_prop + { + \pdfdict_if_empty:nF {g_@@_role/RoleMapNS_##1_dict} + { + \pdf_object_write:nnx {@@/RoleMapNS/##1}{dict} + { + \pdfdict_use:n {g_@@_role/RoleMapNS_##1_dict} + } + \pdfdict_gput:nnx{g_@@_role/Namespace_##1_dict} + {RoleMapNS}{\pdf_object_ref:n {@@/RoleMapNS/##1}} + } + \pdf_object_write:nnx{tag/NS/##1}{dict} + { + \pdfdict_use:n {g_@@_role/Namespace_##1_dict} + } + } + \pdf_object_write:nnx {@@/tree/namespaces}{array} + { + \prop_map_tokens:Nn \g_@@_role_NS_prop{\use_ii:nn} + } + } } % \end{macrocode} % \end{macro} @@ -404,6 +454,7 @@ \bool_if:NT\g_@@_active_tree_bool { \hook_use:n {tagpdf/finish/before} + \@@_tree_final_checks: \@@_tree_write_parenttree: \@@_tree_write_rolemap: \@@_tree_write_classmap: diff --git a/macros/latex/contrib/tagpdf/tagpdf-user.dtx b/macros/latex/contrib/tagpdf/tagpdf-user.dtx index bd2e26ec92..a70c14bb21 100644 --- a/macros/latex/contrib/tagpdf/tagpdf-user.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf-user.dtx @@ -46,7 +46,7 @@ % }^^A % } % -% \date{Version 0.97, released 2022-08-24} +% \date{Version 0.98, released 2022-12-22} % \maketitle % \begin{documentation} % \section{Setup commands} @@ -246,7 +246,7 @@ % \begin{macrocode} %<@@=tag> %<*header> -\ProvidesExplPackage {tagpdf-user} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-user} {2022-12-22} {0.98} {tagpdf - user commands} %</header> % \end{macrocode} @@ -520,8 +520,10 @@ % \subsection{Structure destinations} % In TeXlive 2022 pdftex and luatex will offer support for structure destinations. % The pdfmanagement has already backend support. We activate them if the -% prerequisites are there: The pdf version should be 2.0, structures should be activated, +% prerequisites are there: structures should be activated, % the code in the pdfmanagement must be there. +% Structure destinations are actually PDF 2.0 only but they don't harm in +% older PDF and can improve html export. % \begin{macrocode} \AddToHook{begindocument/before} { @@ -530,7 +532,6 @@ { \g_@@_active_struct_dest_bool } { \g_@@_active_struct_bool } { \cs_if_exist_p:N \pdf_activate_structure_destination: } - { ! \pdf_version_compare_p:Nn < {2.0} } } { \tl_set:Nn \l_pdf_current_structure_destination_tl { __tag/struct/\g__tag_struct_stack_current_tl } @@ -559,15 +560,17 @@ % Probably one should add some checks if we are already in a paragraph. % % -% \begin{macro}{\l_@@_para_bool,\l_@@_para_show_bool,\g_@@_para_int} +% \begin{macro}{\l_@@_para_bool,\l_@@_para_show_bool,\g_@@_para_int,\l_@@_para_tag_default_tl} % At first some variables. % \begin{macrocode} \bool_new:N \l_@@_para_bool \bool_new:N \l_@@_para_show_bool \int_new:N \g_@@_para_begin_int \int_new:N \g_@@_para_end_int +\tl_new:N \l_@@_para_tag_default_tl +\tl_set:Nn \l_@@_para_tag_default_tl { P } \tl_new:N \l_@@_para_tag_tl -\tl_set:Nn \l_@@_para_tag_tl { P } +\tl_set:Nn \l_@@_para_tag_tl { \l_@@_para_tag_default_tl } % \end{macrocode} % \end{macro} % @@ -603,7 +606,7 @@ \llap{\color_select:n{red}\tiny\int_use:N\g_@@_para_begin_int\ } \tag_mc_end: } - \tag_mc_begin:n {tag=\l_@@_para_tag_tl} + \tag_mc_begin:n {} } } \AddToHook{para/end} diff --git a/macros/latex/contrib/tagpdf/tagpdf.dtx b/macros/latex/contrib/tagpdf/tagpdf.dtx index be718ec36d..d9550ab7a7 100644 --- a/macros/latex/contrib/tagpdf/tagpdf.dtx +++ b/macros/latex/contrib/tagpdf/tagpdf.dtx @@ -92,7 +92,7 @@ % \begin{macrocode} %<@@=tag> %<*package> -\ProvidesExplPackage {tagpdf} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf} {2022-12-22} {0.98} { A package to experiment with pdf tagging } \bool_if:nF @@ -118,7 +118,7 @@ % \end{macrocode} %<*debug> % \begin{macrocode} -\ProvidesExplPackage {tagpdf-debug} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-debug} {2022-12-22} {0.98} { debug code for tagpdf } \@ifpackageloaded{tagpdf}{}{\PackageWarning{tagpdf-debug}{tagpdf~not~loaded,~quitting}\endinput} % \end{macrocode} @@ -159,7 +159,7 @@ % \end{macrocode} % \begin{macrocode} %<*base> -\ProvidesExplPackage {tagpdf-base} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdf-base} {2022-12-22} {0.98} {part of tagpdf - provide base, no-op versions of the user commands } %</base> % \end{macrocode} @@ -180,47 +180,6 @@ } %</base> % \end{macrocode} -% \section{Temporary code} -% This is code which will be removed when proper support exists in LaTeX -% It writes only dictionaries! -% \subsection{Faster object writing} -% \begin{macrocode} -%<*package> -\cs_if_free:NT \pdf_object_write:nnn - { - \cs_new_protected:Npn \pdf_object_new:n #1 - { \pdf_object_new:nn{#1}{dict} } - \cs_new_protected:Npn \pdf_object_write:nnn #1#2#3 - { - \pdf_object_write:nn {#1}{#3} - } - \str_if_eq:VnT \c_sys_backend_str {pdftex} - { - \cs_set_protected:Npn \pdf_object_write:nnn #1#2#3 - { - \tex_immediate:D \tex_pdfobj:D - useobjnum ~ - \int_use:c - { c__pdf_backend_object_ \tl_to_str:n {#1} _int } - { << ~ \exp_not:n {#3} ~ >> } - } - } - \str_if_eq:VnT \c_sys_backend_str {luatex} - { - \cs_set_protected:Npn \pdf_object_write:nnn #1#2#3 - { - \tex_immediate:D \tex_pdfextension:D obj ~ - useobjnum ~ - \int_use:c - { c__pdf_backend_object_ \tl_to_str:n {#1} _int } - { << ~ \exp_not:n {#3} ~ >> } - } - } - } -\cs_generate_variant:Nn \pdf_object_write:nnn {nnx} - -%</package> -% \end{macrocode} % \subsection{a LastPage label} % See also issue \#2 in Accessible-xref @@ -276,7 +235,10 @@ % \begin{variable} % { % \l_@@_tmpa_tl, -% \l_@@_tmpb_tl +% \l_@@_tmpb_tl, +% \l_@@_get_tmpc_tl, +% \l_@@_get_parent_tmpa_tl +% \l_@@_get_parent_tmpb_tl % \l_@@_tmpa_str, % \l_@@_tmpa_prop, % \l_@@_tmpa_seq, @@ -290,6 +252,9 @@ % \begin{macrocode} \tl_new:N \l_@@_tmpa_tl \tl_new:N \l_@@_tmpb_tl +\tl_new:N \l_@@_get_tmpc_tl +\tl_new:N \l_@@_get_parent_tmpa_tl +\tl_new:N \l_@@_get_parent_tmpb_tl \str_new:N \l_@@_tmpa_str \prop_new:N \l_@@_tmpa_prop \seq_new:N \l_@@_tmpa_seq @@ -386,7 +351,7 @@ \cs_generate_variant:Nn \pdffile_embed_stream:nnn {nxx,oxx} \cs_generate_variant:Nn \prop_gput:Nnn {Nxx,Nen} \cs_generate_variant:Nn \prop_put:Nnn {Nxx} -\cs_generate_variant:Nn \prop_item:Nn {No} +\cs_generate_variant:Nn \prop_item:Nn {No,Ne} \cs_generate_variant:Nn \ref_label:nn { nv } \cs_generate_variant:Nn \seq_set_split:Nnn{Nne} \cs_generate_variant:Nn \str_set_convert:Nnnn {Nonn, Noon, Nnon } diff --git a/macros/latex/contrib/tagpdf/tagpdf.ins b/macros/latex/contrib/tagpdf/tagpdf.ins index 60c0803cbf..d494018825 100644 --- a/macros/latex/contrib/tagpdf/tagpdf.ins +++ b/macros/latex/contrib/tagpdf/tagpdf.ins @@ -62,6 +62,7 @@ and all files in that bundle must be distributed together. \from{tagpdf-mc-shared.dtx}{shared} \from{tagpdf.dtx}{mcloading} \from{tagpdf-tree.dtx}{package} + \from{tagpdf-data.dtx}{package} %should be before the roles code \from{tagpdf-roles.dtx}{package} \from{tagpdf-struct.dtx}{package}%contains also the attr code. \from{tagpdf-space.dtx}{package} @@ -88,6 +89,24 @@ and all files in that bundle must be distributed together. \generate {\file{tagpdf-mc-code-lua.sty}{\from{tagpdf-mc-luacode.dtx}{luamode}}} +\generate + {\file{tagpdf-ns-pdf.def}{\from{tagpdf-data.dtx}{ns-pdf}}} +\generate + {\file{tagpdf-ns-pdf2.def}{\from{tagpdf-data.dtx}{ns-pdf2}}} +\generate + {\file{tagpdf-ns-mathml.def}{\from{tagpdf-data.dtx}{ns-mathml}}} +\generate + {\file{tagpdf-ns-latex.def}{\from{tagpdf-data.dtx}{ns-latex}}} +\generate + {\file{tagpdf-ns-latex-book.def}{\from{tagpdf-data.dtx}{ns-latex-book}}} +\generate + {\file{tagpdf-ns-latex-inline.def}{\from{tagpdf-data.dtx}{ns-latex-inline}}} +\generate + {\file{tagpdf-parent-child.csv}{\from{tagpdf-data.dtx}{parent-child}}} +\generate + {\file{tagpdf-parent-child-2.csv}{\from{tagpdf-data.dtx}{parent-child-2}}} + + % check later if the mc-code uses debug guards too. \generate {\file{tagpdf-debug.sty} @@ -117,4 +136,6 @@ and all files in that bundle must be distributed together. } \def\currentpostamble{\defaultpostamble}% \generate{\file{tagpdf.lua} {\from{tagpdf-backend.dtx}{lua}}} + + \endbatchfile diff --git a/macros/latex/contrib/tagpdf/tagpdf.pdf b/macros/latex/contrib/tagpdf/tagpdf.pdf Binary files differindex f75c057dbf..a04314fa71 100644 --- a/macros/latex/contrib/tagpdf/tagpdf.pdf +++ b/macros/latex/contrib/tagpdf/tagpdf.pdf diff --git a/macros/latex/contrib/tagpdf/tagpdf.tex b/macros/latex/contrib/tagpdf/tagpdf.tex index 217314035c..c1fe9d2d35 100644 --- a/macros/latex/contrib/tagpdf/tagpdf.tex +++ b/macros/latex/contrib/tagpdf/tagpdf.tex @@ -1,15 +1,15 @@ % !Mode:: "TeX:DE:UTF-8:Main" -%\PassOptionsToPackage{check-declarations,enable-debug}{expl3} +\PassOptionsToPackage{check-declarations,enable-debug}{expl3} %\RequirePackage{pdfmanagement-testphase} -\DocumentMetadata{pdfversion=2.0,lang=en-UK,testphase=phase-II} +\DocumentMetadata{pdfversion=2.0,lang=en-UK,testphase=phase-II,testphase=toc-tagging,testphase=sec-tagging,pdfstandard=a-2b} \makeatletter -\def\UlrikeFischer@package@version{0.97} -\def\UlrikeFischer@package@date{2022-08-24} +\def\UlrikeFischer@package@version{0.98} +\def\UlrikeFischer@package@date{2022-12-22} \makeatother -\documentclass[DIV=12,parskip=half-,bibliography=totoc]{scrartcl} - +\documentclass[DIV=12,parskip=half-,bibliography=totoc,a4paper]{article} +\usepackage{geometry} \usepackage[english]{babel} \usepackage{unicode-math} @@ -20,7 +20,7 @@ %\patchcmd{\MT@get@prot}{\noindent}{\RawNoindent}{}{\fail} \makeatother -\usepackage{scrlayer-scrpage} +%\usepackage{scrlayer-scrpage} \usepackage[autostyle]{csquotes} \usepackage[style=numeric,hyperref=false]{biblatex} @@ -41,7 +41,7 @@ \tikzset{operator/.style = {font=\footnotesize\ttfamily, anchor=base,draw, rounded corners,node distance=4mm and 4mm}} -\usepackage{enumitem,xpatch} +\usepackage{enumitem} \usepackage{listings} \lstset{basicstyle=\ttfamily, columns=fullflexible,language=[LaTeX]TeX, @@ -56,15 +56,12 @@ %show-spaces } -\usepackage{hyperxmp} \usepackage[pdfdisplaydoctitle=true,%hyperfootnotes=false, ]{hyperref} \hypersetup{ pdftitle={The tagpdf package, v\csname UlrikeFischer@package@version\endcsname}, pdfauthor=Ulrike Fischer, - pdfaconformance=a, - pdfuapart=1, colorlinks} \tcbuselibrary{documentation} @@ -79,7 +76,7 @@ \usepackage{tagpdfdocu-patches} -\cfoot*{\pagemark} +%\cfoot*{\pagemark} % @@ -89,28 +86,21 @@ \date{\csname UlrikeFischer@package@date\endcsname} \author{Ulrike Fischer\thanks{fischer@troubleshooting-tex.de}} -\tagpdfsetup{paratagging,paratagging-show} - \usepackage{shortvrb} \MakeShortVerb| \begin{document} \tagstructbegin{tag=Title} -\tagmcbegin{tag=Title} - -\begin{center}%\let\thanks\par +\begin{center}% +\let\thanks\footnote \makeatletter - \usekomafont{title}{\huge \@title \par} - - \vskip .5em - \usekomafont{author}{\@author \par} - - \vskip 1em% - \usekomafont{date}{\@date \par}% - +\huge \@title \par +\vskip .5em +\@author \par +\vskip 1em% +\@date \par \end{center}% - \tagmcend \tagstructend @@ -147,8 +137,12 @@ Issues, comments, suggestions should be added as issues to the github tracker: \tagstructend +\tagtool{sec-add-grouping=false} + \tableofcontents +\tagtool{sec-add-grouping} + \section{Introduction} @@ -250,7 +244,7 @@ It is also possible that validators contradict: that the one says everything is To make the package usable examples are needed: examples that demonstrate how various structures can be tagged and which patches are needed, examples for the test suite, examples that demonstrates problems. -\begin{tcolorbox}[before upper=\tagpdfparaOn] +\begin{tcolorbox} Feedback, contribuations and corrections are welcome! \end{tcolorbox} @@ -299,7 +293,7 @@ I added code to add \enquote{real} space glyphs to the \PDF{}, see section \ref{ \item \textbf{Breaking change!} The argument of \texttt{newattribute} option should no longer add the dictionary bracket \verb+<<..>>+, they are added by the code. -\item \textbf{Breaking change!} The package now requires the new PDF management as provided for now by the package \texttt{pdfmanagement-testphase}. \texttt{pdfmanagement-testphase}, +\item \textbf{Breaking change!} The package now requires the new PDF management as provided for now by the package \pkg{pdfmanagement-testphase}. \pkg{pdfmanagement-testphase}, prepares the ground for better support for tagged PDF in \LaTeX{}. It is part of a larger project to automatically generate tagged PDF \url{https://www.latex-project.org/news/2020/11/30/tagged-pdf-FS-study/} @@ -348,7 +342,8 @@ add annotations to the structure. \item \textbf{More engines supported} The generic mode of \pkg{tagpdf} now works (theoretically, it is not much tested) with all engines supported -by the pdfmanagement. So compilations with Xe\LaTeX{} or with dvips should work. But it should be noted that +by the \PDF\ management. +So compilations with Xe\LaTeX{} or with dvips should work. But it should be noted that these engines and backends don't support the |interspaceword| option. With Xe\LaTeX{} it is perhaps possible implement something with |\XeTeXinterchartoks|, but for the dvips route I don't see an option (apart from lots of manual macros everywhere). @@ -439,11 +434,34 @@ Small bug fixes. \item Small bug fixes. \end{itemize} +\subsection{Changes in version 0.98} + +\begin{itemize} +\item The declarations of tag namespaces have been externalized and are now +read from files when \pkg{tagpdf} is loaded. + +\item The \PDF{} format (and some of the standards) declare various parent-child rules for +structure tags. A first step to implement this rules and check if they are fullfilled have +been done. More information can be found in section~\ref{sec:parent-child}. + +\item As a side effect of the new rule checking, the requirements for new tags + have been tightened: Adding a new tag with add-new-tag now requires that the target role is + defined. Unknown roles error. + +\item |\tagmcbegin| no longer requires that a tag is set, instead if will pick up +the tag name from the surrounding structure. + +\item Structure destination are now created also with \PDF \textless\,2.0. They shouldn't harm and +can improve the html export. + +\end{itemize} + + \subsection{Proof of concept: the tagging of the documentation itself} Starting with version 0.6 the documentation itself has been tagged. The tagging wasn't (and isn't) in no way perfect. The validator from Adobe didn't complain, but PAX3 wanted alternative text for all links (no idea why) and so I put everywhere simple text like \enquote{link} and \enquote{ref}. The links to footnotes gave warnings, so I disabled them. I used types from the \PDF{} version 1.7, mostly as I have no idea what should be used for code in 2.0. Margin notes were simply wrong \ldots -The tagging has been improved and automated over time in sync with improvements and new features in the LaTeX kernel and the pdfmanagement code. +The tagging has been improved and automated over time in sync with improvements and new features in the LaTeX kernel and the \PDF\ management code. But even if the documentation passed the tests of the validators: as mentioned above passing a formal test doesn't mean that the content is really good and usable. I have a lot doubts that the code parts are really readable. The bibliography and the references must be improved. The user commands used for the tagging and also some of the patches used are still rather crude. So there is lot space for improvement. @@ -467,11 +485,16 @@ The \texttt{tagpdf} package can then be loaded and activated by using the \textt the \texttt{testphase} key is documented in \texttt{documentmetadata-support-doc.pdf} which is part of the \texttt{latex-lab} bundle. +Various parts of the code differentiate between \PDF{} version 2.0 and lower versions. If +\PDF{} 2.0 is wanted it is required to set the version early in the \cs{DocumentMetadata} +command so that \pkg{tagpdf} can pick up the correct code path. + \begin{lstlisting} \DocumentMetadata { % testphase = phase-I, % tagging without paragraph tagging testphase = phase-II % tagging with paragraph tagging and other new stuff. + % pdfversion = 2.0 % pdfversion must be set here. } \documentclass{article} \begin{document} @@ -481,11 +504,12 @@ some text \minisec{Activation needed!} -The package can also be loaded normally with |\usepackage| (but it is -still required to use \cs{DocumentMetadata} to load the pdfmanagement) but +It is not recommended anymore, but +the package can also be loaded normally with |\usepackage| (but it is +still required to use \cs{DocumentMetadata} to load the \PDF\ management) but it will then -- apart from loading more packages and defining a lot of things -- not do much. You will have to \emph{activate} it with \verb+\tagpdfsetup+. -The pdfmanagement loaded with \cs{DocumentMetadata} will in any case load \texttt{tagpdf-base} a +The \PDF\ management loaded with \cs{DocumentMetadata} will in any case load \texttt{tagpdf-base} a small package that provides no-op versions of the main tagging commands. Most commands do nothing if tagging is not activated, but in case a test is needed a command (with the usual p,T,F variants) is provided: @@ -500,7 +524,7 @@ The check is true only if \emph{everything} is activated. In all other cases (in %TODO think about tagging of the keys. Aside? Header? -The package has two different modes: The \textbf{generic mode} works (in theory, currently only tested with pdftex and luatex) probably with all engines, the \textbf{lua mode} only with luatex. The differences between both modes will be described later. The mode can be set with package options: +The package has two different modes: The \textbf{generic mode} works (in theory, currently only fully tested with pdflatex) probably with all engines, the \textbf{lua mode} only with lualatex. The differences between both modes will be described later. The mode can be set with package options: \DescribeKey{luamode} @@ -594,29 +618,59 @@ endstream \end{lstlisting} -From this stream one can extract the characters and their placement on the page but not their semantic meaning (the first line is actually a section heading, the last the page number). And while in the example the order is correct there is actually no guaranty that the stream contains the text in the order it should be read. - -Tagging means to enrich the \PDF{} with information about the \emph{semantic} meaning and the \emph{reading order}. (Tagging can do more, one can also store all sorts of layout information like font properties and indentation with tags. But as I already wrote this package concentrates on the part of tagging that is needed to improve accessibility.) - +From this stream one can extract the characters and their placement on the page +but not their semantic meaning (the first line is actually a section heading, +the last the page number). And while in the example the order is correct +there is actually no guaranty that the stream contains the text in the order +it should be read. +Tagging means to enrich the \PDF{} with information about the \emph{semantic} +meaning and the \emph{reading order}. (Tagging can do more, one can also +store all sorts of layout information like font properties and indentation +with tags. But as I already wrote this package concentrates on the part of +tagging that is needed to improve accessibility.) \subsection{Three tasks} - To tag a \PDF{} three tasks must be carried out: \begin{enumerate} \item - \textbf{The mark-content-task}:\sidenote{mc-task} The document must add \enquote{labels} to the page stream which allows to identify and reference the various chunks of text and other content. This is the most difficult part of tagging -- both for the document writer but also for the package code. At first there can be quite many chunks as every one is a leaf node of the structure and so often a rather small unit. At second the chunks must be defined page-wise -- and this is not easy when you don't know where the page breaks are. Also in a standard document a lot text is created automatically, e.g. the toc, references, citations, list numbers etc and it is not always easy to mark them correctly. - -\item - \textbf{The structure-task}:\sidenote{struct-task} The document must declare the structure. This means marking the start and end of semantically connected portions of the document (correctly nested as a tree). This too means some work for the document writer, but less than for the mc-task: at first quite often the mc-task and the structure-task can be combined, e.g. when you mark up a list number or a tabular cell or a section header; at second one doesn't have to worry about page breaks so quite often one can patch standard environments to declare the structure. On the other side a number of structures end in \LaTeX\ only implicitly -- e.g. an item ends at the next item, so getting the \PDF{} structure right still means that additional mark up must be added. - - -\item - \textbf{The tree management}:\sidenote{tree-task} At last the structure must be written into the \PDF{}. For every structure an object of type \texttt{StructElem} must be created and flushed with keys for the parents and the kids. A parenttree must be created to get a reference from the mc-chunks to the parent structure. A rolemap must be written. And a number of dictionary entries. All this is hopefully done automatically and correctly by the package \ldots. + \textbf{The mark-content-task}:\sidenote{mc-task} The document must add + \enquote{labels} to the page stream which allows to identify and reference + the various chunks of text and other content. + This is the most difficult part of tagging -- both for the document writer + but also for the package code. At first there can be quite many + chunks as every one is a leaf node of the structure and so often a rather + small unit. At second the chunks must be defined page-wise -- and + this is not easy when you don't know where the page breaks are. + Also in a standard document a lot text is created automatically, e.g. + the toc, references, citations, list numbers etc and it is not always + easy to mark them correctly. + +\item \textbf{The structure-task}:\sidenote{struct-task} The document must + declare the structure. This means marking the start and end of + semantically connected portions of the document (correctly nested as a + tree). This too means some work for the document writer, but less than + for the mc-task: at first quite often the mc-task and the + structure-task can be combined, e.g. when you mark up a list number or + a tabular cell or a section header; at second one doesn't have to worry + about page breaks so quite often one can patch standard environments to + declare the structure. On the other side a number of structures end in + \LaTeX\ only implicitly -- e.g. an item ends at the next item, so + getting the \PDF{} structure right still means that additional mark up + must be added. + + +\item \textbf{The tree management}:\sidenote{tree-task} At last the + structure must be written into the \PDF{}. For every structure an + object of type \texttt{StructElem} must be created and flushed with + keys for the parents and the kids. A parenttree must be created to get + a reference from the mc-chunks to the parent structure. A rolemap must + be written. And a number of dictionary entries. All this is hopefully + done automatically and correctly by the package \ldots. \end{enumerate} @@ -679,9 +733,9 @@ To tag a \PDF{} three tasks must be carried out: \end{tcolorbox} -\tagstructbegin{tag=Caption}\tagmcbegin{tag=Caption} -\caption{Schematical description of the relation between marked content in the page stream and the structure} -\tagmcend\tagstructend +\tagstructbegin{tag=Caption}\tagmcbegin{tag=Caption} \caption{Schematical +description of the relation between marked content in the page stream and the +structure} \tagmcend\tagstructend \end{figure} @@ -692,10 +746,18 @@ To tag a \PDF{} three tasks must be carried out: -To be able to refer to parts of the text in the structure, the text in the page stream must get \enquote{labels}. In the \PDF{} reference they are called \enquote{marked content}. The three main variants needed here are: +To be able to refer to parts of the text in the structure, the text in the +page stream must get \enquote{labels}. In the \PDF{} reference they are +called \enquote{marked content}. The three main variants needed here are: \begin{description} -\item[Artifacts] They are marked with of a pair of keywords, \texttt{BMC} and \texttt{EMC} which surrounds the text. \texttt{BMC} has a single prefix argument, the fix tag name \texttt{/Artifact}. Artifacts should be used for irrelevant text and page content that should be ignored in the structure. Sadly it is often not possible to leave such text simply unmarked -- the accessibility tests in Acrobat and other validators complain. +\item[Artifacts] They are marked with of a pair of keywords, \texttt{BMC} + and \texttt{EMC} which surrounds the text. \texttt{BMC} has a single + prefix argument, the fix tag name \texttt{/Artifact}. Artifacts should + be used for irrelevant text and page content that should be ignored in + the structure. Sadly it is often not possible to leave such text simply + unmarked -- the accessibility tests in Acrobat and other validators + complain. \begin{lstlisting} @@ -708,8 +770,11 @@ To be able to refer to parts of the text in the structure, the text in the page -\item[Artifacts with a type] -They are marked with of a pair of keywords, \texttt{BDC} and \texttt{EMC} which surrounds the text. \texttt{BDC} has two arguments: again the tag name \texttt{/Artifact} and a following dictionary which allows to specify the suppressed info. Text in header and footer can e.g. be declared as pagination like this: +\item[Artifacts with a type] They are marked with of a pair of keywords, + \texttt{BDC} and \texttt{EMC} which surrounds the text. \texttt{BDC} + has two arguments: again the tag name \texttt{/Artifact} and a + following dictionary which allows to specify the suppressed info. Text + in header and footer can e.g. be declared as pagination like this: \begin{lstlisting} @@ -721,10 +786,19 @@ They are marked with of a pair of keywords, \texttt{BDC} and \texttt{EMC} which -\item[Content] Content is marked also with of a pair of keywords, \texttt{BDC} and \texttt{EMC}. The first argument of \texttt{BDC} is a tag name which describes the structural type of the text\footnote{There is quite some redundancy in the specification here. The structural type is also set in the structure tree. One wonders if it isn't enough to use always \texttt{/SPAN} here.} - Examples are \texttt{/P} (paragraph), \texttt{/H2} (heading), \texttt{/TD} (table cell). The reference mentions a number of standard types but it is possible to add more or to use different names. +\item[Content] Content is marked also with of a pair of keywords, + \texttt{BDC} and \texttt{EMC}. The first argument of \texttt{BDC} is a + tag name which describes the structural type of the text\footnote{There + is quite some redundancy in the specification here. The structural type + is also set in the structure tree. One wonders if it isn't enough to + use always \texttt{/SPAN} here.} Examples are \texttt{/P} (paragraph), + \texttt{/H2} (heading), \texttt{/TD} (table cell). The reference + mentions a number of standard types but it is possible to add more or + to use different names. -In the second argument of \texttt{BDC} -- in the property dictionary -- more data can be stored. \emph{Required} is an \texttt{/MCID}-key which takes an integer as a value: +In the second argument of \texttt{BDC} -- in the property dictionary -- more +data can be stored. \emph{Required} is an \texttt{/MCID}-key which takes an +integer as a value: \begin{lstlisting} @@ -734,10 +808,15 @@ In the second argument of \texttt{BDC} -- in the property dictionary -- more dat \end{lstlisting} -This integer is used to identify the chunk when building the structure tree. The chunks are numbered by page starting with 0. As the numbers are also used as an index in an array they shouldn't be \enquote{holes} in the numbering system -(It is perhaps possible to handle a numbering scheme not starting by 0 and having holes, but it will enlarge the \PDF{} as one would need dummy objects.). +This integer is used to identify the chunk when building the structure +tree. The chunks are numbered by page starting with 0. As the numbers are +also used as an index in an array they shouldn't be \enquote{holes} in the +numbering system (It is perhaps possible to handle a numbering scheme not +starting by 0 and having holes, but it will enlarge the \PDF{} as one would +need dummy objects.). -It is possible to add more entries to the property dictionary, e.g. a title, alternative text or a local language setting. +It is possible to add more entries to the property dictionary, e.g. a +title, alternative text or a local language setting. \end{description} @@ -754,8 +833,11 @@ The needed markers can be added with low level code e.g. like this (in pdftex sy This sounds easy. But there are quite a number of traps, mostly with pdfLaTeX: \begin{enumerate}[beginpenalty=10000] - \item \PDF{} is a page oriented format. And this means that the start \texttt{BDC}/\texttt{BMC} and the corresponding end \texttt{EMC} must be on the same page. - So marking e.g. a section title like in the following example won't always work as the literal before the section could end on the previous page: + \item \PDF{} is a page oriented format. And this means that the start + \texttt{BDC}/\texttt{BMC} and the corresponding end \texttt{EMC} + must be on the same page. So marking e.g. a section title like in the + following example won't always work as the literal before the + section could end on the previous page: \begin{lstlisting} @@ -765,54 +847,86 @@ This sounds easy. But there are quite a number of traps, mostly with pdfLaTeX: \end{lstlisting} - Using the literals \emph{inside} the section argument is better, but then one has to take care that they don't wander into the header and the toc. - - \item - Literals are \enquote{whatsits} nodes and can change spacing, page and line breaking. The literal \emph{behind} the section in the previous example could e.g. lead to a lonely section title at the end of the page. - - \item - The \texttt{/MCID} numbers must be unique on a page. So you can't use the literal in a saved box that you reuse in various places. This is e.\,g. a problem with \texttt{longtable} as it saves the table header and footer in a box. - - \item - The \texttt{/MCID}-chunks are leaf nodes in the structure tree, so they shouldn't be nested. - - \item - Often text in a document is created automatically or moved around: entries in the table of contents, index, bibliography and more. To mark these text chunks correctly one has to analyze the code creating such content to find suitable places to inject the literals. - - \item - The literals are inserted directly and not at shipout. This means that due to the asynchronous page breaking of \TeX\ the MCID-number can be wrong even if the counter is reset at every page. This package uses in generic mode a label-ref-system to get around this problem. This sadly means that often at least three compilations are needed until everything has settled down. - - It can actually be worse: If the text is changed after the MCID-numbers have been assigned, and - a new mc-chunk is inserted in the middle of the page, then all the numbers have to be recalculated - and that requires again a number of compilations until it really settles down again. - Internal references are especially problematic here, as the first compilation - typically creates a non-link |??|, and only the second inserts the structure and the new mc. - When the reference system in LaTeX will be extended, care will be taken to ensure that already the dummy text builds a chunk. - Until then the advice is to first compile the document and resolve all cross-reference and to activate tagging only at the end. - - - \item - There exist environments which process their content more than once -- examples are \texttt{align} and \texttt{tabularx}. + Using the literals \emph{inside} the section argument is better, but then + one has to take care that they don't wander into the header and the toc. + + \item Literals are \enquote{whatsits} nodes and can change spacing, page + and line breaking. The literal \emph{behind} the section in the + previous example could e.g. lead to a lonely section title at the end + of the page. + + \item The \texttt{/MCID} numbers must be unique on a page. So you can't + use the literal in a saved box that you reuse in various places. This + is e.\,g. a problem with \texttt{longtable} as it saves the table + header and footer in a box. + + \item The \texttt{/MCID}-chunks are leaf nodes in the structure tree, so + they shouldn't be nested. + + \item Often text in a document is created automatically or moved around: + entries in the table of contents, index, bibliography and more. To + mark these text chunks correctly one has to analyze the code creating + such content to find suitable places to inject the literals. + + \item The literals are inserted directly and not at shipout. This means + that due to the asynchronous page breaking of \TeX\ the MCID-number + can be wrong even if the counter is reset at every page. This package + uses in generic mode a label-ref-system to get around this problem. + This sadly means that often at least three compilations are needed + until everything has settled down. + + It can actually be worse: If the text is changed after the MCID-numbers + have been assigned, and a new mc-chunk is inserted in the middle of the + page, then all the numbers have to be recalculated and that requires + again a number of compilations until it really settles down again. + Internal references are especially problematic here, as the first + compilation typically creates a non-link |??|, and only the second + inserts the structure and the new mc. When the reference system in LaTeX + will be extended, care will be taken to ensure that already the dummy + text builds a chunk. Until then the advice is to first compile the + document and resolve all cross-reference and to activate tagging only at + the end. + + + \item There exist environments which process their content more than once + -- examples are \texttt{align} and \texttt{tabularx}. So one has to check for doublettes and holes in the counting system. - \item - \PDF{} is a page oriented format. And this means that the start and the end marker must be on the same page \ldots\ \emph{so what to do with normal paragraphs that split over pages??}. This question will be discussed in subsection~\ref{sec:splitpara}. + \item \PDF{} is a page oriented format. And this means that the start and + the end marker must be on the same page \ldots\ \emph{so what to do + with normal paragraphs that split over pages??}. This question will + be discussed in subsection~\ref{sec:splitpara}. \end{enumerate} \subsubsection{Generic mode versus lua mode in the mc-task} -While in generic mode the commands insert the literals directly and so have all the problems described above the lua mode works quite differently: The tagging commands don't insert literals but set some \emph{attributes} which are attached to all the following nodes. When the page is shipped out some lua code is called which wanders through the shipout box and injects the literals at the places where the attributes changes. +While in generic mode the commands insert the literals directly and so have +all the problems described above the lua mode works quite differently: The +tagging commands don't insert literals but set some \emph{attributes} which +are attached to all the following nodes. When the page is shipped out some +lua code is called which wanders through the shipout box and injects the +literals at the places where the attributes changes. This means that quite a number of problems mentioned above are not relevant for the lua mode: -\begin{enumerate} -\item Pagebreaks between start and end of the marker are \emph{not} a problem. So you can mark a complete paragraph. If a pagebreak occur directly after an start marker or before an end marker this can lead to empty chunks in the \PDF{} and so bloat up \PDF{} a bit, but this is imho not really a problem (compared to the size increase by the rest of the tagging). -\item The commands don't insert literals directly and so affect line and page breaking much less. -\item The numbering of the MCID are done at shipout, so no label/ref system is needed. -\item The code can do some marking automatically. Currently everything that has not been marked up by the document is marked as artifact. -\end{enumerate} +\begin{enumerate} +\item Pagebreaks between start and end of the marker are +\emph{not} a problem. So you can mark a complete paragraph. If a pagebreak +occur directly after an start marker or before an end marker this can lead to +empty chunks in the \PDF{} and so bloat up \PDF{} a bit, but this is imho not +really a problem (compared to the size increase by the rest of the tagging). + +\item The commands don't insert literals directly and so affect line and page +breaking much less. + +\item The numbering of the MCID are done at shipout, so no label/ref system + is needed. + +\item The code can do some marking automatically. Currently everything that + has not been marked up by the document is marked as artifact. +\end{enumerate} @@ -820,37 +934,67 @@ This means that quite a number of problems mentioned above are not relevant for \subsubsection{Commands to mark content and chunks} -In generic mode\sidenote{Generic mode only} is vital that the end command is executed on the same page as the begin command. So think carefully how to place them. -For strategies how to handle paragraphs that split over pages see subsection~\ref{sec:splitpara}. +In generic mode\sidenote{Generic mode only} is vital that the end command is +executed on the same page as the begin command. So think carefully how to +place them. For strategies how to handle paragraphs that split over pages see +subsection~\ref{sec:splitpara}. \begin{docCommand}{tagmcbegin}{\marg{key-val-list}}\end{docCommand} \begin{docCommand}{tag_mc_begin:n}{\marg{key-val-list}}\end{docCommand} -These commands insert the begin of the marked content code in the \PDF{}. They don't start a paragraph. \emph{They don't start a group}. -Such markers should not be nested. The command will warn you if this happens. +These commands insert the begin of the marked content code in the \PDF{}. +They don't start a paragraph. \emph{They don't start a group}. Such markers +should not be nested. The command will warn you if this happens. The key-val list understands the following keys: \begin{description} - \item[\PrintKeyName{tag}] - This is required, unless you use the \PrintKeyName{artifact} key. The value of the key is normally one of the standard type listed in section \ref{sec:new-tag} (without a slash at the begin, this is added by the code). It is possible to setup new tags, see the same section. The value of the key is expanded, so it can be a command. The expansion is passed unchanged to the \PDF{}, so it should with a starting slash give a valid \PDF{} name (some ascii with numbers like \texttt{H4} is fine). - - \item[\PrintKeyName{artifact}] - This will setup the marked content as an artifact. The key should be used for content that should be ignored. The key can take one of the values \PrintKeyName{pagination}, \PrintKeyName{pagination/header}, \PrintKeyName{pagination/footer}, \PrintKeyName{layout}, \PrintKeyName{page}, \PrintKeyName{background} and \PrintKeyName{notype} (this is the default). Text in the header and footer should normally be marked with \PrintKeyName{artifact=pagination} or \PrintKeyName{pagination/header}, \PrintKeyName{pagination/footer} but - simply artifact (as it is now done automatically) should be ok too. - - It is not quite clear if rules and other decorative graphical objects needs to be marked up as artifacts. Acrobat seems not to mind if not, but PAC~3 complained. - - The validators complain if some text is not marked up, but it is not quite clear if this is a serious problem. - - The\sidenote{lua mode only} lua mode will mark up everything unmarked as \texttt{artifact=notype}. You can suppress this behaviour by setting the tagpdfsetup key \texttt{tagunmarked} to false. See section \ref{ssec:setup}. - - \item[\PrintKeyName{stash}] - Normally marked content will be stored in the \enquote{current} structure. This may not be what you want. As an example you may perhaps want to put a marginnote behind or before the paragraph it is in the tex-code. With this boolean key the content is marked but not stored in the kid-key of the current structure. - - \item[\PrintKeyName{label}] - This key sets a label by which you can call the marked content later in another structure (if it has been stashed with the previous key). Internally the label name will start with \texttt{tagpdf-}. + \item[\PrintKeyName{tag}] This key is optional. By default the tag name + of the surrounding structure is used, which normally should be fine. + But if needed the name can be set explicitly with this key. The value + of the key is typically one of the standard type listed in section + \ref{sec:new-tag} (without a slash at the begin, this is added by the + code). It is possible to setup new tags, see the same section. The + value of the key is expanded, so it can be a command. The expansion + is passed unchanged to the \PDF{}, so it should with a starting slash + give a valid \PDF{} name (some ascii with numbers like \texttt{H4} + is fine). + + \item[\PrintKeyName{artifact}] This will setup the marked content as an + artifact. The key should be used for content that should be ignored. + The key can take one of the values \PrintKeyName{pagination}, + \PrintKeyName{pagination/header}, \PrintKeyName{pagination/footer}, + \PrintKeyName{layout}, \PrintKeyName{page}, + \PrintKeyName{background} and \PrintKeyName{notype} (this is the + default). Text in the header and footer should normally be marked + with \PrintKeyName{artifact=pagination} or + \PrintKeyName{pagination/header}, \PrintKeyName{pagination/footer} + but simply artifact (as it is now done automatically) should be ok + too. + + It is not quite clear if rules and other decorative graphical objects + needs to be marked up as artifacts. Acrobat seems not to mind if not, but + PAC~3 complained. + + The validators complain if some text is not marked up, but it is not + quite clear if this is a serious problem. + + The\sidenote{lua mode only} lua mode will mark up everything unmarked as + \texttt{artifact=notype}. You can suppress this behaviour by setting the + tagpdfsetup key \texttt{tagunmarked} to false. See section + \ref{ssec:setup}. + + \item[\PrintKeyName{stash}] Normally marked content will be stored in the + \enquote{current} structure. This may not be what you want. As an + example you may perhaps want to put a marginnote behind or before the + paragraph it is in the tex-code. With this boolean key the content is + marked but not stored in the kid-key of the current structure. + + \item[\PrintKeyName{label}] This key sets a label by which you can call + the marked content later in another structure (if it has been stashed + with the previous key). Internally the label name will start with + \texttt{tagpdf-}. \item[\PrintKeyName{alt}] This key inserts an \texttt{/Alt} value in the property dictionary of the BDC operator. See section~\ref{sec:alt}. The value is handled as verbatim string, commands are not expanded but the value will be expanded first once (so works like the key \texttt{alttext-o} in previous versions which has been removed). @@ -865,10 +1009,15 @@ The key-val list understands the following keys: - \item[\PrintKeyName{actualtext}] - This key inserts an \texttt{/ActualText} value in the property dictionary of the BDC operator. See section~\ref{sec:alt}. The value is handled as verbatim string, commands are not expanded but the value will be expanded first once (so works like the key \texttt{actualtext-o} in previous versions which has been removed). + \item[\PrintKeyName{actualtext}] This key inserts an \texttt{/ActualText} + value in the property dictionary of the BDC operator. See + section~\ref{sec:alt}. The value is handled as verbatim string, + commands are not expanded but the value will be expanded first once + (so works like the key \texttt{actualtext-o} in previous versions + which has been removed). - That means that you can do something like in the following listing and and it will insert \verb+X+ (hex encoded) in the \PDF{}. + That means that you can do something like in the following listing and + and it will insert \verb+X+ (hex encoded) in the \PDF{}. \begin{lstlisting} @@ -876,18 +1025,26 @@ The key-val list understands the following keys: \tagmcbegin{tag=Span,actualtext=\myactualtext} \end{lstlisting} - According to the PDF reference, \texttt{/ActualText} should only be used on marked content sequence of type Span. This is not enforced by the code currently. + According to the PDF reference, \texttt{/ActualText} should only be used + on marked content sequence of type Span. This is not enforced by the code + currently. - \item[\PrintKeyName{raw}] - This key allows you to add more entries to the properties dictionary. The value must be correct, low-level \PDF{}. E.g. \verb+raw=/Alt (Hello)+ will insert an alternative Text. + \item[\PrintKeyName{raw}] This key allows you to add more entries to the + properties dictionary. The value must be correct, low-level \PDF{}. + E.g. \verb+raw=/Alt (Hello)+ will insert an alternative Text. \end{description} \begin{docCommand}{tagmcend}{}\end{docCommand} \begin{docCommand}{tag_mc:end}{}\end{docCommand} -These commands insert the end code of the marked content. They don't end a group and it doesn't matter if they are in another group as the starting commands. In generic mode both commands check if there has been a begin marker and issue a warning if not. In luamode it is often possible to omit the command, as the effect of the begin command ends with a new \verb+\tagmcbegin+ anyway. +These commands insert the end code of the marked content. They don't end a +group and it doesn't matter if they are in another group as the starting +commands. In generic mode both commands check if there has been a begin +marker and issue a warning if not. In luamode it is often possible to omit +the command, as the effect of the begin command ends with a new +\verb+\tagmcbegin+ anyway. % no longer relevant @@ -913,7 +1070,9 @@ These commands insert the end code of the marked content. They don't end a group \begin{docCommand}{tag_mc_use:n}{}\end{docCommand} -These commands allow you to record a marked content that you stashed away into the current structure. Be aware that a marked content can be used only once -- the command will warn you if you try to use it a second time. +These commands allow you to record a marked content that you stashed away +into the current structure. Be aware that a marked content can be used only +once -- the command will warn you if you try to use it a second time. \begin{docCommand}{tag_mc_end_push:}{}\end{docCommand} @@ -1205,7 +1364,7 @@ The method is described in Frank Mittelbach's talk at TUG~2021 Please note \begin{itemize} -\item The code requires a pdfmanagement-testphase version v0.95i or newer. +\item The code requires a \pkg{pdfmanagement-testphase} version v0.95i or newer. \item Typically you will need more compilations than previously, don't rely on the rerun messages, but if something looks wrong rerun. \item The code relies on that related |\tagmcbegin| and |\tagmcend| are in the same boxing level. If one is in a box (which hides the marks) and the other in the main galley, things will go wrong. @@ -1324,7 +1483,7 @@ The key-val list understands the following keys: % By default a structure is added as kid to the currently active structure. With the parent key one can choose another parent. The value is a structure number which must refer to an already existing, previously created structure. Such a structure - number can for example be have been stored with \cs{tag\_get:n}, but one can also use + number can have been stored previously with \cs{tag\_get:n}, but one can also use a label on the parent structure and then use \cs{ref\_value:nn}\verb+{tagpdfstruct-label}{tagstruct}+ to retrieve it. @@ -1648,14 +1807,20 @@ In PDF 2.0 the situation is a bit more complicated. At first PDF~2.0 introduced That means that a type can have more than one \enquote{meaning} depending on the name space it belongs to. |section (name space A)| and |section (name space B)| are two different types. -At second PDF 2.0 still requires that a tagged PDF maps all types to a standard type, but now there are three sets of standard types (The meanings of the PDF types can be looked up in the \PDF{}-references \parencite{pdfspec-iso32000-1,pdfspec-iso32000-2_2020}): +At second PDF 2.0 still requires that a tagged PDF maps all types to a +standard type, but now there are three sets of standard types (The meanings +of the PDF types can be looked up in the \PDF{}-references +\parencite{pdfspec-iso32000-1,pdfspec-iso32000-2_2020}): \begin{enumerate} \item The \emph{standard structure namespace for PDF 1.7}, also called the \emph{default standard structure namespace}. The public name of the namespace is |tag/NS/pdf|. This can be used to reference the namespace e.g. in attributes. These are the structure names from PDF 1.7: \ExplSyntaxOn -\clist_use:Nn \c__tag_role_sttags_pdf_pdfII_clist {,\c_space_tl} -,~ \clist_use:Nn\c__tag_role_sttags_only_pdf_clist {,\c_space_tl} +\clist_clear:N\l_tmpa_clist +\prop_map_inline:cn + { g__tag_role_NS_pdf_prop } + { \clist_put_right:Nn \l_tmpa_clist {#1} } +\clist_use:Nn \l_tmpa_clist {,\c_space_tl } \ExplSyntaxOff \item The \emph{standard structure namespace for PDF 2.0}. @@ -1663,11 +1828,27 @@ The public name of the namespace is |tag/NS/pdf2|. This can be used to reference the namespace e.g. in attributes. These are more or less same types as in PDF. The following types have been removed from this set:\\ \ExplSyntaxOn -\clist_use:Nn\c__tag_role_sttags_only_pdf_clist {,\c_space_tl}\\ +\clist_clear:N\l_tmpa_clist +\prop_map_inline:cn { g__tag_role_NS_pdf_prop } + { + \prop_if_in:cnF + { g__tag_role_NS_pdf2_prop } + {#1} + { \clist_put_right:Nn \l_tmpa_clist {#1} } + } +\clist_use:Nn \l_tmpa_clist {,\c_space_tl }\\ \ExplSyntaxOff and the following are new:\\ \ExplSyntaxOn -\clist_use:Nn \c__tag_role_sttags_only_pdfII_clist {,\c_space_tl} +\clist_clear:N\l_tmpa_clist +\prop_map_inline:cn { g__tag_role_NS_pdf2_prop } + { + \prop_if_in:cnF + { g__tag_role_NS_pdf_prop } + {#1} + { \clist_put_right:Nn \l_tmpa_clist {#1} } + } +\clist_use:Nn \l_tmpa_clist {,\c_space_tl } \ExplSyntaxOff \item MathML 3.0 as an \emph{other namespaces}. @@ -1684,10 +1865,16 @@ syntax |section/H1| should in most cases do the right thing. \begin{description} \item[\PrintKeyName{tag}] This is the name of the new type as it should then be used in \cs{tagstructbegin}. -\item[\PrintKeyName{namespace}] This is the namespace of the new type. The value should be a shorthand of a namespace. The allowed values are currently |pdf|, |pdf2|, |mathml| and |user|. The default value (and recommended value for a new tag) is |user|. The public name of the user namespace is |tag/NS/user|. This can be used to reference - the namespace e.g. in attributes. -\item[\PrintKeyName{role}] This is the type the tag should be mapped too. In a PDF 1.7 or earlier this is normally a type from the |pdf| set, in PDF 2.0 from the |pdf|, |pdf2| and |mathml| set. It can also be a user type, or a still unknown type. The PDF format allows mapping to be done transitively. But you should be aware that tagpdf can't (or more precisely won't) check such unusual role mapping. It lies in the responsability of the author to ensure here that every type is correctly role mapped. -\item[\PrintKeyName{role-namespace}] If the role is a known type the default value is the default namespace: |pdf2| for all types in this set, |pdf| for the type which exist only in PDF 1.7, |mathml| for the MathML types, and for previously defined user types whatever namespace has been set there. If the role is unknown, |user| is used and the code hopes that the type will be defined later. +\item[\PrintKeyName{tag-namespace}] This is the namespace of the new type. + The value should be a shorthand of a namespace. The allowed values are + currently |pdf|, |pdf2|, |mathml| and |user|. The default value (and + recommended value for a new tag) is |user|. The public name of the user + namespace is |tag/NS/user|. This can be used to reference the namespace + e.g. in attributes. +\item[\PrintKeyName{role}] This is the type the tag should be mapped too. In a PDF 1.7 or earlier this is normally a type from the |pdf| set, in PDF 2.0 from the |pdf|, |pdf2| or |mathml| set. It can also be a user type, then this user tag must have been declared before. The PDF format allows mapping to be done transitively. But you should be aware that tagpdf can't (or more precisely won't) check if some unusual role mapping makes really sense, this lies in the responsability of the author. + +\item[\PrintKeyName{role-namespace}] The default value is the default namespace of the role: |pdf2| for all types in this set, |pdf| for the type which exist only in PDF 1.7, |mathml| for the MathML types, and for previously defined user types whatever namespace has been set there. With this key the value can be overwritten. + \item[unknown key] An unknown key is interpreted as a |tag/role|, this preserves the old syntax. So this two calls are equivalent: \begin{lstlisting} @@ -1697,10 +1884,71 @@ syntax |section/H1| should in most cases do the right thing. \end{description} -The exact effects of the key depends on the PDF version. With PDF 1.7 or older the namespace keys are ignored, with PDF 2.0 the namespace keys are use to setup the correct rolemaps. The |namespace| key is also used to define the default namespace if the type is used as a role or as tag in a structure. +The exact effects of the keys depend on the PDF version. With PDF 1.7 or older the namespace keys are ignored, with PDF 2.0 the namespace keys are use to setup the correct rolemaps. The |namespace| key is also used to define the default namespace if the type is used as a role or as tag in a structure. + +\subsection{The \texttt{latex} namespace} +Starting with version 0.98 work has started to setup specific latex tags. In +\PDF{} 2.0 in form of a special name space, with \PDF{} 1.7 or older the +tags are role mapped. This is work in progress and bound to change. +\section{Checking parent-child rules}\label{sec:parent-child} + +The \PDF{} references formulate various rules about whether a structure can be +a child of another structure, e.g. a \texttt{Sect} can not be a child of \texttt{P}. +In the \PDF{} 1.7 reference this rules were +rather vage, in the \PDF{} 2.0 reference there is a quite specific matrix, +which sadly misses some of the tags from \PDF{} 1.7. The upcoming +ISO norm 32005 will +address this problem and extend the matrix to cover tags from +\PDF{} 1.7 and 2.0 (but it still misses the \texttt{math} tag and mathml +tags). + +The rules in the matrix are not a simple allowed/not allowed. Instead some rules determine +that structure elements +can appear only once in a parent, or that additional requirements can be found in the +descriptions of the standard structure types, e.g. \texttt{Caption} often has +to be the first element in the parent structure, and elements like \texttt{Part} and \texttt{Div} +inherit restrictions from parent structures. +External standards like \PDF/UA can add more rules. + +Altogether this doesn't make it easy to check if a structure tree is conformant or not without slowing down +the compilation a lot. + +With version 0.98 some first steps to do checks (and to react to the result of a change) +have been implemented. Some checks will led to warning directly, but the majority +will only be visible if the log-level is increased. + +Typical messages will look then like this + +\begin{lstlisting}[mathescape] +Package tagpdf Info: The rule between parent 'Sect (from Sect/pdf2)' +(tagpdf) and child 'H10 (from H10/pdf2)' is '1 (0..n)' + +Package tagpdf Info: The rule between parent 'H2 (from subsection/latex)' +(tagpdf) and child 'H1 (from section/latex)' is '-1 ($\emptyset$)' +\end{lstlisting} + +The descriptions of the parent and childs are rather verbose as the checks have to take role mapping and name spaces into account. The result of a check is a number---negative if the relation is not allowed, +positive if allowed. The text in the parentheses show the symbols used in the \PDF-matrix. + +Be aware +\begin{itemize} +\item This doesn't test all rules, it only implements (hopefully correctly) the matrix. +\item There can be differences between \PDF~1.7 and 2.0, e.g. \texttt{FENote} is rolemapped to \texttt{Note} +in \PDF~1.7 and then has different containment rules. +\item The special tag \texttt{MC} stands for mc-chunks, so \enquote{real content} (the matrix has containments rules for this too). +\item Currently there is as only negative number \texttt{\textminus1} but that is bound to change, depending on if (and how) it is possible to \enquote{repair} a disallowed parent-child relation. +\item Warnings can be wrong. +\end{itemize} + + + + + + + \section{\enquote{Real} space glyphs}\label{sec:spacechars} @@ -1745,20 +1993,20 @@ This is in pdftex a primitive. It inserts the dummy space glyph. \pkg{tagpdf} de \TeX{}live 2022 (earlier in miktex) both engines will knew new keywords which allow to create structure destination easily. - Support for this has been already added to the pdfmanagement and \pkg{tagpdf} will make - use of it if possible. In most cases it should simply work, but one should be aware - that as one now has a destination that is actually tied to the content it gets more - important to actually consider the context and the place where such destinations - are created. It now makes a difference if the destination is created before the - structure is opened or after so in some cases - code that place destinations should be changed to place them inside the structure they - belong too. . - One also has to consider the pages connected to the destinations: The structure destination - is bound to the page where the structure \emph{begins}, if this differ from the page of the page destination - (e.g. if the destination is created by a \verb+\phantomsection+ in the middle of a longer paragraph) - then be necessary to surround - destinations with a dummy structure (a Span or an Artifact) to - get the right page number. + Support for this has been already added to the \PDF\ management and + \pkg{tagpdf} will make use of it if possible. In most cases it should simply + work, but one should be aware that as one now has a destination that is + actually tied to the content it gets more important to actually consider the + context and the place where such destinations are created. It now makes a + difference if the destination is created before the structure is opened or + after so in some cases code that place destinations should be changed to + place them inside the structure they belong too. One also has to consider + the pages connected to the destinations: The structure destination is bound + to the page where the structure \emph{begins}, if this differ from the page + of the page destination (e.g. if the destination is created by a + \verb+\phantomsection+ in the middle of a longer paragraph) then may be + necessary to surround destinations with a dummy structure (a Span or an + Artifact) to get the right page number. @@ -2025,4 +2273,3 @@ This maps 0 to an array and 2 to the object reference \texttt{21 0 R}. Number tr \end{document} %http://msf.mathmlcloud.org/file_formats/8 %sample \PDF{} for math - diff --git a/macros/latex/contrib/tagpdf/tagpdfdocu-patches.sty b/macros/latex/contrib/tagpdf/tagpdfdocu-patches.sty index e01f9fa594..9e39ec1547 100644 --- a/macros/latex/contrib/tagpdf/tagpdfdocu-patches.sty +++ b/macros/latex/contrib/tagpdf/tagpdfdocu-patches.sty @@ -1,145 +1,86 @@ %\RequirePackage[enable-debug]{expl3}[2018/06/14] -\ProvidesExplPackage {tagpdfdocu-patches} {2022-08-24} {0.97} +\ProvidesExplPackage {tagpdfdocu-patches} {2022-12-22} {0.98} {patches/commands for the tagpdf documentation} -\RequirePackage{etoolbox} +\RequirePackage{etoolbox,xpatch} -%%%%% -% tableofcontents -% no internal patches, but KOMA-only solution -% TODO it must be checked how best practice is now for the table of contents. -%%%%% +%vLogos - -\AfterTOCHead{\tag_struct_begin:n{tag=TOC}} -\AfterStartingTOC{\tag_struct_end:} %end TOC -%Marking the toc entries -%around the whole entry so only structure: -\newcommand\tagscrtocentry[1]{\tag_struct_begin:n{tag=TOCI}#1\tag_struct_end:} - -%leaf so structure and mc: -\newcommand\tagscrtocpagenumber[1] - { - \tag_struct_begin:n{tag=Reference} - \tag_mc_begin:n{tag=Reference} - #1 - \tag_mc_end: - \tag_struct_end: - } - - -\DeclareTOCStyleEntry[ - entryformat=\tagscrtocentry, - pagenumberformat=\tagscrtocpagenumber]{tocline}{section} -\DeclareTOCStyleEntry[ - entryformat=\tagscrtocentry, - pagenumberformat=\tagscrtocpagenumber]{tocline}{subsection} -\DeclareTOCStyleEntry[ - entryformat=\tagscrtocentry, - pagenumberformat=\tagscrtocpagenumber]{tocline}{subsubsection} -\DeclareTOCStyleEntry[ - entryformat=\tagscrtocentry, - pagenumberformat=\tagscrtocpagenumber]{tocline}{paragraph} - - -% \IfStr{#3} doesnt like a \label ... -\renewcommand{\addtocentrydefault}[3]{% -% \Ifstr{#3}{}{} -% {% - \Ifstr{#2}{} - {% - \addcontentsline{toc}{#1} - {% - \protect\nonumberline - \tagstructbegin{tag=P}% - \tagmcbegin{tag=P}% - #3% +\ExplSyntaxOn +\DeclareRobustCommand\TeX{ + \tag_mc_end_push: + \tag_struct_begin:n{tag=Span,alt=TeX,actualtext=TeX} + \tagmcbegin{} + \kern-.1667em\lower.5ex\hbox{E}\kern-.125emX\@ + \tagmcend + \tag_struct_end: + \tag_mc_begin_pop:n{}} +\DeclareRobustCommand{\LaTeX}{ + { + \tag_mc_end_push: + \tag_struct_begin:n{tag=Span,alt=LaTeX,actualtext=LaTeX} + \tagmcbegin{} + L\kern-.36em% + \sbox\z@ T% + \vbox to\ht\z@{\hbox{\check@mathfonts + \fontsize\sf@size\z@ + \math@fontsfalse\selectfont + A}% + \vss}% + }% + \kern-.15em% + \TeX \tagmcend - \tagstructend - }% - }% - {% - \addcontentsline{toc}{#1}{% - \tagstructbegin{tag=Lbl}% - \tagmcbegin{tag=Lbl}% - \protect\numberline{#2}% - \tagmcend\tagstructend - \tagstructbegin{tag=P}% - \tagmcbegin{tag=P}% - #3% - \tagmcend - \tagstructend - }% - }% - %} -}% - -% the dots must be marked too - -\renewcommand*{\TOCLineLeaderFill}[1][.]{ - \leaders\hbox{$\m@th - \mkern \@dotsep mu\hbox{\tag_mc_begin:n{artifact}#1\tag_mc_end:}\mkern \@dotsep - mu$}\hfill -} - -%%%%%%%%% -% Sectioning commands -% no internal patches, but KOMA-only solution -%%%%%%%% - -% the structure should be opened rather early to catch the refstepcounter! -\AddtoDoHook{heading/begingroup}{\tagpdfparaOff\use_none:n} -\cs_new_protected:Npn \__tag_struct_section_begin:n #1 {\tagstructbegin{tag=\prop_item:Nn\g_tag_section_level_prop{#1}}} -\AddtoDoHook{heading/postinit}{\__tag_struct_section_begin:n} -\AddtoDoHook{heading/endgroup}{\tagstructend\use_none:n} - -\prop_new:N \g_tag_section_level_prop -\prop_gput:Nnn \g_tag_section_level_prop {section}{H1} -\prop_gput:Nnn \g_tag_section_level_prop {subsection}{H2} -\prop_gput:Nnn \g_tag_section_level_prop {subsubsection}{H3} -\prop_gput:Nnn \g_tag_section_level_prop {paragraph}{H4} - -\renewcommand{\sectionlinesformat}[4] - { - \@hangfrom - {\hskip #2 - %\tl_if_empty:nF{#3} - \tl_if_in:nnF{\@empty}{#3} - { - \tag_mc_begin:n {tag=\prop_item:Nn\g_tag_section_level_prop{#1}} - #3 - \tag_mc_end: - } - } - {\tag_mc_begin:n {tag=\prop_item:Nn\g_tag_section_level_prop{#1}} - #4 - \tag_mc_end:}% - } - -% minisec is simply P, so we let paratagging handle this. -% But we add a strong structure -% TODO: Should changed to H6 as it is no longer required to have H6 only after H5 -% but need to check how to disable paratagging ... -\cs_new:Nn \__tag_docu_tag_minisec:n -{ - \tag_struct_begin:n {tag=Strong} - \tag_mc_begin:n{tag=Strong} - #1 - \tag_mc_end: - \tag_struct_end: -} -% -\addtokomafont{minisec}{\__tag_docu_tag_minisec:n} - - + \tag_struct_end: + \tag_mc_begin_pop:n{} + } + + +% textbf + +\AddToHook{cmd/textbf/before} + {\leavevmode\tag_mc_end_push:\tag_struct_begin:n{tag=Strong}\tagmcbegin{}} + +\AddToHook{cmd/textbf/after} + {\tag_mc_end:\tag_struct_end:\tag_mc_begin_pop:n{}} + + +\newcommand\minisec[1]{% + \if@noskipsec \leavevmode \fi + \par + \@afterindentfalse + \if@nobreak + \everypar{}% + \else + \addpenalty\@secpenalty\addvspace{1.5ex}% + \fi + {\tagpdfsetup{paratag=H10}\parindent \z@ + \setlength{\parfillskip}{\z@ plus 1fil}% + \raggedright\normalfont\bfseries\nobreak + \nobreak\interlinepenalty \@M #1\par\nobreak% + }\nobreak + \@afterheading +} + + + %%%% %% Lists %% patches enumitem internals! %%%% -\AddToHook{env/itemize/begin}{\par\tagstructbegin{tag=L}}%\par is needed to close the paragraph before. +\tagpdfsetup + { + newattribute = + {itemize}{/O /List /ListNumbering/Unordered}, + newattribute = + {enumerate}{/O /List /ListNumbering/Ordered}, + newattribute = + {description}{/O /List /ListNumbering/Description}, + } +\AddToHook{env/itemize/begin}{\par\tagstructbegin{tag=L,attribute-class=itemize}}%\par is needed to close the paragraph before. \AddToHook{env/itemize/end}{\par\tagstructend\tagstructend\tagstructend}%LBody,LI,L -\AddToHook{env/enumerate/begin}{\par\tagstructbegin{tag=L}} +\AddToHook{env/enumerate/begin}{\par\tagstructbegin{tag=L,attribute-class=enumerate}} \AddToHook{env/enumerate/end}{\par\tagstructend\tagstructend\tagstructend}%LBody,LI,L -\AddToHook{env/description/begin}{\par\tagstructbegin{tag=L}} +\AddToHook{env/description/begin}{\par\tagstructbegin{tag=L,attribute-class=description}} \AddToHook{env/description/end}{\par\tagstructend\tagstructend\tagstructend}%LBody,LI,L \newcommand\tag@enit@format@preset[1]{% @@ -151,7 +92,7 @@ \tagstructend\tagstructend} % for the LBody/LI \tagstructbegin{tag=LI} \tagstructbegin{tag=Lbl} - \tagmcbegin{tag=Lbl} + \tagmcbegin{} #1 \tagmcend \tagstructend @@ -161,6 +102,7 @@ \ExplSyntaxOff \xpatchcmd\enit@preset{\@firstofone}{\tag@enit@format@preset}{}{\fail} %close mc from paratagging and reopen ... +% this disturbs someone docCommand which has then a MC outside a P. \xpatchcmd\@item{\box\@labels}{\tagmcend \box\@labels \tagmcbegin{tag=P}}{}{\fail} \ExplSyntaxOn @@ -170,20 +112,22 @@ \def\enit@format{% \tagstructbegin{tag=LI}% \tagstructbegin{tag=Lbl}% - \tagmcbegin{tag=Lbl}% + \tagmcbegin{}% #1\tagmcend}} % listings + verbatim % the paragraph code could create span around the lines ... -% it is quite unclear if Code is a sensible structure as it no longer exists in pdf 2.0 +% we use Code for every line. -\AddToHook{env/lstlisting/begin}{\tagpdfparaOff\tagstructbegin{tag=Code}\tagmcbegin{tag=Code}} -\AddToHook{env/lstlisting/end}{\tagmcend\tagstructend} +\AddToHook{env/lstlisting/begin}{\tagpdfsetup{paratag=Code}} +\AddToHook{env/lstlisting/end}{} -\AddToHook{env/verbatim/begin}{\tagpdfparaOff\tagstructbegin{tag=Code}\tagmcbegin{tag=Code}} -\AddToHook{env/verbatim/end}{\tagmcend\tagstructend} +\AddToHook{env/verbatim/begin}{\tagpdfsetup{paratag=Code}} +\AddToHook{env/verbatim/end}{} +\AddToHook{env/docCommand/begin}{\tagstructbegin{tag=P}\tagpdfsetup{paratag=Code}} +\AddToHook{env/docCommand/end} {\tagstructend} % ======== marginnote ========== % TODO marginnote has a bug (a \par is missing) so it messes up tagging. @@ -209,32 +153,30 @@ %\tcbset{before~upper=\tagpdfparaOn} \AddToHook{env/tcolorbox/begin}{\tagpdfparaOff \tcbset{before~upper=\tagpdfparaOn}} \AddToHook{env/docCommand/begin}{\tagpdfparaOff \tcbset{before~upper=\tagpdfparaOn}} - - %locally for now - - +\ExplSyntaxOff % ======= footnote ======== -% TODO +% done in testphase code % ======= bibliography ======== % biblatex. Creates some empty mc-chunks. % no internal patches, but redefining begentry/finentry is not safe. % better hook is needed. -\newbibmacro*{begentry}{\tagstructbegin{tag=BibEntry}\tagmcbegin{tag=BibEntry}} +\newbibmacro*{begentry}{\tagstructbegin{tag=BibEntry}\tagmcbegin{}} \newbibmacro*{finentry}{\finentry\tagmcend\tagstructend} % ====== hyperref ======== % this should probably go into tagpdf, but it is related to % problem of pdf strings and context .... -\@ifpackageloaded{hyperref}{% -\pdfstringdefDisableCommands{ - \let\tagstructbegin\@gobble - \let\tagmcbegin\@gobble - \let\tagmcend\relax - \let\tagstructend\relax -}}{} +\AddToHook{package/hyperref/after} + {% + \pdfstringdefDisableCommands{% + \let\tagstructbegin\@gobble + \let\tagmcbegin\@gobble + \let\tagmcend\relax + \let\tagstructend\relax + }}{} %====== pictures ======== @@ -243,46 +185,7 @@ \tagpdfsetup { newattribute = - {bbox}{/O /Layout /BBox [0~0~100~100]} + {bbox}{/O /Layout /BBox [0 0 100 100]} } -%%%%% -%======== Role maps ========= -%%%%% -\tagpdfsetup{ add-new-tag=Title/H1,add-new-tag=Strong/Span } - - - \endinput - -% ========================== -% now unneeded commands - -% we need a command which simply starts a paragraph \TagP^, -% a "continuation" variant which closes the previous paragraph (\TagP), -% and a final end. - -\NewDocumentCommand\TagP{t{^}} - { - \tag_mc_if_in:T { \tag_mc_end: } - \IfBooleanF{#1} - { - \tag_struct_end: - } - \tag_struct_begin:n{tag=P} - \tag_mc_begin:n{tag=P} - } -% close a mc and one structure - -\NewDocumentCommand\TagPend{} - { - \tag_mc_if_in:T { \tag_mc_end: } - \tag_struct_end: - } - - - \ExplSyntaxOn -\newcommand\showcurrentstruct{\prop_show:c { g__tag_struct_\g__tag_struct_stack_current_tl _prop }} - - -\ExplSyntaxOff |