summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/pythonimmediate
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex/contrib/pythonimmediate')
-rw-r--r--macros/latex/contrib/pythonimmediate/DEPENDS.txt4
-rw-r--r--macros/latex/contrib/pythonimmediate/README24
-rw-r--r--macros/latex/contrib/pythonimmediate/pythonimmediate.pdfbin0 -> 402484 bytes
-rw-r--r--macros/latex/contrib/pythonimmediate/pythonimmediate.sty183
-rw-r--r--macros/latex/contrib/pythonimmediate/pythonimmediate.tex535
-rwxr-xr-xmacros/latex/contrib/pythonimmediate/pythonimmediate_script_pytotex.py57
-rwxr-xr-xmacros/latex/contrib/pythonimmediate/pythonimmediate_script_textopy.py2295
7 files changed, 3098 insertions, 0 deletions
diff --git a/macros/latex/contrib/pythonimmediate/DEPENDS.txt b/macros/latex/contrib/pythonimmediate/DEPENDS.txt
new file mode 100644
index 0000000000..74052c8330
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/DEPENDS.txt
@@ -0,0 +1,4 @@
+saveenv
+currfile
+l3keys2e
+precattl
diff --git a/macros/latex/contrib/pythonimmediate/README b/macros/latex/contrib/pythonimmediate/README
new file mode 100644
index 0000000000..c164e2ca9b
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/README
@@ -0,0 +1,24 @@
+pythonimmediate -- Library to run Python code
+ \fileinfo.
+
+Released under the LaTeX Project Public License v1.3c or later
+See http://www.latex-project.org/lppl.txt
+
+Report bugs at https://github.com/user202729/TeXlib
+
+========
+
+Copyright 2022 user202729
+
+This work may be distributed and/or modified under the conditions of the
+LaTeX Project Public License (LPPL), either version 1.3c of this license or
+(at your option) any later version. The latest version of this license is in
+the file:
+
+ http://www.latex-project.org/lppl.txt
+
+This work has the LPPL maintenance status `maintained'.
+
+The Current Maintainer of this work is user202729.
+
+This work consists of the files pythonimmediate.sty, pythonimmediate_script_pytotex.py, pythonimmediate_script_textopy.py.
diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate.pdf b/macros/latex/contrib/pythonimmediate/pythonimmediate.pdf
new file mode 100644
index 0000000000..67bb6f1259
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/pythonimmediate.pdf
Binary files differ
diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate.sty b/macros/latex/contrib/pythonimmediate/pythonimmediate.sty
new file mode 100644
index 0000000000..6ed72780bf
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/pythonimmediate.sty
@@ -0,0 +1,183 @@
+% File: pythonimmediate.sty
+% Copyright 2022 user202729
+%
+% This work may be distributed and/or modified under the conditions of the
+% LaTeX Project Public License (LPPL), either version 1.3c of this license or
+% (at your option) any later version. The latest version of this license is in
+% the file:
+%
+% http://www.latex-project.org/lppl.txt
+%
+% This work has the LPPL maintenance status `maintained'.
+%
+% The Current Maintainer of this work is user202729.
+
+\ProvidesExplPackage{pythonimmediate}{2022/12/23}{0.0.0}{Library to run Python code}
+
+\RequirePackage{saveenv}
+\RequirePackage{currfile}
+\RequirePackage{l3keys2e}
+\RequirePackage{precattl}
+
+
+\cs_generate_variant:Nn \str_set:Nn {NV}
+\cs_generate_variant:Nn \str_if_eq:nnT {VnT}
+\cs_generate_variant:Nn \msg_error:nnn {nnV}
+\cs_generate_variant:Nn \str_if_eq:nnF {xnF}
+\cs_generate_variant:Nn \str_range:nnn {Vnn}
+\cs_generate_variant:Nn \str_if_eq:nnF {VnF}
+\cs_generate_variant:Nn \str_if_in:nnF {VnF}
+\cs_generate_variant:Nn \tl_build_put_right:Nn {NV}
+
+%\bench before~rescan.
+%\bench after~rescan.
+
+%\GenerateVariantsFile:n{\rescansynclastfilename}
+
+\str_set:Nn \_pythonimmediate_outputdir{.}
+\tl_set:Nn \_pythonimmediate_textopy_script_path{}
+\str_set:Nn \_pythonimmediate_mode{multiprocessing-network}
+\str_set:Nn \_pythonimmediate_python_executable{python3}
+\keys_define:nn{pythonimmediate}{
+ outputdir.tl_set_x:N=\_pythonimmediate_outputdir,
+ %outputdir.default:n={.}, % huh does not work?
+
+ mode.tl_set:N=\_pythonimmediate_mode,
+ scriptpath.tl_set_x:N=\_pythonimmediate_textopy_script_path,
+
+ python-executable.tl_set:N=\_pythonimmediate_python_executable,
+}
+\ProcessKeysOptions{pythonimmediate}
+
+\str_set:NV \_pythonimmediate_outputdir \_pythonimmediate_outputdir
+\str_set:NV \_pythonimmediate_mode \_pythonimmediate_mode
+\str_set:NV \_pythonimmediate_textopy_script_path \_pythonimmediate_textopy_script_path
+
+\msg_new:nnn {pythonimmediate} {shell-fail} {Shell~command~execution~failed!}
+\msg_new:nnn {pythonimmediate} {cannot-determine-script-path} {Cannot~determine~script~path!}
+\msg_new:nnn {pythonimmediate} {cannot-read-line} {Cannot~read~line!}
+\msg_new:nnn {pythonimmediate} {internal-error} {Internal~error!}
+\msg_new:nnn {pythonimmediate} {invalid-mode} {Invalid~mode:~'#1'.}
+
+
+\bool_new:N \_pythonimmediate_mode_multiprocessing_network
+\bool_new:N \_pythonimmediate_mode_unnamed_pipe
+\str_if_eq:VnT \_pythonimmediate_mode {multiprocessing-network} {\bool_set_true:N \_pythonimmediate_mode_multiprocessing_network}
+\str_if_eq:VnT \_pythonimmediate_mode {unnamed-pipe} {\bool_set_true:N \_pythonimmediate_mode_unnamed_pipe}
+\bool_if:nF {\_pythonimmediate_mode_multiprocessing_network || \_pythonimmediate_mode_unnamed_pipe} {
+ \msg_error:nnV {pythonimmediate} {invalid-mode} \_pythonimmediate_mode
+}
+
+% we need to persistently open the file anyway, so using LaTeX3 stream reference counting doesn't help
+\newread \_pythonimmediate_read_file
+
+
+
+
+%\bench before~setup.
+
+\begingroup
+ \endlinechar=-1~
+ \tl_if_empty:NT \_pythonimmediate_textopy_script_path {
+ % ======== first use kpsewhich to get the _pythonimmediate_script_path here ========
+ % (abuse \_pythonimmediate_read_file variable for this purpose)
+ \openin \_pythonimmediate_read_file=|"kpsewhich~ pythonimmediate_script_textopy.py"~
+ \readline \_pythonimmediate_read_file to \_pythonimmediate_textopy_script_path
+ %\bench after~get~textopy~path.
+ \ifeof \_pythonimmediate_read_file
+ \msg_error:nn {pythonimmediate} {cannot-determine-script-path}
+ \fi
+ }
+
+ \str_if_eq:xnF {\str_range:Vnn \_pythonimmediate_textopy_script_path {-10} {-1}} {textopy.py} {
+ \msg_error:nn {pythonimmediate} {cannot-determine-script-path}
+ }
+
+ \newwrite \_pythonimmediate_write_file
+
+ %\bench before~openin.
+
+ % ======== open persistent pipes to the child processes
+ \bool_if:NTF \_pythonimmediate_mode_unnamed_pipe { % in this case make sure the pipe remains open...
+ \openin \_pythonimmediate_read_file=|"sleep~ 0.5s|\_pythonimmediate_python_executable \space \str_range:Vnn \_pythonimmediate_textopy_script_path {1} {-11} pytotex.py \space \_pythonimmediate_mode"~ % we must use the primitive here to use the pipe file path
+ % TODO sleep infinity causes some resource leak (the process will not exit after TeX exits). Need to fix
+ % but sleep too little might be problematic that it exits before the setup is done
+ % we can just assume machines are not that slow
+ } {
+ \openin \_pythonimmediate_read_file=|"\_pythonimmediate_python_executable \space \str_range:Vnn \_pythonimmediate_textopy_script_path {1} {-11} pytotex.py \space \_pythonimmediate_mode"~ % we must use the primitive here to use the pipe file path
+ }
+
+ %\bench after~openin.
+
+ %\bench before~openout.
+ \immediate\openout \_pythonimmediate_write_file=|"\_pythonimmediate_python_executable \space \_pythonimmediate_textopy_script_path \space \_pythonimmediate_mode"~
+ % both processes must be before the \readline above so that the 2 Python processes are started "in parallel"
+ %\bench after~openout.
+
+ \readline \_pythonimmediate_read_file to \_pythonimmediate_dummy_line % endlinechar still -1
+ %\bench after~read~line.
+ \bool_if:NT \_pythonimmediate_mode_multiprocessing_network {
+ \str_if_eq:VnF \_pythonimmediate_dummy_line {listener-setup-done} {
+ \msg_error:nn {pythonimmediate} {cannot-read-line}
+ }
+ }
+ \bool_if:NT \_pythonimmediate_mode_unnamed_pipe {
+ \str_if_in:VnF \_pythonimmediate_dummy_line {pytotex_pid=} {
+ \msg_error:nn {pythonimmediate} {cannot-read-line}
+ }
+ }
+ \bool_if:NT \_pythonimmediate_mode_unnamed_pipe {
+ \immediate\write\_pythonimmediate_write_file {\_pythonimmediate_dummy_line} % which contains pytotex's pid
+ }
+\endgroup
+
+%\bench after~setup.
+
+
+% read one block of \TeX\ code from Python, store into the specified variable
+% the block is delimited using |surround_delimiter()| in Python i.e. the first and last line are identical
+% new lines are represented with ^^J
+\cs_new_protected:Npn \_pythonimmediate_gread_block:N #1 {
+ \begingroup
+ \endlinechar=10~ % affect \readline
+ \readline \_pythonimmediate_read_file to \_pythonimmediate_delimiter
+
+ \tl_build_gbegin:N #1
+ \readline \_pythonimmediate_read_file to \_pythonimmediate_line
+
+ %\bench read~first~line.
+
+ \bool_do_until:nn {\tl_if_eq_p:NN \_pythonimmediate_delimiter \_pythonimmediate_line} {
+ \tl_build_gput_right:NV #1 \_pythonimmediate_line
+ \ifeof \_pythonimmediate_read_file
+ \msg_error:nn {pythonimmediate} {internal-error}
+ \fi
+ \readline \_pythonimmediate_read_file to \_pythonimmediate_line
+ }
+ \tl_build_gend:N #1
+ \endgroup
+}
+\cs_generate_variant:Nn \tl_build_gput_right:Nn {NV}
+
+\cs_new_protected:Npn \_pythonimmediate_read_block:N #1 {
+ \_pythonimmediate_gread_block:N \_pythonimmediate_block
+ \tl_set_eq:NN #1 \_pythonimmediate_block
+}
+
+
+% read one block of \TeX\ code from Python and |\scantokens|-run it
+% the content inside is the actual TeX code to be executed
+\cs_new_protected:Npn \_pythonimmediate_run_block: {
+ \_pythonimmediate_gread_block:N \_pythonimmediate_code
+ \begingroup
+ \newlinechar=10~
+ \expandafter
+ \endgroup
+ \scantokens \expandafter{\_pythonimmediate_code}
+} % trick described in https://tex.stackexchange.com/q/640274 to scantokens the code with \newlinechar=10
+
+% bootstrap code
+%\bench before~bootstrap.
+\_pythonimmediate_run_block:
+%\bench after~bootstrap.
+
diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate.tex b/macros/latex/contrib/pythonimmediate/pythonimmediate.tex
new file mode 100644
index 0000000000..877bc3a05e
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/pythonimmediate.tex
@@ -0,0 +1,535 @@
+\ProvidesFile{pythonimmediate.tex}[2022/12/23 0.0.0 Library to run Python code]
+\RequirePackage{fvextra}
+\documentclass{l3doc}
+\usepackage{tikz}
+\usetikzlibrary{calc}
+\usetikzlibrary{arrows.meta}
+\EnableCrossrefs
+\CodelineIndex
+\fvset{breaklines=true,gobble=0,tabsize=4,frame=single,numbers=left,numbersep=3pt}
+
+\AtBeginDocument{\DeleteShortVerb\"} % https://tex.stackexchange.com/a/650966/250119
+\MakeOuterQuote{"}
+
+\newcommand\reg{régime}
+\newcommand\DescribePython{\DescribeMacro} % hack
+
+
+\begin{document}
+
+
+% hack to make quotes inside |...| straight
+\edef\temp{\def\noexpand|{\noexpand\Verb\string|}}\temp
+% basically execute \def| { \Verb ⟨catcode other |⟩ }
+
+\GetFileInfo{\jobname.tex}
+
+\title{\pkg{\jobname} --- \fileinfo
+\thanks{This file describes version \fileversion, last revised \filedate.}
+}
+\author{user202729}
+\date{Released \filedate}
+
+\maketitle
+
+\begin{abstract}
+ \fileinfo.
+\end{abstract}
+
+\section{Motivation}
+
+Just like \pkg{Perl\TeX} or \pkg{PyLua\TeX} (and unlike \pkg{Python\TeX} or \pkg{lt3luabridge}),
+this only requires a single run, and variables are persistent throughout the run.
+
+Unlike \pkg{Perl\TeX} or \pkg{PyLua\TeX}, there's no restriction on compiler or script required to run the code.
+
+There's also debugging functionalities -- \TeX\ errors results in Python traceback, and Python error results in \TeX\ traceback.
+Errors in code executed with the |pycode| environment gives the correct traceback point to the Python line of code in the \TeX\ file.
+
+For advanced users, this package allows the user to manipulate the \TeX\ state directly from within Python,
+so you don't need to write a single line of \TeX\ code.
+
+\section{Installation}
+
+%You need to install Python \pkg{daemoniker} and \pkg{psutil} package.
+
+The package should work out of the box with no additional Python package needed.
+
+\subsection{Installation on Overleaf}
+
+At the point of writing, this package can be used on Overleaf.
+
+Instruction:
+
+\begin{itemize}
+ \item Download the following files and place it in the root folder of Overleaf:
+ \begin{itemize}
+ \item \file{saveenv.sty}
+ \item \file{precattl.sty}
+ \item \file{pythonimmediate.sty}
+ \item \file{pythonimmediate_script_textopy.py}
+ \item \file{pythonimmediate_script_pytotex.py}
+ \end{itemize}
+ \item Write the following in the preamble:
+\begin{verbatim}
+\usepackage[abspath]{currfile}
+\usepackage[mode=unnamed-pipe]{pythonimmediate}
+\end{verbatim}
+Refer to \ref{troubleshoot-source-file-not-found} for explanation of the |abspath| option.
+\end{itemize}
+
+For some unknown reason in the default mode on Overleaf (|\nonstopmode|), when there's an error
+the log file might be truncated,
+so in that case consider writing |\errorstopmode|.
+
+Refer to \ref{troubleshoot-Python-error} to read the error traceback in case of Python error.
+
+\section{Usage}
+
+\subsection{Package options}
+
+\DescribeOption{outputdir=}
+Specify the output directory if it's not the default value.
+
+\begin{texnote}
+ The value will be |x|-expanded.
+\end{texnote}
+
+Using the \pkg{outputdir} package, it's possible to determine the output directory automatically, subject to restrictions.
+
+An usage example, if you invoked |pdflatex| with the command
+\begin{verbatim}
+pdflatex --output-directory=/tmp/ a.tex
+\end{verbatim}
+then in the file \file{a.tex} you should have
+\begin{verbatim}
+\usepackage[outputdir=/tmp/]{pythonimmediate}
+\end{verbatim}
+
+\DescribeOption{mode=}
+Method to communicate between \TeX\ and Python. Usually the method will be automatically detected.
+
+Possible values include |multiprocessing-network| and |unnamed-pipe|.
+
+\subsection{\TeX\ interface}
+
+The interface mimics those in popular packages such as \pkg{Python\TeX} or \pkg{PyLua\TeX}.
+
+\subsubsection{Inline commands}
+
+\DescribeMacro{\py}\label{py-command}
+Evaluate some Python expression, consider the result as a string, then execute the result as \TeX\ command.
+
+\begin{texnote}
+ The command is not expandable, and the argument will be fully expanded with the active |~| set to |\relax|,
+ |\set@display@protect| executed and |\escapechar=-1|, then the result
+ passed to Python as a string.
+\end{texnote}
+
+Which, for the users who are not familiar with \TeX\ terminology, roughly means the following:
+\begin{itemize}
+ \item the value can only be used to typeset text,
+ it must not be used to pass "values" to other \LaTeX\ commands.
+
+ The following is legal:
+\begin{verbatim}
+The value of $1+1$ is $\py{1+1}$.
+\end{verbatim}
+The following is illegal, as the result (2) can only be used to typeset text, not passed to another command that expect a "value":
+\begin{verbatim}
+\setcounter{abc}{\py{1+1}}
+\end{verbatim}
+A possible workaround is:
+\begin{verbatim}
+\py{ r'\\setcounter{abc}{' + str(1+1) + '}' }
+\end{verbatim}
+In this example it works without escaping the |{}| characters, but if the Python code has those unbalanced then you can escape them as mentioned below.
+
+ \item Special characters can be "escaped" simply by prefixing the character with backslash.
+
+ For example
+\begin{verbatim}
+\pyc{assert len('\ \ ')==2}
+\pyc{assert ord('\\\\')==0x5c}
+\pyc{assert ord('\%') ==0x25}
+\end{verbatim}
+In the examples above, Python "sees" (i.e. the Python code being executed is)
+\begin{verbatim}
+assert len(' ')==2
+assert ord('\\')==0x5c
+assert ord('%') ==0x25
+\end{verbatim}
+respectively.
+
+ \item Macros will be expanded.
+\begin{verbatim}
+\def\mycode{1+1}
+The value of $1+1$ is $\py{\mycode}$.
+\end{verbatim}
+\end{itemize}
+
+
+\DescribeMacro{\pyc}
+Execute some Python code provided as an argument (the argument will be interpreted as described above).
+
+The command is not expandable -- roughly speaking, you can only use this at "top level".
+
+Any output (as described in \ref{print-to-TeX}) will be typesetted.
+
+The difference between |\py| and |\pyc| is that the argument of |\py| should be a Python expression
+(suitable for passing into |eval()| Python function) while the argument of |\pyc| should be a Python
+statement (suitable for passing into |exec()| Python function).
+
+Therefore,
+\begin{itemize}
+ \item |\py{1+1}| will typeset 2.
+ \item |\pyc{1+1}| is valid, but will do nothing just like |exec("1+1")|.
+ \item |\py{x=1}| is invalid.
+ \item |\pyc{x=1}| is valid and assigns the variable |x| to be 1.
+\end{itemize}
+
+\DescribeMacro{\pycq}
+Same as above, but output (\ref{print-to-TeX}) will not be typesetted.
+
+\DescribeMacro{\pyfile}
+Given an argument being the file name, execute that file.
+
+\DescribeMacro{\pys}
+Performs "string interpolation", the same way as \pkg{Python\TeX}.
+(not yet implemented)
+
+\subsubsection{Environments}
+
+\DescribeEnv{pycode}
+Verbatim-like environment that executes the code inside as Python.
+
+Example usage: The following will typeset |123|
+\begin{verbatim}
+\begin{pycode}
+pythonimmediate.print("123")
+\end{pycode}
+\end{verbatim}
+
+Special note: white spaces at the end of lines are preserved.
+
+Any output (as described in \ref{print-to-TeX}) will be typesetted.
+
+\DescribeEnv{pycodeq}
+Same as above, but output will not be typesetted.
+
+\DescribeEnv{pysub}
+Not yet implemented.
+
+\subsection{Python interface}
+
+The \TeX\ interface is only used to call Python. Apart from that, all the work can be done on the Python side.
+
+All functions in this section should be imported from |pythonimmediate| package, unless specified otherwise.
+
+\subsubsection{Print to \TeX}\label{print-to-TeX}
+
+\DescribePython{.print()}
+\DescribePython{.file}
+These functions are used in |\pyc| command or |pycode| environment.
+
+Unlike most other packages, using |print()| function in Python will print to the console (\TeX\ standard output).
+In order to print \TeX\ code to be executed, you can do one of
+\begin{verbatim}
+pythonimmediate.print(...)
+print(..., file=pythonimmediate.file)
+with contextlib.redirect_stdout(pythonimmediate.file):
+ print(...)
+\end{verbatim}
+Note that in quiet environments, |pythonimmediate.file| is None, the second variant using |print()| will print to stdout
+instead of suppress output. The third variant works as expected.
+
+All output will be buffered until the whole Python code finishes executing.
+In order to typeset the text immediately use one of the advanced commands.
+
+
+
+\DescribePython{.newcommand()}
+\DescribePython{.renewcommand()}
+Same as \LaTeX's |\newcommand| and |\renewcommand|. Can be used as follows:
+
+\begin{verbatim}
+from pythonimmediate import newcommand, renewcommand
+
+@newcommand
+def function():
+ ...
+# define |\function| in TeX
+
+@newcommand("controlsequencename")
+def function():
+ ...
+# define |\controlsequencename| in TeX
+
+def function():
+ ...
+newcommand("controlsequencename", function)
+\end{verbatim}
+
+\DescribePython{.get_arg_str()}
+\DescribePython{.get_optional_arg_str()}
+\DescribePython{.get_verb_arg()}
+\DescribePython{.get_multiline_verb_arg()}
+\DescribePython{.peek_next_char()}
+\DescribePython{.get_next_char()}
+There are those functions that is mostly understandable to an inexperienced \LaTeX\ user,
+and should be sufficient for a lot of programming works.
+
+This is an example of how the functions could be used. The name should be mostly self-explanatory.
+
+\begin{verbatim}
+\documentclass{article}
+\usepackage{pythonimmediate}
+\begin{document}
+\begin{pycode}
+from pythonimmediate import newcommand, peek_next_char, get_next_char, get_arg_str, print
+@newcommand
+def innerproduct():
+ s = get_arg_str() # in the example below this will have the value '\mathbf{a},\mathbf{b}'
+ x, y = s.split(",") # it's just a Python string, manipulate normally (but be careful of comma inside braces, parse the string yourself)
+ print(r"\left\langle" + x + r"\middle|" + y + r"\right\rangle")
+
+@newcommand
+def fx():
+ if peek_next_char() == "_":
+ get_next_char()
+ subscript = get_arg_str()
+ print("f_{" + subscript + "}(x)")
+ else:
+ print("f(x)")
+
+@newcommand
+def sumManyArgs():
+ s = 0
+ while peek_next_char() == "{":
+ i = get_arg_str()
+ s += int(i)
+ print(str(s))
+\end{pycode}
+$1+2+3 = \sumManyArgs{1}{2}{3}$
+
+$\innerproduct{\mathbf{a},\mathbf{b}}=1$
+
+$\fx = 1$, $\fx_i = 2$, $\fx_{ij} = 3$
+\end{document}
+\end{verbatim}
+
+It will typeset:
+
+\begin{quote}
+$1+2+3=6$
+
+$\left\langle\mathbf{a}\middle\vert\mathbf{b}\right\rangle=1$
+
+$f(x)=1$, $f_i(x)=2$, $f_{ij}(x)=3$
+\end{quote}
+
+\DescribePython{.get_arg_estr()}
+\DescribePython{.get_optional_arg_estr()}
+Similar to some functions above, except that the argument is fully expanded and "escapes" of common characters are handled correctly,
+similar to how |\py| command (\ref{py-command}) reads its arguments.
+
+\DescribePython{.execute()}
+Takes a string and execute it immediately. (so that any |.execute()| will be executed before any |.print()|)
+
+Assuming \TeX\ is in errorstopmode (i.e. errors halt \TeX\ execution),
+any error in \TeX\ will create an error in Python and the traceback should point to the correct line of code.
+
+For example, in the following code
+
+\begin{verbatim}
+\documentclass{article}
+\usepackage{tikz}
+\usepackage{pythonimmediate}
+\begin{document}
+
+\begin{tikzpicture}
+\begin{pycode}
+from pythonimmediate import execute
+execute(r'\draw (0, 0) to (1, 1);')
+execute(r'\draw (2, 2) to (p);')
+execute(r'\draw (3, 3) to (4, 4);')
+\end{pycode}
+\end{tikzpicture}
+
+\end{document}
+\end{verbatim}
+each |\draw| command will be executed immediately when the Python |.execute()| function is executed,
+and as the second line throws an error, the Python traceback will point to that line.
+
+\section{Troubleshooting}
+
+\subsection{"Source file not found!" error message}\label{troubleshoot-source-file-not-found}
+
+In order to obtain the exact code with trailing spaces and produce error traceback
+point to the correct \TeX\ file, the Python code need to know the full path to the current
+\TeX\ file for the |pycode| environment.
+
+Nevertheless, this is difficult and does not always work
+(refer to the documentation of \pkg{currfile} for details), so this message is issued
+when the file cannot be found.
+
+In that case try the following fixes:
+
+\begin{itemize}
+ \item Include |\usepackage[abspath]{currfile}| at the start of the document, after the |\documentclass| line.
+ (this option is not included by default because it's easy to get package clash, and usually \pkg{currfile} without
+ the |abspath| option works fine -- unless custom |jobname| is used)
+ \item Explicitly override |currfilename| or |currfileabspath| -- for example
+\begin{verbatim}
+\def\currfilename{main.tex}
+\end{verbatim}
+ Technically this is an abuse of the \pkg{currfile} package API, but it usually works regardless.
+\end{itemize}
+
+\subsection{"Python error" error message}\label{troubleshoot-Python-error}
+
+In case of Python error, the Python traceback is included in the terminal and \TeX\ log file.
+
+Search for "Python error traceback" before the error line in the log file.
+
+On Overleaf, you can either view the log file ("Raw logs" section)
+or the traceback on stderr (download \file{output.stderr} file)
+
+\subsection{"\TeX\ error" error message}\label{troubleshoot-TeX-error}
+
+If an error occur in \TeX, traceback cannot be included in the log file.
+
+Besides, this can only be detected in |\errorstopmode|. Such an error will always halt \TeX,
+and Python will be force-exited after printing the error traceback.
+
+On Overleaf, download \file{output.stderr} file to read the traceback.
+
+\section{Implementation note}
+
+Communication between \TeX\ and Python are done by opening two pseudo-files from the output of a Python process |textopy|
+(similar to |\ior_shell_open:Nn|)
+and to the input of another Python process |pytotex| (this would be |\iow_shell_open:Nn|, if \LaTeX3 have such a function).
+
+There are various methods for the 2 Python child processes to communicate with each other.
+After some initial bootstrapping to setup the communication, we can consider only the |textopy| script, the other
+merely serves as the bridge to send input to \TeX.
+
+The communication protocol is a little complicated, since it must support nesting bidirectional execution of \TeX\ and Python.
+
+Besides, I believe it's not possible to make a "background listener" on the \TeX\ side, so it must keep track of whether a command should be read from Python and executed.
+
+Currently, exception handling (throwing a Python exception in a nested Python function, catch it in the outer Python function) is not supported.
+
+These are some examples of what could happen in the communication protocol.
+
+\ExplSyntaxOn
+\let\fpEval\fp_eval:n
+\ExplSyntaxOff
+
+\begingroup
+
+\tikzset{myarrow/.style={-{Latex[length=2mm]}}}
+
+\def\blockWidth{1}
+\def\blockHeight{7}
+\def\separation{10.5}
+\def\step{0.7}
+
+\def\left #1{
+ \draw [myarrow] (p) ++(\separation, 0) -- node [above] {\small #1} ++(-\separation, 0);
+ \path (p) ++ (0, -\step) coordinate (p);
+}
+\def\right #1{
+ \draw [myarrow] (p) -- node [above] {\small #1} ++(\separation, 0);
+ \path (p) ++ (0, -\step) coordinate (p);
+}
+
+\def\initDrawing{
+ \path (\blockWidth, \fpEval{-\step}) coordinate (p);
+}
+
+\newdimen \tempx
+\newdimen \tempy
+
+\ExplSyntaxOn
+\let \dimToDecimalInUnit \dim_to_decimal_in_unit:nn
+\ExplSyntaxOff
+
+\def\drawBlocks{
+ \path [transform canvas] (p);
+ \pgfgetlastxy{\tempx}{\tempy}
+ \edef\blockHeight{ \fpEval{ - \dimToDecimalInUnit{\tempy}{1cm} } }
+
+ \draw (0, \fpEval{-\blockHeight}) rectangle node [rotate=90] {\TeX} (\blockWidth, 0);
+ \draw (\fpEval{\blockWidth+\separation}, \fpEval{-\blockHeight}) rectangle node [rotate=270] {Python} (\fpEval{\separation+2*\blockWidth}, 0);
+}
+
+\vspace{5pt}
+\begin{tikzpicture}
+\edef\blockHeight{\fpEval{\step*3}}
+\initDrawing
+\right{execute Python code: print(1)}
+\left{execute \TeX\ code: 1}
+\drawBlocks
+\end{tikzpicture}
+
+Nevertheless, there may be more complicated cases where the Python code itself may call \TeX\ code before actually returns:
+
+\vspace{5pt}
+\begin{tikzpicture}
+\edef\blockHeight{\fpEval{\step*5}}
+\initDrawing
+\right{execute Python code: print(var(a)*2)}
+\left{execute \TeX\ code: sendtopy(a); execute another command}
+\right{123}
+\left{execute \TeX\ code: 123123}
+\drawBlocks
+\end{tikzpicture}
+
+Or:
+
+\vspace{5pt}
+\begin{tikzpicture}
+\edef\blockHeight{\fpEval{\step*7}}
+\initDrawing
+\right{execute Python code: tex.exec(a=456); print(var(a)*2)}
+\left{execute \TeX\ code: a=456; sendtopy(done); execute another command}
+\right{done}
+\left{execute \TeX\ code: sendtopy(a); execute another command}
+\right{456}
+\left{456456}
+\drawBlocks
+\end{tikzpicture}
+
+\endgroup
+
+The Python side must not just listen for "done" command back, but must potentially call a nested loop.
+
+The exact protocol is:
+\begin{itemize}
+ \item "execute Python code" sends from \TeX\ to Python has a single line "|i|\meta{handler name}",
+ followed by any number of arguments (depends on the handler).
+
+ Refer to the |define_TeX_call_Python| internal function for details.
+
+ \item "done" sends from \TeX\ to Python has the format "|r|\meta{optional return value as a string in a single line}".
+
+ This is sent by executing \TeX\ command |\pythonimmediatecontinue|, which takes a single argument to be e-expanded using |\write|
+ as the "return value".
+
+ \item "execute \TeX\ code" sends from Python to \TeX\ must only be sent when the \TeX\ side listens for a command.
+ It consist of a single line specify the "command name", which \TeX\ will
+ execute the command named
+ |\__run_|\meta{command name}|:|
+ which must already be defined on the \TeX\ side.
+
+ The command itself might contain additional code to execute more code, e.g. by reading more lines from Python.
+
+ Refer to the |define_Python_call_TeX| internal function for details.
+\end{itemize}
+
+
+
+\PrintIndex
+
+\end{document}
diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate_script_pytotex.py b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_pytotex.py
new file mode 100755
index 0000000000..abb71a2631
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_pytotex.py
@@ -0,0 +1,57 @@
+#!/bin/python3
+"""
+======== Py-to-TeX half ========
+
+receive things that should be passed to TeX from TeX-to-Py half,
+then pass to TeX.
+
+the things that are sent should already be newline-terminated if necessary.
+
+user code are not executed here.
+"""
+
+import sys
+import signal
+signal.signal(signal.SIGINT, signal.SIG_IGN) # when the other half terminates this one will terminates "gracefully"
+
+#debug_file=open(Path(tempfile.gettempdir())/"pythonimmediate_debug_pytotex.txt", "w", encoding='u8', buffering=2)
+#debug=functools.partial(print, file=debug_file, flush=True)
+debug=lambda *args, **kwargs: None
+
+import argparse
+parser=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument("mode", choices=["multiprocessing-network", "unnamed-pipe"])
+args=parser.parse_args()
+
+
+# ======== setup communication method. Just an infinite loop print whatever being sent.
+
+if args.mode=="multiprocessing-network":
+ from multiprocessing.connection import Listener
+
+ address=("localhost", 7348)
+ #address="./pythonimmediate.socket"
+ with Listener(address) as listener:
+ print("listener-setup-done", flush=True)
+ with listener.accept() as connection:
+ debug("accepted a connection")
+ while True:
+ try:
+ data=connection.recv_bytes()
+ debug(" data=", data)
+ sys.__stdout__.buffer.write(data) # will go to TeX
+ sys.__stdout__.buffer.flush()
+ except EOFError: break
+
+elif args.mode=="unnamed-pipe":
+ import os
+ sys.stdout.write("pytotex_pid=" + str(os.getpid()) + "\n")
+ sys.stdout.flush()
+ for line in sys.stdin:
+ sys.stdout.write(line)
+ sys.stdout.flush()
+
+else:
+ assert False
+
+# ========
diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate_script_textopy.py b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_textopy.py
new file mode 100755
index 0000000000..7e4f4a7de9
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/pythonimmediate_script_textopy.py
@@ -0,0 +1,2295 @@
+#!/bin/python3
+"""
+======== TeX-to-Py half ========
+
+receive commands from TeX, then execute it here
+"""
+
+
+#from __future__ import annotations
+import sys
+import os
+import inspect
+import contextlib
+import io
+import functools
+from typing import Optional, Union, Callable, Any, Iterator, Protocol, Iterable, Sequence, Type, Tuple, List, Dict
+import typing
+from abc import ABC, abstractmethod
+from pathlib import Path
+from dataclasses import dataclass
+import tempfile
+import signal
+import traceback
+import re
+import collections
+import enum
+
+
+def user_documentation(x: Union[Callable, str])->Any:
+ return x
+
+
+
+#debug=functools.partial(print, file=sys.stderr, flush=True) # unfortunately this is async ... or so it seems...?
+#debug_file=open(Path(tempfile.gettempdir())/"pythonimmediate_debug_textopy.txt", "w", encoding='u8', buffering=2)
+#debug=functools.partial(print, file=debug_file, flush=True)
+debug=lambda *args, **kwargs: None
+
+
+import argparse
+parser=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument("mode", choices=["multiprocessing-network", "unnamed-pipe"])
+args=parser.parse_args()
+
+expansion_only_can_call_Python=False # normally. May be different in LuaTeX etc.
+
+# ======== setup communication method. Requires raw_readline() and send_raw() methods.
+
+if True:
+ sys.stdin=None # type: ignore
+ # avoid user mistakenly read
+
+ raw_readline=sys.__stdin__.readline # raw_readline() should return "⟨line⟩\n" or "" (if EOF) on each call
+
+if args.mode=="multiprocessing-network":
+ address=("localhost", 7348) # this must be identical to that of the other half-script
+ #address="./pythonimmediate.socket"
+
+ from multiprocessing.connection import Client
+ connection=Client(address)
+ debug("connected")
+
+ def send_raw(s: str)->None: # send_raw() should get pass the s = "⟨line⟩\n"
+ global connection
+ connection.send_bytes(s.encode('u8'))
+
+elif args.mode=="unnamed-pipe":
+ pytotex_pid_line=raw_readline()
+ match_=re.fullmatch("pytotex_pid=(\d+)\n", pytotex_pid_line)
+ assert match_
+ pytotex_pid=int(match_[1])
+
+ connection_=open("/proc/" + str(pytotex_pid) + "/fd/0", "w", encoding='u8',
+ buffering=1 # line buffering
+ )
+
+ def send_raw(s: str)->None:
+ global connection_
+ connection_.write(s)
+ connection_.flush() # just in case
+
+else:
+ assert False
+
+# ======== done.
+
+# https://stackoverflow.com/questions/5122465/can-i-fake-a-package-or-at-least-a-module-in-python-for-testing-purposes
+from types import ModuleType
+pythonimmediate: Any=ModuleType("pythonimmediate")
+pythonimmediate.__file__="pythonimmediate.py"
+sys.modules["pythonimmediate"]=pythonimmediate
+
+pythonimmediate.debugging=True
+
+def export_function_to_module(f: Callable)->Callable:
+ """
+ the functions decorated with this decorator are accessible from user code with
+
+ import pythonimmediate
+ pythonimmediate.⟨function name⟩(...)
+ """
+ setattr(pythonimmediate, f.__name__, f)
+ return f
+
+action_done=False
+
+
+def check_not_finished()->None:
+ global action_done
+ if action_done:
+ raise RuntimeError("can only do one action per block!")
+
+def send_finish(s: str)->None:
+ check_not_finished()
+ global action_done
+ action_done=True
+ send_raw(s)
+
+
+import random
+def surround_delimiter(block: str)->str:
+ while True:
+ delimiter=str(random.randint(0, 10**12))
+ if delimiter not in block: break
+ return delimiter + "\n" + block + "\n" + delimiter + "\n"
+
+bootstrap_code: Optional[str]=""
+def mark_bootstrap(code: str)->None:
+ global bootstrap_code
+ assert bootstrap_code is not None
+ bootstrap_code+=code
+
+def substitute_private(code: str)->str:
+ return (code
+ #.replace("\n", ' ') # because there are comments in code, cannot
+ .replace("__", "_" + "pythonimmediate" + "_")
+ )
+
+def send_bootstrap_code()->None:
+ global bootstrap_code
+ assert bootstrap_code is not None
+ send_raw(surround_delimiter(substitute_private(bootstrap_code)))
+ bootstrap_code = None
+
+# ========
+
+
+# as the name implies, this reads one "command" from Python side and execute it.
+# the command might do additional tasks e.g. read more \TeX\ code.
+#
+# e.g. if `block' is read from the communication channel, run |\__run_block:|.
+
+mark_bootstrap(
+r"""
+\cs_new_protected:Npn \__read_do_one_command: {
+ \begingroup
+ \endlinechar=-1~
+ \readline \__read_file to \__line
+ \expandafter
+ \endgroup % also this will give an error instead of silently do nothing when command is invalid
+ \csname __run_ \__line :\endcsname
+}
+
+% read documentation of |_peek| commands for details what this command does.
+\cs_new_protected:Npn \pythonimmediatecontinue #1 {
+ \immediate\write \__write_file {r #1}
+ \__read_do_one_command:
+}
+
+\cs_new_protected:Npn \pythonimmediatecontinuenoarg {
+ \pythonimmediatecontinue {}
+}
+
+% internal function. Just send an arbitrary block of data to Python.
+\cs_new_protected:Npn \__send_block:e #1 {
+ \immediate\write \__write_file {
+ #1 ^^J
+ pythonimm?""" + '"""' + r"""?'''? % following character will be newline
+ }
+}
+
+\cs_new_protected:Npn \__send_block:n #1 {
+ \__send_block:e {\unexpanded{#1}}
+}
+
+\AtEndDocument{
+ \immediate\write \__write_file {r}
+}
+""")
+
+
+# ========
+
+# when 'i⟨string⟩' is sent from TeX to Python, the function with index ⟨string⟩ in this dict is called
+TeX_handlers: Dict[str, Callable[[], None]]={}
+
+TeXToPyObjectType=Optional[str]
+
+def run_main_loop()->TeXToPyObjectType:
+ while True:
+ line=readline()
+ if not line: return None
+
+ if line[0]=="i":
+ TeX_handlers[line[1:]]()
+ elif line[0]=="r":
+ return line[1:]
+ else:
+ raise RuntimeError("Internal error: unexpected line "+line)
+
+def run_main_loop_get_return_one()->str:
+ line=readline()
+ assert line[0]=="r"
+ return line[1:]
+
+
+
+user_documentation(
+"""
+All exported functions can be accessed through the module as |import pythonimmediate|.
+
+The |_finish| functions are internal functions, which must be called \emph{at most} once in each
+|\pythonimmediate:n| call from \TeX\ to tell \TeX\ what to do.
+
+The |_local| functions simply execute the code. These functions will only return when
+the \TeX\ code finishes executing; nevertheless, the \TeX\ code might recursively execute some Python code
+inside it.
+
+A simple example is |pythonimmediate.run_block_local('123')| which simply typesets |123|.
+
+The |_peek| functions is the same as above; however, the \TeX\ code must contain an explicit command
+|\pythonimmediatecontinue{...}|.
+
+The argument of |\pythonimmediatecontinue| will be |e|-expanded
+by |\write| (note that the written content must not contain any newline character,
+otherwise the behavior is undefined), then returned as a string by the Python code.
+The Python function will only return when |\pythonimmediatecontinue| is called.
+
+In other words, |run_*_local(code)| is almost identical to |run_*_peek(code + "\pythonimmediatecontinue {}")|.
+""")
+
+@export_function_to_module
+def run_block_finish(block: str)->None:
+ send_finish("block\n" + surround_delimiter(block))
+
+
+@user_documentation
+@export_function_to_module
+def execute(block: str)->None:
+ """
+ Run a block of \TeX\ code (might consist of multiple lines).
+ Catcode-changing commands are allowed inside.
+
+ A simple example is |pythonimmediate.run_block_local('123')| which simply typesets |123|.
+
+ A more complicated example is |pythonimmediate.run_block_local(r'\verb+%+')|.
+ """
+ run_block_local(block)
+
+def check_line(line: str, *, braces: bool, newline: bool, continue_: Optional[bool])->None:
+ """
+ check user-provided line before sending to TeX for execution
+ """
+ if braces:
+ assert line.count("{") == line.count("}")
+ if newline:
+ assert '\n' not in line
+ assert '\r' not in line # this is not the line separator but just in case
+ if continue_==True: assert "pythonimmediatecontinue" in line
+ elif continue_==False: assert "pythonimmediatecontinue" not in line
+
+
+
+
+do_run_error_finish=True
+
+
+
+
+
+user_scope: Dict[str, Any]={} # consist of user's local variables etc.
+
+def readline()->str:
+ line=raw_readline()
+ if not line:
+ sys.stderr.write("\n\nTraceback (most recent call last):\n")
+ traceback.print_stack(file=sys.stderr)
+ sys.stderr.write("RuntimeError: Fatal irrecoverable TeX error\n\n")
+ os._exit(1)
+
+
+ assert line[-1]=='\n'
+ line=line[:-1]
+ debug("======== saw line", line)
+ return line
+
+block_delimiter: str="pythonimm?\"\"\"?'''?"
+
+def read_block()->str:
+ """
+ Internal function to read one block sent from \TeX\ (including the final delimiter line,
+ but the delimiter line is not returned)
+ """
+ lines: List[str]=[]
+ while True:
+ line=readline()
+ if line==block_delimiter:
+ return '\n'.join(lines)
+ else:
+ lines.append(line)
+
+
+@export_function_to_module
+class NToken(ABC):
+ """
+ Represent a possibly-notexpanded token.
+ For convenience, a notexpanded token is called a blue token.
+ It's not always possible to determine the notexpanded status of a following token in the input stream.
+ Remark: Token objects must be frozen.
+ """
+
+ @abstractmethod
+ def __str__(self)->str: ...
+
+ @abstractmethod
+ def repr1(self)->str: ...
+
+ @property
+ @abstractmethod
+ def assignable(self)->bool: ...
+
+ def assign(self, other: "NToken")->None:
+ assert self.assignable
+ NTokenList([T.let, self, C.other("="), C.space(' '), other]).execute()
+
+ def assign_future(self)->None:
+ assert self.assignable
+ futurelet_(PTTBalancedTokenList(BalancedTokenList([self.no_blue])))
+
+ def assign_futurenext(self)->None:
+ assert self.assignable
+ futureletnext_(PTTBalancedTokenList(BalancedTokenList([self.no_blue])))
+
+ def meaning_str(self)->str:
+ """
+ get the meaning of this token as a string.
+ """
+ return NTokenList([T.meaning, self]).expand_x().str()
+
+ @property
+ @abstractmethod
+ def blue(self)->"BlueToken": ...
+
+ @property
+ @abstractmethod
+ def no_blue(self)->"Token": ...
+
+ def meaning_equal(self, other: "Token")->bool:
+ return NTokenList([T.ifx, self, other, Catcode.other("1"), T["else"], Catcode.other("0"), T.fi]).expand_x().bool()
+
+ def str(self)->str:
+ """
+ self must represent a character of a TeX string. (i.e. equal to itself when detokenized)
+ return the string content.
+
+ default implementation below. Not necessarily correct.
+ """
+ raise ValueError("Token does not represent a string!")
+
+ def degree(self)->int:
+ """
+ return the imbalance degree for this token ({ -> 1, } -> -1, everything else -> 0)
+
+ default implementation below. Not necessarily correct.
+ """
+ return 0
+
+
+@export_function_to_module
+class Token(NToken):
+ """
+ Represent a TeX token, excluding the notexpanded possibility.
+ See also documentation of NToken.
+ """
+
+ @abstractmethod
+ def serialize(self)->str: ...
+
+ @property
+ def blue(self)->"BlueToken": return BlueToken(self)
+
+ @property
+ def no_blue(self)->"Token": return self
+
+ def __repr__(self)->str:
+ return f"<Token: {self.repr1()}>"
+
+ @staticmethod
+ def deserialize(s: str)->"Token":
+ t=TokenList.deserialize(s)
+ assert len(t)==1
+ return t[0]
+
+ @staticmethod
+ def get_next()->"Token":
+ """
+ Get the following token.
+
+ Note: in LaTeX3 versions without the commit |https://github.com/latex3/latex3/commit/24f7188904d6|
+ sometimes this may error out.
+
+ Note: because of the internal implementation of |\peek_analysis_map_inline:n|, this may
+ tokenize up to 2 tokens ahead (including the returned token),
+ as well as occasionally return the wrong token in unavoidable cases.
+ """
+ return Token.deserialize(str(get_next_()[0]))
+
+ @staticmethod
+ def peek_next()->"Token":
+ """
+ Get the following token without removing it from the input stream.
+
+ Equivalent to get_next() then put_next() immediately. See documentation of get_next() for some notes.
+ """
+ return Token.deserialize(
+ typing.cast(Callable[[], TTPLine], Python_call_TeX_local(
+ r"""
+ \cs_new_protected:Npn \__peek_next_callback: #1 {
+ \immediate\write \__write_file { r^^J #1 }
+ \expandafter % expand the ##1 in (*)
+ \__read_do_one_command:
+ }
+
+ \cs_new_protected:Npn %name% {
+ \peek_analysis_map_inline:n {
+ \peek_analysis_map_break:n {
+ \__tlserialize_char_unchecked:nnNN {##1}{##2}##3 \__peek_next_callback: ##1 % (*)
+ }
+ }
+ }
+ """, recursive=False))()
+ )
+
+ def put_next(self)->None:
+ d=self.degree()
+ if d==0:
+ BalancedTokenList([self]).put_next()
+ else:
+ assert isinstance(self, CharacterToken)
+ if d==1:
+ put_next_bgroup(PTTInt(self.index))
+ else:
+ assert d==-1
+ put_next_egroup(PTTInt(self.index))
+
+
+
+
+
+
+"""
+TeX code for serializing and deserializing a token list.
+Convert a token list from/to a string.
+"""
+
+
+mark_bootstrap(
+r"""
+\precattl_exec:n {
+
+% here #1 is the target token list to store the result to, #2 is a string with the final '.'.
+\cs_new_protected:Npn \__tldeserialize_dot:Nn #1 #2 {
+ \begingroup
+ \tl_set:Nn \__tmp {#2}
+ \tl_replace_all:Nnn \__tmp {~} {\cO\ }
+
+ \def \start ##1 { \csname ##1 \endcsname }
+
+ \def \> ##1 ##2 \cO\ { \csname ##1 \endcsname ##2 \cU\ }
+ \def \\ ##1 \cO\ ##2 { \expandafter \noexpand \csname ##1 \endcsname \csname ##2 \endcsname }
+ \def \1 ##1 ##2 { \char_generate:nn {`##1} {1} \csname ##2 \endcsname }
+ \def \2 ##1 ##2 { \char_generate:nn {`##1} {2} \csname ##2 \endcsname }
+ \def \3 ##1 ##2 { \char_generate:nn {`##1} {3} \csname ##2 \endcsname }
+ \def \4 ##1 ##2 { \char_generate:nn {`##1} {4} \csname ##2 \endcsname }
+ \def \6 ##1 ##2 { #### \char_generate:nn {`##1} {6} \csname ##2 \endcsname }
+ \def \7 ##1 ##2 { \char_generate:nn {`##1} {7} \csname ##2 \endcsname }
+ \def \8 ##1 ##2 { \char_generate:nn {`##1} {8} \csname ##2 \endcsname }
+ \def \A ##1 ##2 { \char_generate:nn {`##1} {10} \csname ##2 \endcsname }
+ \def \B ##1 ##2 { \char_generate:nn {`##1} {11} \csname ##2 \endcsname }
+ \def \C ##1 ##2 { \char_generate:nn {`##1} {12} \csname ##2 \endcsname }
+ \def \D ##1 ##2 { \expandafter \expandafter \expandafter \noexpand \char_generate:nn {`##1} {13} \csname ##2 \endcsname }
+ \def \R ##1 { \cFrozenRelax \csname ##1 \endcsname }
+
+ \let \. \empty
+
+ \exp_args:NNNx
+ \endgroup \tl_set:Nn #1 {\expandafter \start \__tmp}
+}
+
+\cs_new_protected:Npn \__tlserialize_char_unchecked:nnNN #1 #2 #3 #4 {
+ % #1=token, #2=char code, #3=catcode, #4: callback (will be called exactly once and with nothing following the input stream)
+ \int_compare:nNnTF {#2} = {-1} {
+ % token is control sequence
+ \tl_if_eq:onTF {#1} {\cFrozenRelax} {
+ #4 {\cStr{ R }}
+ } {
+ \tl_if_eq:onTF {#1} { \cC{} } {
+ #4 {\cStr{ \\\ }}
+ } {
+ \tl_set:Nx \__name { \expandafter \cs_to_str:N #1 }
+ \exp_args:Nx #4 { \prg_replicate:nn {\str_count_spaces:N \__name} {>} \cStr\\ \__name \cStr\ }
+ }
+ }
+ } {
+ % token is not control sequence
+ % (hex catcode) (character) (or escape sequence with that character)
+ \exp_args:Nx #4 { #3 \expandafter \string #1 }
+ }
+}
+
+}
+
+% deserialize as above but #2 does not end with '.'.
+\cs_new_protected:Npn \__tldeserialize_nodot:Nn #1 #2 {
+ \__tldeserialize_dot:Nn #1 {#2 .}
+}
+
+% serialize token list in #2 store to #1.
+\cs_new_protected:Npn \__tlserialize_nodot_unchecked:Nn #1 #2 {
+ \tl_build_begin:N #1
+ \tl_set:Nn \__tlserialize_callback { \tl_build_put_right:Nn #1 }
+ \tl_analysis_map_inline:nn {#2} {
+ \__tlserialize_char_unchecked:nnNN {##1}{##2}##3 \__tlserialize_callback
+ }
+ \tl_build_end:N #1
+}
+
+% serialize token list in #2 store to #1. Call T or F branch depends on whether serialize is successful.
+% #1 must be different from \__tlserialize_tmp.
+\cs_new_protected:Npn \__tlserialize_nodot:NnTF #1 #2 {
+ \__tlserialize_nodot_unchecked:Nn #1 {#2}
+ \__tldeserialize_nodot:NV \__tlserialize_nodot_tmp #1
+
+ \tl_if_eq:NnTF \__tlserialize_nodot_tmp {#2} % dangling
+}
+
+\cs_new_protected:Npn \__tlserialize_nodot:NnF #1 #2 {
+ \__tlserialize_nodot:NnTF #1 {#2} {} % dangling
+}
+
+\cs_new_protected:Npn \__tlserialize_nodot:NnT #1 #2 #3 { \__tlserialize_nodot:NnTF #1 {#2} {#3} {} }
+
+\msg_new:nnn {pythonimmediate} {cannot-serialize} {Token~list~cannot~be~serialized}
+
+\cs_new_protected:Npn \__tlserialize_nodot:Nn #1 #2{
+ \__tlserialize_nodot:NnF #1 {#2} {
+ \msg_error:nn {pythonimmediate} {cannot-serialize}
+ }
+}
+
+\cs_generate_variant:Nn \__tldeserialize_dot:Nn {NV}
+\cs_generate_variant:Nn \__tldeserialize_nodot:Nn {NV}
+\cs_generate_variant:Nn \__tlserialize_nodot:Nn {NV}
+""")
+
+
+class ControlSequenceTokenMaker:
+ """
+ shorthand to create control sequence objects in Python easier.
+ """
+ def __init__(self, prefix: str)->None:
+ self.prefix=prefix
+ def __getattribute__(self, a: str)->"ControlSequenceToken":
+ return ControlSequenceToken(object.__getattribute__(self, "prefix")+a)
+ def __getitem__(self, a: str)->"ControlSequenceToken":
+ return ControlSequenceToken(object.__getattribute__(self, "prefix")+a)
+
+
+@export_function_to_module
+@dataclass(repr=False, frozen=True)
+class ControlSequenceToken(Token):
+ make=typing.cast(ControlSequenceTokenMaker, None) # some interference makes this incorrect. Manually assign below
+ csname: str
+ @property
+ def assignable(self)->bool:
+ return True
+ def __str__(self)->str:
+ if self.csname=="": return r"\csname\endcsname"
+ return "\\"+self.csname
+ def serialize(self)->str:
+ return ">"*self.csname.count(" ") + "\\" + self.csname + " "
+ def repr1(self)->str:
+ return f"\\{self.csname}"
+
+
+ControlSequenceToken.make=ControlSequenceTokenMaker("")
+
+T=ControlSequenceToken.make
+P=ControlSequenceTokenMaker("_pythonimmediate_") # create private tokens
+
+@export_function_to_module
+class Catcode(enum.Enum):
+ begin_group=bgroup=1
+ end_group=egroup=2
+ math_toggle=math=3
+ alignment=4
+ parameter=param=6
+ math_superscript=superscript=7
+ math_subscript=subscript=8
+ space=10
+ letter=11
+ other=12
+ active=13
+
+ escape=0
+ end_of_line=paragraph=line=5
+ ignored=9
+ comment=14
+ invalid=15
+
+ @property
+ def for_token(self)->bool:
+ """
+ Return whether a token may have this catcode.
+ """
+ return self not in (Catcode.escape, Catcode.line, Catcode.ignored, Catcode.comment, Catcode.invalid)
+
+ def __call__(self, ch: Union[str, int])->"CharacterToken":
+ """
+ Shorthand:
+ Catcode.letter("a") = Catcode.letter(97) = CharacterToken(index=97, catcode=Catcode.letter)
+ """
+ if isinstance(ch, str): ch=ord(ch)
+ return CharacterToken(ch, self)
+
+C=Catcode
+
+@export_function_to_module
+@dataclass(repr=False, frozen=True) # must be frozen because bgroup and egroup below are reused
+class CharacterToken(Token):
+ index: int
+ catcode: Catcode
+ @property
+ def chr(self)->str:
+ return chr(self.index)
+ def __post_init__(self)->None:
+ assert self.catcode.for_token
+ def __str__(self)->str:
+ return self.chr
+ def serialize(self)->str:
+ return f"{self.catcode.value:X}{self.chr}"
+ def repr1(self)->str:
+ cat=str(self.catcode.value).translate(str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉"))
+ return f"{self.chr}{cat}"
+ @property
+ def assignable(self)->bool:
+ return self.catcode==Catcode.active
+ def degree(self)->int:
+ if self.catcode==Catcode.bgroup:
+ return 1
+ elif self.catcode==Catcode.egroup:
+ return -1
+ else:
+ return 0
+ def str(self)->str:
+ catcode=Catcode.space if self.index==32 else Catcode.other
+ if catcode!=self.catcode:
+ raise ValueError("this CharacterToken does not represent a string!")
+ return self.chr
+
+class FrozenRelaxToken(Token):
+ def __str__(self)->str:
+ return r"\relax"
+ def serialize(self)->str:
+ return "R"
+ def repr1(self)->str:
+ return r"[frozen]\relax"
+ @property
+ def assignable(self)->bool:
+ return False
+
+frozen_relax_token=FrozenRelaxToken()
+pythonimmediate.frozen_relax_token=frozen_relax_token
+
+# other special tokens later...
+
+bgroup=Catcode.bgroup("{")
+egroup=Catcode.egroup("}")
+space=Catcode.space(" ")
+
+
+
+@export_function_to_module
+@dataclass(frozen=True)
+class BlueToken(NToken):
+ token: Token
+
+ @property
+ def blue(self)->"BlueToken": return self
+
+ @property
+ def no_blue(self)->"Token": return self.token
+
+ def __str__(self)->str: return str(self.token)
+
+ def repr1(self)->str: return "notexpanded:"+self.token.repr1()
+
+ @property
+ def assignable(self)->bool: return self.token.assignable
+
+ def put_next(self)->None:
+ put_next_blue(PTTBalancedTokenList(BalancedTokenList([self.token])))
+
+
+doc_catcode_table: Dict[int, Catcode]={}
+doc_catcode_table[ord("{")]=Catcode.begin_group
+doc_catcode_table[ord("}")]=Catcode.end_group
+doc_catcode_table[ord("$")]=Catcode.math_toggle
+doc_catcode_table[ord("&")]=Catcode.alignment
+doc_catcode_table[ord("#")]=Catcode.parameter
+doc_catcode_table[ord("^")]=Catcode.math_superscript
+doc_catcode_table[ord("_")]=Catcode.math_subscript
+doc_catcode_table[ord(" ")]=Catcode.space
+doc_catcode_table[ord("~")]=Catcode.active
+for ch in range(ord('a'), ord('z')+1): doc_catcode_table[ch]=Catcode.letter
+for ch in range(ord('A'), ord('Z')+1): doc_catcode_table[ch]=Catcode.letter
+doc_catcode_table[ord("\\")]=Catcode.escape
+doc_catcode_table[ord("%")]=Catcode.comment
+
+e3_catcode_table=dict(doc_catcode_table)
+e3_catcode_table[ord("_")]=Catcode.letter
+e3_catcode_table[ord(":")]=Catcode.letter
+e3_catcode_table[ord(" ")]=Catcode.ignored
+e3_catcode_table[ord("~")]=Catcode.space
+
+
+TokenListType = typing.TypeVar("TokenListType", bound="TokenList")
+
+if typing.TYPE_CHECKING:
+ TokenListBaseClass = collections.UserList[Token]
+else: # Python 3.8 compatibility
+ TokenListBaseClass = collections.UserList
+
+@export_function_to_module
+class TokenList(TokenListBaseClass):
+ @staticmethod
+ def force_token_list(a: Iterable)->Iterable[Token]:
+ for x in a:
+ if isinstance(x, Token):
+ yield x
+ elif isinstance(x, Sequence):
+ yield bgroup
+ child=BalancedTokenList(x)
+ assert child.is_balanced()
+ yield from child
+ yield egroup
+ else:
+ raise RuntimeError(f"Cannot make TokenList from object {x} of type {type(x)}")
+
+ def is_balanced(self)->bool:
+ """
+ check if this is balanced.
+ """
+ degree=0
+ for x in self:
+ degree+=x.degree()
+ if degree<0: return False
+ return degree==0
+
+ def check_balanced(self)->None:
+ """
+ ensure that this is balanced.
+ """
+ if not self.is_balanced():
+ raise ValueError("Token list is not balanced")
+
+ def balanced_parts(self)->"List[Union[BalancedTokenList, Token]]":
+ """
+ split this TokenList into a list of balanced parts and unbalanced {/}tokens
+ """
+ degree=0
+ min_degree=0, 0
+ for i, token in enumerate(self):
+ degree+=token.degree()
+ min_degree=min(min_degree, (degree, i+1))
+ min_degree_pos=min_degree[1]
+
+ left_half: List[Union[BalancedTokenList, Token]]=[]
+ degree=0
+ last_pos=0
+ for i in range(min_degree_pos):
+ d=self[i].degree()
+ degree+=d
+ if degree<0:
+ degree=0
+ if last_pos!=i:
+ left_half.append(BalancedTokenList(self[last_pos:i]))
+ left_half.append(self[i])
+ last_pos=i+1
+ if min_degree_pos!=last_pos:
+ left_half.append(BalancedTokenList(self[last_pos:min_degree_pos]))
+
+ right_half: List[Union[BalancedTokenList, Token]]=[]
+ degree=0
+ last_pos=len(self)
+ for i in range(len(self)-1, min_degree_pos-1, -1):
+ d=self[i].degree()
+ degree-=d
+ if degree<0:
+ degree=0
+ if i+1!=last_pos:
+ right_half.append(BalancedTokenList(self[i+1:last_pos]))
+ right_half.append(self[i])
+ last_pos=i
+ if min_degree_pos!=last_pos:
+ right_half.append(BalancedTokenList(self[min_degree_pos:last_pos]))
+
+ return left_half+right_half[::-1]
+
+ def put_next(self)->None:
+ for part in reversed(self.balanced_parts()): part.put_next()
+
+ @property
+ def balanced(self)->"BalancedTokenList":
+ """
+ return a BalancedTokenList containing the content of this object.
+ it must be balanced.
+ """
+ return BalancedTokenList(self)
+
+ def __init__(self, a: Iterable=())->None:
+ super().__init__(TokenList.force_token_list(a))
+
+ @staticmethod
+ def iterable_from_string(s: str, get_catcode: Callable[[int], Catcode])->Iterable[Token]:
+ """
+ refer to documentation of from_string() for details.
+ """
+ i=0
+ while i<len(s):
+ ch=s[i]
+ i+=1
+ cat=get_catcode(ord(ch))
+ if cat==Catcode.space:
+ yield space
+ # special case: collapse multiple spaces into one but only if character code is space
+ if get_catcode(32) in (Catcode.space, Catcode.ignored):
+ while i<len(s) and s[i]==' ':
+ i+=1
+ elif cat.for_token:
+ yield cat(ch)
+ elif cat==Catcode.ignored:
+ continue
+ else:
+ assert cat==Catcode.escape, f"cannot create TokenList from string containing catcode {cat}"
+ cat=get_catcode(ord(s[i]))
+ if cat!=Catcode.letter:
+ yield ControlSequenceToken(s[i])
+ i+=1
+ else:
+ csname=s[i]
+ i+=1
+ while i<len(s) and get_catcode(ord(s[i]))==Catcode.letter:
+ csname+=s[i]
+ i+=1
+ yield ControlSequenceToken(csname)
+ # special case: remove spaces after control sequence but only if character code is space
+ if get_catcode(32) in (Catcode.space, Catcode.ignored):
+ while i<len(s) and s[i]==' ':
+ i+=1
+
+ @classmethod
+ def from_string(cls: Type[TokenListType], s: str, get_catcode: Callable[[int], Catcode])->TokenListType:
+ """
+ convert a string to a TokenList approximately.
+ The tokenization algorithm is slightly different from TeX's in the following respect:
+
+ * multiple spaces are collapsed to one space, but only if it has character code space (32).
+ * spaces with character code different from space (32) after a control sequence is not ignored.
+ * ^^ syntax are not supported. Use Python's escape syntax as usual.
+ """
+ return cls(TokenList.iterable_from_string(s, get_catcode))
+
+ @classmethod
+ def e3(cls: Type[TokenListType], s: str)->TokenListType:
+ """
+ approximate tokenizer in expl3 catcode, implemented in Python.
+ refer to documentation of from_string() for details.
+ """
+ return cls.from_string(s, lambda x: e3_catcode_table.get(x, Catcode.other))
+
+ @classmethod
+ def doc(cls: Type[TokenListType], s: str)->TokenListType:
+ """
+ approximate tokenizer in document catcode, implemented in Python.
+ refer to documentation of from_string() for details.
+ """
+ return cls.from_string(s, lambda x: doc_catcode_table.get(x, Catcode.other))
+
+ def serialize(self)->str:
+ return "".join(t.serialize() for t in self)
+
+ @classmethod
+ def deserialize(cls: Type[TokenListType], data: str)->TokenListType:
+ result: List[Token]=[]
+ i=0
+ cs_skip_space_count=0
+ while i<len(data):
+ if data[i]==">":
+ cs_skip_space_count+=1
+ i+=1
+ elif data[i]=="\\":
+ j=data.index(' ', i+1)
+ for __ in range(cs_skip_space_count):
+ j=data.index(' ', j+1)
+ cs_skip_space_count=0
+ result.append(ControlSequenceToken(data[i+1:j]))
+ i=j+1
+ elif data[i]=="R":
+ result.append(frozen_relax_token)
+ i+=1
+ else:
+ result.append(CharacterToken(index=ord(data[i+1]), catcode=Catcode(int(data[i], 16))))
+ i+=2
+ return cls(result)
+
+ def __repr__(self)->str:
+ return '<' + type(self).__name__ + ': ' + ' '.join(t.repr1() for t in self) + '>'
+
+ def execute(self)->None:
+ NTokenList(self).execute()
+
+ def expand_x(self)->"BalancedTokenList":
+ return NTokenList(self).expand_x()
+
+ def bool(self)->bool:
+ return NTokenList(self).bool()
+
+ def str(self)->str:
+ return NTokenList(self).str()
+
+
+
+@export_function_to_module
+class BalancedTokenList(TokenList):
+ """
+ Represents a balanced token list.
+ Note that runtime checking is not strictly enforced,
+ use `is_balanced()` method explicitly if you need to check.
+ """
+
+ def __init__(self, a: Iterable=())->None:
+ """
+ constructor. This must check for balanced-ness as balanced() method depends on this.
+ """
+ super().__init__(a)
+ self.check_balanced()
+
+ def expand_o(self)->"BalancedTokenList":
+ return BalancedTokenList(expand_o_(PTTBalancedTokenList(self))[0]) # type: ignore
+ def expand_x(self)->"BalancedTokenList":
+ return BalancedTokenList(expand_x_(PTTBalancedTokenList(self))[0]) # type: ignore
+ def execute(self)->None:
+ execute_(PTTBalancedTokenList(self))
+
+ def put_next(self)->None:
+ put_next_tokenlist(PTTBalancedTokenList(self))
+
+ @staticmethod
+ def get_next()->"BalancedTokenList":
+ """
+ get an (undelimited) argument from the TeX input stream.
+ """
+ return BalancedTokenList(get_argument_tokenlist_()[0]) # type: ignore
+
+
+
+if typing.TYPE_CHECKING:
+ NTokenListBaseClass = collections.UserList[NToken]
+else: # Python 3.8 compatibility
+ NTokenListBaseClass = collections.UserList
+
+@export_function_to_module
+class NTokenList(NTokenListBaseClass):
+ @staticmethod
+ def force_token_list(a: Iterable)->Iterable[NToken]:
+ for x in a:
+ if isinstance(x, NToken):
+ yield x
+ elif isinstance(x, Sequence):
+ yield bgroup
+ child=NTokenList(x)
+ assert child.is_balanced()
+ yield from child
+ yield egroup
+ else:
+ raise RuntimeError(f"Cannot make NTokenList from object {x} of type {type(x)}")
+
+ def __init__(self, a: Iterable=())->None:
+ super().__init__(NTokenList.force_token_list(a))
+
+ def is_balanced(self)->bool:
+ return TokenList(self).is_balanced() # a bit inefficient (need to construct a TokenList) but good enough
+
+ def simple_parts(self)->List[Union[BalancedTokenList, Token, BlueToken]]:
+ """
+ Split this NTokenList into a list of balanced non-blue parts, unbalanced {/} tokens, and blue tokens.
+ """
+ parts: List[Union[TokenList, BlueToken]]=[TokenList()]
+ for i in self:
+ if isinstance(i, BlueToken):
+ parts+=i, TokenList()
+ else:
+ assert isinstance(i, Token)
+ last_part=parts[-1]
+ assert isinstance(last_part, TokenList)
+ last_part.append(i)
+ result: List[Union[BalancedTokenList, Token, BlueToken]]=[]
+ for large_part in parts:
+ if isinstance(large_part, BlueToken):
+ result.append(large_part)
+ else:
+ result+=large_part.balanced_parts()
+ return result
+
+ def put_next(self)->None:
+ for part in reversed(self.simple_parts()): part.put_next()
+
+ def execute(self)->None:
+ """
+ Execute self.
+ """
+ parts=self.simple_parts()
+ if len(parts)==1:
+ x=parts[0]
+ if isinstance(x, BalancedTokenList):
+ x.execute()
+ return
+ NTokenList([*self, T.pythonimmediatecontinue, []]).put_next()
+ continue_until_passed_back()
+
+ def expand_x(self)->BalancedTokenList:
+ """
+ x-expand self. The result must be balanced.
+ """
+ NTokenList([T.edef, P.tmp, bgroup, *self, egroup]).execute()
+ return BalancedTokenList([P.tmp]).expand_o()
+
+ def str(self)->str:
+ """
+ self must represent a TeX string. (i.e. equal to itself when detokenized)
+ return the string content.
+ """
+ return "".join(t.str() for t in self)
+
+ def bool(self)->bool:
+ s=self.str()
+ return {"0": False, "1": True}[s]
+
+
+class TeXToPyData(ABC):
+ @staticmethod
+ @abstractmethod
+ def read()->"TeXToPyData":
+ ...
+ @staticmethod
+ @abstractmethod
+ def send_code(arg: str)->str:
+ pass
+ @staticmethod
+ @abstractmethod
+ def send_code_var(var: str)->str:
+ pass
+
+# tried and failed
+#@typing.runtime_checkable
+#class TeXToPyData(Protocol):
+# @staticmethod
+# def read()->"TeXToPyData":
+# ...
+#
+# #send_code: str
+#
+# #@staticmethod
+# #@property
+# #def send_code()->str:
+# # ...
+
+
+class TTPLine(TeXToPyData, str):
+ send_code=r"\immediate \write \__write_file {{\unexpanded{{ {} }}}}".format
+ send_code_var=r"\immediate \write \__write_file {{\unexpanded{{ {} }}}}".format
+ @staticmethod
+ def read()->"TTPLine":
+ return TTPLine(readline())
+
+# some old commands e.g. \$, \^, \_, \~ require \set@display@protect to be robust.
+# ~ needs to be redefined directly.
+mark_bootstrap(
+r"""
+\precattl_exec:n {
+ \cs_new_protected:Npn \__begingroup_setup_estr: {
+ \begingroup
+ \escapechar=-1~
+ \cC{set@display@protect}
+ \let \cA\~ \relax
+ }
+}
+""")
+
+class TTPELine(TeXToPyData, str):
+ """
+ Same as TTPEBlock, but for a single line only.
+ """
+ send_code=r"\__begingroup_setup_estr: \immediate \write \__write_file {{ {} }} \endgroup".format
+ send_code_var=r"\__begingroup_setup_estr: \immediate \write \__write_file {{ {} }} \endgroup".format
+ @staticmethod
+ def read()->"TTPELine":
+ return TTPELine(readline())
+
+class TTPEmbeddedLine(TeXToPyData, str):
+ @staticmethod
+ def send_code(self)->str:
+ raise RuntimeError("Must be manually handled")
+ @staticmethod
+ def send_code_var(self)->str:
+ raise RuntimeError("Must be manually handled")
+ @staticmethod
+ def read()->"TTPEmbeddedLine":
+ raise RuntimeError("Must be manually handled")
+
+class TTPBlock(TeXToPyData, str):
+ send_code=r"\__send_block:n {{ {} }}".format
+ send_code_var=r"\__send_block:V {}".format
+ @staticmethod
+ def read()->"TTPBlock":
+ return TTPBlock(read_block())
+
+class TTPEBlock(TeXToPyData, str):
+ """
+ A kind of argument that interprets "escaped string" and fully expand anything inside.
+ For example, {\\} sends a single backslash to Python, {\{} sends a single '{' to Python.
+ Done by fully expand the argument in \escapechar=-1 and convert it to a string.
+ Additional precaution is needed, see the note above.
+ """
+ send_code=r"\__begingroup_setup_estr: \__send_block:e {{ {} }} \endgroup".format
+ send_code_var=r"\__begingroup_setup_estr: \__send_block:e {} \endgroup".format
+ @staticmethod
+ def read()->"TTPEBlock":
+ return TTPEBlock(read_block())
+
+class TTPBalancedTokenList(TeXToPyData, BalancedTokenList):
+ send_code=r"\__tlserialize_nodot:Nn \__tmp {{ {} }} \immediate \write \__write_file {{\unexpanded\expandafter{{ \__tmp }}}}".format
+ send_code_var=r"\__tlserialize_nodot:NV \__tmp {} \immediate \write \__write_file {{\unexpanded\expandafter{{ \__tmp }}}}".format
+ @staticmethod
+ def read()->"TTPBalancedTokenList":
+ return TTPBalancedTokenList(BalancedTokenList.deserialize(readline()))
+
+
+class PyToTeXData(ABC):
+ @staticmethod
+ @abstractmethod
+ def read_code(var: str)->str:
+ ...
+ @abstractmethod
+ def write(self)->None:
+ ...
+
+@dataclass
+class PTTVerbatimLine(PyToTeXData):
+ """
+ Represents a line to be tokenized verbatim. Internally the |\readline| primitive is used, as such, any trailing spaces are stripped.
+ The trailing newline is not included, i.e. it's read under |\endlinechar=-1|.
+ """
+ data: str
+ read_code=r"\ior_str_get:NN \__read_file {} ".format
+ def write(self)->None:
+ assert "\n" not in self.data
+ assert self.data.rstrip()==self.data, "Cannot send verbatim line with trailing spaces!"
+ send_raw(self.data+"\n")
+
+@dataclass
+class PTTInt(PyToTeXData):
+ data: int
+ read_code=PTTVerbatimLine.read_code
+ def write(self)->None:
+ PTTVerbatimLine(str(self.data)).write()
+
+@dataclass
+class PTTTeXLine(PyToTeXData):
+ """
+ Represents a line to be tokenized in \TeX's current catcode regime.
+ The trailing newline is not included, i.e. it's tokenized under |\endlinechar=-1|.
+ """
+ data: str
+ read_code=r"\ior_get:NN \__read_file {} ".format
+ def write(self)->None:
+ assert "\n" not in self.data
+ send_raw(self.data+"\n")
+
+@dataclass
+class PTTBlock(PyToTeXData):
+ data: str
+ read_code=r"\__read_block:N {}".format
+ def write(self)->None:
+ send_raw(surround_delimiter(self.data))
+
+@dataclass
+class PTTBalancedTokenList(PyToTeXData):
+ data: BalancedTokenList
+ read_code=r"\ior_str_get:NN \__read_file {0} \__tldeserialize_dot:NV {0} {0}".format
+ def write(self)->None:
+ PTTVerbatimLine(self.data.serialize()+".").write()
+
+
+# ======== define TeX functions that execute Python code ========
+# ======== implementation of |\py| etc. Doesn't support verbatim argument yet. ========
+
+import itertools
+import string
+
+def random_identifiers()->Iterator[str]: # do this to avoid TeX hash collision while keeping the length short
+ for len_ in itertools.count(0):
+ for value in range(1<<len_):
+ for initial in string.ascii_letters:
+ yield initial + f"{value:0{len_}b}".translate({ord("0"): "a", ord("1"): "b"})
+
+random_identifier_iterable=random_identifiers()
+
+def get_random_identifier()->str:
+ return next(random_identifier_iterable)
+
+
+def define_TeX_call_Python(f: Callable[..., None], name: Optional[str]=None, argtypes: Optional[List[Type[TeXToPyData]]]=None, identifier: Optional[str]=None)->str:
+ """
+ This function setups some internal data structure, and
+ returns the \TeX\ code to be executed on the \TeX\ side to define the macro.
+
+ f: the Python function to be executed.
+ It should take some arguments and eventually (optionally) call one of the |_finish| functions.
+
+ name: the macro name on the \TeX\ side. This should only consist of letter characters in |expl3| catcode regime.
+
+ argtypes: list of argument types. If it's None it will be automatically deduced from the function |f|'s signature.
+
+ Returns: some code (to be executed in |expl3| catcode regime) as explained above.
+ """
+ if argtypes is None: argtypes=[p.annotation for p in inspect.signature(f).parameters.values()]
+ if name is None: name=f.__name__
+
+ if identifier is None: identifier=get_random_identifier()
+ assert identifier not in TeX_handlers
+
+ @functools.wraps(f)
+ def g()->None:
+ assert argtypes is not None
+ args=[argtype.read() for argtype in argtypes]
+
+
+ global action_done
+ old_action_done=action_done
+
+ action_done=False
+ try:
+ f(*args)
+ except:
+ if action_done:
+ # error occurred after 'finish' is called, cannot signal the error to TeX, will just ignore (after printing out the traceback)...
+ pass
+ else:
+ # TODO what should be done here? What if the error raised below is caught
+ action_done=True
+ raise
+ finally:
+ if not action_done:
+ run_none_finish()
+
+ action_done=old_action_done
+
+
+ TeX_handlers[identifier]=g
+
+ TeX_argspec = ""
+ TeX_send_input_commands = ""
+ for i, argtype in enumerate(argtypes):
+ if isinstance(argtype, str):
+ raise RuntimeError("string annotation or `from __future__ import annotations' not yet supported")
+ if not issubclass(argtype, TeXToPyData):
+ raise RuntimeError(f"Argument type {argtype} is incorrect, should be a subclass of TeXToPyData")
+ arg = f"#{i+1}"
+ TeX_send_input_commands += argtype.send_code(arg)
+ TeX_argspec += arg
+
+ return """
+ \\cs_new_protected:Npn \\""" + name + TeX_argspec + """ {
+ \immediate \write \__write_file { i """ + identifier + """ }
+ """ + TeX_send_input_commands + """
+ \__read_do_one_command:
+ }
+ """
+
+
+def define_internal_handler(f: Callable)->Callable:
+ mark_bootstrap(define_TeX_call_Python(f))
+ return f
+
+
+import linecache
+
+# https://stackoverflow.com/questions/47183305/file-string-traceback-with-line-preview
+def exec_or_eval_with_linecache(code: str, globals: dict, mode: str)->Any:
+ sourcename: str="<usercode>"
+ i=0
+ while sourcename in linecache.cache:
+ sourcename="<usercode" + str(i) + ">"
+ i+=1
+
+ lines=code.splitlines(keepends=True)
+ linecache.cache[sourcename] = len(code), None, lines, sourcename
+
+ compiled_code=compile(code, sourcename, mode)
+ return (exec if mode=="exec" else eval)(compiled_code, globals)
+
+ #del linecache.cache[sourcename]
+ # we never delete the cache, in case some function is defined here then later are called...
+
+def exec_with_linecache(code: str, globals: Dict[str, Any])->None:
+ exec_or_eval_with_linecache(code, globals, "exec")
+
+def eval_with_linecache(code: str, globals: Dict[str, Any])->Any:
+ return exec_or_eval_with_linecache(code, globals, "eval")
+
+
+@define_internal_handler
+def py(code: TTPEBlock)->None:
+ pythonimmediate.run_block_finish(str(eval_with_linecache(code, user_scope))+"%")
+
+@define_internal_handler
+def pyfile(filename: TTPELine)->None:
+ with open(filename, "r") as f:
+ source=f.read()
+ exec(compile(source, filename, "exec"), user_scope)
+
+def print_TeX(*args, **kwargs)->None:
+ if not hasattr(pythonimmediate, "file"):
+ raise RuntimeError("Internal error: attempt to print to TeX outside any environment!")
+ if pythonimmediate.file is not None:
+ functools.partial(print, file=pythonimmediate.file)(*args, **kwargs) # allow user to override `file` kwarg
+pythonimmediate.print=print_TeX
+
+class RedirectPrintTeX:
+ def __init__(self, t)->None:
+ self.t=t
+
+ def __enter__(self)->None:
+ if hasattr(pythonimmediate, "file"):
+ self.old=pythonimmediate.file
+ pythonimmediate.file=self.t
+
+ def __exit__(self, exc_type, exc_value, tb)->None:
+ if hasattr(self, "old"):
+ pythonimmediate.file=self.old
+ else:
+ del pythonimmediate.file
+
+def run_code_redirect_print_TeX(f: Callable[[], Any])->None:
+ with io.StringIO() as t:
+ with RedirectPrintTeX(t):
+ result=f()
+ if result is not None:
+ t.write(str(result)+"%")
+ content=t.getvalue()
+ if content.endswith("\n"):
+ content=content[:-1]
+ else:
+ #content+=r"\empty" # this works too
+ content+="%"
+ pythonimmediate.run_block_finish(content)
+
+@define_internal_handler
+def pyc(code: TTPEBlock)->None:
+ run_code_redirect_print_TeX(lambda: exec_with_linecache(code, user_scope))
+
+@define_internal_handler
+def pycq(code: TTPEBlock)->None:
+ with RedirectPrintTeX(None):
+ exec_with_linecache(code, user_scope)
+ run_none_finish()
+
+mark_bootstrap(
+r"""
+\NewDocumentCommand\pyv{v}{\py{#1}}
+\NewDocumentCommand\pycv{v}{\pyc{#1}}
+""")
+
+# ======== implementation of |pycode| environment
+mark_bootstrap(
+r"""
+\NewDocumentEnvironment{pycode}{}{
+ \saveenvreinsert \__code {
+ \exp_last_unbraced:Nx \__pycodex {{\__code} {\the\inputlineno} {
+ \ifdefined\currfilename \currfilename \fi
+ } {
+ \ifdefined\currfileabspath \currfileabspath \fi
+ }}
+ }
+}{
+ \endsaveenvreinsert
+}
+""")
+
+def normalize_lines(lines: List[str])->List[str]:
+ return [line.rstrip() for line in lines]
+
+@define_internal_handler
+def __pycodex(code: TTPBlock, lineno_: TTPLine, filename: TTPLine, fileabspath: TTPLine)->None:
+ if not code: return
+
+ lineno=int(lineno_)
+ # find where the code comes from... (for easy meaningful traceback)
+ target_filename: Optional[str] = None
+
+ code_lines_normalized=normalize_lines(code.splitlines(keepends=True))
+
+ for f in (fileabspath, filename):
+ if not f: continue
+ p=Path(f)
+ if not p.is_file(): continue
+ file_lines=p.read_text().splitlines(keepends=True)[lineno-len(code_lines_normalized)-1:lineno-1]
+ if normalize_lines(file_lines)==code_lines_normalized:
+ target_filename=f
+ break
+
+ if not target_filename:
+ raise RuntimeError("Source file not found! (attempted {})".format((fileabspath, filename)))
+
+ with io.StringIO() as t:
+ with RedirectPrintTeX(t):
+ if target_filename:
+ code_=''.join(file_lines) # restore missing trailing spaces
+ code_="\n"*(lineno-len(code_lines_normalized)-1)+code_
+ if target_filename:
+ compiled_code=compile(code_, target_filename, "exec")
+ exec(compiled_code, user_scope)
+ else:
+ exec(code_, user_scope)
+ pythonimmediate.run_block_finish(t.getvalue())
+
+# ======== Python-call-TeX functions
+# ======== additional functions...
+
+user_documentation(
+r"""
+These functions get an argument in the input stream and returns it detokenized.
+
+Which means, for example, |#| are doubled, multiple spaces might be collapsed into one, spaces might be introduced
+after a control sequence.
+
+It's undefined behavior if the message's "string representation" contains a "newline character".
+""")
+
+def template_substitute(template: str, pattern: str, substitute: Union[str, Callable[[re.Match], str]], optional: bool=False)->str:
+ """
+ pattern is a regex
+ """
+ if not optional:
+ #assert template.count(pattern)==1
+ assert len(re.findall(pattern, template))==1
+ return re.sub(pattern, substitute, template)
+
+#typing.TypeVarTuple(PyToTeXData)
+
+#PythonCallTeXFunctionType=Callable[[PyToTeXData], Optional[Tuple[TeXToPyData, ...]]]
+
+class PythonCallTeXFunctionType(Protocol): # https://stackoverflow.com/questions/57658879/python-type-hint-for-callable-with-variable-number-of-str-same-type-arguments
+ def __call__(self, *args: PyToTeXData)->Optional[Tuple[TeXToPyData, ...]]: ...
+
+class PythonCallTeXSyncFunctionType(PythonCallTeXFunctionType, Protocol): # https://stackoverflow.com/questions/57658879/python-type-hint-for-callable-with-variable-number-of-str-same-type-arguments
+ def __call__(self, *args: PyToTeXData)->Tuple[TeXToPyData, ...]: ...
+
+
+@dataclass(frozen=True)
+class Python_call_TeX_data:
+ TeX_code: str
+ recursive: bool
+ finish: bool
+ sync: Optional[bool]
+
+@dataclass(frozen=True)
+class Python_call_TeX_extra:
+ ptt_argtypes: Tuple[Type[PyToTeXData], ...]
+ ttp_argtypes: Union[Type[TeXToPyData], Tuple[Type[TeXToPyData], ...]]
+
+Python_call_TeX_defined: Dict[Python_call_TeX_data, Tuple[Python_call_TeX_extra, Callable]]={}
+
+def Python_call_TeX_local(TeX_code: str, *, recursive: bool=True, sync: Optional[bool]=None, finish: bool=False)->Callable:
+ data=Python_call_TeX_data(
+ TeX_code=TeX_code, recursive=recursive, sync=sync, finish=finish
+ )
+ return Python_call_TeX_defined[data][1]
+
+def build_Python_call_TeX(T: Type, TeX_code: str, *, recursive: bool=True, sync: Optional[bool]=None, finish: bool=False)->None:
+ assert T.__origin__ == typing.Callable[[], None].__origin__ # type: ignore
+ # might be typing.Callable or collections.abc.Callable depends on Python version
+ data=Python_call_TeX_data(
+ TeX_code=TeX_code, recursive=recursive, sync=sync, finish=finish
+ )
+
+ tmp: Any = T.__args__[-1]
+ ttp_argtypes: Union[Type[TeXToPyData], Tuple[Type[TeXToPyData], ...]]
+ if tmp is type(None):
+ ttp_argtypes = ()
+ elif isinstance(tmp, type) and issubclass(tmp, TeXToPyData):
+ # special case, return a single object instead of a tuple of length 1
+ ttp_argtypes = tmp
+ else:
+ ttp_argtypes = tmp.__args__ # type: ignore
+
+ extra=Python_call_TeX_extra(
+ ptt_argtypes=T.__args__[:-1],
+ ttp_argtypes=ttp_argtypes
+ ) # type: ignore
+ if data in Python_call_TeX_defined:
+ assert Python_call_TeX_defined[data][0]==extra
+ else:
+ if isinstance(ttp_argtypes, type) and issubclass(ttp_argtypes, TeXToPyData):
+ # special case, return a single object instead of a tuple of length 1
+ code, result1=define_Python_call_TeX(TeX_code=TeX_code, ptt_argtypes=[*extra.ptt_argtypes], ttp_argtypes=[ttp_argtypes],
+ recursive=recursive, sync=sync, finish=finish,
+ )
+ def result(*args):
+ [tmp]=result1(*args)
+ return tmp
+ else:
+ code, result=define_Python_call_TeX(TeX_code=TeX_code, ptt_argtypes=[*extra.ptt_argtypes], ttp_argtypes=[*ttp_argtypes],
+ recursive=recursive, sync=sync, finish=finish,
+ )
+ mark_bootstrap(code)
+ Python_call_TeX_defined[data]=extra, result
+
+def scan_Python_call_TeX(filename: str)->None:
+ """
+ scan the file in filename for occurrences of typing.cast(T, Python_call_TeX_local(...)), then call build_Python_call_TeX(T, ...) for each occurrence.
+
+ Don't use on untrusted code.
+ """
+ import ast
+ from copy import deepcopy
+ for node in ast.walk(ast.parse(Path(filename).read_text(), mode="exec")):
+ try:
+ if isinstance(node, ast.Call):
+ if (
+ isinstance(node.func, ast.Attribute) and
+ isinstance(node.func.value, ast.Name) and
+ node.func.value.id == "typing" and
+ node.func.attr == "cast"
+ ):
+ T = node.args[0]
+ if isinstance(node.args[1], ast.Call):
+ f_call = node.args[1]
+ if isinstance(f_call.func, ast.Name):
+ if f_call.func.id == "Python_call_TeX_local":
+ f_call=deepcopy(f_call)
+ assert isinstance(f_call.func, ast.Name)
+ f_call.func.id="build_Python_call_TeX"
+ f_call.args=[T]+f_call.args
+ eval(compile(ast.Expression(body=f_call), "<string>", "eval"))
+ except:
+ print("======== error on line", node.lineno, "========", file=sys.stderr)
+ raise
+
+def define_Python_call_TeX(TeX_code: str, ptt_argtypes: List[Type[PyToTeXData]], ttp_argtypes: List[Type[TeXToPyData]],
+ *,
+ recursive: bool=True,
+ sync: Optional[bool]=None,
+ finish: bool=False,
+ )->Tuple[str, PythonCallTeXFunctionType]:
+ r"""
+ |TeX_code| should be some expl3 code that defines a function with name |%name%| that when called should:
+ * run some \TeX\ code (which includes reading the arguments, if any)
+ * do the following if |sync|:
+ * send |r| to Python (equivalently write %sync%)
+ * send whatever needed for the output (as in |ttp_argtypes|)
+ * call |\__read_do_one_command:| iff not |finish|.
+
+ This is allowed to contain the following:
+ * %name%: the name of the function to be defined as explained above.
+ * %read_arg0(\var_name)%, %read_arg1(...)%: will be expanded to code that reads the input.
+ * %send_arg0(...)%, %send_arg1(...)%: will be expanded to code that sends the content.
+ * %send_arg0_var(\var_name)%, %send_arg1_var(...)%: will be expanded to code that sends the content in the variable.
+ * %optional_sync%: expanded to code that writes |r| (to sync), if |sync| is True.
+
+ ptt_argtypes: list of argument types to be sent from Python to TeX (i.e. input of the TeX function)
+
+ ttp_argtypes: list of argument types to be sent from TeX to Python (i.e. output of the TeX function)
+
+ recursive: whether the TeX_code might call another Python function. Default to True.
+ It does not hurt to always specify True, but performance would be a bit slower.
+
+ sync: whether the Python function need to wait for the TeX function to finish.
+ Required if |ttp_argtypes| is not empty.
+ This should be left to be the default None most of the time. (which will make it always sync if |debugging|,
+ otherwise only sync if needed i.e. there's some output)
+
+ finish: Include this if and only if |\__read_do_one_command:| is omitted.
+ Normally this is not needed, but it can be used as a slight optimization; and it's needed internally to implement
+ |run_none_finish| among others.
+ For each TeX-call-Python layer, \emph{exactly one} |finish| call can be made. If the function itself doesn't call
+ any |finish| call (which happens most of the time), then the wrapper will call |run_none_finish|.
+
+ Return some TeX code to be executed, and a Python function object that when called will call the TeX function
+ and return the result.
+
+ Possible optimizations:
+ * the |r| is not needed if not recursive and |ttp_argtypes| is nonempty
+ (the output itself tells Python when the \TeX\ code finished)
+ * the first line of the output may be on the same line as the |r| itself (done, use TTPEmbeddedLine type, although a bit hacky)
+ """
+ if ttp_argtypes!=[]:
+ assert sync!=False
+ sync=True
+
+ if sync is None:
+ sync=pythonimmediate.debugging
+
+ TeX_code=template_substitute(TeX_code, "%optional_sync%",
+ lambda _: r'\immediate\write\__write_file { r }' if sync else '',)
+
+ TeX_code=template_substitute(TeX_code, "%sync%",
+ lambda _: r'\immediate\write\__write_file { r }' if sync else '', optional=True)
+
+ assert sync is not None
+ if ttp_argtypes: assert sync
+ assert ttp_argtypes.count(TTPEmbeddedLine)<=1
+ identifier=get_random_identifier() # TODO to be fair it isn't necessary to make the identifier both ways distinct, can reuse
+
+ TeX_code=template_substitute(TeX_code, "%name%", lambda _: r"\__run_" + identifier + ":")
+
+ for i, argtype_ in enumerate(ptt_argtypes):
+ TeX_code=template_substitute(TeX_code, r"%read_arg" + str(i) + r"\(([^)]*)\)%",
+ lambda match: argtype_.read_code(match[1]),
+ optional=True)
+
+ for i, argtype in enumerate(ttp_argtypes):
+ TeX_code=template_substitute(TeX_code, f"%send_arg{i}" + r"\(([^)]*)\)%",
+ lambda match: argtype.send_code(match[1]),
+ optional=True)
+ TeX_code=template_substitute(TeX_code, f"%send_arg{i}_var" + r"\(([^)]*)\)%",
+ lambda match: argtype.send_code_var(match[1]),
+ optional=True)
+
+ def f(*args)->Optional[Tuple[TeXToPyData, ...]]:
+ assert len(args)==len(ptt_argtypes)
+
+ # send function header
+ check_not_finished()
+ if finish:
+ global action_done
+ action_done=True
+ send_raw(identifier+"\n")
+
+ # send function args
+ for arg, argtype in zip(args, ptt_argtypes):
+ assert isinstance(arg, argtype)
+ arg.write()
+
+ if not sync: return None
+
+ # wait for the result
+ if recursive:
+ result_=run_main_loop()
+ else:
+ result_=run_main_loop_get_return_one()
+
+ result: List[TeXToPyData]=[]
+ if TTPEmbeddedLine not in ttp_argtypes:
+ assert not result_
+ for argtype_ in ttp_argtypes:
+ if argtype_==TTPEmbeddedLine:
+ result.append(TTPEmbeddedLine(result_))
+ else:
+ result.append(argtype_.read())
+ return tuple(result)
+
+ return TeX_code, f
+
+scan_Python_call_TeX(__file__)
+
+def define_Python_call_TeX_local(*args, **kwargs)->PythonCallTeXFunctionType:
+ """
+ used to define "local" handlers i.e. used by this library.
+ The code will be included in mark_bootstrap().
+ """
+ code, result=define_Python_call_TeX(*args, **kwargs)
+ mark_bootstrap(code)
+ return result
+
+# essentially this is the same as the above, but just that the return type is guaranteed to be not None to satisfy type checkers
+def define_Python_call_TeX_local_sync(*args, **kwargs)->PythonCallTeXSyncFunctionType:
+ return define_Python_call_TeX_local(*args, **kwargs, sync=True) # type: ignore
+
+run_none_finish=define_Python_call_TeX_local(
+r"""
+\cs_new_eq:NN %name% \relax
+""", [], [], finish=True, sync=False)
+
+
+"""
+|run_error_finish| is fatal to TeX, so we only run it when it's fatal to Python.
+
+We want to make sure the Python traceback is printed strictly before run_error_finish() is called,
+so that the Python traceback is not interleaved with TeX error messages.
+"""
+run_error_finish=define_Python_call_TeX_local(
+r"""
+\msg_new:nnn {pythonimmediate} {python-error} {Python~error.}
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \wlog{^^JPython~error~traceback:^^J\__data^^J}
+ \msg_error:nn {pythonimmediate} {python-error}
+}
+""", [PTTBlock], [], finish=True, sync=False)
+
+
+put_next_blue=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn \__put_next_blue_tmp {
+ %optional_sync%
+ \expandafter \__read_do_one_command: \noexpand
+}
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__target)%
+ \expandafter \__put_next_blue_tmp \__target
+}
+"""
+ , [PTTBalancedTokenList], [], recursive=False)
+
+
+put_next_tokenlist=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn \__put_next_tmp {
+ %optional_sync%
+ \__read_do_one_command:
+}
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__target)%
+ \expandafter \__put_next_tmp \__target
+}
+"""
+ , [PTTBalancedTokenList], [], recursive=False)
+
+get_next_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% {
+ \peek_analysis_map_inline:n {
+ \peek_analysis_map_break:n {
+ \__tlserialize_char_unchecked:nnNN {##1}{##2}##3 \pythonimmediatecontinue
+ }
+ }
+}
+""", [], [TTPEmbeddedLine], recursive=False)
+
+put_next_bgroup=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__index)%
+ \expandafter \expandafter \expandafter \pythonimmediatecontinuenoarg
+ \char_generate:nn {\__index} {1}
+}
+""", [PTTInt], [], recursive=False)
+
+put_next_egroup=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__index)%
+ \expandafter \expandafter \expandafter \pythonimmediatecontinuenoarg
+ \char_generate:nn {\__index} {2}
+}
+""", [PTTInt], [], recursive=False)
+
+
+get_argument_tokenlist_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% #1 {
+ %sync%
+ %send_arg0(#1)%
+ \__read_do_one_command:
+}
+""", [], [TTPBalancedTokenList], recursive=False)
+
+
+run_tokenized_line_local_=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \__data
+ %optional_sync%
+ \__read_do_one_command:
+}
+""", [PTTTeXLine], [])
+
+@export_function_to_module
+def run_tokenized_line_local(line: str, *, check_braces: bool=True, check_newline: bool=True, check_continue: bool=True)->None:
+ check_line(line, braces=check_braces, newline=check_newline, continue_=(False if check_continue else None))
+ run_tokenized_line_local_(PTTTeXLine(line))
+
+
+
+@export_function_to_module
+def run_tokenized_line_peek(line: str, *, check_braces: bool=True, check_newline: bool=True, check_continue: bool=True)->str:
+ check_line(line, braces=check_braces, newline=check_newline, continue_=(True if check_continue else None))
+ return typing.cast(
+ Callable[[PTTTeXLine], Tuple[TTPEmbeddedLine]],
+ Python_call_TeX_local(
+ r"""
+ \cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \__data
+ }
+ """)
+ )(PTTTeXLine(line))[0]
+
+
+run_block_local_=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \begingroup \newlinechar=10~ \expandafter \endgroup
+ \scantokens \expandafter{\__data}
+ % trick described in https://tex.stackexchange.com/q/640274 to scantokens the code with \newlinechar=10
+
+ %optional_sync%
+ \__read_do_one_command:
+}
+""", [PTTBlock], [])
+
+@export_function_to_module
+def run_block_local(block: str)->None:
+ run_block_local_(PTTBlock(block))
+
+expand_o_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \exp_args:NNV \tl_set:No \__data \__data
+ %sync%
+ %send_arg0_var(\__data)%
+ \__read_do_one_command:
+}
+""", [PTTBalancedTokenList], [TTPBalancedTokenList], recursive=expansion_only_can_call_Python)
+
+expand_x_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \tl_set:Nx \__data {\__data}
+ %sync%
+ %send_arg0_var(\__data)%
+ \__read_do_one_command:
+}
+""", [PTTBalancedTokenList], [TTPBalancedTokenList], recursive=expansion_only_can_call_Python)
+
+execute_=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \__data
+ %optional_sync%
+ \__read_do_one_command:
+}
+""", [PTTBalancedTokenList], [])
+
+futurelet_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \expandafter \futurelet \__data \pythonimmediatecontinuenoarg
+}
+""", [PTTBalancedTokenList], [])
+
+futureletnext_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__data)%
+ \afterassignment \pythonimmediatecontinuenoarg \expandafter \futurelet \__data
+}
+""", [PTTBalancedTokenList], [])
+
+continue_until_passed_back_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_eq:NN %name% \relax
+""", [], [TTPEmbeddedLine])
+
+@export_function_to_module
+def continue_until_passed_back_str()->str:
+ """
+ Usage:
+
+ First put some tokens in the input stream that includes |\pythonimmediatecontinue{...}|
+ (or |%sync% \__read_do_one_command:|), then call |continue_until_passed_back()|.
+
+ The function will only return when the |\pythonimmediatecontinue| is called.
+ """
+ return str(continue_until_passed_back_()[0])
+
+@export_function_to_module
+def continue_until_passed_back()->None:
+ """
+ Same as |continue_until_passed_back_str()| but nothing can be returned from TeX to Python.
+ """
+ result=continue_until_passed_back_str()
+ assert not result
+
+
+@export_function_to_module
+def expand_once()->None:
+ typing.cast(Callable[[], None], Python_call_TeX_local(
+ r"""
+ \cs_new_protected:Npn %name% { \expandafter \pythonimmediatecontinuenoarg }
+ """, recursive=False, sync=True))()
+
+
+@export_function_to_module
+@user_documentation
+def get_arg_str()->str:
+ """
+ Get a mandatory argument.
+ """
+ return typing.cast(Callable[[], TTPEmbeddedLine], Python_call_TeX_local(
+ r"""
+ \cs_new_protected:Npn %name% #1 {
+ \immediate\write\__write_file { \unexpanded {
+ r #1
+ }}
+ \__read_do_one_command:
+ }
+ """, recursive=False))()
+
+get_arg_estr_=define_Python_call_TeX_local_sync(
+r"""
+\cs_new_protected:Npn %name% #1 {
+ %sync%
+ %send_arg0(#1)%
+ \__read_do_one_command:
+}
+""", [], [TTPEBlock], recursive=False)
+@export_function_to_module
+@user_documentation
+def get_arg_estr()->str:
+ return str(get_arg_estr_()[0])
+
+
+get_optional_argument_detokenized_=define_Python_call_TeX_local_sync(
+r"""
+\NewDocumentCommand %name% {o} {
+ \immediate\write \__write_file {
+ r ^^J
+ \IfNoValueTF {#1} {
+ 0
+ } {
+ \unexpanded{1 #1}
+ }
+ }
+ \__read_do_one_command:
+}
+""", [], [TTPLine], recursive=False)
+@export_function_to_module
+@user_documentation
+def get_optional_arg_str()->Optional[str]:
+ """
+ Get an optional argument.
+ """
+ [result]=get_optional_argument_detokenized_()
+ result_=str(result)
+ if result_=="0": return None
+ assert result_[0]=="1", result_
+ return result_[1:]
+
+
+get_optional_arg_estr_=define_Python_call_TeX_local_sync(
+r"""
+\NewDocumentCommand %name% {o} {
+ %sync%
+ \IfNoValueTF {#1} {
+ %send_arg0(0)%
+ } {
+ %send_arg0(1 #1)%
+ }
+ \__read_do_one_command:
+}
+""", [], [TTPEBlock], recursive=False)
+
+@export_function_to_module
+@user_documentation
+def get_optional_arg_estr()->Optional[str]:
+ [result]=get_optional_arg_estr_()
+ result_=str(result)
+ if result_=="0": return None
+ assert result_[0]=="1", result_
+ return result_[1:]
+
+
+get_verbatim_argument_=define_Python_call_TeX_local_sync(
+r"""
+\NewDocumentCommand %name% {v} {
+ \immediate\write\__write_file { \unexpanded {
+ r ^^J
+ #1
+ }}
+ \__read_do_one_command:
+}
+""", [], [TTPLine], recursive=False)
+@export_function_to_module
+@user_documentation
+def get_verb_arg()->str:
+ """
+ Get a verbatim argument. Since it's verbatim, there's no worry of |#| being doubled,
+ but it can only be used at top level.
+ """
+ return str(get_verbatim_argument_()[0])
+
+get_multiline_verbatim_argument_=define_Python_call_TeX_local_sync(
+r"""
+\NewDocumentCommand %name% {+v} {
+ \immediate\write\__write_file { r }
+ \begingroup
+ \newlinechar=13~ % this is what +v argument type in xparse uses
+ \__send_block:n { #1 }
+ \endgroup
+ \__read_do_one_command:
+}
+""", [], [TTPBlock], recursive=False)
+@export_function_to_module
+@user_documentation
+def get_multiline_verb_arg()->str:
+ """
+ Get a multi-line verbatim argument.
+ """
+ return str(get_multiline_verbatim_argument_()[0])
+
+newcommand2=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn %name% {
+ \begingroup
+ \endlinechar=-1~
+ %read_arg0(\__line)%
+ %read_arg1(\__identifier)%
+ \cs_new_protected:cpx {\__line} {
+ \unexpanded{\immediate\write \__write_file} { i \__identifier }
+ \unexpanded{\__read_do_one_command:}
+ }
+ \endgroup
+ %optional_sync%
+ \__read_do_one_command:
+}
+""", [PTTVerbatimLine, PTTVerbatimLine], [], recursive=False)
+
+renewcommand2=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn %name% {
+ \begingroup
+ \endlinechar=-1~
+ \readline \__read_file to \__line
+ \readline \__read_file to \__identifier
+ \exp_args:Ncx \renewcommand {\__line} {
+ \unexpanded{\immediate\write \__write_file} { i \__identifier }
+ \unexpanded{\__read_do_one_command:}
+ }
+ \exp_args:Nc \MakeRobust {\__line} % also make the command global
+ \endgroup
+ %optional_sync%
+ \__read_do_one_command:
+}
+""", [PTTVerbatimLine, PTTVerbatimLine], [], recursive=False)
+
+def check_function_name(name: str)->None:
+ if not re.fullmatch("[A-Za-z]+", name) or (len(name)==1 and ord(name)<=0x7f):
+ raise RuntimeError("Invalid function name: "+name)
+
+def newcommand_(name: str, f: Callable)->Callable:
+ identifier=get_random_identifier()
+
+ newcommand2(PTTVerbatimLine(name), PTTVerbatimLine(identifier))
+
+ _code=define_TeX_call_Python(
+ lambda: run_code_redirect_print_TeX(f),
+ name, argtypes=[], identifier=identifier)
+ # ignore _code, already executed something equivalent in the TeX command
+ return f
+
+def renewcommand_(name: str, f: Callable)->Callable:
+ identifier=get_random_identifier()
+
+ renewcommand2(PTTVerbatimLine(name), PTTVerbatimLine(identifier))
+ # TODO remove the redundant entry from TeX_handlers (although technically is not very necessary, just cause slight memory leak)
+ #try: del TeX_handlers["u"+name]
+ #except KeyError: pass
+
+ _code=define_TeX_call_Python(
+ lambda: run_code_redirect_print_TeX(f),
+ name, argtypes=[], identifier=identifier)
+ # ignore _code, already executed something equivalent in the TeX command
+ return f
+
+
+
+@export_function_to_module
+def newcommand(x: Union[str, Callable, None]=None, f: Optional[Callable]=None)->Callable:
+ """
+ Define a new \TeX\ command.
+ If name is not provided, it's automatically deduced from the function.
+ """
+ if f is not None: return newcommand(x)(f)
+ if x is None: return newcommand # weird design but okay (allow |@newcommand()| as well as |@newcommand|)
+ if isinstance(x, str): return functools.partial(newcommand_, x)
+ return newcommand_(x.__name__, x)
+
+@export_function_to_module
+def renewcommand(x: Union[str, Callable, None]=None, f: Optional[Callable]=None)->Callable:
+ """
+ Redefine a \TeX\ command.
+ If name is not provided, it's automatically deduced from the function.
+ """
+ if f is not None: return newcommand(x)(f)
+ if x is None: return newcommand # weird design but okay (allow |@newcommand()| as well as |@newcommand|)
+ if isinstance(x, str): return functools.partial(renewcommand_, x)
+ return renewcommand_(x.__name__, x)
+
+
+# ========
+
+put_next_TeX_line=define_Python_call_TeX_local(
+r"""
+\cs_new_protected:Npn \__put_next_tmpa {
+ %optional_sync%
+ \__read_do_one_command:
+}
+\cs_new_protected:Npn %name% {
+ %read_arg0(\__target)%
+ \expandafter \__put_next_tmpa \__target
+}
+"""
+ , [PTTTeXLine], [], recursive=False)
+
+@export_function_to_module
+@user_documentation
+def put_next(arg: Union[str, Token, BalancedTokenList])->None:
+ """
+ Put some content forward in the input stream.
+
+ arg: has type |str| (will be tokenized in the current catcode regime, must be a single line),
+ or |BalancedTokenList|.
+ """
+ if isinstance(arg, str): put_next_TeX_line(PTTTeXLine(arg))
+ else: arg.put_next()
+
+
+
+# TODO I wonder which one is faster. Need to benchmark...
+@export_function_to_module
+@user_documentation
+def peek_next_meaning()->str:
+ """
+ Get the meaning of the following token, as a string, using the current |\escapechar|.
+
+ This is recommended over |peek_next_token()| as it will not tokenize an extra token.
+
+ It's undefined behavior if there's a newline (|\newlinechar| or |^^J|, the latter is OS-specific)
+ in the meaning string.
+ """
+ return typing.cast(Callable[[], TTPEmbeddedLine], Python_call_TeX_local(
+ r"""
+ \cs_new_protected:Npn \__peek_next_meaning_callback: {
+
+ \edef \__tmp {\meaning \__tmp} % just in case |\__tmp| is outer, |\write| will not be able to handle it
+ %\immediate\write \__write_file { r \unexpanded\expandafter{\__tmp} }
+ \immediate\write \__write_file { r \__tmp }
+
+ \__read_do_one_command:
+ }
+ \cs_new_protected:Npn %name% {
+ \futurelet \__tmp \__peek_next_meaning_callback:
+ }
+ """, recursive=False))()
+
+
+if 0:
+ peek_next_char_=define_Python_call_TeX_local_sync(
+
+ # first attempt. Slower than peek_next_meaning.
+ r"""
+ \cs_new_protected:Npn \__peek_next_char_callback: {
+ \edef \__tmpb { \expandafter\str_item:nn\expandafter{\meaning \__tmp} {-1} } % \expandafter just in case \__tmp is \outer
+ \if \noexpand\__tmp \__tmpb % is a character
+ \immediate\write \__write_file { r^^J \__tmpb . }
+ \else % is not?
+ \immediate\write \__write_file { r^^J }
+ \fi
+ \__read_do_one_command:
+ }
+ \cs_new_protected:Npn %name% {
+ \futurelet \__tmp \__peek_next_char_callback:
+ }
+ """
+
+ # second attempt. Faster than before but still slower than peek_next_meaning.
+ #r"""
+ #\cs_new_protected:Npn %name% {
+ # \futurelet \__tmp \__peek_next_char_callback:
+ #}
+ #
+ #\cs_new_protected:Npn \__peek_next_char_callback: {
+ # %\if \noexpand\__tmp \c_space_token % there's also this case and that \__tmp is some TeX primitive conditional...
+ # \expandafter \__peek_next_char_callback_b: \meaning \__tmp \relax
+ #}
+ #
+ #\cs_new_protected:Npn \__peek_next_char_callback_b: #1 #2 {
+ # \ifx #2 \relax
+ # \if \noexpand\__tmp #1 % is a character
+ # \immediate\write \__write_file { r^^J #1 }
+ # \else % is not?
+ # \immediate\write \__write_file { r^^J }
+ # \fi
+ # \expandafter \__read_do_one_command:
+ # \else
+ # \expandafter \__peek_next_char_callback_b: \expandafter #2
+ # \fi
+ #}
+ #
+ #"""
+
+ , [], [TTPLine], recursive=False)
+
+
+
+meaning_str_to_catcode: Dict[str, Catcode]={
+ "begin-group character ": Catcode.bgroup,
+ "end-group character ": Catcode.egroup,
+ "math shift character ": Catcode.math,
+ "alignment tab character ": Catcode.alignment,
+ "macro parameter character ": Catcode.parameter,
+ "superscript character ": Catcode.superscript,
+ "subscript character ": Catcode.subscript,
+ "blank space ": Catcode.space,
+ "the letter ": Catcode.letter,
+ "the character ": Catcode.other,
+ }
+
+def parse_meaning_str(s: str)->Optional[Tuple[Catcode, str]]:
+ if s and s[:-1] in meaning_str_to_catcode:
+ return meaning_str_to_catcode[s[:-1]], s[-1]
+ return None
+
+@export_function_to_module
+@user_documentation
+def peek_next_char()->str:
+ """
+ Get the character of the following token, or empty string if it's not a character.
+ Will also return nonempty if the next token is an implicit character token.
+
+ Uses peek_next_meaning() under the hood to get the meaning of the following token. See peek_next_meaning() for a warning on undefined behavior.
+ """
+
+ #return str(peek_next_char_()[0])
+ # too slow (marginally slower than peek_next_meaning)
+
+ r=parse_meaning_str(peek_next_meaning())
+ if r is None:
+ return ""
+ return r[1]
+
+@export_function_to_module
+def get_next_char()->str:
+ result=Token.get_next()
+ assert isinstance(result, CharacterToken), "Next token is not a character!"
+ return result.chr
+
+# ========
+
+try:
+ send_bootstrap_code()
+ run_main_loop() # if this returns cleanly TeX has no error. Otherwise some readline() will reach eof and print out a stack trace
+ assert not raw_readline(), "Internal error: TeX sends extra line"
+
+except:
+ # see also documentation of run_error_finish.
+ sys.stderr.write("\n")
+ traceback.print_exc(file=sys.stderr)
+
+ if do_run_error_finish:
+ action_done=False # force run it
+ run_error_finish(PTTBlock("".join(traceback.format_exc())))
+
+ os._exit(0)
+