1 files changed, 535 insertions, 0 deletions
diff --git a/macros/latex/contrib/pythonimmediate/pythonimmediate.tex b/macros/latex/contrib/pythonimmediate/pythonimmediate.tex
new file mode 100644
index 0000000000..877bc3a05e
--- /dev/null
+++ b/macros/latex/contrib/pythonimmediate/pythonimmediate.tex
@@ -0,0 +1,535 @@
+\ProvidesFile{pythonimmediate.tex}[2022/12/23 0.0.0 Library to run Python code]
+\RequirePackage{fvextra}
+\documentclass{l3doc}
+\usepackage{tikz}
+\usetikzlibrary{calc}
+\usetikzlibrary{arrows.meta}
+\EnableCrossrefs
+\CodelineIndex
+\fvset{breaklines=true,gobble=0,tabsize=4,frame=single,numbers=left,numbersep=3pt}
+
+\AtBeginDocument{\DeleteShortVerb\"}  % https://tex.stackexchange.com/a/650966/250119
+\MakeOuterQuote{"}
+
+\newcommand\reg{régime}
+\newcommand\DescribePython{\DescribeMacro}  % hack
+
+
+\begin{document}
+
+
+% hack to make quotes inside |...| straight
+\edef\temp{\def\noexpand|{\noexpand\Verb\string|}}\temp
+% basically execute \def| { \Verb ⟨catcode other |⟩ }
+
+\GetFileInfo{\jobname.tex}
+
+\title{\pkg{\jobname} --- \fileinfo
+\thanks{This file describes version \fileversion, last revised \filedate.}
+}
+\author{user202729}
+\date{Released \filedate}
+
+\maketitle
+
+\begin{abstract}
+	\fileinfo.
+\end{abstract}
+
+\section{Motivation}
+
+Just like \pkg{Perl\TeX} or \pkg{PyLua\TeX} (and unlike \pkg{Python\TeX} or \pkg{lt3luabridge}),
+this only requires a single run, and variables are persistent throughout the run.
+
+Unlike \pkg{Perl\TeX} or \pkg{PyLua\TeX}, there's no restriction on compiler or script required to run the code.
+
+There's also debugging functionalities -- \TeX\ errors results in Python traceback, and Python error results in \TeX\ traceback.
+Errors in code executed with the |pycode| environment gives the correct traceback point to the Python line of code in the \TeX\ file.
+
+For advanced users, this package allows the user to manipulate the \TeX\ state directly from within Python,
+so you don't need to write a single line of \TeX\ code.
+
+\section{Installation}
+
+%You need to install Python \pkg{daemoniker} and \pkg{psutil} package.
+
+The package should work out of the box with no additional Python package needed.
+
+\subsection{Installation on Overleaf}
+
+At the point of writing, this package can be used on Overleaf.
+
+Instruction:
+
+\begin{itemize}
+	\item Download the following files and place it in the root folder of Overleaf:
+		\begin{itemize}
+			\item \file{saveenv.sty}
+			\item \file{precattl.sty}
+			\item \file{pythonimmediate.sty}
+			\item \file{pythonimmediate_script_textopy.py}
+			\item \file{pythonimmediate_script_pytotex.py}
+		\end{itemize}
+	\item Write the following in the preamble:
+\begin{verbatim}
+\usepackage[abspath]{currfile}
+\usepackage[mode=unnamed-pipe]{pythonimmediate}
+\end{verbatim}
+Refer to \ref{troubleshoot-source-file-not-found} for explanation of the |abspath| option.
+\end{itemize}
+
+For some unknown reason in the default mode on Overleaf (|\nonstopmode|), when there's an error
+the log file might be truncated,
+so in that case consider writing |\errorstopmode|.
+
+Refer to \ref{troubleshoot-Python-error} to read the error traceback in case of Python error.
+
+\section{Usage}
+
+\subsection{Package options}
+
+\DescribeOption{outputdir=}
+Specify the output directory if it's not the default value.
+
+\begin{texnote}
+	The value will be |x|-expanded.
+\end{texnote}
+
+Using the \pkg{outputdir} package, it's possible to determine the output directory automatically, subject to restrictions.
+
+An usage example, if you invoked |pdflatex| with the command
+\begin{verbatim}
+pdflatex --output-directory=/tmp/ a.tex
+\end{verbatim}
+then in the file \file{a.tex} you should have
+\begin{verbatim}
+\usepackage[outputdir=/tmp/]{pythonimmediate}
+\end{verbatim}
+
+\DescribeOption{mode=}
+Method to communicate between \TeX\ and Python. Usually the method will be automatically detected.
+
+Possible values include |multiprocessing-network| and |unnamed-pipe|.
+
+\subsection{\TeX\ interface}
+
+The interface mimics those in popular packages such as \pkg{Python\TeX} or \pkg{PyLua\TeX}.
+
+\subsubsection{Inline commands}
+
+\DescribeMacro{\py}\label{py-command}
+Evaluate some Python expression, consider the result as a string, then execute the result as \TeX\ command.
+
+\begin{texnote}
+	The command is not expandable, and the argument will be fully expanded with the active |~| set to |\relax|,
+	|\set@display@protect| executed and |\escapechar=-1|, then the result
+	passed to Python as a string.
+\end{texnote}
+
+Which, for the users who are not familiar with \TeX\ terminology, roughly means the following:
+\begin{itemize}
+	\item the value can only be used to typeset text,
+		it must not be used to pass "values" to other \LaTeX\ commands.
+
+		The following is legal:
+\begin{verbatim}
+The value of $1+1$ is $\py{1+1}$.
+\end{verbatim}
+The following is illegal, as the result (2) can only be used to typeset text, not passed to another command that expect a "value":
+\begin{verbatim}
+\setcounter{abc}{\py{1+1}} 
+\end{verbatim}
+A possible workaround is:
+\begin{verbatim}
+\py{ r'\\setcounter{abc}{' + str(1+1) + '}' }
+\end{verbatim}
+In this example it works without escaping the |{}| characters, but if the Python code has those unbalanced then you can escape them as mentioned below.
+
+	\item Special characters can be "escaped" simply by prefixing the character with backslash.
+
+		For example
+\begin{verbatim}
+\pyc{assert len('\ \ ')==2}
+\pyc{assert ord('\\\\')==0x5c}
+\pyc{assert ord('\%')  ==0x25}
+\end{verbatim}
+In the examples above, Python "sees" (i.e. the Python code being executed is)
+\begin{verbatim}
+assert len('  ')==2
+assert ord('\\')==0x5c
+assert ord('%') ==0x25
+\end{verbatim}
+respectively.
+
+	\item Macros will be expanded.
+\begin{verbatim}
+\def\mycode{1+1}
+The value of $1+1$ is $\py{\mycode}$.
+\end{verbatim}
+\end{itemize}
+
+
+\DescribeMacro{\pyc}
+Execute some Python code provided as an argument (the argument will be interpreted as described above).
+
+The command is not expandable -- roughly speaking, you can only use this at "top level".
+
+Any output (as described in \ref{print-to-TeX}) will be typesetted.
+
+The difference between |\py| and |\pyc| is that the argument of |\py| should be a Python expression
+(suitable for passing into |eval()| Python function) while the argument of |\pyc| should be a Python
+statement (suitable for passing into |exec()| Python function).
+
+Therefore,
+\begin{itemize}
+	\item |\py{1+1}| will typeset 2.
+	\item |\pyc{1+1}| is valid, but will do nothing just like |exec("1+1")|.
+	\item |\py{x=1}| is invalid.
+	\item |\pyc{x=1}| is valid and assigns the variable |x| to be 1.
+\end{itemize}
+
+\DescribeMacro{\pycq}
+Same as above, but output (\ref{print-to-TeX}) will not be typesetted.
+
+\DescribeMacro{\pyfile}
+Given an argument being the file name, execute that file.
+
+\DescribeMacro{\pys}
+Performs "string interpolation", the same way as \pkg{Python\TeX}.
+(not yet implemented)
+
+\subsubsection{Environments}
+
+\DescribeEnv{pycode}
+Verbatim-like environment that executes the code inside as Python.
+
+Example usage: The following will typeset |123|
+\begin{verbatim}
+\begin{pycode}
+pythonimmediate.print("123")
+\end{pycode}
+\end{verbatim}
+
+Special note: white spaces at the end of lines are preserved.
+
+Any output (as described in \ref{print-to-TeX}) will be typesetted.
+
+\DescribeEnv{pycodeq}
+Same as above, but output will not be typesetted.
+
+\DescribeEnv{pysub}
+Not yet implemented.
+
+\subsection{Python interface}
+
+The \TeX\ interface is only used to call Python. Apart from that, all the work can be done on the Python side.
+
+All functions in this section should be imported from |pythonimmediate| package, unless specified otherwise.
+
+\subsubsection{Print to \TeX}\label{print-to-TeX}
+
+\DescribePython{.print()}
+\DescribePython{.file}
+These functions are used in |\pyc| command or |pycode| environment.
+
+Unlike most other packages, using |print()| function in Python will print to the console (\TeX\ standard output).
+In order to print \TeX\ code to be executed, you can do one of
+\begin{verbatim}
+pythonimmediate.print(...)
+print(..., file=pythonimmediate.file)
+with contextlib.redirect_stdout(pythonimmediate.file):
+	print(...)
+\end{verbatim}
+Note that in quiet environments, |pythonimmediate.file| is None, the second variant using |print()| will print to stdout
+instead of suppress output. The third variant works as expected.
+
+All output will be buffered until the whole Python code finishes executing.
+In order to typeset the text immediately use one of the advanced commands.
+
+
+
+\DescribePython{.newcommand()}
+\DescribePython{.renewcommand()}
+Same as \LaTeX's |\newcommand| and |\renewcommand|. Can be used as follows:
+
+\begin{verbatim}
+from pythonimmediate import newcommand, renewcommand
+
+@newcommand
+def function():
+	...
+# define |\function| in TeX
+
+@newcommand("controlsequencename")
+def function():
+	...
+# define |\controlsequencename| in TeX
+
+def function():
+	...
+newcommand("controlsequencename", function)
+\end{verbatim}
+
+\DescribePython{.get_arg_str()}
+\DescribePython{.get_optional_arg_str()}
+\DescribePython{.get_verb_arg()}
+\DescribePython{.get_multiline_verb_arg()}
+\DescribePython{.peek_next_char()}
+\DescribePython{.get_next_char()}
+There are those functions that is mostly understandable to an inexperienced \LaTeX\ user,
+and should be sufficient for a lot of programming works.
+
+This is an example of how the functions could be used. The name should be mostly self-explanatory.
+
+\begin{verbatim}
+\documentclass{article}
+\usepackage{pythonimmediate}
+\begin{document}
+\begin{pycode}
+from pythonimmediate import newcommand, peek_next_char, get_next_char, get_arg_str, print
+@newcommand
+def innerproduct():
+	s = get_arg_str()    # in the example below this will have the value '\mathbf{a},\mathbf{b}'
+	x, y = s.split(",")  # it's just a Python string, manipulate normally (but be careful of comma inside braces, parse the string yourself)
+	print(r"\left\langle" + x + r"\middle|" + y + r"\right\rangle")
+
+@newcommand
+def fx():
+	if peek_next_char() == "_":
+		get_next_char()
+		subscript = get_arg_str()
+		print("f_{" + subscript + "}(x)")
+	else:
+		print("f(x)")
+
+@newcommand
+def sumManyArgs():
+	s = 0
+	while peek_next_char() == "{":
+		i = get_arg_str()
+		s += int(i)
+	print(str(s))
+\end{pycode}
+$1+2+3 = \sumManyArgs{1}{2}{3}$
+
+$\innerproduct{\mathbf{a},\mathbf{b}}=1$
+
+$\fx = 1$, $\fx_i = 2$, $\fx_{ij} = 3$
+\end{document}
+\end{verbatim}
+
+It will typeset:
+
+\begin{quote}
+$1+2+3=6$
+
+$\left\langle\mathbf{a}\middle\vert\mathbf{b}\right\rangle=1$
+
+$f(x)=1$, $f_i(x)=2$, $f_{ij}(x)=3$
+\end{quote}
+
+\DescribePython{.get_arg_estr()}
+\DescribePython{.get_optional_arg_estr()}
+Similar to some functions above, except that the argument is fully expanded and "escapes" of common characters are handled correctly,
+similar to how |\py| command (\ref{py-command}) reads its arguments.
+
+\DescribePython{.execute()}
+Takes a string and execute it immediately. (so that any |.execute()| will be executed before any |.print()|)
+
+Assuming \TeX\ is in errorstopmode (i.e. errors halt \TeX\ execution),
+any error in \TeX\ will create an error in Python and the traceback should point to the correct line of code.
+
+For example, in the following code
+
+\begin{verbatim}
+\documentclass{article}
+\usepackage{tikz}
+\usepackage{pythonimmediate}
+\begin{document}
+
+\begin{tikzpicture}
+\begin{pycode}
+from pythonimmediate import execute
+execute(r'\draw (0, 0) to (1, 1);')
+execute(r'\draw (2, 2) to (p);')
+execute(r'\draw (3, 3) to (4, 4);')
+\end{pycode}
+\end{tikzpicture}
+
+\end{document}
+\end{verbatim}
+each |\draw| command will be executed immediately when the Python |.execute()| function is executed,
+and as the second line throws an error, the Python traceback will point to that line.
+
+\section{Troubleshooting}
+
+\subsection{"Source file not found!" error message}\label{troubleshoot-source-file-not-found}
+
+In order to obtain the exact code with trailing spaces and produce error traceback
+point to the correct \TeX\ file, the Python code need to know the full path to the current
+\TeX\ file for the |pycode| environment.
+
+Nevertheless, this is difficult and does not always work
+(refer to the documentation of \pkg{currfile} for details), so this message is issued
+when the file cannot be found.
+
+In that case try the following fixes:
+
+\begin{itemize}
+	\item Include |\usepackage[abspath]{currfile}| at the start of the document, after the |\documentclass| line.
+		(this option is not included by default because it's easy to get package clash, and usually \pkg{currfile} without
+		the |abspath| option works fine -- unless custom |jobname| is used)
+	\item Explicitly override |currfilename| or |currfileabspath| -- for example
+\begin{verbatim}
+\def\currfilename{main.tex}
+\end{verbatim}
+		Technically this is an abuse of the \pkg{currfile} package API, but it usually works regardless.
+\end{itemize}
+
+\subsection{"Python error" error message}\label{troubleshoot-Python-error}
+
+In case of Python error, the Python traceback is included in the terminal and \TeX\ log file.
+
+Search for "Python error traceback" before the error line in the log file.
+
+On Overleaf, you can either view the log file ("Raw logs" section)
+or the traceback on stderr (download \file{output.stderr} file)
+
+\subsection{"\TeX\ error" error message}\label{troubleshoot-TeX-error}
+
+If an error occur in \TeX, traceback cannot be included in the log file.
+
+Besides, this can only be detected in |\errorstopmode|. Such an error will always halt \TeX,
+and Python will be force-exited after printing the error traceback.
+
+On Overleaf, download \file{output.stderr} file to read the traceback.
+
+\section{Implementation note}
+
+Communication between \TeX\ and Python are done by opening two pseudo-files from the output of a Python process |textopy|
+(similar to |\ior_shell_open:Nn|)
+and to the input of another Python process |pytotex| (this would be |\iow_shell_open:Nn|, if \LaTeX3 have such a function).
+
+There are various methods for the 2 Python child processes to communicate with each other.
+After some initial bootstrapping to setup the communication, we can consider only the |textopy| script, the other
+merely serves as the bridge to send input to \TeX.
+
+The communication protocol is a little complicated, since it must support nesting bidirectional execution of \TeX\ and Python.
+
+Besides, I believe it's not possible to make a "background listener" on the \TeX\ side, so it must keep track of whether a command should be read from Python and executed.
+
+Currently, exception handling (throwing a Python exception in a nested Python function, catch it in the outer Python function) is not supported.
+
+These are some examples of what could happen in the communication protocol.
+
+\ExplSyntaxOn
+\let\fpEval\fp_eval:n
+\ExplSyntaxOff
+
+\begingroup
+
+\tikzset{myarrow/.style={-{Latex[length=2mm]}}}
+
+\def\blockWidth{1}
+\def\blockHeight{7}
+\def\separation{10.5}
+\def\step{0.7}
+
+\def\left #1{
+	\draw [myarrow] (p) ++(\separation, 0) -- node [above] {\small #1} ++(-\separation, 0);
+	\path (p) ++ (0, -\step) coordinate (p);
+}
+\def\right #1{
+	\draw [myarrow] (p) -- node [above] {\small #1} ++(\separation, 0);
+	\path (p) ++ (0, -\step) coordinate (p);
+}
+
+\def\initDrawing{
+	\path (\blockWidth, \fpEval{-\step}) coordinate (p);
+}
+
+\newdimen \tempx
+\newdimen \tempy
+
+\ExplSyntaxOn
+\let \dimToDecimalInUnit \dim_to_decimal_in_unit:nn 
+\ExplSyntaxOff
+
+\def\drawBlocks{
+    \path [transform canvas] (p);
+	\pgfgetlastxy{\tempx}{\tempy}
+	\edef\blockHeight{ \fpEval{ - \dimToDecimalInUnit{\tempy}{1cm}  } }
+
+	\draw (0, \fpEval{-\blockHeight}) rectangle node [rotate=90] {\TeX} (\blockWidth, 0);
+	\draw (\fpEval{\blockWidth+\separation}, \fpEval{-\blockHeight}) rectangle node [rotate=270] {Python} (\fpEval{\separation+2*\blockWidth}, 0);
+}
+
+\vspace{5pt}
+\begin{tikzpicture}
+\edef\blockHeight{\fpEval{\step*3}}
+\initDrawing
+\right{execute Python code: print(1)}
+\left{execute \TeX\ code: 1}
+\drawBlocks
+\end{tikzpicture}
+
+Nevertheless, there may be more complicated cases where the Python code itself may call \TeX\ code before actually returns:
+
+\vspace{5pt}
+\begin{tikzpicture}
+\edef\blockHeight{\fpEval{\step*5}}
+\initDrawing
+\right{execute Python code: print(var(a)*2)}
+\left{execute \TeX\ code: sendtopy(a); execute another command}
+\right{123}
+\left{execute \TeX\ code: 123123}
+\drawBlocks
+\end{tikzpicture}
+
+Or:
+
+\vspace{5pt}
+\begin{tikzpicture}
+\edef\blockHeight{\fpEval{\step*7}}
+\initDrawing
+\right{execute Python code: tex.exec(a=456); print(var(a)*2)}
+\left{execute \TeX\ code: a=456; sendtopy(done); execute another command}
+\right{done}
+\left{execute \TeX\ code: sendtopy(a); execute another command}
+\right{456}
+\left{456456}
+\drawBlocks
+\end{tikzpicture}
+
+\endgroup
+
+The Python side must not just listen for "done" command back, but must potentially call a nested loop.
+
+The exact protocol is:
+\begin{itemize}
+	\item "execute Python code" sends from \TeX\ to Python has a single line "|i|\meta{handler name}",
+		followed by any number of arguments (depends on the handler).
+
+		Refer to the |define_TeX_call_Python| internal function for details.
+
+	\item "done" sends from \TeX\ to Python has the format "|r|\meta{optional return value as a string in a single line}".
+
+		This is sent by executing \TeX\ command |\pythonimmediatecontinue|, which takes a single argument to be e-expanded using |\write|
+		as the "return value".
+
+	\item "execute \TeX\ code" sends from Python to \TeX\ must only be sent when the \TeX\ side listens for a command.
+		It consist of a single line specify the "command name", which \TeX\ will
+		execute the command named
+		|\__run_|\meta{command name}|:|
+		which must already be defined on the \TeX\ side.
+
+		The command itself might contain additional code to execute more code, e.g. by reading more lines from Python.
+
+		Refer to the |define_Python_call_TeX| internal function for details.
+\end{itemize}
+
+
+
+\PrintIndex
+
+\end{document}