pythonimmediate (18jan24)

git-svn-id: svn://tug.org/texlive/trunk@69487 c570f23f-e606-0410-a88d-b1316a301751

5 files changed, 161 insertions, 41 deletions

diff --git a/Master/texmf-dist/doc/latex/pythonimmediate/README b/Master/texmf-dist/doc/latex/pythonimmediate/README
index 43c413e62c2..9a6d7b9e1a2 100644
--- a/Master/texmf-dist/doc/latex/pythonimmediate/README
+++ b/Master/texmf-dist/doc/latex/pythonimmediate/README
@@ -7,7 +7,7 @@ Report bugs at https://github.com/user202729/pythonimmediate-tex
 
 ========
 
-Copyright 2022-2023 user202729
+Copyright 2022-2024 user202729
 
 This work  may be  distributed and/or  modified under  the conditions  of the
 LaTeX Project Public License (LPPL),  either version 1.3c  of this license or
diff --git a/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.pdf b/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.pdf
index 5b8bf34442d..cfcc852434c 100644
--- a/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.pdf
+++ b/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.pdf
diff --git a/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.tex b/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.tex
index 539a770fc29..9e6b0c1d8b7 100644
--- a/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.tex
+++ b/Master/texmf-dist/doc/latex/pythonimmediate/pythonimmediate.tex
@@ -1,4 +1,4 @@
-\ProvidesFile{pythonimmediate.tex}[2023/03/07 0.4.0 Library to run Python code]
+\ProvidesFile{pythonimmediate.tex}[2024/01/18 0.5.0 Library to run Python code]
 \RequirePackage{fvextra}
 \documentclass{l3doc}
 \usepackage{tikz}
@@ -6,7 +6,8 @@
 \usetikzlibrary{arrows.meta}
 \EnableCrossrefs
 \CodelineIndex
-\fvset{breaklines=true,gobble=0,tabsize=4,frame=single,numbers=left,numbersep=3pt}
+\fvset{breaklines=true,breakanywhere,gobble=0,tabsize=4,frame=single,numbers=left,numbersep=3pt}
+\usepackage{cleveref}
 
 \AtBeginDocument{\DeleteShortVerb\"}  % https://tex.stackexchange.com/a/650966/250119
 \MakeOuterQuote{"}
@@ -17,6 +18,7 @@
 
 \begin{document}
 
+\hfuzz=15pt
 
 % hack to make quotes inside |...| straight
 \edef\temp{\def\noexpand|{\noexpand\Verb\string|}}\temp
@@ -53,7 +55,7 @@ In addition to the \LaTeX\ package, you need the Python \pkg{pythonimmediate-tex
 installation instruction can be found at \url{https://pypi.org/project/pythonimmediate-tex/}.
 
 Note that not all \TeX\ package versions are compatible with all Python package versions.
-This \TeX\ package is compatible with Python package version |0.4.0|.
+This \TeX\ package is compatible with Python package version |0.6.0|.
 
 Remember to enable unrestricted\footnote{
 	There's little point in supporting restricted mode, since it's possible to execute arbitrary shell commands in Python anyway.
@@ -73,39 +75,43 @@ Instruction:
 \begin{itemize}
 	\item Download the following files and place it in the root folder of Overleaf:
 		\begin{itemize}
-			\item \file{saveenv.sty}
-			\item \file{precattl.sty}
 			\item \file{pythonimmediate.sty}
 			\item \file{pythonimmediate_helper.lua}
-			\item \file{pythonimmediate-tex-0.4.0.zip}
+			\item \file{pythonimmediate-tex-0.6.0.zip}
 		\end{itemize}
-		The |0.4.0| part should be replaced with the desired version
+		The |0.6.0| part should be replaced with the desired version
 		of the Python package.
 
 		The |.sty| files can be downloaded from CTAN at 
-		\url{https://ctan.org/pkg/saveenv},
-		\url{https://ctan.org/pkg/precattl},
-		\url{https://ctan.org/pkg/pythonimmediate}
-		respectively.
+		\url{https://ctan.org/pkg/pythonimmediate}.
 
 		The |.zip| file containing Python source code
 		can be downloaded from PyPI: \url{https://pypi.org/project/pythonimmediate-tex/\#files}.
 
+		Remark: Currently the \LaTeX\ package version 0.4.0 is included in \TeX\ Live 2023, you would need to download \LaTeX\ version 0.5.0 from CTAN.
+
+	\item In older \TeX\ Live versions (2022 or less), you will in addition need to download the following files:
+		\begin{itemize}
+			\item \file{saveenv.sty} (from \url{https://ctan.org/pkg/saveenv}),
+			\item \file{precattl.sty} (from \url{https://ctan.org/pkg/precattl}).
+		\end{itemize}
+		Also place in the root folder of Overleaf.
+
 	\item Write the following in the preamble:
 \begin{verbatim}
 \usepackage[abspath]{currfile}
-\usepackage[python-executable={PYTHONPATH=pythonimmediate-tex-0.4.0.zip/pythonimmediate-tex-0.4.0/ python3},args={--mode=unnamed-pipe}]{pythonimmediate}
+\usepackage[python-executable={PYTHONPATH=pythonimmediate-tex-0.6.0.zip/pythonimmediate-tex-0.6.0/ python3},args={--mode=unnamed-pipe}]{pythonimmediate}
 \end{verbatim}
-As above, replace both occurrences of |0.4.0| with the downloaded version specified in the zip file above.
+As above, replace both occurrences of |0.6.0| with the downloaded version specified in the zip file above.
 
-Refer to \ref{troubleshoot-source-file-not-found} for explanation of the |abspath| option.
+Refer to \cref{troubleshoot-source-file-not-found} for explanation of the |abspath| option.
 \end{itemize}
 
 For some unknown reason in the default mode on Overleaf (|\nonstopmode|), when there's an error
 the log file might be truncated,
 so in that case consider writing |\errorstopmode|.
 
-Refer to \ref{troubleshoot-Python-error} to read the error traceback in case of Python error.
+Refer to \cref{troubleshoot-Python-error} to read the error traceback in case of Python error.
 
 Some construct inside the |pycode| block might make the code editor on Overleaf report an error,
 even though the code is valid. Refer to \url{https://www.overleaf.com/learn/how-to/Code_Check#Code_Check_Limitations}.
@@ -131,7 +137,7 @@ The documentation is also available at
 The name/path to the Python executable. Default to |python3|.
 
 Can also be used to provide environment variables to the Python process.
-An example how to do that is explained in \ref{installation-on-overleaf}.
+An example how to do that is explained in \cref{installation-on-overleaf}.
 
 \DescribeOption{python-flags=}
 Flags to be passed to the Python interpreter. For example pass |-O| to disable assertions.
@@ -202,7 +208,7 @@ Execute some Python code provided as an argument (the argument will be interpret
 
 The command is not expandable -- roughly speaking, you can only use this at "top level".
 
-Any output (as described in \ref{print-to-TeX}) will be typesetted.
+Any output (as described in \cref{print-to-TeX}) will be typesetted.
 
 The difference between |\py| and |\pyc| is that the argument of |\py| should be a Python expression
 (suitable for passing into |eval()| Python function) while the argument of |\pyc| should be a Python
@@ -217,7 +223,7 @@ Therefore,
 \end{itemize}
 
 \DescribeMacro{\pycq}
-Same as above, but output (\ref{print-to-TeX}) will not be typesetted.
+Same as above, but output (\cref{print-to-TeX}) will not be typesetted.
 
 \DescribeMacro{\pyfile}
 Given an argument being the file name, execute that file.
@@ -240,7 +246,7 @@ pythonimmediate.print("123")
 
 Special note: white spaces at the end of lines are preserved.
 
-Any output (as described in \ref{print-to-TeX}) will be typesetted.
+Any output (as described in \cref{print-to-TeX}) will be typesetted.
 
 \DescribeEnv{pycodeq}
 Same as above, but output will not be typesetted.
@@ -366,7 +372,7 @@ $f(x)=1$, $f_i(x)=2$, $f_{ij}(x)=3$
 \DescribePython{.get_arg_estr()}
 \DescribePython{.get_optional_arg_estr()}
 Similar to some functions above, except that the argument is fully expanded and "escapes" of common characters are handled correctly,
-similar to how |\py| command (\ref{py-command}) reads its arguments.
+similar to how |\py| command (\cref{py-command}) reads its arguments.
 
 \DescribePython{.execute()}
 Takes a string and execute it immediately. (so that any |.execute()| will be executed before any |.print_TeX()|)
@@ -398,6 +404,11 @@ and as the second line throws an error, the Python traceback will point to that
 
 \section{Troubleshooting}
 
+\subsection{"Incompatible TeX package version" error message}\label{troubleshoot-incompatible-version}
+Exactly what it means. You can upgrade both the Python package and the TeX package to the latest version.
+
+Alternatively, you can also \emph{downgrade} whichever package that is newer. For instance, Python package version 0.5.0 (download from \url{https://pypi.org/project/pythonimmediate-tex/0.5.0/#files}) is compatible with \LaTeX\ package version 0.4.0.
+
 \subsection{"Source file not found!" error message}\label{troubleshoot-source-file-not-found}
 
 In order to obtain the exact code with trailing spaces and produce error traceback
diff --git a/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate.sty b/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate.sty
index 45569738312..1e632aba968 100644
--- a/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate.sty
+++ b/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate.sty
@@ -1,5 +1,5 @@
 % File: pythonimmediate.sty
-% Copyright 2022-2023 user202729
+% Copyright 2022-2024 user202729
 %
 % This work  may be  distributed and/or  modified under  the conditions  of the
 % LaTeX Project Public License (LPPL),  either version 1.3c  of this license or
@@ -12,9 +12,10 @@
 % 
 % The Current Maintainer of this work is user202729.
 
-\ProvidesExplPackage{pythonimmediate}{2023/03/07}{0.4.0}{Library to run Python code}
+\ProvidesExplPackage{pythonimmediate}{2024/01/18}{0.5.0}{Library to run Python code}
 
 \RequirePackage{saveenv}
+\PassOptionsToPackage{abspath}{currfile}  % try to enable abspath if possible (i.e. if package isn't loaded) but otherwise don't give option-clash error
 \RequirePackage{currfile}
 \RequirePackage{l3keys2e}
 \RequirePackage{precattl}
@@ -110,11 +111,14 @@
 
 \bool_if:NTF \_pythonimmediate_child_process {
 	\newwrite \_pythonimmediate_write_file
-	%\immediate\openout \_pythonimmediate_write_file=symlink-to-stderr.txt~
-	% tried this method, it seems to work except that it's buffered when stderr is redirected to a file...
-	\immediate\openout \_pythonimmediate_write_file=|"\_pythonimmediate_python_executable \space \_pythonimmediate_python_flags \space -m ~ pythonimmediate.copy_to_stderr"~
-
-
+	\sys_if_engine_luatex:TF {
+		% use Lua to write to stderr
+		\directlua{ (require "pythonimmediate_helper")() }
+	} {
+		%\immediate\openout \_pythonimmediate_write_file=symlink-to-stderr.txt~
+		% tried this method, it seems to work except that it's buffered when stderr is redirected to a file...
+		\immediate\openout \_pythonimmediate_write_file=|"\_pythonimmediate_python_executable \space \_pythonimmediate_python_flags \space -m ~ pythonimmediate.copy_to_stderr"~
+	}
 } {
 
 	\sys_if_engine_luatex:TF {
diff --git a/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate_helper.lua b/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate_helper.lua
index 8590233242c..f5c54c1239e 100644
--- a/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate_helper.lua
+++ b/Master/texmf-dist/tex/latex/pythonimmediate/pythonimmediate_helper.lua
@@ -1,20 +1,125 @@
 return function(cmd)
-	local process = io.popen(cmd, "w")
+	-- as mentioned in the .sty file this allows explicit flush after each write
+	-- it's not easy to eliminate having to spawn 2 Python processes even with Lua https://stackoverflow.com/questions/8716527/interactive-popen-lua-call
+
+
+	-- if this is a child process, should write to stderr, in which case cmd will be nil
+	local send_content
+	if cmd==nil then
+		send_content=function(str)
+			io.stderr:write(str.."\n")
+			io.stderr:flush()
+		end
+	else
+		process = io.popen(cmd, "w")
+		send_content=function(str)
+			process:write(str.."\n")
+			process:flush()
+		end
+	end
 
 	local function_table=lua.get_functions_table()
 
 	-- https://tex.stackexchange.com/questions/632408/how-can-i-exclude-tex-macros-when-counting-a-strings-characters-in-lua/632464?noredirect=1#comment1623008_632464 this only work in Lua 5.3 or assume it's allocated sequentially
 	local send_content_index=#function_table+1
 	function_table[send_content_index]=function()
-		 process:write(token.scan_string())
-		 process:write("\n")
-		 process:flush() 
-	 end
-
-	 local close_index=#function_table+1
-	 function_table[close_index]=function()
-		 process:close()
-	 end
-
-	 tex.print([[\protected \luadef \_pythonimmediate_send_content:e ]] .. send_content_index .. [[\protected \luadef \_pythonimmediate_close_write: ]] .. close_index .. [[\relax]])
- end
+		send_content(token.scan_string())
+	end
+
+	local close_index=#function_table+1
+	function_table[close_index]=function()
+		process:close()
+	end
+
+	local bgroup=token.create(0x7b, 1)
+	local egroup=token.create(0x7d, 2)
+
+	token.put_next(
+		bgroup,
+		token.create("ifodd"),
+		token.create(0x31, 12),
+		token.create("fi"),
+		egroup
+	)
+	local frozen_relax_tok=token.scan_toks(false, true)[1].tok
+	local null_cs_tok=0x20000000  -- \csname\endcsname
+
+	local cmdname_to_type={
+		left_brace="1",
+		right_brace="2",
+		math_shift="3",
+		tab_mark  ="4",
+		mac_param ="6",
+		sup_mark  ="7",
+		sub_mark  ="8",
+		spacer    ="A",
+		letter    ="B",
+		other_char="C",
+	}
+	local function serialize(tl)
+		local result={}
+		for _, v in ipairs(tl) do
+			local s  -- serialized result of v
+			local function handle_character(cat, index, char)
+				if index<32 then
+					s="^"..cat..utf8.char(index+64)
+				else
+					s=cat..char
+				end
+			end
+			if v.csname~=nil then
+				if v.active then
+					handle_character("D", utf8.codepoint(v.csname), v.csname)
+				elseif v.tok==frozen_relax_tok then
+					s="R"
+				elseif v.tok==null_cs_tok then
+					s="\\ "
+				else
+					local c=v.csname
+					s=""
+					for i=1, #c do
+						if c:byte(i)<33 then s=s.."*" end
+					end
+					s="\\"
+					for i=1, #c do
+						if c:byte(i)<33 then
+							s=s.." "..string.char(c:byte(i)+64)
+						else
+							s=s..c:sub(i,i)
+						end
+					end
+					s=s.." "
+				end
+			else
+				assert(cmdname_to_type[v.cmdname]~=nil)
+				handle_character(cmdname_to_type[v.cmdname], v.mode, utf8.char(v.mode))
+			end
+			result[#result+1]=s
+		end
+		return table.concat(result)
+	end
+
+	local serialize_index=#function_table+1
+	function_table[serialize_index]=function()
+		local result_token=token.get_next()
+		assert(result_token.csname~=nil and not result_token.active)
+		local tl=token.scan_toks()
+		--token.set_macro(-2, result_token.csname, table.concat(result))
+		tex.sprint{token.create "def", result_token, bgroup}
+		tex.sprint(-2, serialize(tl))
+		tex.sprint(egroup)
+	end
+
+	local send_balanced_index=#function_table+1
+	function_table[send_balanced_index]=function()
+		local tl=token.scan_toks()
+		send_content(serialize(tl))
+	end
+
+	tex.print(
+	[[\protected \luadef \_pythonimmediate_send_content:e ]] .. send_content_index ..
+	[[\protected \luadef \_pythonimmediate_close_write: ]] .. close_index ..
+	[[\protected \luadef \_pythonimmediate_tlserialize_nodot_unchecked:Nn ]] .. serialize_index ..
+	[[\protected \luadef \_pythonimmediate_send_balanced_tl:n ]] .. send_balanced_index ..
+	[[\relax]])
+end