diff options
author | Karl Berry <karl@freefriends.org> | 2023-11-09 22:03:40 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2023-11-09 22:03:40 +0000 |
commit | e533cb4b1a8c351864c42ffdfcf70a67cd610b19 (patch) | |
tree | 3978f860de2eed36568b74b6f591d5b7785485a5 | |
parent | 2f55b10b524d38fb118dcc45d26c10ff3ec89248 (diff) |
robust-externalize (9nov23)
git-svn-id: svn://tug.org/texlive/trunk@68757 c570f23f-e606-0410-a88d-b1316a301751
-rw-r--r-- | Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.pdf | bin | 653662 -> 740983 bytes | |||
-rw-r--r-- | Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.tex | 760 | ||||
-rw-r--r-- | Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty | 1335 | ||||
-rwxr-xr-x | Master/tlpkg/bin/tlpkg-ctan-check | 2 | ||||
-rwxr-xr-x | Master/tlpkg/libexec/ctan2tds | 1 | ||||
-rw-r--r-- | Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc | 1 |
6 files changed, 1607 insertions, 492 deletions
diff --git a/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.pdf b/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.pdf Binary files differindex 50b3a2f806b..a00bc94b377 100644 --- a/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.pdf +++ b/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.pdf diff --git a/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.tex b/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.tex index 0a8f57bef94..f02f525626c 100644 --- a/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.tex +++ b/Master/texmf-dist/doc/latex/robust-externalize/robust-externalize.tex @@ -91,10 +91,10 @@ \color{gray}\Huge\ttfamily \char`\{\raisebox{.09em}{\textcolor{red!75!black}{robust\raisebox{-0.1em}{-}externalize}}\char`\}}; \vspace{0.5em} - {\Large\bfseries Cache anything (\tikzname, python…),\\in a robust, efficient and pure way.} + {\Large\bfseries Cache anything (\tikzname, tikz-cd, python…),\\in a robust, efficient and pure way.} \vspace{1em} - {Léo Colisson \quad Version 2.0}\\[3mm] + {Léo Colisson \quad Version 2.1}\\[3mm] {\href{https://github.com/leo-colisson/robust-externalize}{\texttt{github.com/leo-colisson/robust-externalize}}} \end{center} @@ -104,7 +104,7 @@ \textbf{WARNING: This library is young and might lack some testing (and is an important rewrite of a previous version), but v2.0 is definitely usable. Even if we try to stay backward compatible, the only guaranteed way to be immune to changes is to copy/paste the library in your main project folder. Please report any bug to \url{https://github.com/leo-colisson/robust-externalize}!} -\textbf{WARNING 2: the coming version 2.0 improves significantly the compilation time compared with 1.0 that was surprisingly slow. Now, it is fairly well optimized (but the code needs a bit of cleaning and documentation before doing an official release)} +\textbf{WARNING 2: the coming version 2.1 improves significantly the compilation time compared with 2.0 that was surprisingly slow. Now, it is fairly well optimized (but the code needs a bit of cleaning and documentation before doing an official release)} \section{A taste of this library} @@ -177,7 +177,14 @@ And one not cached: % ] ] \end{forest} -\end{codeAndResult} +\end{codeAndResult} +People interested by |tikz-cd| might like this example: +\begin{codeAndResult} +\cacheEnvironment{tikzcd}{tikz, add to preamble={\usepackage{tikz-cd}}} +\begin{tikzcd} + A \rar & B +\end{tikzcd} +\end{codeAndResult} \item For commands: \begin{codeAndResult} % We cache the command \zx @@ -286,16 +293,51 @@ You can also include files from the root folder, if you want, recompile all pict \end{CacheMe} \end{codeexample} +Since v2.1, we also provide easy ways to export counters: +\begin{codeexample}[width=0pt] +\begin{tikzpictureC}<forward counter=page> + \node[rounded corners, fill=red]{The current page is \thepage.}; +\end{tikzpictureC} +\end{codeexample} +or even macros (we can also export evaluated macros): +\begin{codeexample}[width=0pt] +\cacheTikz +\newcommand{\MyNodeWithNewCommand}[2][draw,thick]{\node[rounded corners,fill=red,#1]{#2};} +\begin{tikzpicture}<forward=\MyNodeWithNewCommand> + \MyNodeWithNewCommand{Alice} + \MyNodeWithNewCommand[xshift=2cm]{Bob} +\end{tikzpicture} +\end{codeexample} + +Since manually forwarding elements can be quite verbose, it is also possible to automatically apply some styles depending on the content of the element to cache. For instance, we provide a way to automatically forward some macros: +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{add to preset={tikz}{auto forward}} + +\newcommandAutoForward{\MyNode}[2][draw,thick]{\node[rounded corners,fill=red,#1]{#2};} +\newcommandAutoForward{\MyGreenNode}[2][draw,thick]{\node[rounded corners,fill=green,#1]{#2};} -We also provide many other things, among others: +\begin{tikzpicture} + \MyNode{Recompiled only if MyNode is changed} + \MyNode[xshift=8cm]{but not if the (unused) MyGreenNode is changed.} +\end{tikzpicture}\\ + +\begin{tikzpicture} + \MyGreenNode{Recompiled only if MyGreenNode is changed} + \MyGreenNode[xshift=8cm]{but not if the (unused) MyNode is changed.} +\end{tikzpicture} +\end{codeexample} + +We also provide a number of other functionalities, among others: \begin{itemize} \item To achieve a pleasant and configurable interface (e.g. pass options to a preset with default values), we introduced placeholders, that can be of independent interest. -\item A way to compile a template via:\\ +\item To get maximum purity while minimizing the number of rebuild, we provide functions to forward macros defined in the parent document, and the set of macros that must be forwarded can be automatically detected for each element to cache. It is also possible to conditionally import a library if the element contains a given macro. +\item We provide a way to compile a template via:\\ |new compiled preset={compiled ZX}{templateZX, compile latex template}{}|\\ in order to make the compilation even faster (but you cannot use presets except |add to preamble|) -\item A highly configurable template, compilation and inclusion system, to cache basically anything -\item A way to compile all images in parallel during the first run, so that the first run becomes faster to compile than a normal run -\item Commands to automatically clean the cache while keeping pictures in use +\item We give a highly configurable template, compilation and inclusion system, to cache basically anything +\item We show how to compile all images in parallel using |\robExtConfigure{compile in parallel}| or |\robExtConfigure{compile in parallel after=5}| (make sure to have xargs installed, which is available by default on linux but must be installed on Windows via by installing the lightweight GNU On Windows (Gow) \url{https://github.com/bmatzelle/gow} if you use the default command) even during the first run, so that the first run becomes faster to compile than a normal run(!) +\item We furnish commands to automatically clean the cache while keeping cached elements that are still needed. \item And more\dots \end{itemize} @@ -367,30 +409,21 @@ We completely rewrote the \href{https://github.com/leo-colisson/robust-externali \item with externalization, old library (next runs): 4.16s (so 21x faster) \end{itemize} -\paragraph{Can I compile pictures in parallel to speed up the first run?} If you are worried by the time taken for the first run, you can also run it in manual mode and use for instance GNU |parallel| (or any tool of you choice) to run all commands in the file |\jobname-robExt-compile-missing-figures.sh| in parallel. To do that: -\begin{itemize} -\item First add |\robExtConfigure{enable manual mode}| in your file -\item Create the cache folder, e.g.\ using |mkdir robustExternalize| -\item Compile your file, with, e.g.\ |pdflatex yourfile.tex| (in my benchmark: 4.2s) -\item This should create a file |yourfile-robExt-compile-missing-figures.sh|, with each line containing a compilation command like this (the command should be compatible with Windows, Mac and Linux, if not, let me know on github): -\begin{verbatim} -cd robustExternalize/ && pdflatex -halt-on-error "robExt-4A806941E86C6B657A0CB3D160CFFF3E.tex -\end{verbatim} - \textbf{IMPORTANT}: \LaTeX{} might render \textbf{multiple times} the same picture if it is inside some particular environments (align, tables\dots), so this file can contain duplicates if you inserted cached data inside. To avoid running the same command twice, \textbf{be sure to remove duplicates!} On Linux, this can be done by piping the file into |sort| and |uniq| (if your files can depend on previously compiled elements as explained in \cref{sec:passOutputToOtherTemplate}, you should additionally preserve \mylink{https://unix.stackexchange.com/questions/194780/remove-duplicate-lines-while-keeping-the-order-of-the-lines}{the order of the lines}). Since this is anyway a rather advanced use case, we will not consider that case for simplicity. -\item Run all these command in parallel, for instance on Linux you can install |parallel|, and run the following command (|--bar| displays a progress bar): -\begin{verbatim} -cat yourfile-robExt-compile-missing-figures.sh | sort | uniq | parallel --bar --jobs '200%' -\end{verbatim} -(in my benchmark, it ran during 52s, so 1.6x faster than the original compilation) -\item Recompile the original document with |pdflatex yourfile.tex| -\end{itemize} -In my benchmark, the total time is 2x4.2s + 52s = 60s, so 1.5 times faster than a normal command (of course, this depends a lot on the preamble of the file that you compile, since the loading time of the file is the main bottleneck for the first compilation; the advantage here is that it is easy to include only the necessary things in the preamble of cached pictures, possibly creating different presets if it is easier to manage it this way). +\paragraph{Can I compile pictures in parallel to speed up notably the first run?} + +The first run might be particularly long to compile since many elements are still not cached and re-starting \LaTeX{} on each picture can take quite some time. If you have v2.1 or above, and if you have xargs installed (installed by default on most linux, on Windows you need to install the lightweight GNU On Windows (Gow) \url{https://github.com/bmatzelle/gow}), you can simply compile all images in parallel by typing at the beginning of the file: +\begin{codeexample}[code only] +\robExtConfigure{ + compile in parallel +} +\end{codeexample} +In my benchmark, I observed a compilation 1.5x faster than a normal compilation without any caching involved. We provide more options in \cref{sec:compileParallel}. \paragraph{Can I use version-control to keep the cached files in my repository?} Sure, each cached figure is stored in a few files (typically one pdf and one \LaTeX{} file, plus the source) having the same prefix (the hash), avoiding collision between runs. Just commit these files and you are good to go. \paragraph{Can I deal with baseline position and overlays ?} Yes, the depth of the box is automatically computed and used to include the figure by default, and additional margin is added to the image (30cm by default) to allow overlay text. -\paragraph{How is purity enforced?} Purity is the property that if you remove the cached files and recompile your document, you should end-up with the same output. To enforce purity, we compute the hash of the final program, including the compilation command and the dependency files used for instance in |\input{include.tex}| (unless you prefer not to, for instance to keep parts of the process impure for efficiency reasons), and put the code in a file named based on this hash. Then we compile it if it has not been used before, and include the output. Changing a single character in the file, the tracked dependencies, or the compilation command will lead to a new hash, and therefore to a new generated picture. +\paragraph{How is purity enforced?} Purity is the property that if you remove the cached files and recompile your document, you should end-up with the same output. To enforce purity, we compute the hash of the final program, including the compilation command, the dependency files used for instance in |\input{include.tex}| (unless you prefer not to, for instance to keep parts of the process impure for efficiency reasons), the forwarded macros, and put the code in a file named based on this hash. Then we compile it if it has not been used before, and include the output. Changing a single character in the file, the tracked dependencies, or the compilation command will lead to a new hash, and therefore to a new generated picture. \paragraph{What if I don't want purity for all files?} If you do not want your files to be recompiled if you modify a given file, then just do not add this file to the list of dependencies. @@ -400,21 +433,25 @@ In my benchmark, the total time is 2x4.2s + 52s = 60s, so 1.5 times faster than Pros of \url{https://github.com/sasozivanovic/memoize}: \begin{itemize} -\item The above library is likely to be quicker on the first run since it packs everything in a single file (so you save the time to run latex for each picture). On the other hand, we can load in the preamble exactly what is needed for the picture (we do not load all the preamble of the main file), so our startup time is not huge (it adds 1s per picture in my tests when using the |zx-calculus| library), and you can still either run it in parallel to save time (as far as I know this is not possible with |memoize|), or commit the cached file to help with recompiling the document elsewhere. -\item It seems to be easier to setup as they do not need to specify the preamble of the file to compile (the preamble of the main file is used), and tikz pictures are automatically memoized (we provide |\cacheTikz| for that, but you still need to specify the preamble once). -\item Even if it does not officially support contexts, context might be automatically transmitted (while we need to explicitly pass the context). However, this can break purity (see below). +\item The above library might be easier to setup as they do not need to specify the preamble of the file to compile (the preamble of the main file is used), and tikz pictures are automatically memoized (we provide |\cacheTikz| for that, but you still need to specify the preamble once). +\item This library allows to compile multiple pictures while loading \LaTeX{} only once, saving the time to start \LaTeX{} for each picture. However, it is not clear that it brings a faster compilation time compared to our library for multiple reasons: + \begin{itemize} + \item While during the first run all pictures are compiled with a single \LaTeX{} instance\footnote{We could also do something similar using some scripting to merge multiple cached files together, but this is not yet implemented.} (and inserted at the beginning of the document), they still need to extract each picture in a separate pdf file to include it at the right place in the document. This is done by default using |pdflatex|, which means that you need to call |pdflatex| once per picture, mitigating the advantage of compiling all pictures at once. Note that they also provide a python script to separate the pictures more efficiently, but this needs further manual interventions. + \item We provide an option |compile in parallel| to compile all images in parallel, and the time is in my benchmark 1.5x faster than a normal compilation without any caching. Using this option, we should therefore compile faster than |memoized| that is not able to run jobs in parallel. + \end{itemize} +\item Even if it does not officially support contexts, context might be automatically transmitted (while we need to do a bit of work to manually or semi-automatically define the context needed to compile a picture). However, this can break purity (see below). \end{itemize} Cons of \url{https://github.com/sasozivanovic/memoize}: \begin{itemize} -\item The purity is not enforced so strongly since all images are in the same file. Notably, the hash only depends on the picture, but not on its context. So for instance if you define |\def\mycolor{blue}| before the picture, and use |\mycolor| inside the picture, if you change later the color to, say, |\def\mycolor{red}|, the picture will not be recompiled (so cleaning the cache and recompiling would produce a different result). In our case, the purity is always strictly enforced (unless you choose not to) since the context must be explicitly passed to the compiled file. +\item The purity is not enforced so strongly since all images are in the same file. Notably, the hash only depends on the picture, but not on its context. So for instance if you define |\def\mycolor{blue}| before the picture, and use |\mycolor| inside the picture, if you change later the color to, say, |\def\mycolor{red}|, the picture will not be recompiled (so cleaning the cache and recompiling would produce a different result). In our case, the purity is always strictly enforced (unless you choose not to) since the context must be passed (explicitly or semi-automatically) to the compiled file. \item As a result, the above library has poor support for contexts (in our case, you can easily, for instance, make a picture depend on the current page, and recompile the picture only if the current page changes: you can also do it the other way, and change some counters, say, depending on the cached file). -\item The above library only focuses on \LaTeX{} while our library works for any language -\item The above library can only produce pdf formats, while we can generate any format (text, tex, jpg\dots{} and even videos that I include in my beamer presentations\dots{} but it is another question). +\item The above library only focuses on \LaTeX{} while our library works for any language. +\item The above library can only cache in pdf format, while we can generate any format (text, tex, jpg\dots{} and even videos that I include in my beamer presentations\dots{} but it is another topic!). \item We have an arguably more complete documentation. \end{itemize} -Note that |remember picture| is not working in both libraries. +Note that |remember picture| is not supported in both libraries, but we can provide multiple methods to disable externalizations when |remember picture| is used. \section{Tutorial} @@ -454,17 +491,19 @@ If you forget/do not want to enable |-shell-escape|, this will give you this kin If you only care about \tikzname's picture, you have 3 options: \begin{enumerate} \item Call once |\cacheTikz| that will redefine |tikzpicture| and |\tikz| to use our library (if you use this solution, make sure to read how to disable externalization (\cref{sec:disableExternalization}) as we do not support for instance |remember picture|). Then, configure the default preamble for cached files as explained below by extending the |tikz| or |tikzpicture| presets (that first loads |tikz|). -\item Use the generic commands that allows you to wrap an arbitrary environment or command: +\item Use the generic commands that allows you to wrap an arbitrary environment: \begin{verbatim} -% name environment ---v v----- preset options -\robExtCacheEnvironment{tikzpicture}{tikzpicture} +% name environment ---v v----- preset options +\cacheEnvironment{tikzpicture}{tikzpicture} \end{verbatim} - Note that we do recommend to use |\robExtCacheCommand| for the macro version |\tikz| since |\tikz| has a quite complicated parsing strategy. |\cacheTikz| takes care of this already. + Note that you can also wrap commands (that you call with |\foo| instead of |\begin{foo}...\end{foo}|) using this:\\ + |\cacheCommand{yourcommand}[O{default val}m]{your preset options}|\\ + (|O{default val}m| means that the command accepts one optional argument with default value |default val| and one mandatory argument) but \textbf{we do recommend} to use |\cacheCommand| to cache |\tikz| since |\tikz| has a quite complicated parsing strategy (e.g.\ you can write |\tikz[options] \node{foo};| which has no mandatory argument enclosed in |{}|). |\cacheTikz| takes care of this already and caches both the environment |\begin{tikzpicture}| and the macro |\tikz|. \item Use |tikzpictureC| instead of |tikzpicture| (this is mostly done to easily convert existing code to this library, but works only for |tikz| pictures). \item Use the more general |CacheMe| environment, that can cache \tikzname, \LaTeX{}, python, and much more. \end{enumerate} -These 3 options are illustrated below (note that the newly defined |tikzpicture| and |tikzpictureC| accept a second optional argument that contains the options to pass to |CacheMe| after loading the |tikzpicture| preset, that loads itself the |tikz| preset first): +These 4 options are illustrated below (note that all commands accept a first optional argument enclosed in |<...>| that contains the options to pass to |CacheMe| after loading the |tikzpicture| preset, that loads itself the |tikz| preset first): Option 1: \begin{codeexample}[width=0pt] @@ -475,6 +514,10 @@ Option 1: I am a cached picture: \begin{tikzpicture}[baseline=(A.base)] \node[draw,rounded corners,fill=pink!60](A){Hello World!}; +\end{tikzpicture}. % + +I am a non-cached picture: \begin{tikzpicture}<disable externalization>[baseline=(A.base)] + \node[draw,rounded corners,fill=pink!60](A){Hello World!}; \end{tikzpicture}. \end{codeexample} @@ -569,6 +612,8 @@ Note that if you use the |tikzpictureC| or |tikzpicture| syntax, you want to add \subsection{Dependencies}\label{sec:dependencies} +\textbf{Note: dependencies can sometimes be advantageously replaced with auto-forwarding of arguments, cf.\ \cref{sec:forwardIntro}.} + It might be handy to have a file that is loaded in both the main document and in the cached pictures. For instance, if you have a file |common_inputs.tex| that you want to input in both the main file and in the cached files, that contains, say:\\ |\def\myValueDefinedInCommonInputs{42}|\\ then you can add it as a dependency this way (here we use the |latex| preset that does not wrap the code inside a |tikzpicture| only to illustrate that we can also cache things that are not generated by tikz): @@ -658,8 +703,25 @@ Which gives you { \todo{Check how to use cacheCommand and cacheEnvironment} } +You can also disable externalization on elements that contain a specific string/regex. For instance, you can disable externalization on all elements containing |remember picture|: +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{ + add to preset={tikz}{ + if matches={remember picture}{disable externalization}, + }, +} +\begin{tikzpicture}[remember picture] + \node[fill=green](my node){Point to me}; +\end{tikzpicture} % +Some text % +\begin{tikzpicture}[overlay, remember picture] + \draw[->] (0,0) to[bend left] (my node); +\end{tikzpicture} +\end{codeexample} + -\subsection{Feeding data from the main document to the picture} +\subsection{Feeding data from the main document to the cached documents}\label{sec:forwardIntro} You can feed data from the main document to the cached file using placeholders, since |set placeholder eval={__foo__}{\bar}| will evaluate |\bar| and put the result in |__foo__|. For instance, if the picture depends on the current page, you can do: @@ -669,6 +731,64 @@ You can feed data from the main document to the cached file using placeholders, \end{tikzpictureC} \end{codeexample} +However, since v2.1, we also provide a method to specifically export a counter using |forward counter=my counter| and a macro using |forward=\macroToExport|, possibly by evaluating first using |forward eval=\macroToEval|: +\begin{codeexample}[width=0pt] +\cacheTikz +\NewDocumentCommand{\MyNode}{O{}m}{\node[rounded corners,fill=red,#1]{#2};} +\begin{tikzpicture}<forward=\MyNode> + \MyNode{Alice} + \MyNode[xshift=2cm]{Bob} +\end{tikzpicture} +\end{codeexample} + +To avoid manually writing |forward=...| for each picture, we can instead load |auto forward| and define the macro for instance using |\newcommandAutoForward| (we also provide alternatives for xparse and def-based definitions): +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{add to preset={tikz}{auto forward}} + +\newcommandAutoForward{\MyNode}[2][draw,thick]{\node[rounded corners,fill=red,#1]{#2};} +\newcommandAutoForward{\MyGreenNode}[2][draw,thick]{\node[rounded corners,fill=green,#1]{#2};} + +\begin{tikzpicture} + \MyNode{Recompiled only if MyNode is changed} + \MyNode[xshift=8cm]{but not if the (unused) MyGreenNode is changed.} +\end{tikzpicture}\\ + +\begin{tikzpicture} + \MyGreenNode{Recompiled only if MyGreenNode is changed} + \MyGreenNode[xshift=8cm]{but not if the (unused) MyNode is changed.} +\end{tikzpicture} +\end{codeexample} + +If a macro depends on another macro/package, it is possible to load any additional style like using the last optional argument of |\newcommandAutoForward|: +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{add to preset={tikz}{auto forward}} + +\newcommandAutoForward{\MyName}{Alice} +\newcommandAutoForward{\MyNode}[2][draw,thick]{ + \node[rounded corners,fill=red,#1]{\ding{164} \MyName: #2}; +}[forward=\MyName, add to preamble={\usepackage{pifont}}] + +\begin{tikzpicture} + \MyNode{Recompiled if MyNode or MyName are changed} +\end{tikzpicture} +\end{codeexample} + +If instead of forwarding the macro you want to, say, load a package or any other style, you should use instead |\robExtConfigIfMacroPresent|: +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{add to preset={tikz}{auto forward}} + +\robExtConfigIfMacroPresent{\ding}{add to preamble={\usepackage{pifont}}} + +\begin{tikzpicture} + \node[fill=green, circle]{\ding{164}}; +\end{tikzpicture} +\end{codeexample} + +For more details and functions, see \cref{sec:forward}. + \subsection{Defining macros} You can define macros in the preamble: @@ -700,6 +820,21 @@ We compute this data that is long to compute: We computed the cached value \myLongResult. \end{codeexample} +\subsection{Compile in parallel} +Make sure to have xargs installed (installed by default on most linux, on Windows you need to install the lightweight GNU On Windows (Gow) \url{https://github.com/bmatzelle/gow}), and type in your preamble: +\begin{codeexample}[code only] +\robExtConfigure{ + compile in parallel +} +\end{codeexample} +to compile all figures in parallel (you need to compile your document twice). Your document should build significantly faster, possibly faster than a normal run. If you want to compile in parallel +only if you have more than, say, 5 new elements to cache, do: +\begin{codeexample}[code only] +\robExtConfigure{ + compile in parallel after=5 +} +\end{codeexample} + \subsection{Compile a preset}\label{sec:compilePreset} Long story short: you can compile even faster (around 1.5x in our tests) by compiling presets, but beware that you will not be able to modify the placeholders except |add to preamble| with the default compiler we provide: @@ -928,7 +1063,6 @@ print_file.close() custom include command={ % Useful to replace __MY_TITLE__: \evalPlaceholder{ - % \verbatiminput{\robExtAddCachePathAndName{\robExtFinalHash-code.txt}} \begin{tcolorbox}[colback=red!5!white,colframe=red!75!black,title=__MY_TITLE__] \lstinputlisting[frame=single, breakindent=.5\textwidth, @@ -999,7 +1133,6 @@ print_file.close() custom include command={ % Useful to replace __MY_TITLE__: \evalPlaceholder{ - % \verbatiminput{\robExtAddCachePathAndName{\robExtFinalHash-code.txt}} \begin{tcolorbox}[colback=red!5!white,colframe=red!75!black,title=__MY_TITLE__] % \noindent Code: \lstinputlisting[frame=single,breakindent=.5\textwidth,frame=single,breaklines=true,style=mypython]{\robExtAddCachePathAndName{\robExtFinalHash-code.txt}} @@ -1079,8 +1212,23 @@ plt.savefig("__ROBEXT_OUTPUT_PDF__") \end{CacheMeNoContent}} \end{codeAndResult} +\subsection{Force to recompile or remove a cached item} + +If you want to recompile a file (e.g. an untracked dependency was changed\dots{} and you do not want to track it to avoid recompilation when you change a single line in this file), you can use the |recompile| style since v2.1: + +\begin{codeexample}[vbox] +\cacheEnvironment{tikzcd}{tikz, add to preamble={\usepackage{tikz-cd}}} +\begin{tikzcd}<recompile> + A \rar & B +\end{tikzcd} +\end{codeexample} + +Note that this assumes that your compilation command is idempotent (so running it twice is like running it once) since the aux files are not cached. If you want to clean aux files or if you run an older version, see the documentation of |recompile| in \cref{sec:configureCompilationCommand}. + \section{Documentation} +Before starting this documentation, note that all commands are prefixed with |robExt| and all environments are prefixed with |RobExt|, but we also often define aliases without this prefix. The user is free to use any version, but we recommend to use the non-prefixed version unless a clash with a package forbids you from using it. In the following, we will only print the non-prefixed name when it exists. Note also that we follow the convention that environment names start with an upper case letter while commands start with a lower case letter. + \subsection{How it works} This library must be able to generate 3 elements for any cached content: @@ -1473,11 +1621,15 @@ Note that the if the placeholder content contains at the end the placeholder nam \end{codeexample} The variation |\setPlaceholderRecReplaceFromList| allows us to specify a subset of placeholder that will be allowed to be expanded, and is an alias for |\getPlaceholderInResultReplaceFromList| (except that the optional argument is mandatory): \begin{codeexample}[width=0pt,vbox] -\placeholderFromContent{__MY_PLACEHOLDER__}{Hello __NAME1__, __NAME2__ and __NAME3__, I am a template $\delta_n$.} +\placeholderFromContent{__MY_PLACEHOLDER__}{Hello __NAME1__, __NAME2__ and __NAME3__, + I am a template $\delta_n$. +} \placeholderFromContent{__NAME1__}{Alice} \placeholderFromContent{__NAME2__}{Bob} \placeholderFromContent{__NAME3__}{Charlie} -\setPlaceholderRecReplaceFromList{__MY_PLACEHOLDER__,__NAME2__,__NAME3__}{__OTHER_PLACEHOLDER__}{Here we go: __MY_PLACEHOLDER__} +\setPlaceholderRecReplaceFromList{__MY_PLACEHOLDER__,__NAME2__,__NAME3__}{__OTHER_PLACEHOLDER__}{ + Here we go: __MY_PLACEHOLDER__ +} \printImportedPlaceholdersExceptDefaults \end{codeexample} Note that the star and no import version does NOT import the placeholder it the main group (unless you try to optimize the compilation time you should not need it, but see \cref{sec:importSystem} for details). @@ -1549,8 +1701,8 @@ We provide a list of placeholders that are useful to escape parts of the strings \end{pgfmanualentry} \begin{pgfmanualentry} - \extractcommand\robExtPlaceholderReplaceInplace\marg{placeholder}\marg{from}\marg{to}\@@ - \extractcommand\robExtPlaceholderReplaceInplaceEval\marg{placeholder}\marg{from}\marg{to}\@@ + \extractcommand\placeholderReplaceInplace\marg{placeholder}\marg{from}\marg{to}\@@ + \extractcommand\placeholderReplaceInplaceEval\marg{placeholder}\marg{from}\marg{to}\@@ \makeatletter% \def\extrakeytext{style, } \extractkey/robExt/placeholder replace in place=\marg{placeholder}\marg{from}\marg{to}\@nil @@ -2192,6 +2344,55 @@ plt.ylabel('Number of futurestud.io Tutorials') By default, we compile cached documents only if |-shell-escape| is enabled. However, if the user allowed |pdflatex| (needed to compile latex documents), |cd| (not needed when using |no cache folder|), and |mkdir| (not needed when using |no cache folder| or if the cache folder that defaults to |robustExternalize| is already created) to run in restricted mode (so without enabling |-shell-escape|), then there is no need for |-shell-escape|. In that case, set |force compilation| and this library will compile even if |-shell-escape| is disabled. \end{pgfmanualentry} +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/recompile\@nil + \extractkey/robExt/do not recompile\@nil + \makeatother% + \pgfmanualbody + (new to v2.1) |recompile| will force a compilation even if the image is already cached (mostly useful if you made a change to a non-tracked dependency). Note that since this library does not want to remove any file on the hard drive (we do not want to risk to remove important files in case of a bug), this command will NOT clear the auxiliary files already present, so for maximum purity, ensure that your compiling command is idempotent (so that compiling twice gives the same outcome as compiling once). + +\begin{codeexample}[vbox] +\cacheEnvironment{tikzcd}{tikz, add to preamble={\usepackage{tikz-cd}}} +\begin{tikzcd}<recompile> + A \rar & B +\end{tikzcd} +\end{codeexample} + +If you want to actually remove the aux files, you can either change the compilation command, or add something like this to remove the file (note that it will prompt a message before running the actual command), using the hook |robust-externalize/just-before-writing-files| that we introduced in v2.1: +\begin{codeexample}[code only] +\cacheTikz + +\robExtConfigure{ + clean and recompile/.code={% + \AddToHook{robust-externalize/just-before-writing-files}{% + \edef\command{rm -f "\robExtAddCachePathAndName{\robExtFinalHash.pdf}" && % + rm -f "\robExtAddCachePathAndName{\robExtFinalHash.aux}"}% + \message{You will run the command ``\command''}% + \message{Press X if you do NOT want to run it, otherwise press ENTER.}% + \show\def% this is just used to wait a user input + \immediate\write18{\command}% + }% + },% +} + +\begin{tikzpicture}<clean and recompile> + \node[fill=red] {B}; +\end{tikzpicture} +\end{codeexample} +Note that if you do not want to run a command from \LaTeX{}, or if you are on a version smaller than 2.1, you can also simply print the name of the file in the pdf and remove it manually: + +\begin{codeexample}[width=0cm,vbox] +\begin{CacheMe}{tikzpicture, name output=filetodelete}[baseline=(A.base)] + \node[draw,rounded corners,fill=pink!60](A){Hello World!}; +\end{CacheMe}\\ + +You want to delete the file \texttt{\filetodeleteInCache.pdf}. % +\end{codeexample} + +\end{pgfmanualentry} + \subsubsection{Options to configure the inclusion command} @@ -2312,6 +2513,7 @@ You might want (sometimes or always) to disable externalization, for instance to \makeatletter% \def\extrakeytext{style, } \extractkey/robExt/disable externalization\@nil + \extractkey/robExt/de\@nil \extractkey/robExt/disable externalization now\@nil \extractkey/robExt/enable externalization\@nil \makeatother% @@ -2353,13 +2555,31 @@ You might want (sometimes or always) to disable externalization, for instance to \robExtConfigure{disable externalization now} \tikz\node[fill=green]{The short version of tikz can be confusing.}; } - \end{codeexample} + \end{codeexample} + Since v2.1 you can also automatically disable externalization if the input contains a given string, for instance |remember picture| (that is not cachable with this library): +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{ + add to preset={tikzpicture}{ + % \b means "word boundary", and spaces must be escaped + if matches={remember picture}{disable externalization}, + }, +} +\begin{tikzpicture}[remember picture] + \node[fill=green](my original node){Point to me}; +\end{tikzpicture} % +Some text % +\begin{tikzpicture}[overlay, remember picture] + \draw[->] (0,0) to[bend left] (my original node); +\end{tikzpicture} +\end{codeexample} \end{pgfmanualentry} \begin{pgfmanualentry} \makeatletter% \def\extrakeytext{style, } \extractkey/robExt/command if no externalization\@nil + \extractkey/robExt/command if externalization\@nil \makeatother% \pgfmanualbody You can easily change the command to run if externalization is disabled using by setting the \textbf{.code} of this key. By default, it is configured as:\\ @@ -2368,7 +2588,8 @@ command if no externalization/.code={% \robExtDisableTikzpictureOverwrite\evalPlaceholder{__ROBEXT_MAIN_CONTENT__}% } \end{verbatim} -Unless you know what you are doing, you should include |\robExtDisableTikzpictureOverwrite| as it is often necessary to avoid infinite recursion when externalization is disabled and the original command has been replaced with a cached version (for instance done by |\cacheTikz|). Note that if you write your own style, try to modify |__ROBEXT_MAIN_CONTENT__| so that it can be included as-it in a document: this way you do not need to change this command. + Unless you know what you are doing, you should include |\robExtDisableTikzpictureOverwrite| as it is often necessary to avoid infinite recursion when externalization is disabled and the original command has been replaced with a cached version (for instance done by |\cacheTikz|). Note that if you write your own style, try to modify |__ROBEXT_MAIN_CONTENT__| so that it can be included as-it in a document: this way you do not need to change this command. + You can also use |command if externalization/.append code={...}| if you want to run a code only if externalization is enabled (this is used for instance by the |forward| functionality). \end{pgfmanualentry} \begin{pgfmanualentry} @@ -2419,7 +2640,7 @@ uname -srv > "${outputTxt}" By doing |execute after each externalization={some code}|, you will run some code after the externalization. This might be practical for instance to update a counter (e.g. the number of pages\dots) based on the result of the compiled file. \end{pgfmanualentry} -\subsubsection{Dependencies} +\subsubsection{Dependencies}\label{sec:dependenciesDoc} In order to enforce reproducibility, you should tell what are the files that your code depends on, by adding this file as a dependency. This has the advantage that if this file is changed, your code is automatically recompiled. On the other hand, you might not want this behavior (e.g. if this file often changes in a non-important way): in that case, just don't add the file as a dependency (but keep that in mind as you might not be able to recompile your file if you clear the cache if you introduced breaking changes). @@ -2444,6 +2665,305 @@ In order to enforce reproducibility, you should tell what are the files that you Note that the placeholder |__ROBEXT_WAY_BACK__| contains the path from the cache folder (containing the |.tex| that will be cached) to the root folder.\\ This way, you can easily input files contained in the root folder. \end{pgfmanualentry} +\subsubsection{Forward macros}\label{sec:forward} + +(since v2.1) + +You can also see the tutorial (\cref{sec:forwardIntro}) for some examples. So far, we have seen multiple approaches to use a macro in both the main document and in cached pictures, having different advantages: +\begin{itemize} +\item if we define the macro in a file say |common_inputs.tex|, if we include that file in both files \textbf{and} if we add it to the list of dependencies (\cref{sec:dependenciesDoc}), then you will have great purity (if |common_inputs.tex| changes, all files are recompiled), but this might be a problem if you often change the file |common_inputs.tex| since you will need to recompile all pictures every time you add a new macro +\item If you follow the same procedure, but \textbf{without} adding the file in the list of dependencies, then you will save compilation time (no need to recompile everything if you add a macro in |common_inputs.tex|), but you will lose purity (if you change the definition of a macro in that file, you will need to manually specify the list of figures to recompile, e.g. using |recompile|). +\item You can also create multiple files like +\begin{verbatim} +common_inputs_functionality_A.tex +\end{verbatim} + and +\begin{verbatim} +common_inputs_functionality_B.tex +\end{verbatim} + and use the first solution (maybe by creating a preset that automatically calls |add to preamble| and |add dependencies|), and only input the appropriate file depending on the set of macro that you need. This gives a tradeoff of above approaches (good purity, with less recompilation), but you will still compile often if you put many macros in a single file (and nobody wants to create one macro per file!). +\end{itemize} + +In this section, we provide a third solution that tries to solve the above issue (purity without frequent recompilations), by allowing the user to forward only a specific macro, with something like |forward=\myMacro|. In order to avoid to manually forward all macros used for each picture, we also provide some functions to automatically detect the macros to forward. + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/forward=macro to forward\@nil + \extractkey/robExt/fw=macro to forward\@nil + \extractkey/robExt/forward at letter=macro to forward\@nil + \makeatother% + \pgfmanualbody + |forward| (alias |fw|) will forward the definition of a macro to the picture if externalization is enabled: +\begin{codeexample}[width=0pt] +\cacheTikz +\def\myName{Alice} +\begin{tikzpicture}<forward=\myName> + \node[fill=red]{\myName}; +\end{tikzpicture} +\end{codeexample} +Note that it works for macro defined with |\def|, |\newcommand| (or alike) and |\NewDocumentCommand| (or other xparse commands). For instance with xparse: +\begin{codeexample}[width=0pt] +\cacheTikz +\NewDocumentCommand{\MyNode}{O{}m}{\node[rounded corners,fill=red,#1]{#2};} +\begin{tikzpicture}<forward=\MyNode> + \MyNode{Alice} + \MyNode[xshift=2cm]{Bob} +\end{tikzpicture} +\end{codeexample} +and with |\newcommand|: +\begin{codeexample}[width=0pt] +\cacheTikz +\newcommand{\MyNodeWithNewCommand}[2][draw,thick]{\node[rounded corners,fill=red,#1]{#2};} +\begin{tikzpicture}<forward=\MyNodeWithNewCommand> + \MyNodeWithNewCommand{Alice} + \MyNodeWithNewCommand[xshift=2cm]{Bob} +\end{tikzpicture} +\end{codeexample} +Note that this might not work with quite involved macros. Notably, if the macro is defined by calling other macros, you need to also forward these macros. +\begin{codeexample}[width=0pt] +\cacheTikz +\def\myName{Alice} +\NewDocumentCommand{\MyNodeWithInnerMacros}{O{}m}{\node[rounded corners,fill=red,#1]{\myName: #2};} +\begin{tikzpicture}<forward=\MyNodeWithInnerMacros,forward=\myName> + \MyNodeWithInnerMacros{she is tall} +\end{tikzpicture} +\end{codeexample} +Really involved macros (e.g.\ defined inside libraries) might call many other macros, possibly using the @ letter (used only in module code). If you REALLY want, you can forward such macros if you forward \textbf{all} macros that are used internally, using |forward at letter| that will automatically wrap the macro inside |\makeatletter...\makeatother|, but you should certainly use |add to preamble={\usepackage{yourlib}}| if you need to do something like that: +\begin{codeexample}[width=0pt] +\cacheTikz +\makeatletter +\def\module@name{Alice} +\newcommand{\MyNodeWithAtCommands}[2][draw,thick]{\node[rounded corners,fill=red,#1]{\module@name};} +\begin{tikzpicture}<forward at letter=\MyNodeWithAtCommands,forward at letter=\module@name> + \MyNodeWithAtCommands{Alice} +\end{tikzpicture} +\makeatother +\end{codeexample} +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/forward eval=macro to eval and forward\@nil + \makeatother% + \pgfmanualbody + Evaluate a macro, and defines it using this new value when compiling the function. This might be useful for instance if your macro depends on other macros that you do not want to export: +\begin{codeexample}[width=0pt] +\def\name{Alice} +\def\fullName{\name} +\begin{tikzpictureC}<forward eval=\fullName> + \node[rounded corners, fill=red]{\fullName.}; +\end{tikzpictureC} +\end{codeexample} +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/forward counter=counter to forward\@nil + \makeatother% + \pgfmanualbody + Forward a counter: +\begin{codeexample}[width=0pt] +\begin{tikzpictureC}<forward counter=page> + \node[rounded corners, fill=red]{The current page is \thepage.}; +\end{tikzpictureC} +\end{codeexample} +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/forward color=color to forward\@nil + \extractkey/robExt/auto forward color=\marg{preset}\marg{color to forward}\@nil + \makeatother% + \pgfmanualbody + |forward color| forwards a color (defined with xcolor). It works with colors defined with |\definecolor|: +\begin{codeexample}[width=0pt] +\definecolor{myred}{HTML}{F01122} +\begin{tikzpictureC}<forward color=myred> + \node[fill=myred]{A}; +\end{tikzpictureC} +\end{codeexample} +but also with colors defined with |\colorlet|: +\begin{codeexample}[width=0pt] +\definecolor{myred}{HTML}{F01122} +\colorlet{myviolet}{blue!50!myred} +\begin{tikzpictureC}<forward color=myviolet> + \node[fill=myviolet,yshift=-1cm]{B}; +\end{tikzpictureC} +\end{codeexample} +Note that using |if matches|, you can also automatically forward colors: +\begin{codeexample}[width=0pt] +\definecolor{myred}{HTML}{F01122} +\colorlet{myviolet}{blue!50!myred} +\colorlet{mypink}{pink!90!orange} +\robExtConfigure{ + add to preset={tikz}{ + if matches={mypink}{forward color=mypink}, + }, +} +\begin{tikzpictureC} + \node[fill=mypink,yshift=-1cm]{B}; +\end{tikzpictureC} +\end{codeexample} +but since this is a bit long to type, we provide a shortcut: +\begin{codeexample}[width=0pt] +\definecolor{myred}{HTML}{F01122} +\colorlet{myviolet}{blue!50!myred} +\colorlet{mypink}{pink!90!orange} +\robExtConfigure{ + auto forward color={tikz}{mypink}, +} +\begin{tikzpictureC} + \node[fill=mypink,yshift=-1cm]{B}; +\end{tikzpictureC} +\end{codeexample} +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/auto forward\@nil + \makeatother% + \pgfmanualbody + Enable automatic forwarding of macros marked with |*AutoForward| commands or |\configIfMacroPresent|. This command naively extracts all used macro via a simple regex match and forwards them/execute the style configured in |\configIfMacroPresent| if it exists. +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{add to preset={tikz}{auto forward}} + +\newcommandAutoForward{\MyNode}[2][draw,thick]{\node[rounded corners,fill=red,#1]{#2};} +\newcommandAutoForward{\MyGreenNode}[2][draw,thick]{\node[rounded corners,fill=green,#1]{#2};} + +\begin{tikzpicture} + \MyNode{Recompiled only if MyNode is changed} + \MyNode[xshift=8cm]{but not if the (unused) MyGreenNode is changed.} +\end{tikzpicture}\\ + +\begin{tikzpicture} + \MyGreenNode{Recompiled only if MyGreenNode is changed} + \MyGreenNode[xshift=8cm]{but not if the (unused) MyNode is changed.} +\end{tikzpicture} +\end{codeexample} +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \extractcommand\configIfMacroPresent\marg{macro}\marg{style to run if macro is present}\@@ + \pgfmanualbody + Runs the corresponding style if the macro |macro| is present (make sure to enable |auto forward|). +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{add to preset={tikz}{auto forward}} + +\configIfMacroPresent{\ding}{add to preamble={\usepackage{pifont}}} + +\begin{tikzpicture} + \node[fill=green, circle]{\ding{164}}; +\end{tikzpicture} +\end{codeexample} +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \extractcommand\newcommandAutoForward\marg{macro}\opt{\oarg{nb args}}\opt{\oarg{optional default value}}\marg{definition}\opt{\oarg{additional style}}\@@ + \extractcommand\renewcommandAutoForward\marg{macro}\opt{\oarg{nb args}}\opt{\oarg{optional default value}}\marg{definition}\opt{\oarg{additional style}}\@@ + \extractcommand\providecommandAutoForward\marg{macro}\opt{\oarg{nb args}}\opt{\oarg{optional default value}}\marg{definition}\opt{\oarg{additional style}}\@@ + \extractcommand\NewDocumentCommandAutoForward\marg{macro}\marg{arg spec}\opt{\oarg{additional style}}\marg{definition}\@@ + \extractcommand\RenewDocumentCommandAutoForward\marg{macro}\marg{arg spec}\opt{\oarg{additional style}}\marg{definition}\@@ + \extractcommand\ProvideDocumentCommandAutoForward\marg{macro}\marg{arg spec}\opt{\oarg{additional style}}\marg{definition}\@@ + \extractcommand\DeclareDocumentCommandAutoForward\marg{macro}\marg{arg spec}\opt{\oarg{additional style}}\marg{definition}\@@ + \extractcommand\defAutoForward\marg{macro}\opt{\oarg{def-style arg spec}}\marg{macro}\opt{\oarg{additional style}}\@@ + \pgfmanualbody +\begin{verbatim} +\newcommandAutoForward{\macro}[nb args][default value]{def}[STYLE] +\end{verbatim} + is like +\begin{verbatim} +\newcommand{\macro}[nb args][default value]{def} +\end{verbatim} + but it additionally runs +\begin{verbatim} +\configIfMacroPresent{\macro}{forward=\macro,STYLE} +\end{verbatim} + to automatically forward the macro if it is used. This works similarly for other commands (note that for |\defAutoForward|, you need to put the arguments in a bracket, like |\defAutoForward\myMacro[#1 and #2]{Hey #1, #2}| for |\def\myMacro#1 and #2{Hey #1, #2}|). +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/load auto forward macro config=\marg{macro}\@nil + \makeatother% + \pgfmanualbody + Like |forward| except that it loads the configuration that would have been loaded if the macro was present in the file. This is mostly useful to say that a macro depends on the style of another macro without copy/pasting the style of the second macro inside the first one. +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{add to preset={tikz}{auto forward}} + +\newcommandAutoForward{\MyName}{\ding{164} Alice}[add to preamble={\usepackage{pifont}}] +\newcommandAutoForward{\MyNode}[2][draw,thick]{\node[rounded corners,fill=green,#1]{\MyName: #2};}[ + load auto forward macro config=\MyName +] + +\begin{tikzpicture} + \MyNode{Recompiled only if MyNode is changed} + \MyNode[yshift=-1cm]{or if MyName is changed} +\end{tikzpicture} +\end{codeexample} +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/if matches=\marg{string}\marg{style to apply}\@nil + \extractkey/robExt/if matches regex=\marg{latex3 regex}\marg{style to apply}\@nil + \makeatother% + \pgfmanualbody + Apply the corresponding style if the string (resp.\ regex in \LaTeX{}3 format) matches is matched in the content. The regex version can be more expressive, but is wayyy slower. +\begin{codeexample}[vbox] +\robExtConfigure{ + add to preset={my python}{ + python print code and result, + % \b is for word border, and ( needs to be escaped in regex + if matches={cos(}{add import={from math import cos}}, + if matches={sin(}{add import={from math import sin}}, + }, +} +\begin{CacheMeCode}{my python} +print(cos(1)+sin(2)) +\end{CacheMeCode} +\end{codeexample} +With regex: +\begin{codeexample}[vbox] +\robExtConfigure{ + add to preset={my python}{ + python print code and result, + % \b is for word border, and ( needs to be escaped in regex + if matches regex={\b cos\(}{add import={from math import cos}}, + if matches regex={\b sin\(}{add import={from math import sin}}, + }, +} +\begin{CacheMeCode}{my python} +print(cos(1)+sin(2)) +\end{CacheMeCode} +\end{codeexample} +This can also be practical to disable caching if some pictures uses |remember picture| (which is not supported by this library): +\begin{codeexample}[vbox] +\cacheTikz +\robExtConfigure{ + add to preset={tikzpicture}{ + % \b means "word boundary", and spaces must be escaped + if matches={remember picture}{disable externalization}, + }, +} +\begin{tikzpicture}[remember picture] + \node[fill=green](my original node){Point to me}; +\end{tikzpicture} % +Some text % +\begin{tikzpicture}[overlay, remember picture] + \draw[->] (0,0) to[bend left] (my original node); +\end{tikzpicture} +\end{codeexample} +\end{pgfmanualentry} \subsubsection{Pass compiled file to another template}\label{sec:passOutputToOtherTemplate} @@ -2477,9 +2997,79 @@ Note that if you do not want to display the first cached file, you can use |do n \end{pgfmanualentry} -\subsubsection{Compile even faster}\label{sec:compileFaster} +\subsubsection{Compile in parallel}\label{sec:compileParallel} -Long story short: you can compile even faster (at least 1.5x in our tests, but we expect this to be more visible on larger tests due to the loading time of the preamble of the main document that takes most of the time of the compilation) by compiling presets, but beware that you will not be able to modify the placeholders except |add to preamble| with the default compiler we provide: +(introduced in v2.1, read below on how to do it manually on v2.0) + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/compile in parallel=\opt{nb of pictures to compile normally} (default 0)\@nil + \extractkey/robExt/compile in parallel after=\opt{nb pictures to compile normally} (default 0)\@nil + \makeatother% + \pgfmanualbody + These two commands are alias. Typing in the preamble: +\begin{codeexample}[code only] +\robExtConfigure{ + compile in parallel +} +\end{codeexample} +will cause the cached elements to be compiled in parallel (this requires two compilations of the main project). For this to work out of the box, you need to have GNU |parallel| installed (on linux or Mac) or rush (\url{https://github.com/shenwei356/rush/}) on Windows. Typing |compile in parallel after=3| will start the compilation in parallel only if you have more than 3 new elements to compile, and compile the first 3 elements normally (useful when you work on one picture at a time since you do not need to compile twice the document). +\end{pgfmanualentry} + +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/compile in parallel command=\marg{command to run}\@nil + \extractkey/robExt/compile in parallel with xargs=\opt{number threads} (default 16\%)\@nil + \extractkey/robExt/compile in parallel with gnu parallel=\opt{--jobs option} (default 200\%)\@nil + \extractkey/robExt/compile in parallel with rush=\opt{additional options} (default )\@nil + \makeatother% + \pgfmanualbody + You can customize the compilation command to run via something like: +\begin{codeexample}[code only] +\robExtConfigure{ + compile in parallel command={ + parallel --jobs 200\% :::: '\jobname-\robExtAddPrefixName{compile-missing-figures.sh} + }, +} +\end{codeexample} +The other styles are predefined commands to compile with xargs (by default, already installed in linux, on Windows you can get it by installing the lightweight GNU On Windows (Gow) \url{https://github.com/bmatzelle/gow}), gnu parallel, or rush (\url{https://github.com/shenwei356/rush/}). You can use these commands directly like: +\begin{codeexample}[code only] +\robExtConfigure{ + compile in parallel with gnu parallel +} +\end{codeexample} +or use the optional parameter to configure the number of threads (xargs starts with 16 threads by default, and gnu parallel takes twice the number of CPU threads available), like in: +\begin{codeexample}[code only] +\robExtConfigure{ + compile in parallel with xargs=64,% Compiles with 64 processes at a time +} +\end{codeexample} +\end{pgfmanualentry} + +NB: in version smaller than v2.1 (or if you prefer the manual method), you can instead run it in manual mode and use for instance GNU |parallel| (or any tool of you choice) to run all commands in the file |\jobname-robExt-compile-missing-figures.sh| in parallel. To do that: +\begin{itemize} +\item First add |\robExtConfigure{enable manual mode}| in your file +\item Create the cache folder, e.g.\ using |mkdir robustExternalize| +\item Compile your file, with, e.g.\ |pdflatex yourfile.tex| (in my benchmark: 4.2s) +\item This should create a file |yourfile-robExt-compile-missing-figures.sh|, with each line containing a compilation command like this (the command should be compatible with Windows, Mac and Linux, if not, let me know on github): +\begin{verbatim} +cd robustExternalize/ && pdflatex -halt-on-error "robExt-4A806941E86C6B657A0CB3D160CFFF3E.tex +\end{verbatim} + \textbf{IMPORTANT}: \LaTeX{} might render \textbf{multiple times} the same picture if it is inside some particular environments (align, tables\dots), so this file can contain duplicates if you inserted cached data inside. In version 2.1 this is solved, but on older versions, to avoid running the same command twice, \textbf{be sure to remove duplicates!} On Linux, this can be done by piping the file into |sort| and |uniq| (if your files can depend on previously compiled elements as explained in \cref{sec:passOutputToOtherTemplate}, you should additionally preserve \mylink{https://unix.stackexchange.com/questions/194780/remove-duplicate-lines-while-keeping-the-order-of-the-lines}{the order of the lines}). Since this is anyway a rather advanced use case, we will not consider that case for simplicity. +\item Run all these command in parallel, for instance on Linux you can install |parallel|, and run the following command (|--bar| displays a progress bar): +\begin{verbatim} +cat yourfile-robExt-compile-missing-figures.sh | sort | uniq | parallel --bar --jobs 200% +\end{verbatim} +(note that since v2.1, it is not necessary to remove duplicates using |sort| and |uniq| since the file contains no more duplicates) In my benchmark, it ran during 52s, so 1.6x faster than the original compilation without caching) +\item Recompile the original document with |pdflatex yourfile.tex| +\end{itemize} +In my benchmark, the total time is 2x4.2s + 52s = 60s, so 1.5 times faster than a normal command (of course, this depends a lot on the preamble of the file that you compile, since the loading time of the file is the main bottleneck for the first compilation; the advantage here is that it is easy to include only the necessary things in the preamble of cached pictures, possibly creating different presets if it is easier to manage it this way). + +\subsubsection{Compile a template to compile even faster}\label{sec:compileFaster} + +Long story short: you can compile even faster (at least 1.5x in our tests, but we expect this to be more visible on larger tests due to the loading time of the preamble of the main document that takes most of the time of the compilation) by compiling presets, but beware that you will not be able to modify the placeholders except |add to preamble| with the default compiler we provide. The gain comes from the fact that instead of trying to find all placeholders to replace in the string, we can directly replace a few hard-coded placeholders to save time (yes, \LaTeX{} is not really efficient so it can make a difference\dots). \begin{pgfmanualentry} @@ -2697,7 +3287,11 @@ The next options can be used after calling the |latex| style: \makeatother% \pgfmanualbody |tikz| loads |latex| and then adds |tikz| to the preamble. |tikzpicture| first load |tikz| and wraps the main content within |\begin{tikzpicture}| and |\end{tikzpicture}| using:\\ - |set placeholder={__ROBEXT_MAIN_CONTENT__}{\begin{tikzpicture}__ROBEXT_MAIN_CONTENT_ORIG__\end{tikzpicture}}|\\ +\begin{verbatim} +set placeholder={__ROBEXT_MAIN_CONTENT__}{% + \begin{tikzpicture}__ROBEXT_MAIN_CONTENT_ORIG__\end{tikzpicture}% +} +\end{verbatim} so that the user does not need to type it. See for instance the introduction for examples of use. \end{pgfmanualentry} @@ -2790,6 +3384,28 @@ You can set |__ROBEXT_PYTHON_TCOLORBOX_PROPS__| the options of the tcolorbox,\\ \end{verbatim} \end{pgfmanualentry} +\begin{pgfmanualentry} + \makeatletter% + \def\extrakeytext{style, } + \extractkey/robExt/python/add import\@nil + \makeatother% + \pgfmanualbody + (since v2.1) Add an import statement in the first part of the file. +\begin{codeexample}[vbox] +\robExtConfigure{ + add to preset={my python}{ + python print code and result, + add import={from math import cos}, + add import={from math import sin}, + }, +} +\begin{CacheMeCode}{my python} +print(cos(1)+sin(2)) +\end{CacheMeCode} +\end{codeexample} + +\end{pgfmanualentry} + \subsubsection{Bash} We provide a basic bash template, that sets: @@ -2837,6 +3453,8 @@ This file contains: \verbatiminput{\robExtPathToInput} \end{codeAndResult} +You might also like to use |name output=yourfile| that will create two macros |\yourfile| and |\yourfileInCache|, equal respectively to the prefix |robExt-somehash| and |pathOfCache/robExt-somehash|. + \subsubsection{Custom languages} To add support to new languages, see \cref{sec:otherLanguages} for an example. @@ -2981,6 +3599,19 @@ Here is an example to specify arguments via parenthesis: \tikz(disable externalization) \node[fill=green, rounded corners]{Foo \whereIAm}; \end{codeexample} +You can also change the default style loaded for tikz and tikzpicture (note that you might prefer to modify |tikz| directly using |add to preset|), but you need v2.1 (buggy in v2.0): + +\begin{codeexample}[vbox] +% You might prefer to modify tikz directly using "add to preset" +\cacheTikz[tikz, add to preamble={\def\hello{H}},][tikzpicture, + add to preamble={\def\hello{HH}}, +] + +\tikz[baseline=(A.base)] \node(A){In tikz \hello}; % +\begin{tikzpicture}[baseline=(A.base)] + \node(A){In tikzpicture \hello}; +\end{tikzpicture} +\end{codeexample} \end{pgfmanualentry} @@ -3135,7 +3766,12 @@ and we can disable the externalization: % \zx<disable externalization>{\zxZ{\alpha} \rar & \zxX{\text{Hey}}}. \end{codeAndResult} -You can do a similar trick for environment, by using instead |\NewEnvironmentCopy{robExtEnvironmentOrigNAMEENV}{NAMEENV}| |\robExtAddToEnvironmentResetList{NAMEENV}| and |\def\robExtEnvironmentOrigName{NAMEENV}|. +You can do a similar trick for environment, by using instead +\begin{verbatim} +\NewEnvironmentCopy{robExtEnvironmentOrigNAMEENV}{NAMEENV} +\robExtAddToEnvironmentResetList{NAMEENV} +\def\robExtEnvironmentOrigName{NAMEENV} +\end{verbatim} \subsection{Operations on the cache}\label{sec:operationsCache} @@ -3312,17 +3948,27 @@ In order to know when to stop replacing placeholders earlier (i.e.\ without runn \begin{itemize} \item See how to deal with label, links and cite inside pictures (without disabling externalization). For label and links, I have a proof of concept, I ``just'' need to write it down. \url{https://tex.stackexchange.com/questions/695277/clickable-includegraphics-for-cross-reference-data} \item Deal with remember picture -\item I tried many different options to optimize the compilation time, so the code is a bit cluttered now. Would be cool to reorganize. -\item The doc more or less describes all features, but I might made tons of typo and forgot to document some advanced commands. But should be good enough for normal usage. +\item Since externalizing is all about speed, it would be nice to do more benchmarks. \end{itemize} \section{Acknowledgments} -I am deeply indebted to many users on \url{tex.stackexchange.com} that made the writing of this library possible. I can't list you all, but thank you so much! A big thanks also to the project \url{https://github.com/sasozivanovic/memoize/} from which I borrowed most of the code to automatically wrap commands. +I am deeply indebted to many users on \url{tex.stackexchange.com} that made the writing of this library possible. I can't list you all, but thank you so much! A big thanks also to the project \url{https://github.com/sasozivanovic/memoize/} from which I borrowed most of the code to automatically wrap commands. Thanks to kiryph for providing useful feedbacks. \section{Changelog} \begin{itemize} +\item future v2.1 (master on github, not released yet on CTAN): + \begin{itemize} + \item Fix a bug when compiling the main document with lualatex or xelatex + \item Fix a bug when compiling a template that was not exporting dependencies + \item Add |recompile| to recompile a file even if no dependencies changed. + \item Add a hook |robust-externalize/just-before-writing-files|. + \item Fixed a bug in |\cacheTikz| that was not working with a custom argument for tikzpicture + \item Added options |forward*| to forward macros and counters + \item Added |auto forward| and co + \item Added |compile in parallel| and co + \end{itemize} \item v2.0: \begin{itemize} \item \textbf{not backward compatible}: the preset |tikzpicture| should be used instead of |tikz|: |tikz| does not wrap anymore its content into |\begin{tikzpicture}| in order to be usable inside the command version |\tikz| (but |tikzpicture| does load |tikz| first). Moreover, all arguments to |tikzpictureC| or |tikzpicture| (if you loaded |cacheTikz|) should be specified in the first argument, using |<your options>|. This is used to provide a more uniform interface with the new |\cacheEnvironment| command. Hopefully this should \textbf{not cause too much trouble} since the first version of the library has been published on CTAN only a few days ago. diff --git a/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty b/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty index e61ae871b06..75d55b268c1 100644 --- a/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty +++ b/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty @@ -1,5 +1,5 @@ -\ProvidesPackage{robust-externalize}[2.0 Cache anything (tikz, latex, python) in a robust, efficient and pure way.] -% todo: understand why python scripts with raise NameError("42") do not make latex crash. +\ProvidesPackage{robust-externalize}[2.1 Cache anything (tikz, latex, python) in a robust, efficient and pure way.] +% todo: % change order argument replace from list, it is hard to read this way \RequirePackage{pgfkeys} % We use the /robExt/... path to store our keys. @@ -8,18 +8,13 @@ \RequirePackage{verbatim} % For the \verbatim command, useful for debugging purpose for instance \RequirePackage{xsimverb} % To easily write verbatim code to files \RequirePackage{etoolbox} % To easily write verbatim code to files +\RequirePackage{iftex} % computing md5 sum differs on lualatex/pdflatex/xelatex +\RequirePackage{xparse} % solve https://github.com/leo-colisson/robust-externalize/issues/6 +\RequirePackage{atveryend} % to compile in parallel all images +\usepackage{etl} % For auto forward https://tex.stackexchange.com/a/700844/116348 %% TODO list: % - provide an easy way to use cross-ref, bibtex etc (we just need to add them when writing the file) without recompiling the whole document (we don't want to lose the cache everytime a new bib entry is added) but while preserving. -% - create pre-made settings for tikz, tikz-cd, ... -% - check compatibility with windows -% - write documentation - -%%% Under the hood, this library is quite simple: each picture must, somehow, provide: -%% - \l_robExt_final_file LaTeX3 string containing the content of the final file -%% - -%% Then, the library will hash everything to create a unique name (of the content, the template code, and the set of dependency filenames), -%% it will create a file "MD5.tex" containing the pre-template+content+post-template, and it will compile it. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%% Utils %%%%%%%%%%%% @@ -31,9 +26,7 @@ \def\robExtDebugWarning#1{\message{^^J[robExt warning] #1}} \NewDocumentCommand{\robExtConfigure}{m}{% - \pgfkeys{% - /robExt/.cd,#1%Do not add a space before the #1! - }% + \pgfqkeys{/robExt}{#1}% faster than \pgfkeys{/robExt/.cd,#1} } % https://tex.stackexchange.com/questions/690700/latex3-elegant-way-to-forward-a-variable-outside-of-the-group @@ -52,40 +45,41 @@ %%% Expl3 variants %%% -\cs_generate_variant:Nn \str_replace_all:Nnn { Nnx } -\cs_generate_variant:Nn \str_replace_all:Nnn { Nnv } -\cs_generate_variant:Nn \str_replace_all:Nnn { cnx } -\cs_generate_variant:Nn \str_replace_all:Nnn { cxx } -\cs_generate_variant:Nn \str_replace_all:Nnn { NnV } -\cs_generate_variant:Nn \str_replace_all:Nnn { Nnx } -\cs_generate_variant:Nn \str_replace_all:Nnn { Nnv } +\cs_generate_variant:Nn \str_replace_all:Nnn { Nnx, Nnv, cnx, cxx, NnV, Nnx, Nnv } \cs_generate_variant:Nn \str_set:Nn { NV } \cs_generate_variant:Nn \tl_rescan:nn { nv } -\cs_generate_variant:Nn \tl_set_rescan:Nnn { Nnv } +\cs_generate_variant:Nn \tl_set_rescan:Nnn { Nnv, NnV } \cs_generate_variant:Nn \tl_gset_rescan:Nnn { cnv } -\cs_generate_variant:Nn\seq_remove_all:Nn { NV } +\cs_generate_variant:Nn \seq_remove_all:Nn { NV } -%\cs_generate_variant:Nn\tl_set_rescan:cnn { cnv } -\cs_generate_variant:Nn \iow_now:Nn { NV } -\cs_generate_variant:Nn \iow_now:Nn { Nx } +\cs_generate_variant:Nn \iow_now:Nn { NV,Nx } \cs_generate_variant:Nn \iow_open:Nn { Nx } \cs_generate_variant:Nn \ior_str_get:NN { Nc } \cs_generate_variant:Nn \file_if_exist:nTF { xTF } +\cs_generate_variant:Nn \cs_replacement_spec:N { c } +\cs_generate_variant:Nn \cs_argument_spec:N { c } + +\cs_generate_variant:Nn \regex_extract_all:nnN { VVN, nVN, VVN } +\cs_generate_variant:Nn \regex_extract_all:NnN { NVN } + + +\cs_generate_variant:Nn \regex_match:nnTF { nVTF } + %%% %%% String manipulation %%% -\cs_set:Nn \str_set_hash_robust:Nn { +\cs_set:Nn \robExt_set_hash_robust:Nn { \str_set:Nn {#1} {#2} \str_replace_all:Nnx {#1} { ## } { \c_hash_str } } -\cs_generate_variant:Nn \str_set_hash_robust:Nn { cn } -\cs_generate_variant:Nn \str_set_hash_robust:Nn { cx } +\cs_generate_variant:Nn \robExt_set_hash_robust:Nn { cn } +\cs_generate_variant:Nn \robExt_set_hash_robust:Nn { cx } % Double the number of hashes... quite dirty but cannot find any solution or the user need to double it itself %% See also https://tex.stackexchange.com/questions/695432/latex3-latex-doubles-the-number-of-hashes-when-storing-them-in-string/695460#695460 @@ -95,21 +89,71 @@ } \NewDocumentCommand{\robExtStrSetHashRobust}{mm}{ - \str_set_hash_robust:Nn {#1} {#2} + \robExt_set_hash_robust:Nn {#1} {#2} } \NewDocumentCommand{\robExtRescanHashRobust}{m}{ - \str_set_hash_robust:Nn \l_robExt_tmp_str {#1} - \tl_rescan:nv {}{ l_robExt_tmp_str } + \robExt_set_hash_robust:Nn \l__robExt_tmp_str {#1} + \tl_rescan:nv {}{ l__robExt_tmp_str } +} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%% Error messages %%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% In LaTeX3 error messages are defined here, this allows people to translate them, filter them, change them, +%% make them more verbose etc. +\msg_new:nnn {robExt}{underscore in name}{Error:~all~the~placeholders~should~contain~two~consecutive~underscores~in~their~name.} +\msg_new:nnn {robExt}{placeholder not existing}{[robExtGetPlaceholderInResultFromSinglePlaceholder] ~ The ~ placeholder ~ #3 ~ does ~ not ~ exists.} +\msg_new:nnn {robExt}{forgot template}{You ~ are ~ writing ~ __ROBEXT_TEMPLATE__ ~ to ~ your ~ file: ~ seems ~ like ~ you ~ forgot ~ to ~ define ~ your ~ template ~ or ~ set ~ a ~ preset} +\msg_new:nnn {robExt}{need shell escape}{You ~ need ~ to ~ compile ~ with ~ shell-escape ~ as ~ in: ~ "pdflatex ~ -shell-escape ~ yourfile.tex" ~ to ~ be ~ able ~ to ~ compile ~ automatically ~ the ~ figures} +\msg_new:nnn {robExt}{need shell escape parallel}{You ~ need ~ to ~ compile ~ with ~ shell-escape ~ as ~ in: ~ "pdflatex ~ -shell-escape ~ yourfile.tex" ~ to ~ be ~ able ~ to ~ compile ~ automatically ~ the ~ figures ~ in ~ parallel} +\msg_new:nnn {robExt}{missing compiled pdf}{The ~ pdf ~ file ~ #1.pdf ~ is ~ not ~ present. ~ The ~ compilation ~ command ~ certainly ~ failed, ~ see ~ logs ~ above ~ or ~ in ~ #1.log.} +\msg_new:nnn {robExt}{missing compiled pdf parallel}{The ~ pdf ~ file ~ #1.pdf ~ is ~ not ~ present. ~ The ~ compilation ~ command ~ "#3"~used~to~compile~the~environment~on~line~#2~ certainly ~ failed, ~ see ~ logs ~ above ~ or ~ in ~ #1.log.} +\msg_new:nnn {robExt}{auto forward not in cachemecode}{Auto~forward~is~less~efficient~in~cacheMeCode.} + +% warnings +\msg_new:nnn {robExt}{recommend two underscores in placeholders}{We~recommend~to~use~only~placeholders~containing~two~consecutive~underscores~in~their~name} +\msg_new:nnn {robExt}{enabled parallel no shell escape}{Warning:~you~enabled~parallel~compilation~but~shell-escape~is~disabled.} +\msg_new:nnn {robExt}{rerun because parallel}{Warning:~Compiling~all~missing~figures~in~parallel~with~"#1".~You~need~to~rerun~LaTeX~to~include~them.} + +% dummy placeholders added if image is not present +\def\robExtImagePlaceholderIfManualMode{ + \framebox[\linewidth] + { + \begin{minipage}{\linewidth} + \textbf{Manual ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l__robExt_current_compilation_command_str}\newline or ~ call ~ \newline\texttt{bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} + \end{minipage} + } +} + +\def\robExtImagePlaceholderIfFallbackMode{ + \framebox[\linewidth] + { + \begin{minipage}{\linewidth} + \textbf{Falling ~ back ~ to ~ draft ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l__robExt_current_compilation_command_str}\newline or ~ call ~ \newline\texttt{bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} + \end{minipage} + } } +\def\robExtImagePlaceholderIfParallelCompilation{ + \framebox[\linewidth] + { + \begin{minipage}{\linewidth} + \textbf{The~file:\\ + \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\\ + is~compiled~in~parallel~mode.~You~need~to~recompile~your~document~to~include~it.} + \end{minipage} + } +} + + \ExplSyntaxOff %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%% SCRIPTS %%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% Important: this script is NOT execute by this file, it is only created so that users can remove unused previously cached files by running this script +%%% Important: this script is NOT executed by this file, it is only created so that users can remove unused previously cached files by running this script %%% outside of LaTeX. %%% Create scripts to remove useless files: %%% Note that we don't override the script if it exists on purpose (the user might have changed it to fits their needs) @@ -190,6 +234,19 @@ if __name__ == '__main__': \expandafter \RenewCommandCopy \csname end#1\expandafter\endcsname \csname end#2\endcsname } +%% Depending on the engine, computing md5 is done in different ways +\sys_if_engine_luatex:TF{ + \def\robExt@pdfmdfivesum#1{% + \directlua{tex.print(string.upper(md5.sumhexa("\luaescapestring{#1}")))}% + }% +}{ + \sys_if_engine_xetex:TF {% + \def\robExt@pdfmdfivesum{\mdfivesum}% + }{% + \def\robExt@pdfmdfivesum{\pdfmdfivesum}% + }% +} + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%% Paths %%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -208,13 +265,10 @@ if __name__ == '__main__': \fi#1% } - \NewDocumentCommand{\robExtCheckIfPrefixFolderExists}{}{ % Check if the output directory exists \ifdefined\robExtCacheFolder - %% Not sure how to do an "OR" otherwise - - \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N {\robExtForceCompilation}} + \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N \robExtForceCompilation} { \ifdefined\robExtDoNotMkdirFolder\else \ifdefined\robExtManualMode @@ -229,6 +283,7 @@ if __name__ == '__main__': \fi } + \NewExpandableDocumentCommand{\robExtGetPrefixPath}{}{% \ifdefined\robExtCacheFolder% \robExtCacheFolder% @@ -249,27 +304,37 @@ if __name__ == '__main__': %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Temporary: used when quickly writing to a file -\iow_new:N \g_robExt_write_iow -\ior_new:N \g_robExt_read_ior +\iow_new:N \g__robExt_write_iow +\ior_new:N \g__robExt_read_ior %% This is used to write the full list of figures in a single file (used for instance by Makefile etc...) %% TODO: create a makefile. -\iow_new:N \g_robExt_write_list_all_figures +\iow_new:N \g__robExt_write_list_all_figures_iow %% Create a file robExt-all-figures.txt with the list of .tex files -\iow_open:Nx \g_robExt_write_list_all_figures {\jobname-\robExtAddPrefixName{all-figures.txt}} -\iow_new:N \g_robExt_write_manually_compile_all_missing_figures -\iow_open:Nx \g_robExt_write_manually_compile_all_missing_figures {\jobname-\robExtAddPrefixName{compile-missing-figures.sh}} -% Contains the template: -\str_new:N \l_robExt_template -\str_new:N \l_robExt_final_file +\iow_open:Nx \g__robExt_write_list_all_figures_iow {\jobname-\robExtAddPrefixName{all-figures.txt}} +\iow_new:N \g__robExt_write_manually_compile_all_missing_figures_iow +\iow_open:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\jobname-\robExtAddPrefixName{compile-missing-figures.sh}} % Contains the list of dependency files (useful to compute the final md5sum) -\seq_new:N \l_robExt_dependencies +\seq_new:N \l__robExt_dependencies_str % Contains a string where on each line we have: "md5sum, dependency". The first line has nothing as "dependency" as it is the main fine whose final name is the md5sum of the dependencies. -\str_new:N \l_robExt_dependencies_mdfive +\str_new:N \l__robExt_dependencies_mdfive_str % Contains the current compilation command (including the name of the file to compile). -\str_new:N \l_robExt_currentCompilationCommand - - +\str_new:N \l__robExt_current_compilation_command_str +\str_new:N \l__robExt_tmp_str + + +% To have a code compatible even on older versions +% https://tex.stackexchange.com/questions/615010/how-can-i-use-latex-hooks-in-historical-versions-of-tex-live +\providecommand\RobExtIfFormatAtLeastTF{\@ifl@t@r\fmtversion} +\RobExtIfFormatAtLeastTF{2020-10-01}% +{ + % https://ctan.math.illinois.edu/macros/latex/base/lthooks-doc.pdf + \NewHook{robust-externalize/just-before-writing-files} + \def\robExtUseHookJustBeforeWritingFiles{\UseHook{robust-externalize/just-before-writing-files}}% +}% +{ + \def\robExtUseHookJustBeforeWritingFiles{} +} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%% Placeholders %%%%%%%%%%%% @@ -277,22 +342,22 @@ if __name__ == '__main__': %% Placeholders are strings like "__MYLIBRARY__" or "__MAINCONTENT__" that will be replaced by a given content. %% In practice: -%% - for every placeholder MYPLACEHOLDER, a macro \l_robExt_placeholder_MYPLACEHOLDER_str is created, containing +%% - for every placeholder MYPLACEHOLDER, a macro \l__robExt_placeholder_MYPLACEHOLDER_str is created, containing %% the string to use to replace it -%% - a sequence \l_robExt_placeholder_group_main_seq that is a list of string, contains the list of all placeholders, +%% - a sequence \l__robExt_placeholder_group_main_seq that is a list of string, contains the list of all placeholders, %% in a string, like [MYLIBRARY, MAINCONTENT] etc... %% One issue is that LaTeX does not keep some symbols (e.g. % etc...) when used inside a macro, so we define %% different commands depending on whether they can be used in a macro or not, whether they should be taken %% from an external file etc... -\seq_clear_new:N \l_robExt_placeholder_group_main_seq +\seq_clear_new:N \l__robExt_placeholder_group_main_seq %%% %%% Debug in terminal %%% \NewDocumentCommand{\robExtShowPlaceholder}{sm}{ - \cs_if_exist:cTF {l_robExt_placeholder_#2_str} { - \message{Placeholder ~ #2 ~ contains:^^J~ \use:c{l_robExt_placeholder_#2_str}} + \cs_if_exist:cTF {l__robExt_placeholder_#2_str} { + \message{Placeholder ~ #2 ~ contains:^^J~ \use:c{l__robExt_placeholder_#2_str}} }{ \message{Placeholder ~ #2 ~ does ~ not ~ exist.} } @@ -301,13 +366,13 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtShowPlaceholders}{s}{ \message{List ~ of ~ placeholders:} - \seq_map_inline:Nn \l_robExt_placeholder_group_main_seq {\message{##1,}} + \seq_map_inline:Nn \l__robExt_placeholder_group_main_seq {\message{##1,}} \IfBooleanTF{#1}{}{\show\def} } \NewDocumentCommand{\robExtShowPlaceholdersContents}{s}{ \message{List ~ of ~ placeholders:} - \seq_map_inline:Nn \l_robExt_placeholder_group_main_seq {\robExtShowPlaceholder*{##1}} + \seq_map_inline:Nn \l__robExt_placeholder_group_main_seq {\robExtShowPlaceholder*{##1}} \IfBooleanTF{#1}{}{\show\def} } @@ -318,11 +383,11 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtPrintPlaceholderNoReplacement}{sm}{% % For some reasons, newlines are displayed as \Omega. We need to replace them with \\ % https://tex.stackexchange.com/questions/694716/print-latex3-string-verbatim/694717 - \tl_set_eq:Nc \l_robExt_tmp_str { l_robExt_placeholder_#2_str } - \tl_replace_all:Nnn \l_robExt_tmp_str {^^J} { \par } - \tl_replace_all:Nnn \l_robExt_tmp_str { ~ } { \ } - \IfBooleanTF{#1}{\texttt{\use:c{l_robExt_placeholder_#2_str}}}{\begin{flushleft}\ttfamily% - \l_robExt_tmp_str + \tl_set_eq:Nc \l__robExt_tmp_str { l__robExt_placeholder_#2_str } + \tl_replace_all:Nnn \l__robExt_tmp_str {^^J} { \par } + \tl_replace_all:Nnn \l__robExt_tmp_str { ~ } { \ } + \IfBooleanTF{#1}{\texttt{\use:c{l__robExt_placeholder_#2_str}}}{\begin{flushleft}\ttfamily% + \l__robExt_tmp_str \end{flushleft}% } } @@ -332,11 +397,11 @@ if __name__ == '__main__': \robExtGetPlaceholderInResult{#2} % For some reasons, newlines are displayed as \Omega. We need to replace them with \\ % https://tex.stackexchange.com/questions/694716/print-latex3-string-verbatim/694717 - \tl_set_eq:NN \l_robExt_tmp_str \l_robExt_result_str - \tl_replace_all:Nnn \l_robExt_tmp_str {^^J} { \par } - \tl_replace_all:Nnn \l_robExt_tmp_str { ~ } { \ } - \IfBooleanTF{#1}{\texttt{\l_robExt_tmp_str}}{\begin{flushleft}\ttfamily% - \l_robExt_tmp_str + \tl_set_eq:NN \l__robExt_tmp_str \l_robExt_result_str + \tl_replace_all:Nnn \l__robExt_tmp_str {^^J} { \par } + \tl_replace_all:Nnn \l__robExt_tmp_str { ~ } { \ } + \IfBooleanTF{#1}{\texttt{\l__robExt_tmp_str}}{\begin{flushleft}\ttfamily% + \l__robExt_tmp_str \end{flushleft}% } } @@ -345,7 +410,7 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtPrintAllPlaceholdersExceptDefaults}{s}{ List ~ of ~ placeholders:\\ - \seq_map_inline:Nn \l_robExt_placeholder_group_main_seq { + \seq_map_inline:Nn \l__robExt_placeholder_group_main_seq { % We hide the elements starting with __ROBEXT_ \str_if_in:nnTF { ##1 } { __ROBEXT_ } { \IfBooleanTF {#1} { @@ -361,7 +426,7 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtPrintAllPlaceholders}{}{ List ~ of ~ placeholders:\\ - \seq_map_inline:Nn \l_robExt_placeholder_group_main_seq {- ~ Placeholder ~ called ~ \texttt{\tl_to_str:n {##1}} ~ contains: \robExtPrintPlaceholderNoReplacement{##1}} + \seq_map_inline:Nn \l__robExt_placeholder_group_main_seq {- ~ Placeholder ~ called ~ \texttt{\tl_to_str:n {##1}} ~ contains: \robExtPrintPlaceholderNoReplacement{##1}} } \let\printAllPlaceholders\robExtPrintAllPlaceholders \let\printImportedPlaceholders\robExtPrintAllPlaceholders @@ -374,43 +439,43 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtEvalPlaceholderNoReplacement}{m}{ % scantokens add an empty space at the end, so we need to add \empty to avoid it having effects % https://tex.stackexchange.com/questions/213659/could-someone-further-elucidate-expansion-catcodes-and-scantokens - \tl_rescan:nv {}{ l_robExt_placeholder_#1_str } + \tl_rescan:nv {}{ l__robExt_placeholder_#1_str } } \let\evalPlaceholderNoReplacement\robExtEvalPlaceholderNoReplacement \NewDocumentCommand{\robExtGetPlaceholderNoReplacement}{m}{ - \str_use:c { l_robExt_placeholder_#1_str } + \str_use:c { l__robExt_placeholder_#1_str } } \let\getPlaceholderNoReplacement\robExtGetPlaceholderNoReplacement \NewDocumentCommand{\robExtRescanPlaceholderInVariableNoReplacement}{mm}{ - \tl_gset_rescan:cnv {#1} {} { l_robExt_placeholder_#2_str } + \tl_gset_rescan:cnv {#1} {} { l__robExt_placeholder_#2_str } } \let\rescanPlaceholderInVariableNoReplacement\robExtRescanPlaceholderInVariableNoReplacement -% Make sure that the placeholder is in the list \l_robExt_placeholder_group_main_seq. +% Make sure that the placeholder is in the list \l__robExt_placeholder_group_main_seq. % This should automatically be called by other tools \NewDocumentCommand{\robExtAddPlaceholderToList}{m}{ \cs_if_exist:NTF {\robExtCompletelyDisableWholeImportSystem}{}{ % Make sure we have a string here: - \str_set_hash_robust:Nn \l_robExt_tmp_str { #1 } + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #1 } % First we remove existing occurrences (useful to avoid listing the same macro more than once % if we redefine a macro): - \seq_remove_all:NV \l_robExt_placeholder_group_main_seq \l_robExt_tmp_str - \seq_put_right:NV \l_robExt_placeholder_group_main_seq \l_robExt_tmp_str + \seq_remove_all:NV \l__robExt_placeholder_group_main_seq \l__robExt_tmp_str + \seq_put_right:NV \l__robExt_placeholder_group_main_seq \l__robExt_tmp_str } } \let\robExtImportPlaceholder\robExtAddPlaceholderToList \let\importPlaceholder\robExtAddPlaceholderToList \NewDocumentCommand{\robExtClearImportedPlaceholders}{}{ - \seq_clear:N \l_robExt_placeholder_group_main_seq + \seq_clear:N \l__robExt_placeholder_group_main_seq } \let\clearImportedPlaceholders\robExtClearImportedPlaceholders \NewDocumentCommand{\robExtRemoveImportedPlaceholder}{m}{ - \str_set_hash_robust:Nn \l_robExt_tmp_str { #1 } - \seq_remove_all:NV \l_robExt_placeholder_group_main_seq \l_robExt_tmp_str + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #1 } + \seq_remove_all:NV \l__robExt_placeholder_group_main_seq \l__robExt_tmp_str } \let\removeImportedPlaceholder\robExtRemoveImportedPlaceholder @@ -418,11 +483,11 @@ if __name__ == '__main__': % add it first in the list \NewDocumentCommand{\robExtAddPlaceholderToListFirst}{m}{ % Make sure we have a string here: - \str_set_hash_robust:Nn \l_robExt_tmp_str { #1 } + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #1 } % First we remove existing occurrences (useful to avoid listing the same macro more than once % if we redefine a macro): - \seq_remove_all:NV \l_robExt_placeholder_group_main_seq \l_robExt_tmp_str - \seq_put_left:NV \l_robExt_placeholder_group_main_seq \l_robExt_tmp_str + \seq_remove_all:NV \l__robExt_placeholder_group_main_seq \l__robExt_tmp_str + \seq_put_left:NV \l__robExt_placeholder_group_main_seq \l__robExt_tmp_str } \let\robExtImportPlaceholderFirst\robExtAddPlaceholderToListFirst \let\importPlaceholderFirst\robExtAddPlaceholderToListFirst @@ -430,15 +495,15 @@ if __name__ == '__main__': % add it first in the list \NewDocumentCommand{\robExtAddPlaceholdersToListFirst}{m}{ - \str_set_hash_robust:Nn \l_robExt_tmp_str { #1 } - \seq_set_from_clist:NN \l_tmp_seq \l_robExt_tmp_str + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #1 } + \seq_set_from_clist:NN \l_tmp_seq \l__robExt_tmp_str % First we remove existing occurrences (useful to avoid listing the same macro more than once % if we redefine a macro): \seq_map_inline:Nn \l_tmp_seq { % This seems to be required to ensure catcodes are correct before removing the elements: - \seq_remove_all:Nn \l_robExt_placeholder_group_main_seq {##1} + \seq_remove_all:Nn \l__robExt_placeholder_group_main_seq {##1} } - \seq_put_left:NV \l_robExt_placeholder_group_main_seq \l_robExt_tmp_str + \seq_put_left:NV \l__robExt_placeholder_group_main_seq \l__robExt_tmp_str } \let\robExtImportPlaceholdersFirst\robExtAddPlaceholdersToListFirst \let\importPlaceholdersFirst\robExtAddPlaceholdersToListFirst @@ -447,11 +512,11 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtRemovePlaceholder}{m}{ % This seems to be required to ensure catcodes are correct before removing the elements: - \str_set_hash_robust:Nn \l_robExt_tmp_str { #1 } + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #1 } % First we check if it exists, no need to loop over everything otherwise - \str_if_exist:cTF { l_robExt_placeholder_#1_str } { - \seq_remove_all:NV \l_robExt_placeholder_group_main_seq \l_robExt_tmp_str - \expandafter \let \csname l_robExt_placeholder_#1_str \endcsname \undefined + \str_if_exist:cTF { l__robExt_placeholder_#1_str } { + \seq_remove_all:NV \l__robExt_placeholder_group_main_seq \l__robExt_tmp_str + \expandafter \let \csname l__robExt_placeholder_#1_str \endcsname \undefined } { } } \let\removePlaceholder\robExtRemovePlaceholder @@ -459,11 +524,11 @@ if __name__ == '__main__': \NewDocumentCommand{\checkIfPlaceholderNameIsLegal}{m}{ \cs_if_exist:NTF {\robExtPlaceholderOnlyWithUnderscores} { \str_if_in:nnTF {#1}{__}{}{ - \PackageError{robExt}{Error:~all~the~placeholders~should~contain~two~consecutive~underscores~in~their~name.}{} + \msg_error:nn{robExt}{underscore in name} } } { \str_if_in:nnTF {#1}{__}{}{ - \message{WARNING:~we~recommend~to~use~only~placeholders~containing~two~consecutive~underscores~in~their~name.} + \msg_warning:nn{robExt}{recommend two underscores in placeholders} } } } @@ -474,7 +539,7 @@ if __name__ == '__main__': %% Tested! %% The star version does NOT import the placeholder (useful for efficiency reasons) \NewDocumentCommand{\robExtPlaceholderFromContent}{smm}{ - \str_set_hash_robust:cn { l_robExt_placeholder_#2_str } {#3} + \robExt_set_hash_robust:cn { l__robExt_placeholder_#2_str } {#3} \checkIfPlaceholderNameIsLegal{#2} \ifdefined\robExtCompletelyDisableWholeImportSystem\else% Mostly for time optimizations... \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}}% @@ -486,7 +551,7 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtPlaceholderFromContentFirst}{smm}{ - \str_set_hash_robust:cn { l_robExt_placeholder_#2_str } {#3} + \robExt_set_hash_robust:cn { l__robExt_placeholder_#2_str } {#3} \checkIfPlaceholderNameIsLegal{#2} \IfBooleanTF {#1} {} {\robExtAddPlaceholderToListFirst{#2}} } @@ -496,7 +561,7 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtPlaceholderFromString}{smm}{ - \str_set_eq:cN { l_robExt_placeholder_#2_str } {#3} + \str_set_eq:cN { l__robExt_placeholder_#2_str } {#3} \checkIfPlaceholderNameIsLegal{#2} \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } @@ -504,7 +569,7 @@ if __name__ == '__main__': \let\setPlaceholderFromString\robExtPlaceholderFromString \NewDocumentCommand{\robExtPlaceholderFromStringExpanded}{smm}{ - \str_set:cx { l_robExt_placeholder_#2_str } {#3} + \str_set:cx { l__robExt_placeholder_#2_str } {#3} \checkIfPlaceholderNameIsLegal{#2} \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } @@ -513,7 +578,7 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtCopyPlaceholder}{smm}{ - \str_set_eq:cc { l_robExt_placeholder_#2_str } { l_robExt_placeholder_#3_str } + \str_set_eq:cc { l__robExt_placeholder_#2_str } { l__robExt_placeholder_#3_str } \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } \let\copyPlaceholder\robExtCopyPlaceholder @@ -522,24 +587,24 @@ if __name__ == '__main__': %% Add something to the right of the placeholder %% By default it adds a space in between, unless we use the star version \NewDocumentCommand{\robExtAddToPlaceholder}{smm}{ - \str_if_exist:cTF { l_robExt_placeholder_#2_str } { - \str_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first - \IfBooleanTF{#1}{}{\str_put_right:cn { l_robExt_placeholder_#2_str } { ~ } } - \str_put_right:cV { l_robExt_placeholder_#2_str } \l_tmp_str + \str_if_exist:cTF { l__robExt_placeholder_#2_str } { + \robExt_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first + \IfBooleanTF{#1}{}{\str_put_right:cn { l__robExt_placeholder_#2_str } { ~ } } + \str_put_right:cV { l__robExt_placeholder_#2_str } \l_tmp_str }{ - \str_set_hash_robust:cn { l_robExt_placeholder_#2_str } {#3} + \robExt_set_hash_robust:cn { l__robExt_placeholder_#2_str } {#3} \robExtAddPlaceholderToList{#2} } } \let\addToPlaceholder\robExtAddToPlaceholder \NewDocumentCommand{\robExtAddToPlaceholderNoImport}{smm}{ - \str_if_exist:cTF { l_robExt_placeholder_#2_str } { - \str_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first - \IfBooleanTF{#1}{}{\str_put_right:cn { l_robExt_placeholder_#2_str } { ~ } } - \str_put_right:cV { l_robExt_placeholder_#2_str } \l_tmp_str + \str_if_exist:cTF { l__robExt_placeholder_#2_str } { + \robExt_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first + \IfBooleanTF{#1}{}{\str_put_right:cn { l__robExt_placeholder_#2_str } { ~ } } + \str_put_right:cV { l__robExt_placeholder_#2_str } \l_tmp_str }{ - \str_set_hash_robust:cn { l_robExt_placeholder_#2_str } {#3} + \robExt_set_hash_robust:cn { l__robExt_placeholder_#2_str } {#3} } } \let\addToPlaceholderNoImport\robExtAddToPlaceholderNoImport @@ -548,12 +613,12 @@ if __name__ == '__main__': %% Add something to the right of the placeholder %% By default it adds a space in between, unless we use the star version \NewDocumentCommand{\robExtAddBeforePlaceholder}{smm}{ - \str_if_exist:cTF { l_robExt_placeholder_#2_str } { - \str_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first - \IfBooleanTF{#1}{}{\str_put_left:cn { l_robExt_placeholder_#2_str } { ~ } } - \str_put_left:cV { l_robExt_placeholder_#2_str } \l_tmp_str + \str_if_exist:cTF { l__robExt_placeholder_#2_str } { + \robExt_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first + \IfBooleanTF{#1}{}{\str_put_left:cn { l__robExt_placeholder_#2_str } { ~ } } + \str_put_left:cV { l__robExt_placeholder_#2_str } \l_tmp_str }{ - \str_set_hash_robust:cn { l_robExt_placeholder_#2_str } {#3} + \robExt_set_hash_robust:cn { l__robExt_placeholder_#2_str } {#3} \robExtAddPlaceholderToList{#2} } } @@ -561,12 +626,12 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtAddBeforePlaceholderNoImport}{smm}{ - \str_if_exist:cTF { l_robExt_placeholder_#2_str } { - \str_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first - \IfBooleanTF{#1}{}{\str_put_left:cn { l_robExt_placeholder_#2_str } { ~ } } - \str_put_left:cV { l_robExt_placeholder_#2_str } \l_tmp_str + \str_if_exist:cTF { l__robExt_placeholder_#2_str } { + \robExt_set_hash_robust:Nn \l_tmp_str {#3} %% needed as put_right does not convert to string first + \IfBooleanTF{#1}{}{\str_put_left:cn { l__robExt_placeholder_#2_str } { ~ } } + \str_put_left:cV { l__robExt_placeholder_#2_str } \l_tmp_str }{ - \str_set_hash_robust:cn { l_robExt_placeholder_#2_str } {#3} + \robExt_set_hash_robust:cn { l__robExt_placeholder_#2_str } {#3} } } \let\addBeforePlaceholderNoImport\robExtAddBeforePlaceholderNoImport @@ -593,20 +658,20 @@ if __name__ == '__main__': \XSIMfilewritestart{\jobname-robExt-tmp-file-you-can-remove.tmp}% }{% \XSIMfilewritestop% - \ior_open:Nn \g_robExt_read_ior {\jobname-robExt-tmp-file-you-can-remove.tmp}% - %% Put the file in l_robExt_tmp_contain_file - \str_clear_new:N \l_robExt_tmp% - \ior_str_map_inline:Nn \g_robExt_read_ior {% - \str_gput_right:Nx \l_robExt_tmp {\tl_to_str:N{##1}^^J}% + \ior_open:Nn \g__robExt_read_ior {\jobname-robExt-tmp-file-you-can-remove.tmp}% + %% Put the file in l__robExt_tmp_contain_file_str + \str_clear_new:N \l__robExt_tmp_str% + \ior_str_map_inline:Nn \g__robExt_read_ior {% + \str_gput_right:Nx \l__robExt_tmp_str {\tl_to_str:n{##1}^^J}% }% - \str_set_eq:cN {l_robExt_placeholder_#2_str} \l_robExt_tmp% + \str_set_eq:cN {l__robExt_placeholder_#2_str} \l__robExt_tmp_str% \IfBooleanTF {#1} {} { \robExtAddPlaceholderToList{#2} %% Otherwise they will be lost when the environment ends - \robExtKeepaftergroup{l_robExt_placeholder_group_main_seq}% + \robExtKeepaftergroup{l__robExt_placeholder_group_main_seq}% } %% for other variable - \robExtKeepaftergroup{l_robExt_placeholder_#2_str}% + \robExtKeepaftergroup{l__robExt_placeholder_#2_str}% }% \let\PlaceholderFromCode\RobExtPlaceholderFromCode \let\endPlaceholderFromCode\endRobExtPlaceholderFromCode @@ -614,7 +679,7 @@ if __name__ == '__main__': \let\endSetPlaceholderCode\endRobExtPlaceholderFromCode \NewDocumentCommand{\robExtKeepPlaceholderAfterGroup}{m}{ - \robExtKeepaftergroup{l_robExt_placeholder_#1_str}% + \robExtKeepaftergroup{l__robExt_placeholder_#1_str}% } \let\keepPlaceholderAfterGroup\robExtKeepPlaceholderAfterGroup @@ -624,22 +689,21 @@ if __name__ == '__main__': %% MYLIBPATH = robExt-abcmylib.py %% Tested! \NewDocumentCommand{\robExtPlaceholderPathFromFilename}{smm}{ - \ior_open:Nn \g_robExt_read_ior {#3} - %% Put the file in l_robExt_tmp_contain_file - \str_clear_new:N \l_robExt_tmp_contain_file - \ior_str_map_inline:Nn \g_robExt_read_ior { - \str_put_right:Nx \l_robExt_tmp_contain_file {\tl_to_str:N{##1}^^J} + \ior_open:Nn \g__robExt_read_ior {#3} + %% Put the file in l__robExt_tmp_contain_file_str + \str_clear_new:N \l__robExt_tmp_contain_file_str + \ior_str_map_inline:Nn \g__robExt_read_ior { + \str_put_right:Nx \l__robExt_tmp_contain_file_str {\tl_to_str:n{##1}^^J} } %% computes the new filename based on the md5 - \str_clear_new:N \l_robExt_tmp_filename - \str_set:Nx \l_robExt_tmp_filename_no_prefix {\pdfmdfivesum{\l_robExt_tmp_contain_file}#3} + \str_set:Nx \l__robExt_tmp_filename_no_prefix_str {\robExt@pdfmdfivesum{\l__robExt_tmp_contain_file_str}#3} %% Writes the content to the file \robExtCheckIfPrefixFolderExists - \iow_open:Nx \g_robExt_write_iow {\robExtAddCachePathAndName{\l_robExt_tmp_filename_no_prefix}} - \iow_now:NV \g_robExt_write_iow \l_robExt_tmp_contain_file - \iow_close:N \g_robExt_write_iow + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\l__robExt_tmp_filename_no_prefix_str}} + \iow_now:NV \g__robExt_write_iow \l__robExt_tmp_contain_file_str + \iow_close:N \g__robExt_write_iow %% sets the template name to the relative path to the file - \str_set:cx { l_robExt_placeholder_#2_str } {\robExtPrefixFilename\l_robExt_tmp_filename_no_prefix} + \str_set:cx { l__robExt_placeholder_#2_str } {\robExtPrefixFilename\l__robExt_tmp_filename_no_prefix_str} \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } \let\placeholderPathFromFilename\robExtPlaceholderPathFromFilename @@ -650,14 +714,14 @@ if __name__ == '__main__': %% Tested! \NewDocumentCommand{\robExtPlaceholderFromFileContent}{smm}{ \checkIfPlaceholderNameIsLegal{#2} - \ior_open:Nn \g_robExt_read_ior {#3} - %% Put the file in l_robExt_tmp_contain_file - \str_clear_new:N \l_robExt_tmp_contain_file - \ior_str_map_inline:Nn \g_robExt_read_ior { - \str_put_right:Nx \l_robExt_tmp_contain_file {\tl_to_str:N{##1}^^J} + \ior_open:Nn \g__robExt_read_ior {#3} + %% Put the file in l__robExt_tmp_contain_file_str + \str_clear_new:N \l__robExt_tmp_contain_file_str + \ior_str_map_inline:Nn \g__robExt_read_ior { + \str_put_right:Nx \l__robExt_tmp_contain_file_str {\tl_to_str:n{##1}^^J} } %% sets the template name to the relative path to the file - \str_set_eq:cN { l_robExt_placeholder_#2_str } \l_robExt_tmp_contain_file + \str_set_eq:cN { l__robExt_placeholder_#2_str } \l__robExt_tmp_contain_file_str \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } \let\placeholderFromFileContent\robExtPlaceholderFromFileContent @@ -669,17 +733,16 @@ if __name__ == '__main__': %% MYLIBPATH = robExt-abc.py %% Tested! \NewDocumentCommand{\robExtPlaceholderPathFromContent}{smO{.tex}m}{ - \str_set_hash_robust:Nn \l_robExt_tmp_contain_file {#4} + \robExt_set_hash_robust:Nn \l__robExt_tmp_contain_file_str {#4} %% computes the new filename based on the md5 - \str_clear_new:N \l_robExt_tmp_filename - \str_set:Nx \l_robExt_tmp_filename_no_prefix {\pdfmdfivesum{\l_robExt_tmp_contain_file}#3} + \str_set:Nx \l__robExt_tmp_filename_no_prefix_str {\robExt@pdfmdfivesum{\l__robExt_tmp_contain_file_str}#3} %% Writes the content to the file \robExtCheckIfPrefixFolderExists - \iow_open:Nx \g_robExt_write_iow {\robExtAddCachePathAndName{\l_robExt_tmp_filename_no_prefix}} - \iow_now:NV \g_robExt_write_iow \l_robExt_tmp_contain_file - \iow_close:N \g_robExt_write_iow + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\l__robExt_tmp_filename_no_prefix_str}} + \iow_now:NV \g__robExt_write_iow \l__robExt_tmp_contain_file_str + \iow_close:N \g__robExt_write_iow %% sets the template name to the relative path to the file - \str_set:cx { l_robExt_placeholder_#2_str } {\robExtPrefixFilename\l_robExt_tmp_filename_no_prefix} + \str_set:cx { l__robExt_placeholder_#2_str } {\robExtPrefixFilename\l__robExt_tmp_filename_no_prefix_str} \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } \let\placeholderPathFromContent\robExtPlaceholderPathFromContent @@ -695,53 +758,45 @@ if __name__ == '__main__': \XSIMfilewritestart*{\jobname-robExt-tmp-file-you-can-remove.tmp} }{ \XSIMfilewritestop - \ior_open:Nn \g_robExt_read_ior {\jobname-robExt-tmp-file-you-can-remove.tmp} - %% Put the file in \l_robExt_tmp_contain_file - \str_clear_new:N \l_robExt_tmp_contain_file - \ior_str_map_inline:Nn \g_robExt_read_ior { - \str_gput_right:Nx \l_robExt_tmp_contain_file {\tl_to_str:N{##1}^^J} + \ior_open:Nn \g__robExt_read_ior {\jobname-robExt-tmp-file-you-can-remove.tmp} + %% Put the file in \l__robExt_tmp_contain_file_str + \str_clear_new:N \l__robExt_tmp_contain_file_str + \ior_str_map_inline:Nn \g__robExt_read_ior { + \str_gput_right:Nx \l__robExt_tmp_contain_file_str {\tl_to_str:n{##1}^^J} } %% computes the new filename based on the md5 - \str_clear_new:N \l_robExt_tmp_filename - \str_set:Nx \l_robExt_tmp_filename_no_prefix {\pdfmdfivesum{\l_robExt_tmp_contain_file}#2} + \str_set:Nx \l__robExt_tmp_filename_no_prefix_str {\robExt@pdfmdfivesum{\l__robExt_tmp_contain_file_str}#2} %% Writes the content to the file \robExtCheckIfPrefixFolderExists - \iow_open:Nx \g_robExt_write_iow {\robExtAddCachePathAndName{\l_robExt_tmp_filename_no_prefix}} - \iow_now:NV \g_robExt_write_iow \l_robExt_tmp_contain_file - \iow_close:N \g_robExt_write_iow + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\l__robExt_tmp_filename_no_prefix_str}} + \iow_now:NV \g__robExt_write_iow \l__robExt_tmp_contain_file_str + \iow_close:N \g__robExt_write_iow %% sets the template name to the relative path to the file - \str_set:cx { l_robExt_placeholder_#3_str } {\robExtPrefixFilename\l_robExt_tmp_filename_no_prefix} + \str_set:cx { l__robExt_placeholder_#3_str } {\robExtPrefixFilename\l__robExt_tmp_filename_no_prefix_str} \IfBooleanTF {#1} {} { \robExtAddPlaceholderToList{#3} - \robExtKeepaftergroup{l_robExt_placeholder_group_main_seq} + \robExtKeepaftergroup{l__robExt_placeholder_group_main_seq} } %% Otherwise they will be lost when the environment ends - \robExtKeepaftergroup{l_robExt_placeholder_#3_str} + \robExtKeepaftergroup{l__robExt_placeholder_#3_str} } \let\PlaceholderPathFromCode\RobExtPlaceholderPathFromCode \let\endPlaceholderPathFromCode\endRobExtPlaceholderPathFromCode -%%% Evaluate a string by replacing the placeholders until there is none left -%%% the result will be in \robExtResult -% \cs_set:Nn \replace_until_impossible:N { -% \str_set:Nn \l_robExt_result_str { ls } -% } - -% backup, we are gona test it % %%% Evaluate a string by replacing the placeholders until there is none left % %%% the result will be in \robExtResult -\cs_set:Nn \replace_until_impossible:N { - \robExtDebugMessage{We needed to run replace_until_impossible} +\cs_set:Nn \__robExt_replace_until_impossible: { + \robExtDebugMessage{We~needed~to~run~replace_until_impossible} % Try to replace directly if single placeholder - \str_if_exist:cTF { l_robExt_placeholder_ \l_robExt_result_str _str }{ - \str_set_eq:Nc {\l_robExt_result_str}{ l_robExt_placeholder_ \l_robExt_result_str _str } + \str_if_exist:cTF { l__robExt_placeholder_ \l_robExt_result_str _str }{ + \str_set_eq:Nc {\l_robExt_result_str}{ l__robExt_placeholder_ \l_robExt_result_str _str } \str_set:Nn \l_robext_tmp_before {} } { \str_set_eq:NN \l_robext_tmp_before \l_robExt_result_str - \seq_map_inline:Nn \l_robExt_placeholder_group_main_seq { + \seq_map_inline:Nn \l__robExt_placeholder_group_main_seq { \robExtDebugMessage{Trying ~ to ~ replace ##1^^J} - \str_if_exist:cTF { l_robExt_placeholder_##1_str } { - \str_replace_all:Nnv \l_robExt_result_str { ##1 } { l_robExt_placeholder_##1_str } + \str_if_exist:cTF { l__robExt_placeholder_##1_str } { + \str_replace_all:Nnv \l_robExt_result_str { ##1 } { l__robExt_placeholder_##1_str } }{ \robExtDebugWarning{WARNING: ~ the ~ placeholder ~ ##1 ~ does ~ not ~ exist.} } @@ -756,7 +811,7 @@ if __name__ == '__main__': }{ % The strings are different: we restart %\message{The strings are different, we restart: It ~ used ~ to ~ be ~\l_robext_tmp_before^^J^^J now it is^^J^^J \l_robExt_result_str} - \replace_until_impossible:N { } + \__robExt_replace_until_impossible: } }{ % We found no __ so we stop before even if a change was made @@ -764,25 +819,25 @@ if __name__ == '__main__': }{ % We compare the result \str_compare:eNeTF \l_robext_tmp_before = \l_robExt_result_str { - % \str_set_eq:cN { l_robExt_placeholder_#1_str } \l_robExt_result_str + % \str_set_eq:cN { l__robExt_placeholder_#1_str } \l_robExt_result_str % \robExtAddPlaceholderToList{#1} }{ % The strings are different: we restart - \replace_until_impossible:N { } + \__robExt_replace_until_impossible: } } } \NewDocumentCommand{\robExtGetPlaceholderInResult}{sO{}m}{ - \str_set_hash_robust:Nn \l_robExt_result_str { #3 } - \robExtDebugMessage{Begin to evaluate placeholder^^J \l_robExt_result_str} - \replace_until_impossible:N { } - \robExtDebugMessage{Ended the replacement^^J \l_robExt_result_str} + \robExt_set_hash_robust:Nn \l_robExt_result_str { #3 } + \robExtDebugMessage{Begin~to~evaluate~placeholder^^J~\l_robExt_result_str} + \__robExt_replace_until_impossible: + \robExtDebugMessage{Ended~the~replacement^^J~\l_robExt_result_str} \tl_if_blank:nTF {#2} {} { % To avoid infinite recursion later and allow concatenation to a placeholder that does not exists % we remove the name of the placeholder at the end \str_remove_all:Nn \l_robExt_result_str { #2 } - \str_set_eq:cN { l_robExt_placeholder_#2_str } \l_robExt_result_str + \str_set_eq:cN { l__robExt_placeholder_#2_str } \l_robExt_result_str \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } } @@ -791,19 +846,19 @@ if __name__ == '__main__': %% like robExtGetPlaceholderInResult put assumes that there is a single placeholder name. %% It is used mainly for efficiency as it avoids a loop on all placeholder. \NewDocumentCommand{\robExtGetPlaceholderInResultFromSinglePlaceholder}{sO{}m}{ - \cs_if_exist:cTF { l_robExt_placeholder_#3_str }{ - \str_set_eq:Nc { \l_robExt_result_str }{ l_robExt_placeholder_#3_str } + \cs_if_exist:cTF { l__robExt_placeholder_#3_str }{ + \str_set_eq:Nc { \l_robExt_result_str }{ l__robExt_placeholder_#3_str } }{ - \PackageError{robExt}{[robExtGetPlaceholderInResultFromSinglePlaceholder] ~ The ~ placeholder ~ #3 ~ does ~ not ~ exists.} + \msg_error:nn{robExt}{placeholder not existing} } - \robExtDebugMessage{Starting to replace placeholder #3 to get \l_robExt_result_str} - \replace_until_impossible:N { } - \robExtDebugMessage{Ended up with \l_robExt_result_str} + \robExtDebugMessage{Starting~to~replace~placeholder~#3~to~get~\l_robExt_result_str} + \__robExt_replace_until_impossible: + \robExtDebugMessage{Ended~up~with~\l_robExt_result_str} \tl_if_blank:nTF {#2} {} { % To avoid infinite recursion later and allow concatenation to a placeholder that does not exists % we remove the name of the placeholder at the end \str_remove_all:Nn \l_robExt_result_str { #2 } - \str_set_eq:cN { l_robExt_placeholder_#2_str } \l_robExt_result_str + \str_set_eq:cN { l__robExt_placeholder_#2_str } \l_robExt_result_str \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#2}} } } @@ -811,13 +866,13 @@ if __name__ == '__main__': \cs_set:Nn \__replace_from_list:N { \str_set_eq:NN \l_robext_tmp_before \l_robExt_result_str \clist_map_inline:Nn {#1} { - \str_if_exist:cTF { l_robExt_placeholder_##1_str }{ - \str_replace_all:Nnv \l_robExt_result_str { ##1 } { l_robExt_placeholder_##1_str } + \str_if_exist:cTF { l__robExt_placeholder_##1_str }{ + \str_replace_all:Nnv \l_robExt_result_str { ##1 } { l__robExt_placeholder_##1_str } } { } } % We compare the result \str_compare:eNeTF \l_robext_tmp_before = \l_robExt_result_str { - % \str_set_eq:cN { l_robExt_placeholder_#1_str } \l_robExt_result_str + % \str_set_eq:cN { l__robExt_placeholder_#1_str } \l_robExt_result_str % \robExtAddPlaceholderToList{#1} }{ % The strings are different: we restart @@ -826,12 +881,12 @@ if __name__ == '__main__': } \NewDocumentCommand{\robExtGetPlaceholderInResultReplaceFromList}{smO{}m}{ - \str_set_hash_robust:Nn \l_robExt_result_str { #4 } - \clist_set:Nn \l_robExt_list_placeholder_replace_clist {#2} - \__replace_from_list:N { \l_robExt_list_placeholder_replace_clist } + \robExt_set_hash_robust:Nn \l_robExt_result_str { #4 } + \clist_set:Nn \l__robExt_list_placeholder_replace_clist {#2} + \__replace_from_list:N \l__robExt_list_placeholder_replace_clist \tl_if_blank:nTF {#3} {} { \str_remove_all:Nn \l_robExt_result_str { #3 } - \str_set_eq:cN { l_robExt_placeholder_#3_str } \l_robExt_result_str + \str_set_eq:cN { l__robExt_placeholder_#3_str } \l_robExt_result_str \IfBooleanTF {#1} {} {\robExtAddPlaceholderToList{#3}} } } @@ -844,8 +899,8 @@ if __name__ == '__main__': %%%% Same version, but replaces only one (useful sometimes) \cs_set:Nn \__replace_n_times:n { \str_set_eq:NN \l_robext_tmp_before \l_robExt_result_str - \seq_map_inline:Nn \l_robExt_placeholder_group_main_seq { - \str_replace_all:Nnv \l_robExt_result_str { ##1 } { l_robExt_placeholder_##1_str } + \seq_map_inline:Nn \l__robExt_placeholder_group_main_seq { + \str_replace_all:Nnv \l_robExt_result_str { ##1 } { l__robExt_placeholder_##1_str } } \int_compare:nTF { #1 > 1}{ \__replace_n_times:n {\int_eval:n {#1-1}} @@ -881,7 +936,7 @@ if __name__ == '__main__': %%% Evaluate a string by replacing the placeholders until there is none left %%% the result will be in \robExtResult \NewDocumentCommand{\robExtEvalPlaceholder}{m}{ - \str_set_hash_robust:Nn \l_test_str {#1} + \robExt_set_hash_robust:Nn \l_test_str {#1} \robExtGetPlaceholderInResult{#1} \tl_rescan:nv {} { l_robExt_result_str } } @@ -890,7 +945,7 @@ if __name__ == '__main__': %%% Evaluate a string by replacing the placeholders until there is none left %%% the result will be in \robExtResult \NewDocumentCommand{\robExtEvalPlaceholderReplaceFromList}{mm}{ - \str_set_hash_robust:Nn \l_test_str {#2} + \robExt_set_hash_robust:Nn \l_test_str {#2} \robExtGetPlaceholderInResultReplaceFromList{#1}{#2} \tl_rescan:nv {} { l_robExt_result_str } } @@ -899,21 +954,21 @@ if __name__ == '__main__': %%% Evaluate a placeholder by first trying some strings, then checking if a template is still present (if the %%% mode is enabled), and then it continues normally. Mostly for performance reasons. \NewDocumentCommand{\robExtEvalPlaceholderStartFromList}{mm}{ - \str_set_hash_robust:Nn \l_test_str {#2} + \robExt_set_hash_robust:Nn \l_test_str {#2} \robExtGetPlaceholderInResultReplaceFromList{#1}{#2} \cs_if_exist:NTF {\robExtPlaceholderOnlyWithUnderscores} { %% there might be non-important placeholders that are replaced later: - \str_set_eq:NN \l_robExt_result_minus_str \l_robExt_result_str - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_OUTPUT_PREFIX__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_SOURCE_FILE__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_OUTPUT_PDF__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_WAY_BACK__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_CACHE_FOLDER__} - \str_if_in:NnTF { \l_robExt_result_minus_str } { __ } { - \replace_until_impossible:N { } + \str_set_eq:NN \l__robExt_result_minus_str \l_robExt_result_str + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_OUTPUT_PREFIX__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_SOURCE_FILE__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_OUTPUT_PDF__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_WAY_BACK__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_CACHE_FOLDER__} + \str_if_in:NnTF { \l__robExt_result_minus_str } { __ } { + \__robExt_replace_until_impossible: }{ } - }{ \replace_until_impossible:N { } } + }{ \__robExt_replace_until_impossible: } \tl_rescan:nv {} { l_robExt_result_str } } \let\evalPlaceholderStartFromList\robExtEvalPlaceholderStartFromList @@ -921,37 +976,37 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtEvalPlaceholderInplace}{m}{ \robExtGetPlaceholderInResult{#1} - \tl_set_rescan:Nnx \l_robExt_tmp_tl {} { \l_robExt_result_str } - \str_set_hash_robust:cx { l_robExt_placeholder_#1_str } { \l_robExt_tmp_tl } + \tl_set_rescan:Nnx \l__robExt_tmp_tl {} { \l_robExt_result_str } + \robExt_set_hash_robust:cx { l__robExt_placeholder_#1_str } { \l__robExt_tmp_tl } } \let\evalPlaceholderInplace\robExtEvalPlaceholderInplace \NewDocumentCommand{\robExtEvalPlaceholderInplaceFromSinglePlaceholder}{m}{ \robExtGetPlaceholderInResultFromSinglePlaceholder{#1} - \tl_set_rescan:Nnx \l_robExt_tmp_tl {} { \l_robExt_result_str } - \str_set_hash_robust:cx { l_robExt_placeholder_#1_str } { \l_robExt_tmp_tl } + \tl_set_rescan:Nnx \l__robExt_tmp_tl {} { \l_robExt_result_str } + \robExt_set_hash_robust:cx { l__robExt_placeholder_#1_str } { \l__robExt_tmp_tl } } \let\evalPlaceholderInplaceFromSinglePlaceholder\robExtEvalPlaceholderInplaceFromSinglePlaceholder % Sometimes two symbols ## (with a different catcode than a normal hash) are added instead of a single # \NewDocumentCommand{\robExtPlaceholderDoubleNumberHashesInplace}{m}{ - \str_replace_all:cxx { l_robExt_placeholder_#1_str } { \c_hash_str } { \c_hash_str \c_hash_str } + \str_replace_all:cxx { l__robExt_placeholder_#1_str } { \c_hash_str } { \c_hash_str \c_hash_str } } \let\placeholderDoubleNumberHashesInplace\robExtPlaceholderDoubleNumberHashesInplace \NewDocumentCommand{\robExtPlaceholderHalveNumberHashesInplace}{m}{ - \str_replace_all:cxx { l_robExt_placeholder_#1_str } { \c_hash_str \c_hash_str } { \c_hash_str } + \str_replace_all:cxx { l__robExt_placeholder_#1_str } { \c_hash_str \c_hash_str } { \c_hash_str } } \let\placeholderHalveNumberHashesInplace\robExtPlaceholderHalveNumberHashesInplace \NewDocumentCommand{\robExtPlaceholderReplaceInplace}{mmm}{ - \str_replace_all:cnn { l_robExt_placeholder_#1_str } { #2 } { #3 } + \str_replace_all:cnn { l__robExt_placeholder_#1_str } { #2 } { #3 } } \let\placeholderReplaceInplace\robExtPlaceholderReplaceInplace \NewDocumentCommand{\robExtPlaceholderReplaceInplaceEval}{mmm}{ - \str_replace_all:cxx { l_robExt_placeholder_#1_str } { #2 } { #3 } + \str_replace_all:cxx { l__robExt_placeholder_#1_str } { #2 } { #3 } } \let\placeholderReplaceInplaceEval\robExtPlaceholderReplaceInplaceEval @@ -971,78 +1026,78 @@ if __name__ == '__main__': \NewDocumentCommand{\robExtAddPlaceholdersToGroup}{mm}{ % Make sure we have a string here: - \str_set_hash_robust:Nn \l_robExt_tmp_str { #2 } - \seq_set_from_clist:NN \l_robExt_tmp_seq \l_robExt_tmp_str - \cs_if_exist:cTF {l_robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l_robExt_placeholder_group_#1_seq}} + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #2 } + \seq_set_from_clist:NN \l__robExt_tmp_seq \l__robExt_tmp_str + \cs_if_exist:cTF {l__robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l__robExt_placeholder_group_#1_seq}} % First we remove existing occurrences (useful to avoid listing the same macro more than once % if we redefine a macro): - \seq_map_inline:Nn \l_robExt_tmp_seq { + \seq_map_inline:Nn \l__robExt_tmp_seq { \message{Adding ##1 to #1} - \seq_remove_all:cn {l_robExt_placeholder_group_#1_seq} {##1} - \seq_put_right:cn {l_robExt_placeholder_group_#1_seq} {##1} + \seq_remove_all:cn {l__robExt_placeholder_group_#1_seq} {##1} + \seq_put_right:cn {l__robExt_placeholder_group_#1_seq} {##1} } } \let\addPlaceholdersToGroup\robExtAddPlaceholdersToGroup \NewDocumentCommand{\robExtEnsureGroupPlaceholdersExists}{m}{ - \cs_if_exist:cTF {l_robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l_robExt_placeholder_group_#1_seq}} + \cs_if_exist:cTF {l__robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l__robExt_placeholder_group_#1_seq}} } \let\ensureGroupPlaceholdersExists\robExtEnsureGroupPlaceholdersExists \NewDocumentCommand{\robExtAddPlaceholdersToGroupBefore}{mm}{ % Make sure we have a string here: - \str_set_hash_robust:Nn \l_robExt_tmp_str { #2 } - \seq_set_from_clist:NN \l_robExt_tmp_seq \l_robExt_tmp_str - \seq_reverse:N \l_robExt_tmp_seq - \cs_if_exist:cTF {l_robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l_robExt_placeholder_group_#1_seq}} + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #2 } + \seq_set_from_clist:NN \l__robExt_tmp_seq \l__robExt_tmp_str + \seq_reverse:N \l__robExt_tmp_seq + \cs_if_exist:cTF {l__robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l__robExt_placeholder_group_#1_seq}} % First we remove existing occurrences (useful to avoid listing the same macro more than once % if we redefine a macro): - \seq_map_inline:Nn \l_robExt_tmp_seq { - \seq_remove_all:cn {l_robExt_placeholder_group_#1_seq} {##1} - \seq_put_left:cn {l_robExt_placeholder_group_#1_seq} {##1} + \seq_map_inline:Nn \l__robExt_tmp_seq { + \seq_remove_all:cn {l__robExt_placeholder_group_#1_seq} {##1} + \seq_put_left:cn {l__robExt_placeholder_group_#1_seq} {##1} } } \let\addPlaceholdersToGroupBefore\robExtAddPlaceholdersToGroupBefore \NewDocumentCommand{\robExtRemovePlaceholdersFromGroup}{mm}{ % Make sure we have a string here: - \str_set_hash_robust:Nn \l_robExt_tmp_str { #2 } - \seq_set_from_clist:NN \l_robExt_tmp_seq \l_robExt_tmp_str - \seq_reverse:N \l_robExt_tmp_seq - \cs_if_exist:cTF {l_robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l_robExt_placeholder_group_#1_seq}} + \robExt_set_hash_robust:Nn \l__robExt_tmp_str { #2 } + \seq_set_from_clist:NN \l__robExt_tmp_seq \l__robExt_tmp_str + \seq_reverse:N \l__robExt_tmp_seq + \cs_if_exist:cTF {l__robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l__robExt_placeholder_group_#1_seq}} % First we remove existing occurrences (useful to avoid listing the same macro more than once % if we redefine a macro): - \seq_map_inline:Nn \l_robExt_tmp_seq { - \seq_remove_all:cn {l_robExt_placeholder_group_#1_seq} {##1} + \seq_map_inline:Nn \l__robExt_tmp_seq { + \seq_remove_all:cn {l__robExt_placeholder_group_#1_seq} {##1} } } \let\removePlaceholdersFromGroup\robExtRemovePlaceholdersFromGroup \let\removePlaceholderFromGroup\robExtRemovePlaceholdersFromGroup \NewDocumentCommand{\robExtClearGroupPlaceholders}{m}{ - \seq_clear_new:c {l_robExt_placeholder_group_#1_seq} + \seq_clear_new:c {l__robExt_placeholder_group_#1_seq} } \let\clearGroupPlaceholders\robExtClearGroupPlaceholders \NewDocumentCommand{\robExtCopyGroupPlaceholders}{mm}{ - \seq_set_eq:cc {l_robExt_placeholder_group_#1_seq} {l_robExt_placeholder_group_#2_seq} + \seq_set_eq:cc {l__robExt_placeholder_group_#1_seq} {l__robExt_placeholder_group_#2_seq} } \let\copyGroupPlaceholders\robExtCopyGroupPlaceholders \NewDocumentCommand{\robExtAppendGroupPlaceholders}{mm}{ - \seq_map_inline:cn {l_robExt_placeholder_group_#2_seq} { - \seq_remove_all:cn {l_robExt_placeholder_group_#1_seq} {##1} - \seq_put_right:cn {l_robExt_placeholder_group_#1_seq} {##1} + \seq_map_inline:cn {l__robExt_placeholder_group_#2_seq} { + \seq_remove_all:cn {l__robExt_placeholder_group_#1_seq} {##1} + \seq_put_right:cn {l__robExt_placeholder_group_#1_seq} {##1} } } \let\appendGroupPlaceholders\robExtAppendGroupPlaceholders \NewDocumentCommand{\robExtAppendBeforeGroupPlaceholders}{mm}{ - \seq_eq:Nc \l_robExt_tmp_seq {l_robExt_placeholder_group_#2_seq} - \seq_reverse:N \l_robExt_tmp_seq - \seq_map_inline:Nn \l_robExt_tmp_seq { - \seq_remove_all:cn {l_robExt_placeholder_group_#1_seq} {##1} - \seq_put_left:cn {l_robExt_placeholder_group_#1_seq} {##1} + \seq_eq:Nc \l__robExt_tmp_seq {l__robExt_placeholder_group_#2_seq} + \seq_reverse:N \l__robExt_tmp_seq + \seq_map_inline:Nn \l__robExt_tmp_seq { + \seq_remove_all:cn {l__robExt_placeholder_group_#1_seq} {##1} + \seq_put_left:cn {l__robExt_placeholder_group_#1_seq} {##1} } } \let\appendBeforeGroupPlaceholders\robExtAppendBeforeGroupPlaceholders @@ -1053,7 +1108,7 @@ if __name__ == '__main__': \let\importPlaceholdersFromGroup\robExtImportPlaceholdersFromGroup \NewDocumentCommand{\robExtImportAllPlaceholders}{}{ - \seq_map_inline:Nn \l_robExt_list_groups_seq { + \seq_map_inline:Nn \l__robExt_list_groups_seq { \robExtImportPlaceholdersFromGroup{##1} } } @@ -1062,24 +1117,24 @@ if __name__ == '__main__': %%%% List groups, mostly for debugging purpose %%%% -\seq_new:N \l_robExt_list_groups_seq % contains the list of all groups +\seq_new:N \l__robExt_list_groups_seq % contains the list of all groups \NewDocumentCommand{\robExtRegisterGroupPlaceholders}{m}{ - \seq_if_exist:cTF {l_robExt_placeholder_group_#1_seq}{}{ - \seq_clear_new:c {l_robExt_placeholder_group_#1_seq} + \seq_if_exist:cTF {l__robExt_placeholder_group_#1_seq}{}{ + \seq_clear_new:c {l__robExt_placeholder_group_#1_seq} } - \seq_put_right:Nn \l_robExt_list_groups_seq {#1} + \seq_put_right:Nn \l__robExt_list_groups_seq {#1} } \let\registerGroupPlaceholders\robExtRegisterGroupPlaceholders \let\newGroupPlaceholders\robExtRegisterGroupPlaceholders \NewDocumentCommand{\robExtShowAllRegisteredGroupsAndPlaceholders}{s}{% - \seq_map_inline:Nn \l_robExt_list_groups_seq {% + \seq_map_inline:Nn \l__robExt_list_groups_seq {% \message{^^J- Group ##1: ^^J}% - \seq_map_inline:cn {l_robExt_placeholder_group_##1_seq} {% + \seq_map_inline:cn {l__robExt_placeholder_group_##1_seq} {% \message{, ####1 }% \IfBooleanTF{#1}{ - \message{Content:\use:c{l_robExt_placeholder_####1_str}} + \message{Content:\use:c{l__robExt_placeholder_####1_str}} }{} }% \message{^^J} @@ -1089,16 +1144,16 @@ if __name__ == '__main__': \let\showAllRegisteredGroupsAndPlaceholders\robExtShowAllRegisteredGroupsAndPlaceholders \NewDocumentCommand{\robExtPrintAllRegisteredGroups}{s}{% - \seq_map_inline:Nn \l_robExt_list_groups_seq {% + \seq_map_inline:Nn \l__robExt_list_groups_seq {% \texttt{##1}\par } } \let\printAllRegisteredGroups\robExtPrintAllRegisteredGroups \NewDocumentCommand{\robExtPrintAllRegisteredGroupsAndPlaceholders}{s}{% - \seq_map_inline:Nn \l_robExt_list_groups_seq {% + \seq_map_inline:Nn \l__robExt_list_groups_seq {% - ~ Group \texttt{##1}:\par - \seq_map_inline:cn {l_robExt_placeholder_group_##1_seq} {% + \seq_map_inline:cn {l__robExt_placeholder_group_##1_seq} {% Placeholder \texttt{####1}% \IfBooleanTF {#1}{\robExtPrintPlaceholderNoReplacement{####1}}{\par} }% @@ -1110,7 +1165,7 @@ if __name__ == '__main__': % The star version displays the content as well \NewDocumentCommand{\robExtPrintGroupPlaceholders}{sm}{% Content~ of~ group~ \texttt{#2}:\par% - \seq_map_inline:cn {l_robExt_placeholder_group_#2_seq} {% + \seq_map_inline:cn {l__robExt_placeholder_group_#2_seq} {% -~Placeholder ~ \texttt{##1}\par \IfBooleanTF {#1}{\robExtPrintPlaceholderNoReplacement{##1}}{} }% } @@ -1121,23 +1176,23 @@ if __name__ == '__main__': %%%%%%%%%%% Dependencies %%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \NewDocumentCommand{\robExtResetDependencies}{m}{ - \seq_clear:N \l_robExt_dependencies + \seq_clear:N \l__robExt_dependencies_str } \NewDocumentCommand{\robExtAddDependency}{m}{ - \seq_put_left:Nx \l_robExt_dependencies {#1} + \seq_put_left:Nx \l__robExt_dependencies_str {#1} } \NewDocumentCommand{\robExtDebugDependency}{}{ - \show\l_robExt_dependencies + \show\l__robExt_dependencies_str } % Useful when compiling commands \NewDocumentCommand{\robExtSaveDependencies}{m}{ - \seq_set_eq:cN {#1} \l_robExt_dependencies + \seq_gset_eq:cN {#1} \l__robExt_dependencies_str } \NewDocumentCommand{\robExtRestoreDependencies}{m}{ - \seq_set_eq:Nc \l_robExt_dependencies {#1} + \seq_set_eq:Nc \l__robExt_dependencies_str {#1} } @@ -1163,22 +1218,22 @@ if __name__ == '__main__': % then do a more normal thing), third is the name of the placeholder \cs_set:Nn \try_to_evaluate_single_placeholder_without_going_to_replace_until_impossible:NNn { %%%%% Take care of the compilation command: might seem a bit weird and convoluted, but this is needed to get really good efficiency - %%%%% since \replace_until_impossible:N { } is quite costly, we try to avoid it + %%%%% since \__robExt_replace_until_impossible: is quite costly, we try to avoid it \cs_if_exist:NTF {#1} { % We only replace fixed state of placeholders - \str_set_eq:Nc \l_robExt_result_str {l_robExt_placeholder_#3_str} + \str_set_eq:Nc \l_robExt_result_str {l__robExt_placeholder_#3_str} \seq_map_inline:Nn {#1} { - \robExtDebugMessage{replacing ##1 in \l_robExt_result_str} - \str_replace_all:Nnv \l_robExt_result_str {##1} {l_robExt_placeholder_##1_str} + \robExtDebugMessage{replacing~##1~in~\l_robExt_result_str} + \str_replace_all:Nnv \l_robExt_result_str {##1} {l__robExt_placeholder_##1_str} } - \robExtDebugMessage{Cool, we avoided the costly replace (only try).} + \robExtDebugMessage{Cool,~we~avoided~the~costly~replace~(only~try).} }{ \cs_if_exist:NTF {#2} { % We start by replacing a fixed state of placeholders - \str_set_eq:Nc \l_robExt_result_str {l_robExt_placeholder_#3_str} - \robExtDebugMessage{We start by replacing a fixed state of placeholders} - %\show#2 + \str_set_eq:Nc \l_robExt_result_str {l__robExt_placeholder_#3_str} + \robExtDebugMessage{We~start~by~replacing~a~fixed~state~of~placeholders} \seq_map_inline:Nn {#2} { - \str_if_exist:cTF {l_robExt_placeholder_##1_str} { - \str_replace_all:Nnv \l_robExt_result_str {##1} {l_robExt_placeholder_##1_str} + \str_if_exist:cTF {l__robExt_placeholder_##1_str} { + \robExtDebugMessage{Replacing~##1~into~\use:c{l__robExt_placeholder_##1_str}}% + \str_replace_all:Nnv \l_robExt_result_str {##1} {l__robExt_placeholder_##1_str} } { \robExtDebugMessage{Warning: ~ trying~to~replace~##1~that~does~not~exist.} } @@ -1188,28 +1243,28 @@ if __name__ == '__main__': \cs_if_exist:NTF {\robExtPlaceholderOnlyWithUnderscores} { % all templates have underscores % first check if some non-important elements are here: for this we first remove them - \str_set_eq:NN \l_robExt_result_minus_str \l_robExt_result_str - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_OUTPUT_PREFIX__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_SOURCE_FILE__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_OUTPUT_PDF__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_WAY_BACK__} - \str_remove_all:Nn \l_robExt_result_minus_str {__ROBEXT_CACHE_FOLDER__} + \str_set_eq:NN \l__robExt_result_minus_str \l_robExt_result_str + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_OUTPUT_PREFIX__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_SOURCE_FILE__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_OUTPUT_PDF__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_WAY_BACK__} + \str_remove_all:Nn \l__robExt_result_minus_str {__ROBEXT_CACHE_FOLDER__} % Now, we check if some __ are left: - \str_if_in:NnTF { \l_robExt_result_minus_str } { __ } { + \str_if_in:NnTF { \l__robExt_result_minus_str } { __ } { % Some templates are left - \robExtDebugMessage{Still some templates are left.} - \replace_until_impossible:N { } + \robExtDebugMessage{Still~some~templates~are~left.} + \__robExt_replace_until_impossible: } { % No more template: the end - \robExtDebugMessage{Cool, we avoided the costly replace.} + \robExtDebugMessage{Cool,~we~avoided~the~costly~replace: \l_robExt_result_str.} } }{ % no way to know, must do it anyway - \robExtDebugMessage{Seems like not all templates contain underscore. Too bad, it will be slower} - \replace_until_impossible:N { } + \robExtDebugMessage{Seems~like~not~all~templates~contain~underscore.~Too~bad,~it~will~be~slower} + \__robExt_replace_until_impossible: } }{ - \robExtDebugMessage{No advice on how to start: doing it the long way} + \robExtDebugMessage{No~advice~on~how~to~start:~doing~it~the~long~way} \robExtGetPlaceholderInResultFromSinglePlaceholder{#3} } } @@ -1236,9 +1291,9 @@ if __name__ == '__main__': \let\firstPlaceholdersInTemplate\robExtFirstPlaceholdersInTemplate % for use outside of expl3 (pgf) -\def\robExtCurrentCompilationCommand{\l_robExt_currentCompilationCommand} +\def\robExtCurrentCompilationCommand{\l__robExt_current_compilation_command_str} \NewDocumentCommand{\robExtGetNearlyFinalValueTemplateAndCompilationCommand}{}{ - %% oututs the compilation in \l_robExt_currentCompilationCommand and the template in \l_robExt_result_str + %% oututs the compilation in \l__robExt_current_compilation_command_str and the template in \l_robExt_result_str %% This is for efficiency reasons: we only allow a few placeholders: %% In compilation command: %% @@ -1255,88 +1310,46 @@ if __name__ == '__main__': %% - __ROBEXT_WAY_BACK__ %% - __ROBEXT_CACHE_FOLDER__ \ifdefined\robExtDisablePlaceholders% - \robExtDebugMessage{You have disabled placeholders (this is likely a compiled template)}% - \str_set_eq:NN \l_robExt_currentCompilationCommand \l_robExt_placeholder___ROBEXT_COMPILATION_COMMAND___str - \str_set_eq:NN \l_robExt_result_str \l_robExt_placeholder___ROBEXT_TEMPLATE___str% - \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_MAIN_CONTENT__} {l_robExt_placeholder___ROBEXT_MAIN_CONTENT___str} - \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_LATEX_PREAMBLE__} {l_robExt_placeholder___ROBEXT_LATEX_PREAMBLE___str} - \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_MAIN_CONTENT_ORIG__} {l_robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str} + \robExtDebugMessage{You~have~disabled~placeholders~(this~is~likely~a~compiled~template)}% + \str_set_eq:NN \l__robExt_current_compilation_command_str \l__robExt_placeholder___ROBEXT_COMPILATION_COMMAND___str + \str_set_eq:NN \l_robExt_result_str \l__robExt_placeholder___ROBEXT_TEMPLATE___str% + \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_MAIN_CONTENT__} {l__robExt_placeholder___ROBEXT_MAIN_CONTENT___str} + \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_LATEX_PREAMBLE__} {l__robExt_placeholder___ROBEXT_LATEX_PREAMBLE___str} + \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_MAIN_CONTENT_ORIG__} {l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str} \else \ifdefined\robExtEnableImportMechanism\else\robExtImportAllPlaceholders\fi% %%%%% Take care of the compilation command: might seem a bit weird and convoluted, but this is needed to get really good efficiency - %%%%% since \replace_until_impossible:N { } is quite costly, we try to avoid it + %%%%% since \__robExt_replace_until_impossible: is quite costly, we try to avoid it \try_to_evaluate_single_placeholder_without_going_to_replace_until_impossible:NNn \robExtOnlyTryThisSequenceOnCompilationCommand \robExtFirstTryThisSequenceOnCompilationCommand {__ROBEXT_COMPILATION_COMMAND__} - \str_set_eq:NN \l_robExt_currentCompilationCommand \l_robExt_result_str + \str_set_eq:NN \l__robExt_current_compilation_command_str \l_robExt_result_str %%%%% Take care of \try_to_evaluate_single_placeholder_without_going_to_replace_until_impossible:NNn \robExtOnlyTryThisSequenceOnTemplate \robExtFirstTryThisSequenceOnTemplate {__ROBEXT_TEMPLATE__} \fi } \NewDocumentCommand{\robExtSetBackCompilationCommandAndTemplate}{}{ - \robExtPlaceholderFromString{__ROBEXT_COMPILATION_COMMAND__}{\l_robExt_currentCompilationCommand}% + \robExtPlaceholderFromString{__ROBEXT_COMPILATION_COMMAND__}{\l__robExt_current_compilation_command_str}% \robExtPlaceholderFromString{__ROBEXT_TEMPLATE__}{\l_robExt_result_str}% } -%%% \l_robExt_final_file must contain before calling this function the content of the final file. -%%% \l_robExt_dependencies must contain the extensions (list). -%%% \l_robExt_currentCompilationCommand contains the compilation command to use. -%%% Note that we do note parse them as input to allow more flexibility on the way the user -%%% defines them, and to limit issues with expansion. \NewDocumentCommand{\robExtWriteFile}{m}{ - %%% First we get all dependencies stored in \l_robExt_dependencies to create a csv-like file: - \str_clear:N \l_robExt_dependencies_mdfive + %%% First we get all dependencies stored in \l__robExt_dependencies_str to create a csv-like file: + \str_clear:N \l__robExt_dependencies_mdfive_str \robExtGetNearlyFinalValueTemplateAndCompilationCommand - % \ifdefined\robExtDisablePlaceholders% - % %% Compilation - - % \str_set_eq:NN \l_robExt_currentCompilationCommand \l_robExt_placeholder___ROBEXT_COMPILATION_COMMAND___str - % \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_LATEX_CREATE_OUT_FILE__} {l_robExt_placeholder___ROBEXT_LATEX_CREATE_OUT_FILE___str} - - % \str_set_eq:NN \l_robExt_result_str \l_robExt_placeholder___ROBEXT_TEMPLATE___str% - % \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_LATEX_CREATE_OUT_FILE__} {l_robExt_placeholder___ROBEXT_LATEX_CREATE_OUT_FILE___str} - % \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_LATEX_WRITE_DEPTH_TO_OUT_FILE__} {l_robExt_placeholder___ROBEXT_LATEX_WRITE_DEPTH_TO_OUT_FILE___str} - % % \show\l_robExt_placeholder___ROBEXT_MAIN_CONTENT___str - % \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_MAIN_CONTENT__} {l_robExt_placeholder___ROBEXT_MAIN_CONTENT___str} - % \str_replace_all:Nnv \l_robExt_result_str {__ROBEXT_MAIN_CONTENT_ORIG__} {l_robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str} - % \else% - % %% Make sure to remove these placeholders as they should not be replaced. - % %% Not that we cannot just give them their final value here, as it cannot yet be determined without - % %% first computing the md5 hash. - % \ifdefined\robExtEnableImportMechanism\else\robExtImportAllPlaceholders\fi% - % % \robExtConfigure{run ~ early ~ before ~ compilation} - % % \robExtConfigure{show ~imported~ placeholders}% debug - % % \robExtRemovePlaceholder{__ROBEXT_SOURCE_FILE__} - % % \robExtRemovePlaceholder{__ROBEXT_OUTPUT_PDF__} - % % \robExtRemovePlaceholder{__ROBEXT_OUTPUT_PREFIX__} - % % it seems to take non negligible time let us replace them manually - % % \robExtPlaceholderFromString{__ROBEXT_WAY_BACK__}{\robExtCacheFolderWayBack} - % % \robExtPlaceholderFromString{__ROBEXT_CACHE_FOLDER__}{\robExtCacheFolder} - % %%% We rescan the string in order to evaluate stuff like \myframes into "12,45,56". - % % \robExtConfigure{run ~ before ~ compilation} - % \robExtGetPlaceholderInResultFromSinglePlaceholder{__ROBEXT_COMPILATION_COMMAND__} - % % \robExtConfigure{run ~ after ~ compilation} - % % \ifdefined\robExtDoNotRescanFirstTime - % \str_set_eq:NN \l_robExt_currentCompilationCommand \l_robExt_result_str - % % \else - % % \tl_set_rescan:Nnx \l_robExt_currentCompilationCommand {} { \l_robExt_result_str } - % % \fi% - % %% We get the template - % % \robExtConfigure{run ~ before ~ template} - % \robExtGetPlaceholderInResultFromSinglePlaceholder{__ROBEXT_TEMPLATE__} - % % \robExtConfigure{run ~ after ~ template} - % \fi% - %\str_set_eq:NN \l_robExt_final_file_minus_hash_str \l_robExt_result_str % We first add on the first line the compilation command, and on the second line the template file. - \str_set:Nx \l_robExt_dependencies_mdfive {command,\l_robExt_currentCompilationCommand^^J\pdfmdfivesum{\l_robExt_result_str ^^J},^^J} %% ^^J is a newline: LaTeX will automatically add a new line when writing the file - \seq_map_inline:Nn \l_robExt_dependencies { - \str_put_right:Nx \l_robExt_dependencies_mdfive {\file_mdfive_hash:n{##1},##1^^J} %% ^^J is a newline + \str_set:Nx \l__robExt_dependencies_mdfive_str {command,\l__robExt_current_compilation_command_str^^J\robExt@pdfmdfivesum{\l_robExt_result_str ^^J},^^J} %% ^^J is a newline: LaTeX will automatically add a new line when writing the file + \seq_map_inline:Nn \l__robExt_dependencies_str { + \str_put_right:Nx \l__robExt_dependencies_mdfive_str {\file_mdfive_hash:n{##1},##1^^J} %% ^^J is a newline } %% %% Compute the final hash (the hash of all dependencies, including the current picture that is on the first line): %% The last newline is needed as the write operation automatically adds a newline. - \tl_set:Nx \robExtFinalHash {\pdfmdfivesum{\l_robExt_dependencies_mdfive^^J}} + \tl_set:Nx \robExtFinalHash {\robExt@pdfmdfivesum{\l__robExt_dependencies_mdfive_str^^J}} + % Might be useful to clean the previously compiled files, or change the MD5 sum, for instance if we do + % not want to recompile when the file changes + \robExtUseHookJustBeforeWritingFiles% %% We add the figure in the list of files. - \iow_now:Nx \g_robExt_write_list_all_figures {\robExtAddPrefixName{\robExtFinalHash.tex}} + \iow_now:Nx \g__robExt_write_list_all_figures_iow {\robExtAddPrefixName{\robExtFinalHash.tex}} %% We can now set the placeholders, and recompute the final value of the file: %% I was doing before \robExtPlaceholderFromContent + \robExtEvalPlaceholderInplace, but it is too slow \str_set:Nx \l_tmp_output_prefix_str {\robExtAddPrefixName{\robExtFinalHash}} @@ -1346,22 +1359,21 @@ if __name__ == '__main__': \str_replace_all:Nnx \l_robExt_result_str {__ROBEXT_OUTPUT_PDF__}{\l_tmp_output_prefix_str .pdf} \str_replace_all:Nnx \l_robExt_result_str {__ROBEXT_WAY_BACK__}{\robExtCacheFolderWayBack} \str_replace_all:Nnx \l_robExt_result_str {__ROBEXT_CACHE_FOLDER__}{\robExtCacheFolder} - %\str_set_eq:NN \l_robExt_final_file_str \l_robExt_result_str \file_if_exist:xTF{\robExtAddCachePathAndName{\robExtFinalHash.tex}}{ \message{The\space file\space \robExtAddCachePathAndName{\robExtFinalHash.tex} \space already\space exists.^^J} }{ \str_if_eq:VnTF { \l_robExt_result_str }{__ROBEXT_TEMPLATE__} { - \PackageError{robExt}{You ~ are ~ writing ~ __ROBEXT_TEMPLATE__ ~ to ~ your ~ file: ~ seems ~ like ~ you ~ forgot ~ to ~ define ~ your ~ template ~ or ~ set ~ a ~ preset}{} + \msg_error:nn{robExt}{forgot template} }{ % Check if the output directory exists \robExtCheckIfPrefixFolderExists - \iow_open:Nx \g_robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.deps}} - \iow_now:NV \g_robExt_write_iow \l_robExt_dependencies_mdfive - \iow_close:N \g_robExt_write_iow + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.deps}} + \iow_now:NV \g__robExt_write_iow \l__robExt_dependencies_mdfive_str + \iow_close:N \g__robExt_write_iow %% Save the final file: - \iow_open:Nx \g_robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.tex}} - \iow_now:NV \g_robExt_write_iow \l_robExt_result_str - \iow_close:N \g_robExt_write_iow + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.tex}} + \iow_now:NV \g__robExt_write_iow \l_robExt_result_str + \iow_close:N \g__robExt_write_iow \message{Source ~ saved ~ in ~ \robExtAddCachePathAndName{\robExtFinalHash.tex}.} } } @@ -1372,38 +1384,82 @@ if __name__ == '__main__': % Think about the number of compilations. \NewDocumentCommand{\robExtCompileFile}{m}{ \file_if_exist:xTF{\robExtAddCachePathAndName{\robExtFinalHash.pdf}}{ + \cs_if_exist:NTF {\robExtForceRecompilation}{ + % We cannot do an OR with file_if_exist since it is not expandable, hence this trick + \let\l__robExt_do_not_compile\undefined + }{ + \def\l__robExt_do_not_compile{} + } + }{ + \let\l__robExt_do_not_compile\undefined + } + \cs_if_exist:NTF {\l__robExt_do_not_compile}{ \message{No ~ need ~ to ~ recompile ~ \robExtAddCachePathAndName{\robExtFinalHash.pdf}^^J} }{ + % Used to know later if we have run a compilation command at that step or not + \cs_undefine:N \l__robExt_I_just_ran_a_compilation_command: % No need to re-evaluate it % \robExtGetPlaceholderInResultFromSinglePlaceholder{__ROBEXT_COMPILATION_COMMAND__} - \str_replace_all:Nnx \l_robExt_currentCompilationCommand {__ROBEXT_OUTPUT_PREFIX__}{\l_tmp_output_prefix_str} - \str_replace_all:Nnx \l_robExt_currentCompilationCommand {__ROBEXT_SOURCE_FILE__}{\l_tmp_output_prefix_str .tex} - \str_replace_all:Nnx \l_robExt_currentCompilationCommand {__ROBEXT_OUTPUT_PDF__}{\l_tmp_output_prefix_str .pdf} - \str_replace_all:Nnx \l_robExt_currentCompilationCommand {__ROBEXT_WAY_BACK__}{\robExtCacheFolderWayBack} - \str_replace_all:Nnx \l_robExt_currentCompilationCommand {__ROBEXT_CACHE_FOLDER__}{\robExtCacheFolder} - % \ifdefined\robExtDoNotRescanSecondTime - % \str_set_eq:NN \l_robExt_finalCompilationCommand \l_robExt_currentCompilationCommand - % \else - % \tl_set_rescan:Nnx \l_robExt_finalCompilationCommand {} { \l_robExt_currentCompilationCommand } - % \fi% + \str_replace_all:Nnx \l__robExt_current_compilation_command_str {__ROBEXT_OUTPUT_PREFIX__}{\l_tmp_output_prefix_str} + \str_replace_all:Nnx \l__robExt_current_compilation_command_str {__ROBEXT_SOURCE_FILE__}{\l_tmp_output_prefix_str .tex} + \str_replace_all:Nnx \l__robExt_current_compilation_command_str {__ROBEXT_OUTPUT_PDF__}{\l_tmp_output_prefix_str .pdf} + \str_replace_all:Nnx \l__robExt_current_compilation_command_str {__ROBEXT_WAY_BACK__}{\robExtCacheFolderWayBack} + \str_replace_all:Nnx \l__robExt_current_compilation_command_str {__ROBEXT_CACHE_FOLDER__}{\robExtCacheFolder} % Make sure this command is run from the cache folder \ifdefined\robExtCacheFolder - \str_put_left:Nx \l_robExt_currentCompilationCommand {cd ~ \robExtCacheFolder \space && ~ } + \str_put_left:Nx \l__robExt_current_compilation_command_str {cd ~ \robExtCacheFolder \space && ~ } + \fi% + %%% We enable manual mode if we enabled "compile in parallel after=N" and we compiled more than N elements + % Check if we want to run stuff in parallel + \ifdefined\robExt@compile@parallel@after + % TODO: **** + \int_gset:Nn \g__robExt_number_figures_just_compiled_or_to_compile {\g__robExt_number_figures_just_compiled_or_to_compile + 1} + % We check if the number of figures that we already compiled is large enough to start parallel compilation + \int_compare:nNnTF { + \g__robExt_number_figures_just_compiled_or_to_compile} > {\robExt@compile@parallel@after + } { + \def\robExtManualMode + } {} \fi \ifdefined\robExtManualMode - \message{[robExt] Manual mode enabled: please, manually compile the images using \l_robExt_currentCompilationCommand or run 'bash \jobname-\robExtAddPrefixName{compile-missing-figures.sh}'.} - \iow_now:Nx \g_robExt_write_manually_compile_all_missing_figures {\l_robExt_currentCompilationCommand}% a new line is automatically added + \message{[robExt] Manual mode enabled: please, manually compile the images using \l__robExt_current_compilation_command_str or run 'bash \jobname-\robExtAddPrefixName{compile-missing-figures.sh}'.} + % Avoid writing the same command multiple times (might occur in environments that execute the + % same command multiple times like align etc) + \cs_if_exist:cTF {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :}{}{ + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\l__robExt_current_compilation_command_str}% a new line is automatically added + \cs_gset:cn {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :} {} + \ifdefined\robExt@compile@parallel@after + \gdef\robExt@compile@parallel@must@compile{} + \seq_gput_right:Nx \g__robExt_files_compiled_in_parallel {\robExtAddCachePathAndName{\robExtFinalHash}} + % Gets the line with the error for easier debugging + \cs_gset:cx {g__robExt_lines_error_\robExtAddCachePathAndName{\robExtFinalHash}:} {\msg_line_number:} + \cs_gset:cx {g__robExt_compilation_command_\robExtAddCachePathAndName{\robExtFinalHash}:} {\l__robExt_current_compilation_command_str} + \fi + } \else - \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N {\robExtForceCompilation} } + \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N \robExtForceCompilation } { - \message{[robExt] We ~ will ~ start ~ the ~ compilation using: ~ \l_robExt_currentCompilationCommand.} - \sys_shell_now:x {\l_robExt_currentCompilationCommand} % The ~ are used in ExplSyntaxOn to add space + \message{[robExt] We ~ will ~ start ~ the ~ compilation using: ~ \l__robExt_current_compilation_command_str.} + \sys_shell_now:x {\l__robExt_current_compilation_command_str} % The ~ are used in ExplSyntaxOn to add space + %% We notify that we are running the compilation command, this way it is possible to check later + %% if the file is not present because of some compilation error or because we are in manual mode + %% or fallback + \cs_set:Nn \l__robExt_I_just_ran_a_compilation_command: {} }{ \ifdefined\robExtFallbackManualMode - \message{[robExt] Fallback to manual mode: please, manually compile the images using \l_robExt_currentCompilationCommand or run 'bash \jobname-\robExtAddPrefixName{compile-missing-figures.sh}'.} - \iow_now:Nx \g_robExt_write_manually_compile_all_missing_figures {\l_robExt_currentCompilationCommand}% a new line is automatically added + % Avoid writing the same command multiple times (might occur in environments that execute the + % same command multiple times like align etc) + \cs_if_exist:cTF {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :}{}{ + \message{[robExt] Fallback to manual mode: please, manually compile the images using \l__robExt_current_compilation_command_str or run 'bash \jobname-\robExtAddPrefixName{compile-missing-figures.sh}'.} + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\l__robExt_current_compilation_command_str}% a new line is automatically added + \cs_gset:cn {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :} {} + \ifdefined\robExt@compile@parallel@after + \gdef\robExt@compile@parallel@must@compile{} + \seq_gput_right:Nx \g__robExt_files_compiled_in_parallel {\robExtAddCachePathAndName{\robExtFinalHash}} + \fi + } \else - \PackageError{robExt}{You ~ need ~ to ~ compile ~ with ~ shell-escape ~ as ~ in: ~ "pdflatex ~ -shell-escape ~ yourfile.tex" ~ to ~ be ~ able ~ to ~ compile ~ automatically ~ the ~ figures}{} + \msg_error:nn{robExt}{need shell escape} \fi } \fi @@ -1457,29 +1513,28 @@ if __name__ == '__main__': \fi% %\robExtConfigure{run ~ after ~ include ~ command}% }{ - \ifdefined\robExtManualMode - \framebox[\linewidth] - { - \begin{minipage}{\linewidth} - \textbf{Draft ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l_robExt_currentCompilationCommand}\newline or ~ call ~ \newline\texttt{bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} - \end{minipage} - } - %\fbox{\textbf{Draft ~ Mode: ~ you ~ are ~ in ~ manual ~ mode:}\par\textbf{ ~ please ~ compile ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ \ifdefined\robExtCacheFolder cd \robExtCacheFolder; \fi bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}} - \message{[robExt] ~ You ~ are ~ in ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}} - \else - \ifdefined\robExtFallbackManualMode - \framebox[\linewidth] - { - \begin{minipage}{\linewidth} - \textbf{Falling ~ back ~ to ~ draft ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l_robExt_currentCompilationCommand}\newline or ~ call ~ \newline\texttt{bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} - \end{minipage} + % Check if we tried to run a compilation command at the previous step: + \cs_if_exist:NTF \l__robExt_I_just_ran_a_compilation_command: { + % We ran a compilation command but it failed: we print an error message + \msg_error:nnxxx{robExt}{missing compiled pdf parallel}{\robExtAddCachePathAndName{\robExtFinalHash}}{\msg_line_number:}{\l__robExt_current_compilation_command_str} + } { % We ran no compilation command: we print instead a placeholder depending on the context: + \cs_if_exist:NTF \robExt@compile@parallel@must@compile { + % we are supposed to compile the current picture in parallel, let'us check if it will be possible: + \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N \robExtForceCompilation} { + \robExtImagePlaceholderIfParallelCompilation + }{ + \robExtImagePlaceholderIfManualMode } - % \fbox{\textbf{Draft ~ Mode: ~ you ~ are ~ in ~ manual ~ mode:}\par\textbf{ ~ please ~ compile ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ \ifdefined\robExtCacheFolder cd \robExtCacheFolder; \fi bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}} - \message{[robExt] ~ You ~ are ~ falling ~ back ~ to ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}} - \else - \PackageError{robExt}{For ~ an ~ unknown ~ reason ~ the ~ pdf ~ file ~ \robExtAddCachePathAndName{\robExtFinalHash.pdf} ~ is ~ not ~ present. ~ The ~ compilation ~ command ~ certainly ~ failed, ~ see ~ logs ~ above.}{} - \fi - \fi + }{ + \cs_if_exist:NTF \robExtManualMode { + \robExtImagePlaceholderIfManualMode + \message{[robExt] ~ You ~ are ~ in ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}} + }{ + \robExtImagePlaceholderIfFallbackMode + \message{[robExt] ~ You ~ are ~ falling ~ back ~ to ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}} + } + } + } }% }% \fi% @@ -1538,6 +1593,178 @@ if __name__ == '__main__': \fi% } +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Automatically forward elements +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% \robExtSetCodeToRunIfMacroPresent{\hello}{some code} will execute "some code" if \hello is present in +% the main content to build and if "auto forward" is enabled. +\NewDocumentCommand{\robExtSetCodeToRunIfMacroPresent}{mm}{ + % we need to remove the leading space + % \str_set:Nx \l__robExt_tmp_str {\cs_to_str:N #1} + \expandafter\def \csname l__robExt_execute_if_macro_present\string#1\endcsname {#2}% +} + +% \robExtRunCodeToRunForMacroPresent{\hello} will run the code that was configured when calling +% \robExtSetCodeToRunIfMacroPresent{\hello}{some code} +\NewDocumentCommand{\robExtRunCodeToRunForMacroPresent}{m}{ + % \str_set:Nx \l__robExt_tmp_str {\cs_to_str:N ##1} + \cs_if_exist_use:c {l__robExt_execute_if_macro_present\string#1} +} + +% \autoForwardMacro{\hello}[\firstMacroDeps,\secondMacroDeps] will be used to tell to "auto forward" to forward \hello, \firstMacroDeps, and \secondMacroDeps if \hello is present in the main content to build. +\NewDocumentCommand{\robExtConfigIfMacroPresent}{mm}{% + \robExtSetCodeToRunIfMacroPresent{#1}{% + \pgfkeysalso{#2}% + }% +} +\let\configIfMacroPresent\robExtConfigIfMacroPresent + +\NewDocumentCommand{\robExtAutoForwardMacro}{mO{}}{% + \robExtConfigIfMacroPresent{#1}{ + forward=#1,#2 + }%a +} +\let\autoForwardMacro\robExtAutoForwardMacro + +\NewDocumentCommand{\robExtLoadAutoForwardMacroConfig}{m}{ + %\str_set:Nx \l__robExt_tmp_str {\cs_to_str:N #1} + \cs_if_exist:cTF {l__robExt_execute_if_macro_present\string#1}{ + % if the macro is present twice, we want to run \forward only once + \cs_if_exist:cTF {l__robExt_execute_if_macro_present_already_forwarded\string#1 :}{}{ + \use:c {l__robExt_execute_if_macro_present\string#1} + % define it so that we do not import twice next time + \cs_set:cx {l__robExt_execute_if_macro_present_already_forwarded\string#1 :} {} + } + }{} +} + + + +%% Old version: too inneficient +% \regex_const:Nn \l__robExt_macro_regex { \\[A-Za-z]+ } +% \NewDocumentCommand{\robExtAutoForward}{}{ +% \seq_clear_new:N \l__robExt_matches_seq +% \regex_extract_all:NVN \l__robExt_macro_regex \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str \l__robExt_matches_seq% +% \seq_map_inline:Nn \l__robExt_matches_seq { +% \robExtLoadAutoForwardMacroConfig{##1} +% } +% } + +%% "auto forward" +\NewDocumentCommand{\robExtAutoForward}{}{ + \ifdefined\robExtUserInputCacheMe\else + \msg_warning:nn{robExt}{auto forward not in cachemecode} + \tl_set_rescan:NnV \robExtUserInputCacheMe {} \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str + \fi + \robExt@normalBraces\robExtUserInputCacheMe + \expandafter\robExt@getfromstringA\robExtUserInputCacheMe\end +} + +% https://tex.stackexchange.com/questions/700834/efficiently-program-search-of-macro-in-string/700844 +\def\robExt@getfromstringA#1{\ifx#1\end \else + %\ifcsname L:\string#1\endcsname \addto\listmacros{#1}\fi + \robExtLoadAutoForwardMacroConfig{#1}% + \expandafter\robExt@getfromstringA\fi +} + +\def\robExt@normalBraces#1{{\catcode`{=12 \catcode`}=12 \everyeof={\endfile}% + \expandafter\robExt@normalBracesA\expandafter#1\scantokens\expandafter{#1}}} +\def\robExt@normalBracesA#1#2\endfile{\gdef#1{#2}} + +%%% To automatically define and forward +\NewDocumentCommand{\newcommandAutoForward}{moomO{}}{ + \IfNoValueTF{#2}{ + \IfNoValueTF{#3}{ + \newcommand{#1}{#4} + }{ + \newcommand{#1}[#3]{#4} + } + }{ + \IfNoValueTF{#3}{ + \newcommand{#1}[#2]{#4} + }{ + \newcommand{#1}[#2][#3]{#4} + } + } + \robExtAutoForwardMacro{#1}[#5] +} + +%%% To automatically define and forward +\NewDocumentCommand{\renewcommandAutoForward}{moomO{}}{ + \IfNoValueTF{#2}{ + \IfNoValueTF{#3}{ + \renewcommand{#1}{#4} + }{ + \renewcommand{#1}[#3]{#4} + } + }{ + \IfNoValueTF{#3}{ + \renewcommand{#1}[#2]{#4} + }{ + \renewcommand{#1}[#2][#3]{#4} + } + } + \robExtAutoForwardMacro{#1}[#5] +} + +%%% To automatically define and forward +\NewDocumentCommand{\providecommandAutoForward}{moomO{}}{ + \IfNoValueTF{#2}{ + \IfNoValueTF{#3}{ + \providecommand{#1}{#4} + }{ + \providecommand{#1}[#3]{#4} + } + }{ + \IfNoValueTF{#3}{ + \providecommand{#1}[#2]{#4} + }{ + \providecommand{#1}[#2][#3]{#4} + } + } + \robExtAutoForwardMacro{#1}[#5] +} + +\NewDocumentCommand{\NewDocumentCommandAutoForward}{mmO{}m}{ + \NewDocumentCommand{#1}{#2}{#4} + \robExtAutoForwardMacro{#1}[#3] +} + +\NewDocumentCommand{\RenewDocumentCommandAutoForward}{mmO{}m}{ + \RenewDocumentCommand{#1}{#2}{#4} + \robExtAutoForwardMacro{#1}[#3] +} + +\NewDocumentCommand{\ProvideDocumentCommandAutoForward}{mmO{}m}{ + \ProvideDocumentCommand{#1}{#2}{#4} + \robExtAutoForwardMacro{#1}[#3] +} + +\NewDocumentCommand{\DeclareDocumentCommandAutoForward}{mmO{}m}{ + \DeclareDocumentCommand{#1}{#2}{#4} + \robExtAutoForwardMacro{#1}[#3] +} + +\NewDocumentCommand{\defAutoForward}{mO{}mO{}}{ + \def#1#2{#3} + \robExtAutoForwardMacro{#1}[#4] +} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Automatically forward (regex-based) +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% called by style "if matches". +% \robExtRegexMatches{my regex}{my style} will apply "my style" if "__ROBEXT_MAIN_CONTENT_ORIG__" matches +% "my regex". +\NewDocumentCommand{\robExtIfMatchesRegex}{mm}{ + \regex_match:nVTF {#1} \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str {\pgfkeysalso{#2}} {} +} + +\NewDocumentCommand{\robExtIfMatchesString}{mm}{ + \str_if_in:NnTF \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str {#1} {\pgfkeysalso{#2}} {} +} \ExplSyntaxOff @@ -1545,10 +1772,10 @@ if __name__ == '__main__': %%% Interface %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % We create interface into pgfkeys in order to allow easier creation of content via style -\pgfkeys{ - /robExt/.cd, +\pgfkeys{% + /robExt/.cd,% % We create a default style that will be loaded (mostly for the user) - default style/.style={}, + default style/.style={},% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Code to create new styles %%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1576,7 +1803,7 @@ if __name__ == '__main__': % \robExtShowPlaceholder*{__ROBEXT_TMP__} \robExtEvalPlaceholderReplaceFromList{__ROBEXT_TMP__}{% \pgfkeys{% - /robExt/.cd, + /robExt/.cd,% #1/.style={__ROBEXT_TMP__},% }% }% @@ -1629,8 +1856,8 @@ if __name__ == '__main__': \robExtKeepPlaceholderAfterGroup{__ROBEXT_COMPILED_#1_INCLUDE_COMMAND__}% \robExtRescanPlaceholderInVariableNoReplacement{robExtCompiledIncludeCommand#1}{__ROBEXT_INCLUDE_COMMAND__}% \robExtKeepaftergroup{robExtCompiledIncludeCommand#1}% - \robExtKeepaftergroup{l_robExt_placeholder___ROBEXT_COMPILED_#1_INCLUDE_COMMAND___str}% - \robExtKeepaftergroup{l_robExt_placeholder___ROBEXT_COMPILED_#1_TEMPLATE___str}% + \robExtKeepaftergroup{l__robExt_placeholder___ROBEXT_COMPILED_#1_INCLUDE_COMMAND___str}% + \robExtKeepaftergroup{l__robExt_placeholder___ROBEXT_COMPILED_#1_TEMPLATE___str}% \robExtSaveDependencies{robExtCompiledDependencies#1}% \endgroup% }, @@ -1810,6 +2037,10 @@ if __name__ == '__main__': % allow commands like mkdir/cd/pdflatex to run in restricted mode. force compilation/.code={\def\robExtForceCompilation{}}, do not force compilation/.code={\let\robExtForceCompilation\undefined}, + % Recompile the file even if it has already been compiled (we do NOT clean the file as we do not feel + % safe to remove files from this code, but this can also be done by the user hooking into + recompile/.code={\def\robExtForceRecompilation{}}, + do not recompile/.code={\let\robExtForceRecompilation\undefined}, set compilation command/.code={\robExtSetCompilationCommand{#1}}, add argument to compilation command/.code={\robExtAddArgumentToCompilationCommand{#1}}, add arguments to compilation command/.style={ @@ -1872,15 +2103,90 @@ if __name__ == '__main__': %% Note: this does not work reliably for now %% TODO: fix this! disable externalization/.code={\def\robExtDisableExternalization{}}, + de/.style={disable externalization}, disable externalization now/.code={\robExtDisableTikzpictureOverwrite\def\robExtDisableExternalization{}}, enable externalization/.code={\let\robExtDisableExternalization\undefined}, % Useful to wrap, for instance, text + command if no externalization/.code={}, command if no externalization/.code={\robExtDisableTikzpictureOverwrite\evalPlaceholder{__ROBEXT_MAIN_CONTENT__}}, print verbatim if no externalization/.style={ command if no externalization/.code={% \robExtPrintPlaceholder{__ROBEXT_MAIN_CONTENT__}% }, }, + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%% Forward macros if externalization is enabled %%% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + command if externalization/.code={}, + fw/.style={forward=#1}, + forward/.style={% + command if externalization/.append code={% + \robExtGetCommandDefinitionInMacro{#1}% + \expanded{% + \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% + \expandonce{\robExtDefinitionCommand}% + }% + }% + }, + }, + forward at letter/.style={% + command if externalization/.append code={% + \robExtGetCommandDefinitionInMacro{#1}% + \expanded{% + \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% + \noexpand\makeatletter% + \expandonce{\robExtDefinitionCommand}% + \noexpand\makeatother% + }% + }% + }, + }, + forward eval/.style={% + command if externalization/.append code={% + \expanded{% + \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% + \noexpand\def\noexpand#1{#1}% + }% + }% + },% + }, + forward counter/.style={% + command if externalization/.append code={% + \expanded{% + \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% + \noexpand\makeatletter% + % Make sure the counter exists: + \noexpand\ifcsname c@#1\noexpand\endcsname\noexpand\else\noexpand\newcounter{#1}\noexpand\fi% + \noexpand\setcounter{#1}{\the\value{#1}}% + \noexpand\makeatother + }% + }% + },% + },% + forward color/.style={ + command if externalization/.append code={% + \extractcolorspecs{#1}{\zx@tmp@model}{\zx@tmp@cmd}% + \expanded{% + \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% + \noexpand\definecolor{#1}{\zx@tmp@model}{\zx@tmp@cmd}% + }% + }% + },% + },% + %% This will try to automatically forward some macros. For this, you must first define + %% + auto forward/.code={\robExtAutoForward}, + auto forward color/.style 2 args={% + add to preset={#1}{ + % if matches={\b #2\b}{forward color=#2}, + % the Regex is maybe more precise (avoid unnecessary forwards) but is way faster + if matches={#2}{forward color=#2}, + }, + }, + load auto forward macro config/.code={\robExtLoadAutoForwardMacroConfig{#1}}, + %% This will + if matches regex/.code 2 args={\robExtIfMatchesRegex{#1}{#2}}, + if matches/.code 2 args={\robExtIfMatchesString{#1}{#2}}, %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Run code before/after inclusion %%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1901,17 +2207,93 @@ if __name__ == '__main__': \expandafter\xdef\csname #1InCache\endcsname{\robExtAddCachePathAndName{\robExtFinalHash}}% }% }, + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%% Compile in parallel automatically + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + compile in parallel/.style={compile in parallel after=#1}, + compile in parallel/.default=0, + compile in parallel after/.code={% + \gdef\robExt@compile@parallel@after{#1}% + }, + compile in parallel after/.default=0, + compile in parallel command/.store in=\robExt@compile@parallel@command, + % Different backends to compile in parallel + compile in parallel with gnu parallel/.style={ + compile in parallel command={parallel --jobs #1 :::: '\jobname-\robExtAddPrefixName{compile-missing-figures.sh}'}, + }, + compile in parallel with gnu parallel/.default={200\%}, + compile in parallel with xargs/.style={ + compile in parallel command={xargs -t -a '\jobname-\robExtAddPrefixName{compile-missing-figures.sh}' -I "{}" -P #1 \robExtParallelShell\space '{}'}, + }, + compile in parallel with xargs/.default={16}, + % By default, we compile with xargs and 16 processes: + compile in parallel with xargs, + compile in parallel with rush/.style={ + compile in parallel command={rush -i "\jobname-\robExtAddPrefixName{compile-missing-figures.sh}" #1 "\robExtParallelShell\space '{}'"}, + }, + compile in parallel with rush/.default={}, } +\ExplSyntaxOn +\int_new:N \g__robExt_number_figures_just_compiled_or_to_compile +\int_set:Nn \g__robExt_number_figures_just_compiled_or_to_compile {0} +% Will hold the list of input files to compile in parallel. This is needed to +% check at the end to find the processes that failed to build. +\seq_new:N \g__robExt_files_compiled_in_parallel + +\AfterLastShipout{ + % Check if we want to run stuff in parallel + \ifdefined\robExt@compile@parallel@after + % Check if we must compile + \ifdefined\robExt@compile@parallel@must@compile + % Check if we can compile + \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N \robExtForceCompilation} + { + % Check if a command was provided + \ifdefined\robExtParallelShell\else + \sys_if_platform_windows:TF { + \def\robExtParallelShell{cmd~/C} + }{ + \def\robExtParallelShell{sh~-c} + } + \fi + % We close the file or reading it is empty + \iow_close:N \g__robExt_write_manually_compile_all_missing_figures_iow + \msg_warning:nnx{robExt}{rerun because parallel}{\robExt@compile@parallel@command} + \message{\robExt@compile@parallel@command} + \sys_shell_now:x {\robExt@compile@parallel@command} + % We check if some files failed to compile + \seq_map_inline:Nn \g__robExt_files_compiled_in_parallel { + \message{#1} + \file_if_exist:xTF{#1.pdf}{}{ + \msg_error:nnxxx{robExt}{missing compiled pdf parallel}{#1}{\use:c {g__robExt_lines_error_#1:}}{\use:c {g__robExt_compilation_command_#1:}} + } + } + } { + \ifdefined\robExtFallbackManualMode + \msg_warning:nn{robExt}{enabled parallel no shell escape} + \else + \msg_error:nn{robExt}{need shell escape parallel} + \fi + } + \fi + \fi +} +\ExplSyntaxOff + % Not really made for the end user % It assumes that __ROBEXT_COMPILATION_COMMAND__ and __ROBEXT_TEMPLATE__ is set \NewDocumentCommand{\robExtEvaluateCompileAndInclude}{}{% \ifdefined\robExtDisableExternalization% - \pgfkeys{ + \pgfkeys{% /robExt/.cd, command if no externalization, }% \else% + \pgfkeys{% + /robExt/.cd, + command if externalization, + }% \ifdefined\robExtExecuteBefore\robExtExecuteBefore\fi% \robExtWriteFile{}% \robExtCompileFile{}% @@ -1926,10 +2308,12 @@ if __name__ == '__main__': %% #1: Arguments, #2: content to externalize \NewDocumentCommand{\robExtCacheMe}{O{}m}{% {% Group + %% We store the input in a non-string element for efficiently implementing "auto forward" + \def\robExtUserInputCacheMe{#2} \pgfkeys{% - /robExt/.cd, - set placeholder={__ROBEXT_MAIN_CONTENT_ORIG__}{#2}, - default style, + /robExt/.cd,% + set placeholder={__ROBEXT_MAIN_CONTENT_ORIG__}{#2},% + default style,% #1, }% \robExtEvaluateCompileAndInclude% @@ -1949,8 +2333,8 @@ if __name__ == '__main__': }{% \endRobExtPlaceholderFromCode% \pgfkeys{% - /robExt/.cd, - #1, + /robExt/.cd,% + #1,% }% \robExtEvaluateCompileAndInclude% } @@ -1964,6 +2348,102 @@ if __name__ == '__main__': \let\endCacheMeNoContent\endRobExtCacheMeNoContent +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%% Forward a macro %%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% In this section, we provide commands to forward a macro, like +%% \def\myMacro{stuff} +%% \cacheMe<forward=\myMacro> +%% Note that this will only be forwarded if externalization is enabled (otherwise the macro already exists) +%% I published this code in this answer, thanks to egreg for some pointers +%% https://tex.stackexchange.com/questions/697120/how-to-extract-the-definition-of-a-macro-to-write-in-a-file-for-instance +\ExplSyntaxOn + +\cs_new:Nn \__robExt_getCommandDefinitionFromXparse:N +{ + \str_clear_new:N \l__robExt_definition_str + \GetDocumentCommandArgSpec { #1 } + \str_set:Nx \l__robExt_definition_str { + \c_backslash_str DeclareDocumentCommand + \c_left_brace_str + \c_backslash_str \cs_to_str:N #1 + \c_right_brace_str + \c_left_brace_str \ArgumentSpecification \c_right_brace_str + \c_left_brace_str + \cs_replacement_spec:c { \cs_to_str:N #1 ~ code } + \c_right_brace_str + }% +} + +\cs_new:Nn \__robExt_getCommandDefinitionFromDef:N +{ + \str_clear_new:N \l__robExt_definition_str + \str_set:Nx \l__robExt_definition_str { + \c_backslash_str def \c_backslash_str \cs_to_str:N #1 + \cs_argument_spec:c { \cs_to_str:N #1 } + \c_left_brace_str + \cs_replacement_spec:c { \cs_to_str:N #1 } + \c_right_brace_str + }% +} + +\def\robExt@extractDefaultNewcommand#1#2#3#4{ + #4% +}% + +\cs_new:Nn \__robExt_getCommandDefinitionFromNewcommand:N +{ + \str_clear_new:N \l__robExt_definition_str + \str_clear_new:N \l__robExt_tmp + %% \l__robExt_tmp_str will look like [#1]#2#3#4#5#6#7: + \str_set:Nx \l__robExt_tmp_str {\cs_argument_spec:c { \c_backslash_str \cs_to_str:N #1 }}% + %% We remove the brackets: + \str_replace_all:Nnn \l__robExt_tmp_str {[} {} + \str_replace_all:Nnn \l__robExt_tmp_str {]} {} + \str_set:Nx \l__robExt_definition_str { + %% Make sure the command does not exist + \c_backslash_str providecommand \c_backslash_str \cs_to_str:N #1 \c_left_brace_str \c_right_brace_str + %% this line will be like "\newdocumentcommand{\mymacro}" + \c_backslash_str renewcommand \c_left_brace_str \c_backslash_str \cs_to_str:N #1 \c_right_brace_str + %% It must have at least one argument, since elements without optional arguments are turned into \def + %% Moreover, macros can't have more than 1 argument, so it will be easier to parse, we just need the last digit + [\str_range:Nnn \l__robExt_tmp_str {-1} {-1}] %% <- this is the number of mandatory arguments + [\expandafter \robExt@extractDefaultNewcommand #1 ] %% <- this is the value of the default argument + \c_left_brace_str + \cs_replacement_spec:c { \c_backslash_str \cs_to_str:N #1 } + \c_right_brace_str + }% +} + +% \robExtGetCommandDefinitionInMacro{\myMacro} will populate +% \l__robExt_definition_str and \robExtDefinitionCommand +% with a string to execute to write it properly. +\NewDocumentCommand{\robExtGetCommandDefinitionInMacro}{m}{ + \cs_if_exist:NTF #1 { + \cs_if_exist:cTF {\cs_to_str:N #1 ~ code}{ + % xparse-based definition + \__robExt_getCommandDefinitionFromXparse:N #1% + } { + \cs_if_exist:cTF {\c_backslash_str \cs_to_str:N #1}{ + % \newcommand-based definition + \__robExt_getCommandDefinitionFromNewcommand:N #1 + }{ + % \def-based definition + \__robExt_getCommandDefinitionFromDef:N #1 + } + } + }{ + \msg_error:nn{robExt}{The~macro~#1~does~not~exist} + } + \let\robExtDefinitionCommand\l__robExt_definition_str +} + +\NewDocumentCommand{\robExtGetCommandDefinition}{m}{% + \robExtGetCommandDefinitionInMacro{#1}% + \l__robExt_definition_str% +} +\ExplSyntaxOff + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%% Default presets %%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2237,6 +2717,9 @@ finished_with_no_error() force python3/.style={ set placeholder={__ROBEXT_PYTHON_EXEC__}{python3} }, + add import/.style={ + add to placeholder no space={__ROBEXT_PYTHON_IMPORT__}{##1^^J}, + }, } } @@ -2480,7 +2963,7 @@ touch "${outputPdf}" } \NewDocumentCommand{\robExtExternalizeAllTikzpictures}{O{tikz}O{tikzpicture}O{<>}}{% - \robExtCacheEnvironment[#3]{#2}{tikzpicture}% + \robExtCacheEnvironment[#3]{tikzpicture}{#2}% % Tikz has a more complicated parsing system \DeclareCommandCopy{\robExtCommandOrigtikz}{\tikz}% we save the function for later reset \robExtAddToCommandResetList{tikz}% we add it to the list of functions to reset @@ -2494,22 +2977,6 @@ touch "${outputPdf}" } \let\cacheTikz\robExtExternalizeAllTikzpictures -% \NewDocumentCommand{\robExtExternalizeAllTikzpictures}{O{<>}O{tikz}O{tikzpicture}}{% -% \robExtCacheEnvironment[#1]{#3}{tikzpicture}% -% % \robExtCacheCommand[#1]{tikz}[O{}m]{tikz} -% % Tikz has a more complicated parsing system -% \DeclareCommandCopy{\robExtCommandOrigtikz}{\tikz}% we save the function for later reset -% \robExtAddToCommandResetList{tikz}% we add it to the list of functions to reset -% \DeclareDocumentCommand{\tikz}{D#1{}}{% -% \def\robExtCommandOrigName{tikz}% we specify the name of the current environment -% \edef\robExt@tmp@args{\detokenize{##1}}% we specify the name of the current environment -% \edef\robExt@tmp@orig@args{\detokenize{#2}}% we specify the name of the current environment -% % we start the parsing -% \robExt@tikz% -% }% -% } -% \let\cacheTikz\robExtExternalizeAllTikzpictures - %% The cached version \DeclareDocumentEnvironment{tikzpictureC}{D<>{}O{}O{}}{% \begin{CacheMe}{tikzpicture,#1}[#2]% @@ -2529,7 +2996,7 @@ touch "${outputPdf}" \def\robExtEnvironmentOrigName{#2}% \CacheMe{% #3,% - set placeholder no import={__ROBEXT_MAIN_CONTENT__}{\begin{#2}__ROBEXT_MAIN_CONTENT_ORIG__\end{#2}}, + set placeholder no import={__ROBEXT_MAIN_CONTENT__}{\begin{#2}__ROBEXT_MAIN_CONTENT_ORIG__\end{#2}},% ##1% }% }{\endCacheMe}% diff --git a/Master/tlpkg/bin/tlpkg-ctan-check b/Master/tlpkg/bin/tlpkg-ctan-check index 34e397b9933..79a6517455e 100755 --- a/Master/tlpkg/bin/tlpkg-ctan-check +++ b/Master/tlpkg/bin/tlpkg-ctan-check @@ -438,7 +438,7 @@ my @TLP_working = qw( interchar interfaces interpreter interval intopdf intro-scientific inversepath invoice invoice-class invoice2 - iodhbwm ionumbers iopart-num ipaex ipaex-type1 is-bst iscram iso + iodhbwm ionumbers iopart-num ipaex ipaex-type1 ipsum is-bst iscram iso iso10303 isodate isodoc isomath isonums isopt isorot isosafety isotope isphysicalmath issuulinks istgame itnumpar iwhdp iwona iwonamath diff --git a/Master/tlpkg/libexec/ctan2tds b/Master/tlpkg/libexec/ctan2tds index c131dbcce1c..ceaa5c5ab15 100755 --- a/Master/tlpkg/libexec/ctan2tds +++ b/Master/tlpkg/libexec/ctan2tds @@ -1755,6 +1755,7 @@ chomp (my $ctan_root = `tlpkginfo --ctan-root`); 'impatient' => '&POSTimpatient', 'impatient-fr' => '&POSTimpatient_fr', 'interactiveworkbook' => '&POSTinteractiveworkbook', + 'ipsum' => '&POST_onelevel', 'knitting' => '&POSTknitting', 'knuth-lib' => '&POSTknuth_lib', 'koma-script' => '&POSTkoma_script', #'&POST_rmsymlink', diff --git a/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc b/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc index 83ccc1df3fd..e0df9dbc8d7 100644 --- a/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc +++ b/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc @@ -715,6 +715,7 @@ depend inversepath depend invoice depend invoice-class depend invoice2 +depend ipsum depend iso depend iso10303 depend isodate |