diff options
author | Karl Berry <karl@freefriends.org> | 2024-01-16 21:28:43 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2024-01-16 21:28:43 +0000 |
commit | bb337ccdb3aef768019c346a86e49f151a1c380a (patch) | |
tree | c599bd887898b2e5fd0dd3b5763da899ffd72e8b /Master/texmf-dist/tex | |
parent | 34da3a6782c9cee95f81eda0f55cde49d5571d06 (diff) |
robust-externalize (16jan24)
git-svn-id: svn://tug.org/texlive/trunk@69459 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex')
-rw-r--r-- | Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty | 1071 |
1 files changed, 967 insertions, 104 deletions
diff --git a/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty b/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty index 3bf14a9f955..0a36647946d 100644 --- a/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty +++ b/Master/texmf-dist/tex/latex/robust-externalize/robust-externalize.sty @@ -1,4 +1,4 @@ -\ProvidesPackage{robust-externalize}[2.2 Cache anything (tikz, latex, python) in a robust, efficient and pure way.] +\ProvidesPackage{robust-externalize}[2.5 Cache anything (tikz, latex, python) in a robust, efficient and pure way.] % todo: % change order argument replace from list, it is hard to read this way @@ -11,7 +11,7 @@ \RequirePackage{iftex} % computing md5 sum differs on lualatex/pdflatex/xelatex \RequirePackage{xparse} % solve https://github.com/leo-colisson/robust-externalize/issues/6 \RequirePackage{atveryend} % to compile in parallel all images -\usepackage{etl} % For auto forward https://tex.stackexchange.com/a/700844/116348 +%\usepackage{etl} % For auto forward https://tex.stackexchange.com/a/700844/116348 %% TODO list: % - provide an easy way to use cross-ref, bibtex etc (we just need to add them when writing the file) without recompiling the whole document (we don't want to lose the cache everytime a new bib entry is added) but while preserving. @@ -20,14 +20,127 @@ %%%%%%%%%%% Utils %%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% If the file contains a space, the jobname gets quotes around... +% https://tex.stackexchange.com/questions/418670/avoid-quotation-marks-when-using-jobname-or-currfilename +\newcommand*{\jobnameNoQuotes}{} +\newcommand*{\setjobnameNoQuotes}{% + \edef\jobnameNoQuotes{\jobname}% + \expandafter\expandafter\expandafter\setjobnameNoQuotesaux + \expandafter\jobnameNoQuotes\expandafter"\jobnameNoQuotes"\relax +} +\newcommand*{\setjobnameNoQuotesaux}{} +\def\setjobnameNoQuotesaux#1"#2"#3\relax{\def\jobnameNoQuotes{#2}} +\setjobnameNoQuotes + \ExplSyntaxOn \NewDocumentCommand{\robExtIfWindowsTF}{mm}{% \sys_if_platform_windows:TF {#1}{#2}% } + +% This will be printed in from of the error message, mostly useful in texstudio that only print lines +% containing errors +\def\robExtPrefixLogMessage{} + +% Prints the next 4 lines after an error: +\def\robExtLinesAfterError{3} +\def\robExtRemoveLineNumber{} + +%% Needed for studio to prepend ! in front of all lines, including big ones. +%% https://tex.stackexchange.com/questions/705502/message-add-symbol-in-front-of-new-lines-even-if-there-is-a-line-break +\def\robExtMessageWithPrefixNumberLines{^^J}% +\cs_new:Nn \robExt__message_with_prefix:n { + \str_set:Nn \l_tmpa_str {#1} + \int_compare:nNnTF {\str_count:N \l_tmpa_str + \str_count:N \robExtPrefixLogMessage} > {78} { + \typeout{\robExtPrefixLogMessage \str_range:Nnn \l_tmpa_str {1} {78 - \str_count:N \robExtPrefixLogMessage}\robExtMessageWithPrefixNumberLines} + \robExt__message_with_prefix:x {\str_range:Nnn \l_tmpa_str {78 - \str_count:N \robExtPrefixLogMessage + 1} {-1}} + }{ + \typeout{\robExtPrefixLogMessage \l_tmpa_str \robExtMessageWithPrefixNumberLines} + } +} +\cs_generate_variant:Nn \robExt__message_with_prefix:n { x } + +% We might want to remove l.XX in front of lines as emacs interpret this as a line of error and goes to a wrong place. +\cs_new:Nn \robExt__clean_line_if_needed:n { + \ifdefined\robExtRemoveLineNumber% + \str_compare:eNeTF {\str_range:nnn {#1} {1} {2}} = {l.} { + \str_range:nnn {#1} {3} {-1}% + } { + #1 + } + \else + #1 + \fi +} +\cs_generate_variant:Nn \robExt__clean_line_if_needed:n { V } + +\cs_new:Nn \robExt_get_errors_from_file:n { + \ior_open:Nn \g__robExt_read_ior {#1}% + \cs_undefine:N \l__robExt_I_saw_an_error: + \str_clear_new:N \l__robExt_tmp_str% + \int_set:Nn \l_tmpa_int {0} + \ior_str_map_inline:Nn \g__robExt_read_ior {% + \ifdefined\robExtPrintWholeFile + \str_gput_right:Nx \l__robExt_tmp_str {\tl_to_str:n{##1}^^J}% + \cs_set:Nn \l__robExt_I_saw_an_error: {} + \int_set:Nn \l_tmpa_int {\robExtLinesAfterError} + \else + \str_if_in:xnTF {\str_lowercase:n{##1}} {error} { + \str_gput_right:Nx \l__robExt_tmp_str {\robExt__clean_line_if_needed:V{\tl_to_str:n{##1}^^J}}% + \cs_set:Nn \l__robExt_I_saw_an_error: {} + \int_set:Nn \l_tmpa_int {\robExtLinesAfterError} + } { + \str_if_in:xnTF {\str_lowercase:n{##1}} {invalid} { + \str_gput_right:Nx \l__robExt_tmp_str {\robExt__clean_line_if_needed:V{\tl_to_str:n{##1}^^J}}% + \cs_set:Nn \l__robExt_I_saw_an_error: {} + \int_set:Nn \l_tmpa_int {\robExtLinesAfterError} + } { + % Check if it starts with a ! (usually latex errors start with this symbol) + \str_compare:eNeTF {\str_range:nnn {##1} {1} {1}} = {!} { + \str_gput_right:Nx \l__robExt_tmp_str {\robExt__clean_line_if_needed:V{\tl_to_str:n{##1}^^J}}% + \cs_set:Nn \l__robExt_I_saw_an_error: {} + \int_set:Nn \l_tmpa_int {\robExtLinesAfterError} + }{ + % Check if we saw an error not too long ago: + \int_compare:nNnTF {\l_tmpa_int} > {0} { + \str_gput_right:Nx \l__robExt_tmp_str {\robExt__clean_line_if_needed:V{\tl_to_str:n{##1}^^J}}% + } { + \str_set:Nn \l_tmp_str {##1} + } + \int_set:Nn \l_tmpa_int {\l_tmpa_int - 1} + } + } + } + \fi + }% + \cs_if_exist:NTF \l__robExt_I_saw_an_error: {} { + % We print the last line if we found no error. Might be useful for errors like: + % sh: ligne 1: gnuplot : commande introuvable. + \str_set_eq:NN \l__robExt_tmp_str \l_tmp_str + } +} + +\cs_new:Nn \robExt_write_file_in_log:n { + \ior_open:Nn \g__robExt_read_ior {#1}% + \ior_str_map_inline:Nn \g__robExt_read_ior {% + \ifdefined\robExtPrintWholeFile + \ifdefempty{\robExtPrefixLogMessage}{ + \typeout{\tl_to_str:n{##1}}% + }{ + \robExt__message_with_prefix:x {\tl_to_str:n{##1}} + } + \else + \typeout{\tl_to_str:n{##1}}% + \fi + }% +} + +\cs_generate_variant:Nn \robExt_write_file_in_log:n { x } + \ExplSyntaxOff % Debug function: by default, do nothing \def\robExtDebugMessage#1{} +\def\robExtDebugInfo#1{\message{#1}} \def\robExtDebugWarning#1{\message{^^J[robExt warning] #1}} \NewDocumentCommand{\robExtConfigure}{m}{% @@ -53,6 +166,7 @@ \cs_generate_variant:Nn \str_replace_all:Nnn { Nnx, Nnv, cnx, cxx, NnV, Nnx, Nnv } \cs_generate_variant:Nn \str_set:Nn { NV } +\cs_generate_variant:Nn \str_if_in:nnTF { xnTF } \cs_generate_variant:Nn \tl_rescan:nn { nv } \cs_generate_variant:Nn \tl_set_rescan:Nnn { Nnv, NnV } @@ -65,6 +179,7 @@ \cs_generate_variant:Nn \ior_str_get:NN { Nc } \cs_generate_variant:Nn \file_if_exist:nTF { xTF } +\cs_generate_variant:Nn \file_mdfive_hash:n { x } \cs_generate_variant:Nn \cs_replacement_spec:N { c } \cs_generate_variant:Nn \cs_argument_spec:N { c } @@ -110,25 +225,33 @@ %% In LaTeX3 error messages are defined here, this allows people to translate them, filter them, change them, %% make them more verbose etc. \msg_new:nnn {robExt}{underscore in name}{Error:~all~the~placeholders~should~contain~two~consecutive~underscores~in~their~name.} -\msg_new:nnn {robExt}{placeholder not existing}{[robExtGetPlaceholderInResultFromSinglePlaceholder] ~ The ~ placeholder ~ #3 ~ does ~ not ~ exists.} +\msg_new:nnn {robExt}{placeholder not existing}{[robExtGetPlaceholderInResultFromSinglePlaceholder] ~ The ~ placeholder ~ #1 ~ does ~ not ~ exists.} \msg_new:nnn {robExt}{forgot template}{You ~ are ~ writing ~ __ROBEXT_TEMPLATE__ ~ to ~ your ~ file: ~ seems ~ like ~ you ~ forgot ~ to ~ define ~ your ~ template ~ or ~ set ~ a ~ preset} \msg_new:nnn {robExt}{need shell escape}{You ~ need ~ to ~ compile ~ with ~ shell-escape ~ as ~ in: ~ "pdflatex ~ -shell-escape ~ yourfile.tex" ~ to ~ be ~ able ~ to ~ compile ~ automatically ~ the ~ figures} +\msg_new:nnn {robExt}{need shell escape with}{You ~ need ~ to ~ compile ~ with ~ shell-escape ~ as ~ in: ~ "pdflatex ~ -shell-escape ~ yourfile.tex" ~ to ~ be ~ able ~ to ~ #1} \msg_new:nnn {robExt}{need shell escape parallel}{You ~ need ~ to ~ compile ~ with ~ shell-escape ~ as ~ in: ~ "pdflatex ~ -shell-escape ~ yourfile.tex" ~ to ~ be ~ able ~ to ~ compile ~ automatically ~ the ~ figures ~ in ~ parallel} -\msg_new:nnn {robExt}{missing compiled pdf}{The ~ pdf ~ file ~ #1.pdf ~ is ~ not ~ present. ~ The ~ compilation ~ command ~ certainly ~ failed, ~ see ~ logs ~ above ~ or ~ in ~ #1.log.} -\msg_new:nnn {robExt}{missing compiled pdf parallel}{The ~ pdf ~ file ~ #1.pdf ~ is ~ not ~ present. ~ The ~ compilation ~ command ~ "#3"~used~to~compile~the~environment~on~line~#2~ certainly ~ failed, ~ see ~ logs ~ above ~ or ~ in ~ #1.log.} +\msg_new:nnn {robExt}{missing compiled pdf parallel}{The~compilation~of~block~at~line~#2~failed~as~the~following ~ file~is~missing: ~ #1.pdf. ~ The ~ compilation ~ command ~ "#3"~used~to~compile~the~environment~on~line~#2~ certainly ~ failed, ~ see ~ logs ~ above ~ or ~ in ~ #1.log.} +\msg_new:nnn {robExt}{dependency does not exist}{The~dependency~#1~does~not~exist.} + +\msg_new:nnn {robExt}{missing compiled pdf parallel with log}{The~compilation~of~the~code~block~at~line~#2~failed:~the~following ~ file~is~indeed~missing: ~ #1.pdf. ~ The ~ compilation ~ command ~ "#3"~used~to~compile~the~environment~on~line~#2~ certainly ~ failed~with~errors:^^Jvvvvvv^^J\l__robExt_tmp_str^^J\string^\string^\string^\string^\string^\string^ ^^JSee~full~logs~#4~ or ~ in ~ #1-compilation.log.} + + \msg_new:nnn {robExt}{auto forward not in cachemecode}{Auto~forward~is~less~efficient~in~cacheMeCode.} +\msg_new:nnn {robExt}{file does not exist}{The~file~#1~does~not~exist.} + % warnings \msg_new:nnn {robExt}{recommend two underscores in placeholders}{We~recommend~to~use~only~placeholders~containing~two~consecutive~underscores~in~their~name} \msg_new:nnn {robExt}{enabled parallel no shell escape}{Warning:~you~enabled~parallel~compilation~but~shell-escape~is~disabled.} \msg_new:nnn {robExt}{rerun because parallel}{Warning:~Compiling~all~missing~figures~in~parallel~with~"#1".~You~need~to~rerun~LaTeX~to~include~them.} +\msg_new:nnn {robExt}{gpgetvar recompilation needed}{Warning:~you~need~to~recompile~as~the~gpgetvar~variable~"#1"~does~not~exist~yet.} % dummy placeholders added if image is not present \def\robExtImagePlaceholderIfManualMode{ \framebox[\linewidth] { \begin{minipage}{\linewidth} - \textbf{Manual ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l__robExt_current_compilation_command_str}\newline or ~ call ~ \newline\texttt{bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} + \textbf{Manual ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l__robExt_current_compilation_command_str}\newline or ~ call ~ \newline\texttt{bash ~ \jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} \end{minipage} } } @@ -137,7 +260,7 @@ \framebox[\linewidth] { \begin{minipage}{\linewidth} - \textbf{Falling ~ back ~ to ~ draft ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l__robExt_current_compilation_command_str}\newline or ~ call ~ \newline\texttt{bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} + \textbf{Falling ~ back ~ to ~ draft ~ mode: ~ either ~ compile ~ with ~ \texttt{-shell-escape} ~ or ~ compile:\newline \texttt{\robExtAddCachePathAndName{\robExtFinalHash.tex}}\newline via ~ \newline \texttt{\l__robExt_current_compilation_command_str}\newline or ~ call ~ \newline\texttt{bash ~ \jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}}\newline to ~ compile ~ all ~ missing ~ figures.} \end{minipage} } } @@ -163,9 +286,10 @@ %%% Important: this script is NOT executed by this file, it is only created so that users can remove unused previously cached files by running this script %%% outside of LaTeX. %%% Create scripts to remove useless files: -%%% Note that we don't override the script if it exists on purpose (the user might have changed it to fits their needs) -\begin{filecontents}[noheader]{robExt-remove-old-figures.py} +\begin{filecontents}[noheader,overwrite]{robExt-remove-old-figures.py} #!/usr/bin/env python3 +## This file is automatically generated, so do not change that file directly +## Instead, if you need to change it, copy it with a different name and edit the copy import os import re import glob @@ -214,6 +338,63 @@ if __name__ == '__main__': main() \end{filecontents} +\begin{filecontents}[noheader,overwrite]{robExt-prepare-for-arxiv.py} +#!/usr/bin/env python3 +## This file is automatically generated, so do not change that file directly +## Instead, if you need to change it, copy it with a different name and edit the copy + +## This file must be called when sending stuff to the arxiv website, that would otherwise remove the .pdf +## in presence of a .tex. +## This script will simply rename the .tex files in robustExternalize into .tex-backup. +## Then, you should call "rename backup files for arxiv" to rename back the files. +import os +import re +import glob +# Just run this script in order to remove all old figures not listed in robExt-all-figures.txt. + +# Note that this part is not extracted from the pdf file since it might be different on a previous run. You can however hardcode +# it here, your updated script will not be overriden unless you remove it yourself. +prefixes = [ "robExt-" ] +folders = [ "robustExternalize" ] + +def main(): + listOfFilesToMove = [] + listOfFilesAlreadyAdded = set() + for folder in folders: + for root, dirs, files in os.walk(folder): + for f in files: + if f.endswith(".tex"): + pdf = os.path.splitext(f)[0] + ".pdf" + if os.path.exists(os.path.join(root,pdf)): + listOfFilesToMove.append((os.path.join(root,f), os.path.join(root,f + "-backup"))) + if f.endswith(".tex-backup"): + tex = os.path.splitext(f)[0] + ".tex" + listOfFilesAlreadyAdded.add(os.path.join(root,tex)) + for (f,f2) in listOfFilesToMove: + print(f"-- {f} moved to {f2}") + listOfFilesAlreadyAdded.discard(f) + for f in listOfFilesAlreadyAdded: + print(f"-- {f} already moved") + print(f"Above are the files to move or already moved, are you sure you want to proceed? [y/N] (based on prefixes {prefixes})") + x = input().strip() + if x not in ["y", "Y"]: + print("All right, we abort.") + exit(1) + for (f,f2) in listOfFilesToMove: + os.replace(f, f2) + print(f"Moved {f} to {f2}") + with open('robExt-arxiv-files-to-rename.txt', 'w') as fd: + for (f,f2) in listOfFilesToMove: + fd.write(f'{f}\n') + for f in listOfFilesAlreadyAdded: + fd.write(f'{f}\n') + print('The files have been written to robExt-arxiv-files-to-rename.txt.') + print('Add \\robExtConfigure{rename backup files for arxiv} in your tex file.') + +if __name__ == '__main__': + main() +\end{filecontents} + \ExplSyntaxOn %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -258,6 +439,9 @@ if __name__ == '__main__': %%%%%%%%%%% Paths %%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% For support of --output-directory +\def\robExtPrefixAllCompilationCommands{\ifdefined\robExtOutputDirectory cd~"\robExtOutputDirectory"~&&~\fi} + \def\robExtPrefixFilename{robExt-} \NewExpandableDocumentCommand{\robExtAddCachePathAndName}{m}{% @@ -272,6 +456,48 @@ if __name__ == '__main__': \fi#1% } +% mv in unix and move in windows: +\robExtIfWindowsTF{ + \def\robExtMv{move} + \def\robExtCp{copy} +}{ + \def\robExtMv{mv} + \def\robExtCp{cp} +} + +\NewDocumentCommand{\robExtBackupSource}{m}{% + % arXiv will remove the .pdf file if the .tex is present. Solution: we move the .tex into .tex-backup + % and restore it during compilation. We could also backup the pdf but usually this file is bigger and will contain + % binary data that might not be so easy to copy in pure tex (in arxiv shell escape is disabled so we cannot + % use cp). + % \show\inBackupSource + \robExtRunCmdIfPossible{% + \robExtMv\space"\robExtAddCachePath{#1}"~% + "\robExtAddCachePath{#1-backup}"% + }% +} + +\NewDocumentCommand{\robExtRunCmdIfPossible}{m}{% + % Check if the output directory exists + \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N \robExtForceCompilation} + { + \ifdefined\robExtManualMode + \message{Warning:~running~in~manual~mode~so~we~cannot~run~the~command~``#1''.} + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {#1} + \else + \message{Running:~\robExtPrefixAllCompilationCommands #1}% + \sys_shell_now:x {\robExtPrefixAllCompilationCommands #1} + \fi + }{ + \ifdefined\robExtFallbackManualMode + \message{Warning:~shell~escape~fallback~to~manual~mode~so~cannot~run~#1.} + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {#1} + \else + \msg_error:nnx{robExt}{need shell escape with}{run~#1} + \fi + } +} + \NewDocumentCommand{\robExtCheckIfPrefixFolderExists}{}{ % Check if the output directory exists \ifdefined\robExtCacheFolder @@ -279,17 +505,74 @@ if __name__ == '__main__': { \ifdefined\robExtDoNotMkdirFolder\else \ifdefined\robExtManualMode - \message{If ~ you ~ get~ an~ error,~ make ~ sure ~ to ~ enable ~ pdflatex ~ -shell-escape ~ or ~ to ~ manually ~ create ~ the ~ folder ~ \robExtCacheFolder.} + \message{If ~ you ~ get~ an~ error,~ make ~ sure ~ to ~ enable ~ pdflatex ~ -shell-escape ~ or ~ to ~ MANUALLY ~ CREATE ~ THE ~ FOLDER ~ \robExtCacheFolder.} \else - \sys_shell_now:x {mkdir ~ -p ~ \robExtCacheFolder} + \sys_shell_now:x {\robExtPrefixAllCompilationCommands mkdir ~ -p ~ \robExtCacheFolder} \fi \fi }{ - \message{If ~ you ~ get~ an~ error,~ make ~ sure ~ to ~ enable ~ pdflatex ~ -shell-escape ~ or ~ to ~ manually ~ CREATE ~ THE ~ FOLDER ~ \robExtCacheFolder.} + \message{Warning: If ~ you ~ get~ an~ error,~ make ~ sure ~ to ~ enable ~ pdflatex ~ -shell-escape ~ or ~ to ~ manually ~ CREATE ~ THE ~ FOLDER ~ \robExtCacheFolder.} } \fi } +\NewDocumentCommand{\robExtCopyFileToCache}{m}{ + \robExtCheckIfPrefixFolderExists% + \file_if_exist:xTF {#1}{}{ + \msg_error:nnx{robExt}{file does not exist}{#1~(to~copy~to~cache)} + } + \file_if_exist:xTF {\robExtAddCachePath{#1}}{ + \robExtDebugInfo{The~file~\robExtAddCachePath{#1}~already~exists,~let~us~compare~md5~sum.} + \str_set:Nx \l_tmpa_str {\file_mdfive_hash:n{#1}} + \str_set:Nx \l_tmpb_str {\file_mdfive_hash:n{\robExtAddCachePath{#1}}} + \str_compare:eNeTF {\l_tmpa_str} = {\l_tmpb_str} { + \robExtDebugInfo{The~file~\robExtAddCachePath{#1}~has~the~good~md5~hash.} + }{ + \robExtDebugInfo{The~md5~hashes~of~#1~and~\robExtAddCachePath{#1}~are~different(\l_tmpa_str \space vs \space \l_tmpb_str),~let~us~copy~the~file.}% + \robExtRunCmdIfPossible{\robExtCp \space "#1"~"\robExtAddCachePath{#1}"}% + } + }{ + \robExtDebugInfo{The~file~\robExtAddCachePath{#1}~does~not~exist,~let~us~copy~it:} + \robExtRunCmdIfPossible{\robExtCp \space "#1"~"\robExtAddCachePath{#1}"}% + } +} +\let\copyFileToCache\robExtCopyFileToCache + +% \robExtRenameBackupFilesForArxiv{robExt-arxiv-files-to-rename.txt} takes all elements listed in the .tex file +% (like robustExternalize/robExt-F83AB245D8043E653A89489C2E25AA6A.tex) and rename the +% robustExternalize/robExt-F83AB245D8043E653A89489C2E25AA6A.tex-backup into +% robustExternalize/robExt-F83AB245D8043E653A89489C2E25AA6A.tex +% (needed for arxiv) +% This is done, importantly, without shell escape as arxiv does not have shell escape. +\NewDocumentCommand{\robExtRenameBackupFilesForArxiv}{O{-backup}m}{ + \file_if_exist:xTF{#2} { + \robExtDebugInfo{We~will~read~the~file~#2.} + \ior_open:Nn \g__robExt_read_ior {#2}% + %% We avoid creating a new ior since the number of available files is limited in tex + \seq_clear:N \l_tmpa_seq + \ior_str_map_inline:Nn \g__robExt_read_ior {% + \file_if_exist:xTF{##1#1} { + \seq_put_right:Nn \l_tmpa_seq {##1} + }{ + \robExtDebugInfo{Warning:~##1#1~does~not~exist,~so~we~will~not~rename~back~this~file~for~arXiv.} + } + }% + \ior_close:N \g__robExt_read_ior + \seq_map_inline:Nn \l_tmpa_seq { + \robExtDebugInfo{We~copy~the~source~##1#1~to~##1^^J}% + % First we read the file: + \ior_open:Nn \g__robExt_read_ior {##1#1}% + \iow_open:Nx \g__robExt_write_iow {##1} + \ior_str_map_inline:Nn \g__robExt_read_ior {% + \iow_now:Nx \g__robExt_write_iow {\tl_to_str:n{####1}}% + }% + \iow_close:N \g__robExt_write_iow + \ior_close:N \g__robExt_read_ior + } + }{ + \robExtDebugInfo{Warning:~#2~does~not~exist,~so~we~will~not~rename~back~elements~for~arXiv.} + } +} \NewExpandableDocumentCommand{\robExtGetPrefixPath}{}{% \ifdefined\robExtCacheFolder% @@ -317,9 +600,9 @@ if __name__ == '__main__': %% TODO: create a makefile. \iow_new:N \g__robExt_write_list_all_figures_iow %% Create a file robExt-all-figures.txt with the list of .tex files -\iow_open:Nx \g__robExt_write_list_all_figures_iow {\jobname-\robExtAddPrefixName{all-figures.txt}} +\iow_open:Nx \g__robExt_write_list_all_figures_iow {\jobnameNoQuotes-\robExtAddPrefixName{all-figures.txt}} \iow_new:N \g__robExt_write_manually_compile_all_missing_figures_iow -\iow_open:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\jobname-\robExtAddPrefixName{compile-missing-figures.sh}} +\iow_open:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}} % Contains the list of dependency files (useful to compute the final md5sum) \seq_new:N \l__robExt_dependencies_str @@ -450,7 +733,7 @@ if __name__ == '__main__': } \let\evalPlaceholderNoReplacement\robExtEvalPlaceholderNoReplacement -\NewDocumentCommand{\robExtGetPlaceholderNoReplacement}{m}{ +\NewExpandableDocumentCommand{\robExtGetPlaceholderNoReplacement}{m}{ \str_use:c { l__robExt_placeholder_#1_str } } \let\getPlaceholderNoReplacement\robExtGetPlaceholderNoReplacement @@ -545,7 +828,7 @@ if __name__ == '__main__': %% Warning: only LaTeX-friendly code should be placed here, as LaTeX does not preserve some symbols and adds spaces %% Tested! %% The star version does NOT import the placeholder (useful for efficiency reasons) -\NewDocumentCommand{\robExtPlaceholderFromContent}{smm}{ +\NewDocumentCommand{\robExtPlaceholderFromContent}{sm+m}{ \robExt_set_hash_robust:cn { l__robExt_placeholder_#2_str } {#3} \checkIfPlaceholderNameIsLegal{#2} \ifdefined\robExtCompletelyDisableWholeImportSystem\else% Mostly for time optimizations... @@ -662,10 +945,10 @@ if __name__ == '__main__': %% https://tex.stackexchange.com/a/680259/116348 might work and be more efficient, but it might be less reliable %% and definitely more complicated and error prone. Instead, we write to a file and read the result. %% TODO: try to do it using verbatim, might be trivial or complicated, not sure, maybe see https://tex.stackexchange.com/questions/555359/reading-lines-verbatim-into-a-sequence-variable - \XSIMfilewritestart{\jobname-robExt-tmp-file-you-can-remove.tmp}% + \XSIMfilewritestart{\jobnameNoQuotes-robExt-tmp-file-you-can-remove.tmp}% }{% \XSIMfilewritestop% - \ior_open:Nn \g__robExt_read_ior {\jobname-robExt-tmp-file-you-can-remove.tmp}% + \ior_open:Nn \g__robExt_read_ior {\jobnameNoQuotes-robExt-tmp-file-you-can-remove.tmp}% %% Put the file in l__robExt_tmp_contain_file_str \str_clear_new:N \l__robExt_tmp_str% \ior_str_map_inline:Nn \g__robExt_read_ior {% @@ -762,10 +1045,10 @@ if __name__ == '__main__': %% This will copy "some code" in the cache, and set MYLIBPATH to the name of the file in the cache like %% MYLIBPATH = robExt-abc.py \NewDocumentEnvironment{RobExtPlaceholderPathFromCode}{sO{}m}{ - \XSIMfilewritestart*{\jobname-robExt-tmp-file-you-can-remove.tmp} + \XSIMfilewritestart*{\jobnameNoQuotes-robExt-tmp-file-you-can-remove.tmp} }{ \XSIMfilewritestop - \ior_open:Nn \g__robExt_read_ior {\jobname-robExt-tmp-file-you-can-remove.tmp} + \ior_open:Nn \g__robExt_read_ior {\jobnameNoQuotes-robExt-tmp-file-you-can-remove.tmp} %% Put the file in \l__robExt_tmp_contain_file_str \str_clear_new:N \l__robExt_tmp_contain_file_str \ior_str_map_inline:Nn \g__robExt_read_ior { @@ -856,7 +1139,7 @@ if __name__ == '__main__': \cs_if_exist:cTF { l__robExt_placeholder_#3_str }{ \str_set_eq:Nc { \l_robExt_result_str }{ l__robExt_placeholder_#3_str } }{ - \msg_error:nn{robExt}{placeholder not existing} + \msg_error:nnx{robExt}{placeholder not existing}{#3} } \robExtDebugMessage{Starting~to~replace~placeholder~#3~to~get~\l_robExt_result_str} \__robExt_replace_until_impossible: @@ -1039,7 +1322,7 @@ if __name__ == '__main__': % First we remove existing occurrences (useful to avoid listing the same macro more than once % if we redefine a macro): \seq_map_inline:Nn \l__robExt_tmp_seq { - \message{Adding ##1 to #1} + \robExtDebugMessage{Adding ##1 to #1} \seq_remove_all:cn {l__robExt_placeholder_group_#1_seq} {##1} \seq_put_right:cn {l__robExt_placeholder_group_#1_seq} {##1} } @@ -1126,6 +1409,7 @@ if __name__ == '__main__': \seq_new:N \l__robExt_list_groups_seq % contains the list of all groups +%%%% \robExtRegisterGroupPlaceholders{bash} adds "bash" to the list of existing groups \NewDocumentCommand{\robExtRegisterGroupPlaceholders}{m}{ \seq_if_exist:cTF {l__robExt_placeholder_group_#1_seq}{}{ \seq_clear_new:c {l__robExt_placeholder_group_#1_seq} @@ -1187,7 +1471,11 @@ if __name__ == '__main__': } \NewDocumentCommand{\robExtAddDependency}{m}{ - \seq_put_left:Nx \l__robExt_dependencies_str {#1} + \file_if_exist:nTF {#1} { + \seq_put_left:Nx \l__robExt_dependencies_str {#1} + } { + \msg_error:nnx{robExt}{dependency does not exist}{#1} + } } \NewDocumentCommand{\robExtDebugDependency}{}{ @@ -1356,7 +1644,12 @@ if __name__ == '__main__': % not want to recompile when the file changes \robExtUseHookJustBeforeWritingFiles% %% We add the figure in the list of files. - \iow_now:Nx \g__robExt_write_list_all_figures_iow {\robExtAddPrefixName{\robExtFinalHash.tex}} + % Avoid writing the same command multiple times (might occur in environments that execute the + % same command multiple times like align etc) + \cs_if_exist:cTF {l__robExt_write_list_all_figures_\robExtAddPrefixName{\robExtFinalHash.tex}:}{}{ + \iow_now:Nx \g__robExt_write_list_all_figures_iow {\robExtAddPrefixName{\robExtFinalHash.tex}} + \cs_gset:cn {l__robExt_write_list_all_figures_\robExtAddPrefixName{\robExtFinalHash.tex}:} {} + } %% We can now set the placeholders, and recompute the final value of the file: %% I was doing before \robExtPlaceholderFromContent + \robExtEvalPlaceholderInplace, but it is too slow \str_set:Nx \l_tmp_output_prefix_str {\robExtAddPrefixName{\robExtFinalHash}} @@ -1367,21 +1660,45 @@ if __name__ == '__main__': \str_replace_all:Nnx \l_robExt_result_str {__ROBEXT_WAY_BACK__}{\robExtCacheFolderWayBack} \str_replace_all:Nnx \l_robExt_result_str {__ROBEXT_CACHE_FOLDER__}{\robExtCacheFolder} \file_if_exist:xTF{\robExtAddCachePathAndName{\robExtFinalHash.tex}}{ - \message{The\space file\space \robExtAddCachePathAndName{\robExtFinalHash.tex} \space already\space exists.^^J} + \robExtDebugInfo{The\space file\space \robExtAddCachePathAndName{\robExtFinalHash.tex} \space already\space exists.^^J} }{ - \str_if_eq:VnTF { \l_robExt_result_str }{__ROBEXT_TEMPLATE__} { - \msg_error:nn{robExt}{forgot template} + % arXiv removes the .pdf if the .tex is present... so we need to manually remove the .tex file. + \file_if_exist:xTF{\robExtAddCachePathAndName{\robExtFinalHash.pdf}}{ + \robExtDebugInfo{The\space file\space \robExtAddCachePathAndName{\robExtFinalHash.pdf} \space already\space exists.^^J} + % Sometimes we still want to refer to the original tex file, for instance to input the source. + % So if a file .tex-backup exists, we copy it back to .tex: + \file_if_exist:xTF{\robExtAddCachePathAndName{\robExtFinalHash.tex-backup}} { + \robExtDebugInfo{We~copy~the~source~\robExtFinalHash.tex-backup~to~\robExtFinalHash.tex^^J}% + % First we read the file: + \ior_open:Nn \g__robExt_read_ior {\robExtAddCachePathAndName{\robExtFinalHash.tex-backup}}% + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.tex}} + \ior_str_map_inline:Nn \g__robExt_read_ior {% + \iow_now:Nx \g__robExt_write_iow {\tl_to_str:n{##1}}% + }% + \iow_close:N \g__robExt_write_iow + \ior_close:N \g__robExt_read_ior + }{} }{ - % Check if the output directory exists - \robExtCheckIfPrefixFolderExists - \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.deps}} - \iow_now:NV \g__robExt_write_iow \l__robExt_dependencies_mdfive_str - \iow_close:N \g__robExt_write_iow - %% Save the final file: - \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.tex}} - \iow_now:NV \g__robExt_write_iow \l_robExt_result_str - \iow_close:N \g__robExt_write_iow - \message{Source ~ saved ~ in ~ \robExtAddCachePathAndName{\robExtFinalHash.tex}.} + \str_if_eq:VnTF { \l_robExt_result_str }{__ROBEXT_TEMPLATE__} { + \msg_error:nn{robExt}{forgot template} + }{ + % Check if the output directory exists + \robExtCheckIfPrefixFolderExists + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.deps}} + \iow_now:NV \g__robExt_write_iow \l__robExt_dependencies_mdfive_str + \iow_close:N \g__robExt_write_iow + %% Save the final file: + \iow_open:Nx \g__robExt_write_iow {\robExtAddCachePathAndName{\robExtFinalHash.tex}} + \iow_now:NV \g__robExt_write_iow \l_robExt_result_str + \iow_close:N \g__robExt_write_iow + \robExtDebugInfo{Source ~ saved ~ in ~ \robExtAddCachePathAndName{\robExtFinalHash.tex}.} + \ifdefined\robExtPrintSourceWhenSaving + \typeout{Content~of~the~deps~file~\robExtAddCachePathAndName{\robExtFinalHash.deps}:^^J} + \robExt_write_file_in_log:x {\robExtAddCachePathAndName{\robExtFinalHash.deps}} + \typeout{Content~of~the~source~file~\robExtAddCachePathAndName{\robExtFinalHash.tex}:^^J} + \robExt_write_file_in_log:x {\robExtAddCachePathAndName{\robExtFinalHash.tex}} + \fi + } } } } @@ -1401,7 +1718,7 @@ if __name__ == '__main__': \let\l__robExt_do_not_compile\undefined } \cs_if_exist:NTF {\l__robExt_do_not_compile}{ - \message{No ~ need ~ to ~ recompile ~ \robExtAddCachePathAndName{\robExtFinalHash.pdf}^^J} + \robExtDebugInfo{No ~ need ~ to ~ recompile ~ \robExtAddCachePathAndName{\robExtFinalHash.pdf}^^J} }{ % Used to know later if we have run a compilation command at that step or not \cs_undefine:N \l__robExt_I_just_ran_a_compilation_command: @@ -1429,25 +1746,37 @@ if __name__ == '__main__': } {} \fi \ifdefined\robExtManualMode - \message{[robExt] Manual mode enabled: please, manually compile the images using \l__robExt_current_compilation_command_str or run 'bash \jobname-\robExtAddPrefixName{compile-missing-figures.sh}'.} + \message{[robExt] Manual mode enabled: please, manually compile the images using \l__robExt_current_compilation_command_str or run 'bash \jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}'.} % Avoid writing the same command multiple times (might occur in environments that execute the % same command multiple times like align etc) \cs_if_exist:cTF {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :}{}{ - \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\l__robExt_current_compilation_command_str}% a new line is automatically added + \ifdefined\robExtDoNotRedirectOutputInManuallyCompileMissingFigures% + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\l__robExt_current_compilation_command_str}% a new line is automatically added + \else + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {(~\l__robExt_current_compilation_command_str~)~>~\robExtAddCachePathAndName{\robExtFinalHash-compilation.log}~2>&1}% a new line is automatically added + \fi \cs_gset:cn {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :} {} \ifdefined\robExt@compile@parallel@after \gdef\robExt@compile@parallel@must@compile{} \seq_gput_right:Nx \g__robExt_files_compiled_in_parallel {\robExtAddCachePathAndName{\robExtFinalHash}} % Gets the line with the error for easier debugging \cs_gset:cx {g__robExt_lines_error_\robExtAddCachePathAndName{\robExtFinalHash}:} {\msg_line_number:} + \ifdefined\robExtPrintWholeFile + \cs_gset:cx {g__robExt_print_whole_file_\robExtAddCachePathAndName{\robExtFinalHash}:} {} + \fi \cs_gset:cx {g__robExt_compilation_command_\robExtAddCachePathAndName{\robExtFinalHash}:} {\l__robExt_current_compilation_command_str} \fi } - \else + \else% \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N \robExtForceCompilation } { - \message{[robExt] We ~ will ~ start ~ the ~ compilation using: ~ \l__robExt_current_compilation_command_str.} - \sys_shell_now:x {\l__robExt_current_compilation_command_str} % The ~ are used in ExplSyntaxOn to add space + % For a better debugging, we create a new file: + \ifdefined\robExtDoNotRedirectOutput% + \sys_shell_now:x {\robExtPrefixAllCompilationCommands \l__robExt_current_compilation_command_str}% + \else + \sys_shell_now:x {\robExtPrefixAllCompilationCommands (~\l__robExt_current_compilation_command_str~)~>~\robExtAddCachePathAndName{\robExtFinalHash-compilation.log}~2>&1}% + \fi + \message{[robExt] We ~ will ~ start ~ the ~ compilation ~ using: ~ \l__robExt_current_compilation_command_str.}% %% We notify that we are running the compilation command, this way it is possible to check later %% if the file is not present because of some compilation error or because we are in manual mode %% or fallback @@ -1457,8 +1786,12 @@ if __name__ == '__main__': % Avoid writing the same command multiple times (might occur in environments that execute the % same command multiple times like align etc) \cs_if_exist:cTF {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :}{}{ - \message{[robExt] Fallback to manual mode: please, manually compile the images using \l__robExt_current_compilation_command_str or run 'bash \jobname-\robExtAddPrefixName{compile-missing-figures.sh}'.} - \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\l__robExt_current_compilation_command_str}% a new line is automatically added + \message{[robExt] Fallback to manual mode: please, manually compile the images using \l__robExt_current_compilation_command_str or run 'bash \jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}'.} + \ifdefined\robExtDoNotRedirectOutputInManuallyCompileMissingFigures% + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {\l__robExt_current_compilation_command_str}% a new line is automatically added + \else + \iow_now:Nx \g__robExt_write_manually_compile_all_missing_figures_iow {(~\l__robExt_current_compilation_command_str~)~>~\robExtAddCachePathAndName{\robExtFinalHash-compilation.log}~2>&1}% a new line is automatically added + \fi \cs_gset:cn {l__robExt_missing_figure_\l__robExt_current_compilation_command_str :} {} \ifdefined\robExt@compile@parallel@after \gdef\robExt@compile@parallel@must@compile{} @@ -1466,6 +1799,9 @@ if __name__ == '__main__': \fi } \else + \ifdefined\robExtPrintSourceWhenSaving + \typeout{Compilation~command~that~we~would~have~run:^^J\l__robExt_current_compilation_command_str^^J} + \fi \msg_error:nn{robExt}{need shell escape} \fi } @@ -1523,10 +1859,29 @@ if __name__ == '__main__': % Check if we tried to run a compilation command at the previous step: \cs_if_exist:NTF \l__robExt_I_just_ran_a_compilation_command: { % We ran a compilation command but it failed: we print an error message - \msg_error:nnxxx{robExt}{missing compiled pdf parallel}{\robExtAddCachePathAndName{\robExtFinalHash}}{\msg_line_number:}{\l__robExt_current_compilation_command_str} + % We first check if an error file has been produced: + \file_if_exist:nTF {\robExtAddCachePathAndName{\robExtFinalHash-compilation.log}} { + % We print the error twice as some editor might display only the first error while in others + % it might be easier to see the last error. For instance, in emacs if we do not do that + % then it first prints the error message in a file in the cache (but it forgets the + % robustExternalize prefix) + % But first we get the lines containing the word error: + \robExt_get_errors_from_file:n {\robExtAddCachePathAndName{\robExtFinalHash-compilation.log}}% + % We print the message before the log because otherwise emacs is confused and tries to open the + % auxiliary cached file, and I don't want that. + \ifdefined\robExtHideFirstErrorMessage\else + \msg_error:nnxxxx{robExt}{missing compiled pdf parallel with log}{\robExtAddCachePathAndName{\robExtFinalHash}}{\msg_line_number:}{\l__robExt_current_compilation_command_str}{below~(you~might~need~to~press~ENTER~to~go~to~the~next~error)} + \fi + \message{--------~We~print~now~the~full~log~--------^^J} + \robExt_write_file_in_log:n {\robExtAddCachePathAndName{\robExtFinalHash-compilation.log}}% + \message{--------~End~of~the~full~log~--------^^J} + \msg_error:nnxxxx{robExt}{missing compiled pdf parallel with log}{\robExtAddCachePathAndName{\robExtFinalHash}}{\msg_line_number:}{\l__robExt_current_compilation_command_str}{above} + }{ + \msg_error:nnxxx{robExt}{missing compiled pdf parallel}{\robExtAddCachePathAndName{\robExtFinalHash}}{\msg_line_number:}{\l__robExt_current_compilation_command_str} + }% } { % We ran no compilation command: we print instead a placeholder depending on the context: \cs_if_exist:NTF \robExt@compile@parallel@must@compile { - % we are supposed to compile the current picture in parallel, let'us check if it will be possible: + % we are supposed to compile the current picture in parallel, let us check if it will be possible: \bool_if:nTF { \sys_if_shell_unrestricted_p: || \cs_if_exist_p:N \robExtForceCompilation} { \robExtImagePlaceholderIfParallelCompilation }{ @@ -1535,10 +1890,10 @@ if __name__ == '__main__': }{ \cs_if_exist:NTF \robExtManualMode { \robExtImagePlaceholderIfManualMode - \message{[robExt] ~ You ~ are ~ in ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}} + \message{[robExt] ~ You ~ are ~ in ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}} }{ \robExtImagePlaceholderIfFallbackMode - \message{[robExt] ~ You ~ are ~ falling ~ back ~ to ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobname-\robExtAddPrefixName{compile-missing-figures.sh}} + \message{[robExt] ~ You ~ are ~ falling ~ back ~ to ~ manual ~ mode: ~ please ~ compile ~ yourself ~ \robExtAddCachePathAndName{\robExtFinalHash.tex} ~ or ~ use ~ the ~ bash ~ \jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}} } } } @@ -1634,7 +1989,7 @@ if __name__ == '__main__': } \let\autoForwardMacro\robExtAutoForwardMacro -\NewDocumentCommand{\robExtLoadAutoForwardMacroConfig}{m}{ +\NewDocumentCommand{\robExtLoadAutoForwardMacroConfig}{+m}{ %\str_set:Nx \l__robExt_tmp_str {\cs_to_str:N #1} \cs_if_exist:cTF {l__robExt_execute_if_macro_present\string#1}{ % if the macro is present twice, we want to run \forward only once @@ -1660,16 +2015,21 @@ if __name__ == '__main__': %% "auto forward" \NewDocumentCommand{\robExtAutoForward}{}{ - \ifdefined\robExtUserInputCacheMe\else - \msg_warning:nn{robExt}{auto forward not in cachemecode} - \tl_set_rescan:NnV \robExtUserInputCacheMe {} \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str + % some code will automatically add auto forward, but we don't want to run it twice. + \robExtDebugInfo{Running auto forward.} + \ifdefined\robExt@already@ran@autoforward\robExtDebugInfo{Auto forward already ran}\else + \ifdefined\robExtUserInputCacheMe\else + \msg_warning:nn{robExt}{auto forward not in cachemecode} + \tl_set_rescan:NnV \robExtUserInputCacheMe {} \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str + \fi + \robExt@normalBraces\robExtUserInputCacheMe + \expandafter\robExt@getfromstringA\robExtUserInputCacheMe\robExt@getfromstringA%\robExt@getfromstringA is added to know when to stop + \def\robExt@already@ran@autoforward{}% \fi - \robExt@normalBraces\robExtUserInputCacheMe - \expandafter\robExt@getfromstringA\robExtUserInputCacheMe\robExt@getfromstringA%\robExt@getfromstringA is added to know when to stop } % https://tex.stackexchange.com/questions/700834/efficiently-program-search-of-macro-in-string/700844 -\def\robExt@getfromstringA#1{\ifx#1\robExt@getfromstringA \else +\long\def\robExt@getfromstringA#1{\ifx#1\robExt@getfromstringA \else %\ifcsname L:\string#1\endcsname \addto\listmacros{#1}\fi \robExtLoadAutoForwardMacroConfig{#1}% \expandafter\robExt@getfromstringA\fi @@ -1677,7 +2037,7 @@ if __name__ == '__main__': \def\robExt@normalBraces#1{{\catcode`{=12 \catcode`}=12 \everyeof={\endfile}% \expandafter\robExt@normalBracesA\expandafter#1\scantokens\expandafter{#1}}} -\def\robExt@normalBracesA#1#2\endfile{\gdef#1{#2}} +\long\def\robExt@normalBracesA#1#2\endfile{\gdef#1{#2}} %%% To automatically define and forward \NewDocumentCommand{\newcommandAutoForward}{moomO{}}{ @@ -1774,33 +2134,57 @@ if __name__ == '__main__': } \NewDocumentCommand{\defAutoForward}{mO{}mO{}}{ - \def#1#2{#3} - \robExtAutoForwardMacro{#1}[#4] + \def#1#2{#3}% + \robExtAutoForwardMacro{#1}[#4]% } -% \robExtGenericAutoForward[tikz]{string to match}[additional style to run]{ +% \robExtGenericAutoForward[tikz][namespace]{string to match}[additional style to run]{ % code to run in cached file if string matches, and to always run in current folder if not using % the star version % } \ExplSyntaxOff -\NewDocumentCommand{\robExtGenericAutoForward}{sO{latex}mO{}m}{% - \IfBooleanTF{#1}{}{#5}% We run the code right now - \robExt@set@hash@robust\zxTmpMacro{#5}% + +\NewDocumentCommand{\definecolorAutoForward}{mmmO{}}{% + \definecolor{#1}{#2}{#3}% + \robExtConfigure{if matches word={#1}{forward color=#1,#4}}% +} + +\NewDocumentCommand{\colorletAutoForward}{mmO{}}{% + \colorlet{#1}{#2}% + \robExtConfigure{if matches word={#1}{forward color=#1,#3}}% +} + + +\NewDocumentCommand{\robExtRunHereAndInPreambleOfCachedFiles}{O{latex}m}{% + #2% \robExtConfigure{% - add to preset={#2}{% - if matches word={#3}{ - command if externalization/.append code={ - \expanded{% - \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% - \expandonce{\zxTmpMacro}% - % #1%% - }% + add to preset={#1}{% + add to preamble={#2}, + },% + }% +} +\let\runHereAndInPreambleOfCachedFiles\robExtRunHereAndInPreambleOfCachedFiles + +\NewDocumentCommand{\robExtGenericAutoForward}{sO{latex}O{}mO{}m}{% + \IfBooleanTF{#1}{}{#6}% We run the code right now + \robExt@set@hash@robust\zxTmpMacro{#6}% + \expanded{% + \noexpand\robExtConfigure{% + register word with namespace={#3}{#4}{% + /utils/exec={% + \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% + \expandonce{\zxTmpMacro}% }% - },#4% + },% + #5% + },% + add to preset={#2}{% + auto forward words namespace={#3},% }% }% - }% + }% } + \let\genericAutoForward\robExtGenericAutoForward % uses "if matches" instead of "if matches word" @@ -1810,7 +2194,7 @@ if __name__ == '__main__': \robExtConfigure{% add to preset={#2}{% if matches={#3}{ - command if externalization/.append code={ + run command if externalization={ \expanded{% \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% \expandonce{\zxTmpMacro}% @@ -1840,8 +2224,120 @@ if __name__ == '__main__': \str_if_in:NnTF \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str {#1} {\pgfkeysalso{#2}} {} } +% \robExtRegisterWord {namespace} {word} {style} +\NewDocumentCommand{\robExtRegisterWord}{mmm}{ + % \robExt_register_match_word:nnn {#1} {#2} {\pgfkeysalso{#3}} + \robExtRegisterWordCode{#1}{#2}{\pgfkeysalso{#3}}% +} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Automatically forward (word-based) +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% For efficiency reasons, we first extract in a single pass all words in the text (a word is basically [A-Za-z]+) +% and run a function on these words + +%%% I tried this, but it is really slow (500x slower than if_string_matches, overall 20% increase of +%%% compilation time) +% % \__robExt_auto_forward_words:N \commandToRunOnEachWord \stringToSearchOn +% \cs_set:Nn \__robExt_auto_forward_words:NN { +% % \l_tmpa_str will contain the current word read so far +% \str_set:Nn \l_tmpa_str {}% +% \str_map_inline:Nn #2 { +% % \token_case_charcode:NnTF ##1 {} {} {} +% \__robExt_if_letter:nTF {##1} { +% \str_put_right:Nn \l_tmpa_str {##1} +% }{ +% \str_if_empty:NTF \l_tmpa_str { } { +% % if the string is empty, we run the command on the string +% #1 \l_tmpa_str% +% \str_set:Nn \l_tmpa_str {}% we reset its value +% } +% } +% } +% } + +% %% \__robExt_if_letter:nTF {char} {true} {false} tests if an element is a letter +% %% https://tex.stackexchange.com/a/700864/116348 +% \prg_new_conditional:Npnn \__robExt_if_letter:n #1 { TF } +% { +% \bool_lazy_or:nnTF +% { +% \bool_lazy_and_p:nn +% { \int_compare_p:nNn { `#1 } > { `a - 1 } } +% { \int_compare_p:nNn { `#1 } < { `z + 1 } } +% } +% { +% \bool_lazy_and_p:nn +% { \int_compare_p:nNn { `#1 } > { `A - 1 } } +% { \int_compare_p:nNn { `#1 } < { `Z + 1 } } +% } +% \prg_return_true: +% \prg_return_false: +% } + +% % \robExt_register_match_word {namespace that defaults to empty} {word} {code to run if word is present} +% \cs_set:Nn \robExt_register_match_word:nnn { +% \cs_set:cn {l__robExt_execute_if_word_present_#1_#2:} {#3} +% } + +% % \robExt_try_to_execute_if_match_word:nn {namespace} {word} +% \cs_set:Nn \robExt_try_to_execute_if_match_word:nn { +% \cs_if_exist:cTF {l__robExt_execute_if_word_present_#1_#2:} {% +% \cs_if_exist:cTF {l__robExt_execute_if_word_present_#1_#2__already_forwarded:}{\message{Already forwarded}}{ +% \use:c {l__robExt_execute_if_word_present_#1_#2:}% +% % define it so that we do not import twice next time +% \cs_set:cx {l__robExt_execute_if_word_present_#1_#2__already_forwarded:} {} +% } +% } { } +% } +% \cs_generate_variant:Nn \robExt_try_to_execute_if_match_word:nn { nV } + +% \NewDocumentCommand{\robExtAutoForwardWords}{O{}}{ +% \cs_set:Nn \__robExt_tmp_fct:N { +% \robExt_try_to_execute_if_match_word:nV {#1} ##1 +% } +% \__robExt_auto_forward_words:NN \__robExt_tmp_fct:N \l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str +% } + +% \robExtAutoForwardWords[namespace] +\NewDocumentCommand{\robExtAutoForwardWords}{m}{% + % needed or the original string will be changed + \let\robExt@tmpString\l__robExt_placeholder___ROBEXT_MAIN_CONTENT_ORIG___str% + \robExt@scanmacro@find@word{#1}\robExt@tmpString% +} + \ExplSyntaxOff +% Thanks a lot wipet for this optimized version!! +% https://tex.stackexchange.com/questions/701351/more-efficient-string-extraction-of-words/701441#701441 +\def\robExtWordSeparators{{ };:."?!@+-/*,=\{\}[]\\'()&|~_^<>} + +% Not for user, use \robExtAutoForwardWords +% \robExt@scanmacro@find@word{namespace}\stringToParse +\def\robExt@scanmacro@find@word#1#2{% + \def\robExt@namespace{#1}% + % we will change the lccode (basically ascii code) of some characters like +- to delimit what a word is + % supposed to be. Since we do not want to affect other stuff, we put it in a bgroup. + \bgroup \expandafter\robExt@set@to@comma\robExtWordSeparators\relax% + \ifdefined\robExtAdditionalCodeInScanMacroFindWord\robExtAdditionalCodeInScanMacroFindWord\fi% + % This seems to set the string in lower case, so fill or Fill or get triggered. But it is not what we want here. + % \lowercase\expandafter{\expandafter\gdef\expandafter#2\expandafter{#2}}% + \lowercase\expandafter{\expandafter\gdef\expandafter#2\expandafter{#2}}% + \edef#2{\detokenize\expandafter{#2}}% + % \message{\string#2: \meaning#2} % prints the modified format of the scanned macro + \expandafter\egroup% + \expandafter\robExt@wordscan@aux#2,\relax,% +} +\def\robExt@set@to@comma #1{\ifx\relax#1\else \lccode`#1=`, \expandafter\robExt@set@to@comma\fi} +\def\robExt@wordscan@aux#1,{\ifx\relax#1\empty\else% + %\message{{#1}} % prints each scanned "word" + \ifcsname robExt@action@to@run@on@word:\robExt@namespace:#1\endcsname \csname robExt@action@to@run@on@word:\robExt@namespace:#1\endcsname \fi% + \expandafter\robExt@wordscan@aux\fi% +} +%\robExtRegisterWordCode{namespace}{word}{code} +\def\robExtRegisterWordCode#1#2#3{\expandafter\gdef\csname robExt@action@to@run@on@word:\string#1:\string#2\endcsname{#3}} + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Interface %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1864,10 +2360,16 @@ if __name__ == '__main__': \robExtEvalPlaceholderReplaceFromList{__ROBEXT_TMP__}{% \pgfkeys{% /robExt/.cd, - #1/.append style={__ROBEXT_TMP__},% + #1/.append style={% + % Some styles run differently if inside a preset or not, + % like if matches word. This macro helps with detecting it. + /utils/exec={\def\robExtCurrentlyDefiningPreset{}},% + __ROBEXT_TMP__% + },% }% }% \robExtRemovePlaceholder{__ROBEXT_TMP__}% let us clean our variables + \let\robExtCurrentlyDefiningPreset\undefined% }, new preset/.code 2 args={% \robExtStrSetDoubleHash{\robExtTmpStr}{#2}% @@ -1878,10 +2380,16 @@ if __name__ == '__main__': \robExtEvalPlaceholderReplaceFromList{__ROBEXT_TMP__}{% \pgfkeys{% /robExt/.cd,% - #1/.style={__ROBEXT_TMP__},% + #1/.style={% + % Some styles run differently if inside a preset or not, + % like if matches word. This macro helps with detecting it. + /utils/exec={\def\robExtCurrentlyDefiningPreset{}},% + __ROBEXT_TMP__% + },% }% }% \robExtRemovePlaceholder{__ROBEXT_TMP__}% let us clean our variables + \let\robExtCurrentlyDefiningPreset\undefined% }, % new compiled preset={latex compiled}{ % code to compile in order to produce the expected strings @@ -2025,8 +2533,8 @@ if __name__ == '__main__': %%%%%%%%%%%%% %%% Debug %%% %%%%%%%%%%%%% - more logs/.code={\def\robExtDebugMessage#1{\message{^^J[robExt] #1}}}, - less logs/.code={\def\robExtDebugMessage#1{}}, + more logs/.code={\def\robExtDebugMessage##1{\message{^^J[robExt] ##1}}\def\robExtDebugInfo##1{^^J[robExt] ##1}}, + less logs/.code={\def\robExtDebugMessage##1{}\def\robExtDebugInfo##1{}}, show placeholder/.code={\robExtShowPlaceholder{#1}}, show placeholders/.code={\robExtShowPlaceholders}, show placeholders contents/.code={\robExtShowPlaceholdersContents}, @@ -2116,6 +2624,27 @@ if __name__ == '__main__': recompile/.code={\def\robExtForceRecompilation{}}, do not recompile/.code={\let\robExtForceRecompilation\undefined}, set compilation command/.code={\robExtSetCompilationCommand{#1}}, + % like "set compilation command" but moves the "__ROBEXT_OUTPUT_PDF__-tmp" to "__ROBEXT_OUTPUT_PDF__" if + % there is no error. + set compilation command move if no error/.style={ + set compilation command/.expanded={#1 && \robExtMv\space"__ROBEXT_OUTPUT_PDF__-tmp" "__ROBEXT_OUTPUT_PDF__"} + }, + set compilation command create if no error/.code={\robExtSetCompilationCommand{#1 && echo "" > __ROBEXT_OUTPUT_PDF__}}, + do not redirect compilation output/.code={\def\robExtDoNotRedirectOutput{}\def\robExtDoNotRedirectOutputInManuallyCompileMissingFigures{}}, + print whole file in error message/.code={\def\robExtPrintWholeFile{}}, + do not print whole file in error message/.code={\let\robExtPrintWholeFile\undefined}, + prefix log message with/.code={\def\robExtPrefixLogMessage{#1}}, + spacing between lines in log/.code={\def\robExtMessageWithPrefixNumberLines{#1}}, + % This adds a ! in front of the code if \robExtPrintWholeFile is defined + logs should show as errors/.style={prefix log message with={!\space}}, + texstudio/.style={ + logs should show as errors, + spacing between lines in log={^^J^^J(sorry, these 4 empty lines are ugly but needed for texstudio to consider them as separate errors, customize me "with spacing between lines in log")^^J}, + }, + remove line number/.code={\def\robExtRemoveLineNumber{}}, % Removes the l. if the log error line starts with l.42 or it will disturb emacs. + do not remove line number/.code={\let\robExtRemoveLineNumber\undefined}, + remove line number, + nb lines after error to show/.code={\def\robExtLinesAfterError{#1}}, add argument to compilation command/.code={\robExtAddArgumentToCompilationCommand{#1}}, add arguments to compilation command/.style={ add argument to compilation command/.list={#1} @@ -2143,6 +2672,24 @@ if __name__ == '__main__': % The default include command includes the pdf, making sure it is raised depending on its depth, % but you can override it: custom include command/.code={\def\robExtIncludeCommand{#1}}, + include command is input/.style={ + custom include command={#1\input{\robExtAddCachePathAndName{\robExtFinalHash.pdf}}}, + }, + include command is input/.default={}, + % Sometimes (gnuplot), we need to include an image located in the cache folder: + % https://tex.stackexchange.com/questions/171587/append-entries-to-an-existing-graphicspath + add cache to graphicspath/.code={% + \appto\Ginput@path{\robExtCacheFolder}% + },% + %% This is needed notably if the cached elements are nested, like the include command uses itself a tikz picture + %% etc cached via \cacheTikz... It it hard to reset everything efficiently (like we might not want to reset + %% all compilation commands etc), so you can add here stuff that might need to be restored later. + reset/.code={% + \let\robExtIncludeCommandAdvanced\undefined% + \let\robExtIncludeCommand\undefined% + \let\robExtPrintWholeFile\undefined% + \setPlaceholder{__ROBEXT_INCLUDEGRAPHICS_FILE__}{\robExtAddCachePathAndName{\robExtFinalHash.pdf}}% + }, %% Use this when we do not want to include anything (e.g. the video will be processed later in the chain): do not include pdf/.style={ custom include command={}% @@ -2153,12 +2700,22 @@ if __name__ == '__main__': disable manual mode/.code={\let\robExtManualMode\undefined}, enable fallback to manual mode/.code={\def\robExtFallbackManualMode{}}, disable fallback to manual mode/.code={\let\robExtFallbackManualMode\undefined}, + % For the arxiv, we need to rename the source .tex into .tex-backup or it will be removed + backup source for arxiv/.code={\def\robExtEnableBackupSource{}}, + do not backup source for arxiv/.code={\let\robExtEnableBackupSource\undefined}, + rename backup files for arxiv/.code={\robExtRenameBackupFilesForArxiv{#1}}, + rename backup files for arxiv/.default={robExt-arxiv-files-to-rename.txt}, + % print in the log the file. Useful to debug in arxiv where we do not have any access to the logs + print source when saving/.code={\def\robExtPrintSourceWhenSaving{}}, + copy file to cache/.code={\robExtCopyFileToCache{#1}}, %% Arguments to include graphics include graphics args/.code={\def\robExtIncludeGraphicsArgs{#1}}, %% The role of this command is to set \l_robExt_result_str, that will contain the final string. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Configuration of the cache %%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % For support of --output-directory + set output directory/.code={\def\robExtOutputDirectory{#1}}, %% Configure the prefix (default to "robExt-") set filename prefix/.code={\def\robExtPrefixFilename{#1}}, % first argument is subfolder, second is how to get from subfolder to the folder containing the source: @@ -2171,6 +2728,13 @@ if __name__ == '__main__': % Change this before starting to cache any library, and if you change it mid-document, be aware % that you will not be able to refer to elements in the old folder. set subfolder and way back={robustExternalize/}{../}, + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %%% Enable externalization for tikz %%% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + cache tikz/.code={\robExtExternalizeAllTikzpictures{}}, + do not cache tikz/.code={\robExtDoNotExternalizeAllTikzpictures{}}, + cache tikz 2 args/.code 2 args={\robExtExternalizeAllTikzpictures[#1][#2]{}}, + cache tikz 3 args/.code n args={3}{\robExtExternalizeAllTikzpictures[#1][#2][3]{}}, %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Disable externalization %%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2191,9 +2755,20 @@ if __name__ == '__main__': %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Forward macros if externalization is enabled %%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - command if externalization/.code={}, + command if externalization/.code={\def\robExt@we@are@already@in@command@if@externalization{}}, + % Doing "command if externalization/.append code={}" is not always good since we don't always know if the style + % will always be run before running "command if externalization". In particular this arrives when using + % something like "if matches word={mypink}{forward color=mypink}" + run command if externalization/.code={ + \ifdefined\robExt@we@are@already@in@command@if@externalization% + % we are already running command if externalization + #1% + \else% + \pgfkeysalso{command if externalization/.append code={#1}}% + \fi% + }, fw/.style={forward=#1}, - run code before main content if externalization enabled/.code={\message{#1}}, + run code before main content if externalization enabled/.code={}, % run code before main content if externalization enabled/.code={ % \message{aaa #1} % \def\zx@tmp{#1}% @@ -2213,7 +2788,7 @@ if __name__ == '__main__': % }, % }, forward/.style={% - command if externalization/.append code={% + run command if externalization={% \robExtGetCommandDefinitionInMacro{#1}% \expanded{% \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% @@ -2223,7 +2798,7 @@ if __name__ == '__main__': }, }, forward at letter/.style={% - command if externalization/.append code={% + run command if externalization={% \robExtGetCommandDefinitionInMacro{#1}% \expanded{% \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% @@ -2235,7 +2810,7 @@ if __name__ == '__main__': }, }, forward eval/.style={% - command if externalization/.append code={% + run command if externalization={% \expanded{% \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% \noexpand\def\noexpand#1{#1}% @@ -2244,7 +2819,7 @@ if __name__ == '__main__': },% }, forward counter/.style={% - command if externalization/.append code={% + run command if externalization={% \expanded{% \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% \noexpand\makeatletter% @@ -2256,8 +2831,23 @@ if __name__ == '__main__': }% },% },% + % Useful when we want to force the value of a forwarded counter. Useful for arXiv, e.g. + % if the page number is not the good one. + forward counter force value/.style 2 args={% + run command if externalization={% + \expanded{% + \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% + \noexpand\makeatletter% + % Make sure the counter exists: + \noexpand\ifcsname c@#1\noexpand\endcsname\noexpand\else\noexpand\newcounter{#1}\noexpand\fi% + \noexpand\setcounter{#1}{#2}% + \noexpand\makeatother + }% + }% + },% + },% forward color/.style={ - command if externalization/.append code={% + run command if externalization={% \extractcolorspecs{#1}{\zx@tmp@model}{\zx@tmp@cmd}% \expanded{% \noexpand\robExtAddBeforePlaceholder*{__ROBEXT_MAIN_CONTENT__}{% @@ -2268,19 +2858,47 @@ if __name__ == '__main__': },% %% This will try to automatically forward some macros. For this, you must first define %% - auto forward/.code={\robExtAutoForward}, + auto forward only macros/.code={\robExtAutoForward}, + auto forward/.code={\robExtAutoForward\pgfkeysalso{auto forward words}}, auto forward color/.style 2 args={% - add to preset={#1}{ - % if matches={\b #2\b}{forward color=#2}, - % the Regex is maybe more precise (avoid unnecessary forwards) but is way faster - if matches={#2}{forward color=#2}, + add to preset={#1}{% + if matches word={#2}{forward color=#2}, }, }, load auto forward macro config/.code={\robExtLoadAutoForwardMacroConfig{#1}}, + %% Auto forward words + auto forward words namespace/.code={% + \ifdefined\robExt@autoforward@enabled\else% + \pgfkeysalso{command if externalization/.append code={% + \ifdefined\robExt@autoforward@enabled% + \expanded{\noexpand\robExtAutoForwardWords{\robExt@autoforward@enabled}}% + \fi% + }% + }% + \fi% + \def\robExt@autoforward@enabled{#1}% + },%%% todo: finish + auto forward words/.style={auto forward words namespace={}}, %% This will if matches regex/.code 2 args={\robExtIfMatchesRegex{#1}{#2}}, - if matches word/.code 2 args={\robExtIfMatchesString{#1}{#2}}, if matches/.code 2 args={\robExtIfMatchesString{#1}{#2}}, + if matches word/.code 2 args={% + \robExtRegisterWord{}{#1}{#2}% + % If ran inside a preset, we want to enable it, otherwise we enable it on the latex preset + \ifdefined\robExtCurrentlyDefiningPreset% + \pgfkeysalso{auto forward words}% + \else% + \pgfkeysalso{/robExt/latex/.append style={auto forward words}}% + \fi% + }, + % more efficient since we can register the words before the preset, but make sure to call `auto forward words namespace`. + % You can use an empty namespace. + register word with namespace/.code n args={3}{% + \robExtRegisterWord{#1}{#2}{#3}% + }, + register word/.style 2 args={% + register word with namespace={}{#1}{#2},% + }, %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Run code before/after inclusion %%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2320,12 +2938,12 @@ if __name__ == '__main__': compile in parallel command/.store in=\robExt@compile@parallel@command, % Different backends to compile in parallel compile in parallel with gnu parallel/.style={ - compile in parallel command={parallel --jobs #1 :::: '\jobname-\robExtAddPrefixName{compile-missing-figures.sh}'}, + compile in parallel command={parallel --jobs #1 :::: '\jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}'}, }, compile in parallel with gnu parallel/.default={200\%}, compile in parallel with xargs/.style={ % Use " instead of ' as Windows does not consider ' as a valid surrounding for strings - compile in parallel command={xargs -t -I "{}" -P #1 \robExtParallelShell\space "{}" < "\jobname-\robExtAddPrefixName{compile-missing-figures.sh}"}, + compile in parallel command={xargs -t -I "{}" -P #1 \robExtParallelShell\space "{}" < "\jobnameNoQuotes-\robExtAddPrefixName{compile-missing-figures.sh}"}, }, % With 0 it spawns as many shells at possible, i.e. it compiles all 200 pictures at the same time, making the % system a laggy, and it is actually *slower* (2:05mn vs 1:15mn) than when using 16 threads. @@ -2361,12 +2979,36 @@ if __name__ == '__main__': \iow_close:N \g__robExt_write_manually_compile_all_missing_figures_iow \msg_warning:nnx{robExt}{rerun because parallel}{\robExt@compile@parallel@command} \message{\robExt@compile@parallel@command} - \sys_shell_now:x {\robExt@compile@parallel@command} + % Note that we do not save this into a file because it might help to see in real time where we are in + % the compilation process. This also means that we get poorer error message if this specific + % command fails in texstudio since it only prints the log file and not the whole terminal... + % Since this should anyway be quite rare (user has not xargs installed basically), this should be fine. + \sys_shell_now:x {\robExtPrefixAllCompilationCommands\robExt@compile@parallel@command} % We check if some files failed to compile \seq_map_inline:Nn \g__robExt_files_compiled_in_parallel { - \message{#1} \file_if_exist:xTF{#1.pdf}{}{ - \msg_error:nnxxx{robExt}{missing compiled pdf parallel}{#1}{\use:c {g__robExt_lines_error_#1:}}{\use:c {g__robExt_compilation_command_#1:}} + \file_if_exist:nTF {#1-compilation.log} { + % We print the error twice as some editor might display only the first error while in others + % it might be easier to see the last error. For instance, in emacs if we do not do that + % then it first prints the error message in a file in the cache (but it forgets the + % robustExternalize prefix) + % But first we get the lines containing the word error: + \let\robExtPrintWholeFile\undefined + \cs_if_exist:cTF {g__robExt_print_whole_file_#1:} {\def\robExtPrintWholeFile{}} {} + \robExt_get_errors_from_file:n {#1-compilation.log}% + % We print the message before the log because otherwise emacs is confused and tries to open the + % auxiliary cached file, and I don't want that. + \ifdefined\robExtHideFirstErrorMessage\else + \msg_error:nnxxxx{robExt}{missing compiled pdf parallel with log}{#1}{\use:c {g__robExt_lines_error_#1:}}{\use:c {g__robExt_compilation_command_#1:}}{below~(you~might~need~to~press~ENTER~to~go~to~the~next~error)} + \fi + \message{--------~We~print~now~the~full~log~--------^^J} + \robExt_write_file_in_log:n {#1-compilation.log}% + \message{--------~End~of~the~full~log~--------^^J} + \msg_error:nnxxxx{robExt}{missing compiled pdf parallel with log}{#1}{\use:c {g__robExt_lines_error_#1:}}{\use:c {g__robExt_compilation_command_#1:}}{above} + \let\robExtPrintWholeFile\undefined + }{ + \msg_error:nnxxx{robExt}{missing compiled pdf parallel}{#1}{\use:c {g__robExt_lines_error_#1:}}{\use:c {g__robExt_compilation_command_#1:}} + } } } } { @@ -2378,6 +3020,18 @@ if __name__ == '__main__': } \fi \fi + % \robExtBackupSource moves the source .tex into .tex-backup, needed for the arXiv website that removes the + % .pdf if there is a .tex file. + % We need to run it at the very end, as if we move it right after a cacheMe, we cannot refer anymore to the + % input file of this block later. + \iow_close:N \g__robExt_write_list_all_figures_iow + \ifdefined\robExtEnableBackupSource% + \ior_open:Nn \g__robExt_read_ior {\jobnameNoQuotes-\robExtAddPrefixName{all-figures.txt}}% + \ior_str_map_inline:Nn \g__robExt_read_ior {% + \robExtBackupSource{#1}% + }% + \ior_close:N \g__robExt_read_ior + \fi% } \ExplSyntaxOff @@ -2401,17 +3055,22 @@ if __name__ == '__main__': \ifdefined\robExtExecuteNamedOutput\robExtExecuteNamedOutput\fi% \ifdefined\robExtExecuteAfter\robExtExecuteAfter\fi% \fi% - \message{Finished to include the file.}% + \robExtDebugInfo{Finished to include the file.}% } %% #1: Arguments, #2: content to externalize -\NewDocumentCommand{\robExtCacheMe}{O{}m}{% +\NewDocumentCommand{\robExtCacheMe}{O{}+m}{% {% Group %% We store the input in a non-string element for efficiently implementing "auto forward" \edef\robExtUserInputCacheMe{\unexpanded{#2}}% \unexpanded is needed if the macro contains a #1 \pgfkeys{% /robExt/.cd,% + %% This is needed notably if the cached elements are nested, like the include command uses itself a tikz + %% picture etc cached via \cacheTikz... It it hard to reset everything efficiently (like we might not + %% want to reset all compilation commands etc), so you can add here stuff that might need to be restored + %% later. + reset, set placeholder={__ROBEXT_MAIN_CONTENT_ORIG__}{#2},% default style,% #1, @@ -2637,12 +3296,16 @@ __ROBEXT_LATEX_WRITE_DEPTH_TO_OUT_FILE__% enable placeholders, % This way it appears before orig in the list: more efficient import placeholders from group={latex}, + %/utils/exec={\message{I am in the latex style!!!!!!}}, set placeholder={__ROBEXT_MAIN_CONTENT__}{__ROBEXT_MAIN_CONTENT_ORIG__}, import placeholder={__ROBEXT_MAIN_CONTENT_ORIG__}, first placeholders latex, set template={__ROBEXT_LATEX__}, set compilation command={__ROBEXT_LATEX_COMPILATION_COMMAND__}, %% Configure the latex compilation engine + add to compilation command options/.style={ + add to placeholder={__ROBEXT_LATEX_COMPILATION_COMMAND_OPTIONS__}{#1}, + }, use latexmk/.style={ set placeholder={__ROBEXT_LATEX_ENGINE__}{latexmk}, }, @@ -2787,9 +3450,9 @@ def finished_with_no_error(): created (otherwise robust-externalize will think that the compilation failed) ''' if not os.path.exists(get_filename_from_extension(".pdf")): - # we create an empty path + # we create a nearly empty pdf (not empty or arxiv will remove it) with open(get_filename_from_extension(".pdf"), 'w') as f: - pass + f.write("ok") ### Starting main content __ROBEXT_MAIN_CONTENT__ @@ -2810,6 +3473,7 @@ finished_with_no_error() \robExtConfigure{ python/.style={ enable placeholders, + print whole file in error message, import placeholders={__ROBEXT_PYTHON__,__ROBEXT_PYTHON_IMPORT__,__ROBEXT_PYTHON_MAIN_CONTENT_WRAPPED__,__ROBEXT_PYTHON_FINISHED_WITH_NO_ERROR__,__ROBEXT_PYTHON_EXEC__}, set compilation command={__ROBEXT_PYTHON_EXEC__ "__ROBEXT_SOURCE_FILE__"}, set template={__ROBEXT_PYTHON__}, @@ -2906,13 +3570,24 @@ print_file.close() \robExtClearGroupPlaceholders{main} +%% Inspired by +%% https://tex.stackexchange.com/questions/259247/rescaling-gnuplottex-to-fit-in-subfigure/259271 +\ExplSyntaxOn +\DeclareExpandableDocumentCommand{\robExtLenToCm}{ O{cm} m } +{ + \dim_to_decimal_in_unit:nn { #2 } { 1 #1 } #1 +} +\ExplSyntaxOff +\let\lenToCm\robExtLenToCm + \robExtConfigure{ verbatim text/.style={ enable placeholders, set template={__ROBEXT_MAIN_CONTENT__}, custom include command={\evalPlaceholder{\verbatiminput{\robExtAddCachePathAndName{\robExtFinalHash.tex}}}}, %% Apparently this works on windows as well https://stackoverflow.com/questions/1702762/how-can-i-create-an-empty-file-at-the-command-line-in-windows - set compilation command={echo "" > __ROBEXT_OUTPUT_PDF__}, + %% We do not want it to be empty or arXiv will remove the file. + set compilation command={echo "ok" > __ROBEXT_OUTPUT_PDF__}, }, verbatim text no include/.style={ verbatim text, @@ -2928,6 +3603,94 @@ print_file.close() %%%%%% +%%%%%% Group "gnuplot" +%%%%%% + +% We clean the "main" group, that we will copy later to gnuplot. +\robExtClearGroupPlaceholders{main} + +\begin{PlaceholderFromCode}*{__GNUPLOT_TEMPLATE__} +set terminal __ROBEXT_GNUPLOT_TERMINAL__ +set output "__ROBEXT_GNUPLOT_OUTPUTFILE__" +set loadpath "__ROBEXT_WAY_BACK__" +__ROBEXT_GNUPLOT_PRELUDE__ +__ROBEXT_MAIN_CONTENT__ +\end{PlaceholderFromCode} + +\begin{PlaceholderFromCode}{__ROBEXT_GNUPLOT_PRELUDE__} +\end{PlaceholderFromCode} + +\robExtCopyGroupPlaceholders{gnuplot}{main} +\robExtRegisterGroupPlaceholders{gnuplot} + +\robExtConfigure{ + new preset={gnuplot}{ + enable placeholders, + print whole file in error message, + import placeholders from group={gnuplot}, + set compilation command={gnuplot -c "__ROBEXT_SOURCE_FILE__" && echo "ok" > "__ROBEXT_OUTPUT_PDF__"}, + set terminal/.style={ + set placeholder={__ROBEXT_GNUPLOT_TERMINAL__}{#1}, + }, + % The extension is important. Notably, the cairolatex expects a .tex. + set filename/.style={ + set placeholder={__ROBEXT_GNUPLOT_OUTPUTFILE__}{__ROBEXT_OUTPUT_PREFIX__-gnuplot-#1}, + set placeholder={__ROBEXT_INCLUDEGRAPHICS_FILE__}{\robExtAddCachePathAndName{\robExtFinalHash-gnuplot-#1}}, + }, + pdf terminal/.style={ + set terminal={pdf #1}, + set filename={.pdf}, + }, + pdf terminal/.default={}, + pdf terminal, + is tex output/.style={ + set filename={.tex}, + add cache to graphicspath, % At least in cairolatex, we first need to input a file that includegraphics an image in the cache. + custom include command={\message{We will input the file \robExtAddCachePathAndName{\robExtFinalHash-gnuplot-.tex}}\input{\robExtAddCachePathAndName{\robExtFinalHash-gnuplot-.tex}}}, + }, + tikz terminal/.style={ + set terminal={tikz #1}, + is tex output, + }, + tikz terminal/.default={}, + % Warning: you cannot define this option with the name "tikz" if you try to nest elements, for instance + % if the include command calls an \input that calls a tikz picture, that is cached by \cacheTikz and therefore + % call the "tikz" preset that is now changed. + cairolatex terminal/.style={ + set terminal={cairolatex #1}, + is tex output, + }, + cairolatex terminal/.default={}, + set template={__GNUPLOT_TEMPLATE__}, + print verbatim if no externalization, + }, +} + + +% Like \gpgetvar but do not produce an error if the variable does not exist yet +% Fix https://github.com/leo-colisson/robust-externalize/issues/17 +\ExplSyntaxOn +\NewDocumentCommand{\robExtGpgetvar}{m}{% + \ifcsname gp@var@#1\endcsname% + \gpgetvar{#1}% + \else% + \msg_warning:nnx{robExt}{gpgetvar recompilation needed}{#1}% + \emph{Please~recompile~to~load~ variable:~}% + \texttt{\tl_to_str:n {#1}}% + \fi% +} +% Is useful if you want a number even if there is an error https://github.com/leo-colisson/robust-externalize/issues/17#issuecomment-1862556771 +% It must be expandable. +\NewExpandableDocumentCommand{\robExtGpgetvarNb}{sO{404}m}{% + \ifcsname gp@var@#3\endcsname% + \gpgetvar{#3}% + \else% We cannot print a waring or siunit will complain. + \sys_if_engine_luatex:TF{\directlua{texio.write_nl('Warning:~the~variable~#3~used~in~robExtGpgetvarNb~does~not~exist~yet.~Please~recompile.')}}{}#2% + \fi% +} +\ExplSyntaxOff + +%%%%%% %%%%%% Group "bash" %%%%%% @@ -2941,7 +3704,7 @@ outputTex="__ROBEXT_OUTPUT_PREFIX__-out.tex" outputPdf="__ROBEXT_OUTPUT_PDF__" __ROBEXT_MAIN_CONTENT__ # Create the pdf file to certify that no compilation error occured -touch "${outputPdf}" +echo "ok" > "${outputPdf}" \end{RobExtPlaceholderFromCode} \robExtSetPlaceholder{__ROBEXT_BASH_SHELL__}{bash} @@ -2949,6 +3712,7 @@ touch "${outputPdf}" \robExtConfigure{ bash/.style={ enable placeholders, + print whole file in error message, import placeholders={__ROBEXT_BASH_TEMPLATE__,__ROBEXT_BASH_SHELL__}, set compilation command={__ROBEXT_BASH_SHELL__ "__ROBEXT_SOURCE_FILE__"}, set template={__ROBEXT_BASH_TEMPLATE__}, @@ -2960,6 +3724,50 @@ touch "${outputPdf}" \robExtRegisterGroupPlaceholders{bash} %%%%%% +%%%%%% Group "web image", to download images automatically +%%%%%% + +\robExtClearGroupPlaceholders{main} + +% \begin{RobExtPlaceholderFromCode}{__ROBEXT_BASH_TEMPLATE__} +% # Quit if there is an error +% set -e +% outputTxt="__ROBEXT_OUTPUT_PREFIX__-out.txt" +% outputTex="__ROBEXT_OUTPUT_PREFIX__-out.tex" +% outputPdf="__ROBEXT_OUTPUT_PDF__" +% __ROBEXT_MAIN_CONTENT__ +% # Create the pdf file to certify that no compilation error occured +% echo "ok" > "${outputPdf}" +% \end{RobExtPlaceholderFromCode} + +\robExtConfigure{ + web image/.style={ + enable placeholders, + print whole file in error message, + wget/.style={ + set compilation command move if no error={wget "__ROBEXT_MAIN_CONTENT__" -O "__ROBEXT_OUTPUT_PDF__-tmp"}, + }, + curl/.style={ + set compilation command move if no error={curl "__ROBEXT_MAIN_CONTENT__" -L -o "__ROBEXT_OUTPUT_PDF__-tmp"}, + }, + if unix={ + wget + }, + if windows={ + curl + }, + set template={}, + } +} + +\NewDocumentCommand{\robExtIncludegraphicsWeb}{D<>{}O{}m}{\robExtCacheMe[web image,set includegraphics options={#2},#1]{#3}} +\let\includegraphicsWeb\robExtIncludegraphicsWeb + +\robExtCopyGroupPlaceholders{web image}{main} +\robExtRegisterGroupPlaceholders{web image} + + +%%%%%% %%%%%% Group "default", available in all styles %%%%%% @@ -3063,6 +3871,7 @@ touch "${outputPdf}" } \NewDocumentCommand{\robExtExternalizeAllTikzpictures}{O{tikz}O{tikzpicture}O{<>}}{% + \robExtDoNotExternalizeAllTikzpictures% \robExtCacheEnvironment[#3]{tikzpicture}{#2}% % Tikz has a more complicated parsing system \DeclareCommandCopy{\robExtCommandOrigtikz}{\tikz}% we save the function for later reset @@ -3077,12 +3886,65 @@ touch "${outputPdf}" } \let\cacheTikz\robExtExternalizeAllTikzpictures +\NewDocumentCommand{\robExtDoNotExternalizeAllTikzpictures}{}{% + \ifdefined\robExtCommandOrigtikz% + \DeclareCommandCopy{\tikz}{\robExtCommandOrigtikz}% + \let\robExtCommandOrigtikz\undefined + \fi% + \ifdefined\robExtEnvironmentOrigtikzpicture% + \DeclareEnvironmentCopy{tikzpicture}{robExtEnvironmentOrigtikzpicture}% + \let\robExtEnvironmentOrigtikzpicture\undefined + \fi% +} +\let\doNotCacheTikz\robExtDoNotExternalizeAllTikzpictures + + %% The cached version \DeclareDocumentEnvironment{tikzpictureC}{D<>{}O{}O{}}{% \begin{CacheMe}{tikzpicture,#1}[#2]% }{\end{CacheMe}}% %%%%%% +%%%%%% Cache tikzit +%%%%%% + +\NewDocumentCommand{\robExtCacheTikzit}{O{tikzit}}{% + \ifdefined\robExtCommandOrigtikzfig\else% + \DeclareCommandCopy{\robExtCommandOrigtikzfig}{\tikzfig}% + \fi% + \robExtAddToCommandResetList{tikzfig}% + \DeclareDocumentCommand{\tikzfig}{D<>{}m}{% + % tikzit tries first ##2.tikz and fallsback to figures/##2.tikz otherwise. + \IfFileExists{##2.tikz}{\def\robExtTikzfigFile{##2}}{\def\robExtTikzfigFile{figures/##2}}% + \expanded{\noexpand\cacheMe[ + #1, + add dependencies/.expanded={\robExtTikzfigFile.tikz}, + ##1, + command if no externalization/.code={\noexpand\robExtDisableTikzpictureOverwrite\noexpand\robExtCommandOrigtikzfig{##2}}, + ]{\noexpand\tikzfig{__ROBEXT_WAY_BACK__\robExtTikzfigFile}}}% + }% +} +\let\cacheTikzit\robExtCacheTikzit + +\NewDocumentCommand{\robExtCacheTikzitWithStyle}{O{tikzit}m}{% + \robExtCacheTikzit[#1]% + \robExtConfigure{ + % First, we copy the tikzit-related files to the cache: + copy file to cache={tikzit.sty}, + copy file to cache={#2}, + % We create the tikzit preset loaded by default: + new preset={#1}{ + latex, + add to preamble={ + \usepackage{tikzit} + \input{#2} + }, + }, + }% +} +\let\cacheTikzitWithStyle\robExtCacheTikzitWithStyle + +%%%%%% %%%%%% The generic functions %%%%%% @@ -3090,7 +3952,8 @@ touch "${outputPdf}" \NewDocumentCommand{\robExtCacheEnvironment}{O{<>}mm}{% %% We need to save the original environment to avoid infinite recursion if we disable externalization %% https://tex.stackexchange.com/questions/695391/why-isnt-my-environment-restored/695398 - \NewEnvironmentCopy{robExtEnvironmentOrig#2}{#2}% + \ifcsname robExtEnvironmentOrig#2\endcsname\PackageWarning{Your are trying to cache an environment #2 that seems to be already cached... Expect weird things to happen.}{}\fi + \DeclareEnvironmentCopy{robExtEnvironmentOrig#2}{#2}% \robExtAddToEnvironmentResetList{#2}% \DeclareDocumentEnvironment{#2}{D#1{}}{% \def\robExtEnvironmentOrigName{#2}% |