From 92effab22b73c432933e7eb8e12c1cefda221df9 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Wed, 3 Jan 2024 03:03:02 +0000 Subject: CTAN sync 202401030303 --- macros/generic/advice/CHANGELOG.md | 9 + macros/generic/advice/advice.edtx | 129 +- macros/generic/collargs/CHANGELOG.md | 25 + macros/generic/collargs/README.md | 11 + macros/generic/collargs/collargs.edtx | 107 +- macros/generic/collargs/collargs.ins | 10 +- macros/generic/memoize/CHANGELOG.md | 59 + macros/generic/memoize/Makefile | 56 +- macros/generic/memoize/doc/memoize-clean.1.md | 4 +- macros/generic/memoize/doc/memoize-code.pdf | Bin 621070 -> 626766 bytes macros/generic/memoize/doc/memoize-code.sty | 27 +- macros/generic/memoize/doc/memoize-code.tex | 34 +- macros/generic/memoize/doc/memoize-doc.mst | 1 + macros/generic/memoize/doc/memoize-doc.pdf | Bin 0 -> 2180931 bytes macros/generic/memoize/doc/memoize-doc.sty | 6 +- macros/generic/memoize/doc/memoize-doc.tex | 9314 ++++++++++++++++++++ macros/generic/memoize/doc/memoize-extract.1.md | 82 +- macros/generic/memoize/doc/memoize.mst | 1 - macros/generic/memoize/doc/memoize.pdf | Bin 2164795 -> 0 bytes macros/generic/memoize/doc/memoize.tex | 9094 ------------------- macros/generic/memoize/memoize-clean.pl | 25 +- macros/generic/memoize/memoize-clean.py | 16 +- macros/generic/memoize/memoize-extract.pl | 859 +- macros/generic/memoize/memoize-extract.py | 674 +- macros/generic/memoize/memoize.edtx | 760 +- macros/generic/memoize/memoize.ins | 1 + macros/latex/contrib/changebar/CATALOG | 2 +- macros/latex/contrib/changebar/README | 4 +- macros/latex/contrib/changebar/changebar.bug | 11 + macros/latex/contrib/changebar/changebar.dtx | 28 +- macros/latex/contrib/changebar/changebar.pdf | Bin 483608 -> 484332 bytes macros/latex/contrib/eolang/eolang.dtx | 88 +- macros/latex/contrib/eolang/eolang.pdf | Bin 1935887 -> 1937780 bytes .../latex/contrib/nicematrix/nicematrix-code.dtx | 90 +- .../latex/contrib/nicematrix/nicematrix-code.pdf | Bin 0 -> 973857 bytes .../latex/contrib/nicematrix/nicematrix-french.pdf | Bin 596600 -> 596978 bytes .../latex/contrib/nicematrix/nicematrix-french.tex | 16 +- macros/latex/contrib/nicematrix/nicematrix.ins | 4 +- macros/latex/contrib/nicematrix/nicematrix.pdf | Bin 612159 -> 611795 bytes macros/latex/contrib/nicematrix/nicematrix.tex | 17 +- .../latex/contrib/proflycee/doc/ProfLycee-doc.pdf | Bin 1444492 -> 1450770 bytes .../latex/contrib/proflycee/doc/ProfLycee-doc.tex | 4 +- macros/latex/contrib/proflycee/tex/ProfLycee.sty | 5 +- .../proflycee/tex/proflycee-tools-analyse.tex | 80 +- .../proflycee/tex/proflycee-tools-arithm.tex | 24 +- .../proflycee/tex/proflycee-tools-complexes.tex | 609 ++ 46 files changed, 12285 insertions(+), 10001 deletions(-) create mode 100644 macros/generic/collargs/CHANGELOG.md create mode 100644 macros/generic/memoize/CHANGELOG.md create mode 100644 macros/generic/memoize/doc/memoize-doc.mst create mode 100644 macros/generic/memoize/doc/memoize-doc.pdf create mode 100644 macros/generic/memoize/doc/memoize-doc.tex delete mode 100644 macros/generic/memoize/doc/memoize.mst delete mode 100644 macros/generic/memoize/doc/memoize.pdf delete mode 100644 macros/generic/memoize/doc/memoize.tex create mode 100644 macros/latex/contrib/nicematrix/nicematrix-code.pdf create mode 100644 macros/latex/contrib/proflycee/tex/proflycee-tools-complexes.tex (limited to 'macros') diff --git a/macros/generic/advice/CHANGELOG.md b/macros/generic/advice/CHANGELOG.md index 63de5163a9..f112324840 100644 --- a/macros/generic/advice/CHANGELOG.md +++ b/macros/generic/advice/CHANGELOG.md @@ -14,6 +14,15 @@ those cases, please see [Memoize's GitHub repository](https://github.com/sasozivanovic/memoize) for the recent development history. +## 2024/01/02 v1.1.0 +* Implement `\AdviceCsnameGetOriginal`. +* Change `\AdviceGetOriginal` to expand to the internal control sequence + holding the original definition of the advised command under full expansion. +* Add the missing info messages for environment (de)activation. +* Add the original command meaning to trace. + +Manual: Memoize 2024/01/02 v1.1.0 + ## 2023/10/25 v1.0.1 * Require package `xparse`, as `\GetDocumentCommandArgSpec` is being dropped from the LaTeX kernel. diff --git a/macros/generic/advice/advice.edtx b/macros/generic/advice/advice.edtx index dcbdfb297e..415d9da79b 100644 --- a/macros/generic/advice/advice.edtx +++ b/macros/generic/advice/advice.edtx @@ -29,14 +29,14 @@ % \relax % %<*main> -%\ProvidesPackage{advice}[2023/10/25 v1.0.1 Extend commands and environments] +%\ProvidesPackage{advice}[2024/01/02 v1.1.0 Extend commands and environments] %%D \module[ %%D file=t-advice.tex, -%%D version=1.0.1, +%%D version=1.1.0, %%D title=Advice, %%D subtitle=Extend commands and environments, %%D author=Saso Zivanovic, -%%D date=2023-10-25, +%%D date=2024-01-02, %%D copyright=Saso Zivanovic, %%D license=LPPL, %%D ] @@ -96,7 +96,9 @@ % namespace is hard-coded into these keys via |#1|; their arguments are % the command/environment (cs)name, and setup keys belonging to path % \meta{installation keypath}|/\meta{setup key name}|. + % \indentmacrocode #2/.code 2 args={% + % \noindentmacrocode % Call the internal setup macro, wrapping the received keylist into a % |pgfkeys| invocation. \AdviceSetup{#1}{#2}{##1}{\pgfqkeys{#1/#2}{##2}}% @@ -151,7 +153,7 @@ % defined right away, as they don't change with the type of activation % (immediate vs.\ deferred). activate csname/.style={activate/.expand once={\csname##1\endcsname}}, - deactivate csname/.style={activate/.expand once={\csname##1\endcsname}}, + deactivate csname/.style={deactivate/.expand once={\csname##1\endcsname}}, % \end{key} % % \begin{key}{activate key, deactivate key} @@ -178,7 +180,8 @@ % These keys are used to setup the handling of the command or % environment. The storage macros (|\AdviceRunConditions| etc.) have % public names as they also play a crucial role in the handler - % definitions, see section~\ref{sec:code:advice:handle}. \indentmacrocode + % definitions, see section~\ref{sec:code:advice:handle}. + % \indentmacrocode[-0.5em] run conditions/.store in=\AdviceRunConditions, bailout handler/.store in=\AdviceBailoutHandler, outer handler/.store in=\AdviceOuterHandler, @@ -215,7 +218,7 @@ reset/.code={\csname\advice@setup@init@\AdviceType\endcsname}, % \end{mmzautokey} % - % \begin{mmzautokey} + % \begin{mmzautokey}{after setup} % The code given here will be executed once we exit the setup group. % |integrated driver| of Memoize uses it to declare a conditional. after setup/.code={\appto\AdviceAfterSetup{##1}}, @@ -257,6 +260,7 @@ % Open a group, so that we allow for embedded |auto| invocations. \begingroup \def\AdviceName{#3}% + \advice@def@AdviceCsname % Command, complain, or environment? \collargs@cs@cases{#3}{% \def\AdviceType{command}% @@ -399,7 +403,8 @@ \noexpand\csdef{advice@I#1//\string#2}{% \def\noexpand\AdviceOuterHandler{\expandonce\AdviceOuterHandler}% \def\noexpand\AdviceCollector{\expandonce\AdviceCollector}% - \def\noexpand\AdviceRawCollectorOptions{\expandonce\AdviceRawCollectorOptions}% + \def\noexpand\AdviceRawCollectorOptions{% + \expandonce\AdviceRawCollectorOptions}% \def\noexpand\AdviceCollectorOptions{\expandonce\AdviceCollectorOptions}% \def\noexpand\AdviceArgs{\expandonce\AdviceArgs}% \def\noexpand\AdviceInnerHandler{\expandonce\AdviceInnerHandler}% @@ -412,9 +417,9 @@ % \label{sec:code:advice:activation} % % \begin{advicekey}{activation/immediate, activation/deferred} -% The two subkeys of |/advice/activation| install the immediate and the +% These two subkeys of |/advice/activation| install the immediate and the % deferred activation code into the installation keypath. They are invoked -% by \meta{installation keypath}|/activation=|\meta{type}. +% by key \meta{installation keypath}|/activation=|\meta{type}. % % Under the deferred activation regime, the commands are not (de)activated % right away. Rather, the (de)activation calls are collected in style @@ -490,9 +495,27 @@ \def\advice@original@cs#1#2{\csname advice@o#1//\string#2\endcsname} \def\AdviceGetOriginal#1#2{% \ifcsname advice@o#1//\string#2\endcsname - \csname advice@o#1//\string#2\expandafter\endcsname + \expandonce{\csname advice@o#1//\string#2\expandafter\endcsname\expandafter}% \else - \expandafter#2% + \expandafter\noexpand\expandafter#2% + \fi +} +% \end{macro} +% +% \begin{macro}{\AdviceCsnameGetOriginal} +% A version of |\AdviceGetOriginal| which accepts a control sequence name as +% the second argument. +\begingroup +\catcode`\/=0 +\catcode`\\=12 +/gdef/advice@backslash@other{\}% +/endgroup +\def\AdviceCsnameGetOriginal#1#2{% + \ifcsname advice@o#1//\advice@backslash@other#2\endcsname + \expandonce{\csname advice@o#1//\advice@backslash@other#2\expandafter\endcsname + \expandafter}% + \else + \expandonce{\csname#2\expandafter\endcsname\expandafter}% \fi } % \end{macro} @@ -696,6 +719,7 @@ \def\advice@handle@outer#1#2{% \def\AdviceNamespace{#1}% \def\AdviceName{#2}% + \advice@def@AdviceCsname \let\AdviceReplaced\AdviceName \def\AdviceOriginal{\AdviceGetOriginal{#1}{#2}}% % We then load the stage-two settings. This defines the following macros: @@ -724,6 +748,14 @@ % All prepared, we execute the outer handler. \AdviceOuterHandler } +\def\advice@def@AdviceCsname{% + \begingroup + \escapechar=-1 + \expandafter\expandafter\expandafter\endgroup + \expandafter\expandafter\expandafter\def + \expandafter\expandafter\expandafter\AdviceCsname + \expandafter\expandafter\expandafter{\expandafter\string\AdviceName}% +} % \end{macro} % % \begin{macro}{\ifAdviceRun} @@ -745,18 +777,22 @@ % CollArgs' command |\CollectArgumentsRaw|. It will provide that command % with: % \begin{itemize} -% \item the collector options, given in the raw format: the caller -% (|\collargsCaller|), the raw options (|\AdviceRawCollectorOptions|) and the -% user options (|\AdviceRawCollectorOptions|, wrapped in |\collargsSet|; +% \item the collector options, given in the raw format: +% \begin{itemize} +% \item the caller (|\collargsCaller|), +% \item the raw options (|\AdviceRawCollectorOptions|), and +% \item the user options (|\AdviceRawCollectorOptions|, wrapped in +% |\collargsSet|; +% \end{itemize} % \item the argument specification |\AdviceArgs| of the handled command; and % \item the inner handler |\AdviceInnerHandler| to execute after collecting the % arguments; the inner handler receives the collected arguments as a single % braced argument. % \end{itemize} -% If the argument specification is not defined (either the user did not set -% it, or has reset it by writing |args| without a value), it is assumed that -% the handled command was defined by |xparse| and |\AdviceArgs| will be -% retrieved by |\GetDocumentCommandArgSpec|. +% \hangindent=0pt If the argument specification is not defined (either the user +% did not set it, or has reset it by writing |args| without a value), it is +% assumed that the handled command was defined by |xparse| and |\AdviceArgs| +% will be retrieved by |\GetDocumentCommandArgSpec|. % \begin{listingregion}{_advice-CollectArgumentsRaw.tex} \def\advice@CollectArgumentsRaw{% \AdviceIfArgs{}{% @@ -845,6 +881,7 @@ \advice@activate@error@activated{#1}{#2}{Environment}{already}% }{% \csdef{\advice@original@csname{#1}{#2}}{}% + \PackageInfo{advice (#1)}{Activated environment "#2"}% }% } \def\advice@deactivate@env#1#2{% @@ -852,6 +889,7 @@ \csundef{\advice@original@csname{#1}{#2}}{}% }{% \advice@activate@error@activated{#1}{#2}{Environment}{not yet}% + \PackageInfo{advice (#1)}{Dectivated environment "#2"}% }% } % \end{macro} @@ -911,6 +949,7 @@ \def\advice@begin@env@outer#1#2#3{% \def\AdviceNamespace{#1}% \def\AdviceName{#3}% + \let\AdviceCsname\advice@undefined \def\AdviceReplaced{#2{#3}}% \def\AdviceOriginal{\AdviceGetOriginal{#1}{#2}{#3}}% \advice@init@I{#1}{#3}% @@ -1000,6 +1039,8 @@ % Install the tracing code. \def\advice@trace@init@i#1#2{% \advice@trace{Advising \detokenize\expandafter{\string#2} (\detokenize{#1})}% + \advice@trace{\space\space Original command meaning: + \expandafter\expandafter\expandafter\meaning\advice@original@cs{#1}{#2}}% \advice@setup@init@i{#1}{#2}% \edef\AdviceRunConditions{% % We first execute the original run conditions, so that we can show the @@ -1032,13 +1073,13 @@ Executing collector: \detokenize\expandafter{\AdviceCollector}}% \noexpand\advice@trace{\space\space\space\space - argument specification: + Argument specification: \detokenize\expandafter{\AdviceArgs}}% \noexpand\advice@trace{\space\space\space\space - options: + Options: \detokenize\expandafter{\AdviceCollectorOptions}}% \noexpand\advice@trace{\space\space\space\space - raw options: + Raw options: \detokenize\expandafter{\AdviceRawCollectorOptions}}% \expandonce\AdviceCollector }% @@ -1052,7 +1093,7 @@ Received arguments: \noexpand\detokenize{##1}}% \noexpand\advice@trace{\space\space\space\space - options: + Options: \detokenize\expandafter{\AdviceOptions}}% \expandonce{\AdviceInnerHandler}{##1}% }% @@ -1079,38 +1120,38 @@ % The \TikZ; code resides in a special file. It is meant to be |\input| at any % time, so we need to temporarily assign |@| category code 11. %<*tikz> -\edef\mmzresetatcatcode{\catcode`\noexpand\@\the\catcode`\@\relax}% +\edef\adviceresetatcatcode{\catcode`\noexpand\@\the\catcode`\@\relax}% \catcode`\@=11 \def\AdviceCollectTikZArguments{% -% We initialize the token register which will hold the collected arguments, and -% start the collection. Nothing of note happens until \dots - \mmz@temptoks={}% - \mmz@tikz@anim + % We initialize the token register which will hold the collected arguments, + % and start the collection. Nothing of note happens until \dots + \toks0={}% + \advice@tikz@anim } -\def\mmz@tikz@anim{% - \pgfutil@ifnextchar[{\mmz@tikz@opt}{% - \pgfutil@ifnextchar:{\mmz@tikz@anim@a}{% - \mmz@tikz@code}}%] +\def\advice@tikz@anim{% + \pgfutil@ifnextchar[{\advice@tikz@opt}{% + \pgfutil@ifnextchar:{\advice@tikz@anim@a}{% + \advice@tikz@code}}%] } -\def\mmz@tikz@anim@a#1=#2{% - \toksapp\mmz@temptoks{#1={#2}}% - \mmz@tikz@anim +\def\advice@tikz@anim@a#1=#2{% + \toksapp0{#1={#2}}% + \advice@tikz@anim } -\def\mmz@tikz@opt[#1]{% - \toksapp\mmz@temptoks{[#1]}% - \mmz@tikz@code +\def\advice@tikz@opt[#1]{% + \toksapp0{[#1]}% + \advice@tikz@code } -\def\mmz@tikz@code{% - \pgfutil@ifnextchar\bgroup\mmz@tikz@braced\mmz@tikz@single +\def\advice@tikz@code{% + \pgfutil@ifnextchar\bgroup\advice@tikz@braced\advice@tikz@single } -\long\def\mmz@tikz@braced#1{\toksapp\mmz@temptoks{{#1}}\mmz@tikz@done} -\def\mmz@tikz@single#1;{\toksapp\mmz@temptoks{#1;}\mmz@tikz@done} +\long\def\advice@tikz@braced#1{\toksapp0{{#1}}\advice@tikz@done} +\def\advice@tikz@single#1;{\toksapp0{#1;}\advice@tikz@done} % \dots\ we finish collecting the arguments, when we execute the inner handler, % with the (braced) collected arguments is its sole argument. -\def\mmz@tikz@done{% - \expandafter\AdviceInnerHandler\expandafter{\the\mmz@temptoks}% +\def\advice@tikz@done{% + \expandafter\AdviceInnerHandler\expandafter{\the\toks0}% } -\mmzresetatcatcode +\adviceresetatcatcode % % \end{macrocode} % diff --git a/macros/generic/collargs/CHANGELOG.md b/macros/generic/collargs/CHANGELOG.md new file mode 100644 index 0000000000..e3e1204b2d --- /dev/null +++ b/macros/generic/collargs/CHANGELOG.md @@ -0,0 +1,25 @@ +# CollArgs changelog + +This package was developed as an auxiliary package of +[Memoize](https://ctan.org/pkg/memoize), and is documented alongside that +package. Note, however, that the Memoize manual, `memoize-doc.pdf`, is not +reissued for patch-level releases of CollArgs; the version of the Memoize manual +fully applicable to the particular version of CollArgs is indicated at each +version. + +Whenever the date of the Memoize manual is older than the release date of a +particular version of CollArgs, the documented source of CollArgs shown in that +version of `memoize-code.pdf` of course does not reflect the latest changes; in +those cases, please see [Memoize's GitHub +repository](https://github.com/sasozivanovic/memoize) for the recent +development history. + +## 2024/01/02 v1.1.0 +* Implement `brace collected`. + +Manual: Memoize 2024/01/02 v1.1.0 + +## 2023/10/10 v1.0.0 +* The initial release. + +Manual: Memoize 2023/10/10 v1.0.0 diff --git a/macros/generic/collargs/README.md b/macros/generic/collargs/README.md index f36abea78a..235225f08a 100644 --- a/macros/generic/collargs/README.md +++ b/macros/generic/collargs/README.md @@ -6,3 +6,14 @@ This package was developed as an auxiliary package of [Memoize](https://ctan.org/pkg/memoize). This is why it is, somewhat unconventionally, documented alongside that package. This applies to both the manual and the documented code listing. + +# LICENCE + +This work may be distributed and/or modified under the conditions of the LaTeX +Project Public License, either version 1.3c of this license or (at your option) +any later version. The latest version of this license is in +https://www.latex-project.org/lppl.txt and version 1.3c or later is part of all +distributions of LaTeX version 2008 or later. + +This work has the LPPL maintenance status `maintained'. The Current Maintainer +of this work is Sašo Živanović (saso.zivanovic@guest.arnes.si). diff --git a/macros/generic/collargs/collargs.edtx b/macros/generic/collargs/collargs.edtx index 40a102c1a8..6cd39ad4ca 100644 --- a/macros/generic/collargs/collargs.edtx +++ b/macros/generic/collargs/collargs.edtx @@ -10,11 +10,11 @@ %% (Sa\v{s}o \v{Z}ivanovi\'{c}) %% %% This work may be distributed and/or modified under the conditions of the -%% LaTeX Project Public License, either version 1.3 of this license or (at your -%% option) any later version. The latest version of this license is in -%% http://www.latex-project.org/lppl.txt and version 1.3 or later is part of -%% all distributions of LaTeX version 2005/12/01 or later. -%% +%% LaTeX Project Public License, either version 1.3c of this license or (at +%% your option) any later version. The latest version of this license is in +%% https://www.latex-project.org/lppl.txt and version 1.3c or later is part of +%% all distributions of LaTeX version 2008 or later. +%% %% This work has the LPPL maintenance status `maintained'. %% The Current Maintainer of this work is Saso Zivanovic. %% @@ -46,14 +46,14 @@ % % % \paragraph{Identification} -%\ProvidesPackage{collargs}[2023/10/10 v1.0.0 Collect arguments of any command] +%\ProvidesPackage{collargs}[2024/01/02 v1.1.0 Collect arguments of any command] %%D \module[ %%D file=t-collargs.tex, -%%D version=1.0.0, +%%D version=1.1.0, %%D title=CollArgs, %%D subtitle=Collect arguments of any command, %%D author=Saso Zivanovic, -%%D date=2023-10-10, +%%D date=2024-01-02, %%D copyright=Saso Zivanovic, %%D license=LPPL, %%D ] @@ -70,24 +70,38 @@ %\catcode`\@11\relax % % \begin{macro}{\toksapp,\gtoksapp,\etoksapp,\xtoksapp} -% Our macros for appending to a token register only accept a control sequence -% defined by |\toksdef| (like |\mytoks|) but not an explicit register -% designation like |\toks0|, so we only define them if noone else did; the -% names of the macros match the \hologo{LuaTeX} primitives, so they surely -% won't be defined there. -\ifdefined\toksapp\else - \long\def\toksapp#1#2{#1\expandafter{\the#1#2}}% -\fi -\ifdefined\etoksapp\else - \long\def\etoksapp#1#2{#1\expandafter{\expanded{\the#1#2}}}% -\fi -\ifdefined\gtoksapp\else - \long\def\gtoksapp#1#2{\global#1\expandafter{\the#1#2}}% -\fi -\ifdefined\xtoksapp\else - \long\def\xtoksapp#1#2{\global#1\expandafter{\expanded{\the#1#2}}}% +% Macros for appending to a token register. We don't have to define them in +% \hologo{LuaTeX}, where they exist as primitives. Same as these primitives, +% out macros accept either a register number or a |\toksdef|fed control +% sequence as the (unbraced) |#1|; |#2| is the text to append. +\ifdefined\luatexversion +\else + \def\toksapp{\toks@cs@or@num\@toksapp} + \def\gtoksapp{\toks@cs@or@num\@gtoksapp} + \def\etoksapp{\toks@cs@or@num\@etoksapp} + \def\xtoksapp{\toks@cs@or@num\@xtoksapp} + \def\toks@cs@or@num#1#2#{% + % Test whether |#2| (the original |#1|) is a number or a control sequence. + \ifnum-2>-1#2 + % It is a number. |\toks@cs@or@num@num| will gobble |\toks@cs@or@num@cs| + % below. + \expandafter\toks@cs@or@num@num + % The register control sequence in |#2| is skipped over in the false + % branch. + \fi + \toks@cs@or@num@cs{#1}{#2}% + } + % |#1| is one of |\@toksapp| and friends. The second macro prefixes the + % register number by |\toks|. + \def\toks@cs@or@num@cs#1#2{#1{#2}} + \def\toks@cs@or@num@num\toks@cs@or@num@cs#1#2{#1{\toks#2 }} + % Having either |\tokscs| or |\toks| in |#1|, we can finally do the + % real job. + \long\def\@toksapp#1#2{#1\expandafter{\the#1#2}}% + \long\def\@etoksapp#1#2{#1\expandafter{\expanded{\the#1#2}}}% + \long\def\@gtoksapp#1#2{\global#1\expandafter{\the#1#2}}% + \long\def\@xtoksapp#1#2{\global#1\expandafter{\expanded{\the#1#2}}}% \fi -\ifdefined\toks@\else\toksdef\toks@=0 \fi % \end{macro} % % \begin{macro}{\CollectArguments,\CollectArgumentsRaw} @@ -521,7 +535,19 @@ \newif\ifcollargsNoDelimiters % \end{macro} % \end{collargskey} -% + +% \begin{collargskey}{brace collected} +% \begin{macro}{\ifcollargsBraceCollected} +% When this conditional is set to false, the collected arguments are not +% enclosed in braces when passed on to \meta{next-code}. +\collargsSet{% + brace collected/.is if=collargsBraceCollected, +} +\newif\ifcollargsBraceCollected +\collargsBraceCollectedtrue +% \end{macro} +% \end{collargskey} +% % \subsubsection{The central loop} % \label{sec:code:collargs:central-loop} % @@ -678,7 +704,12 @@ \expanded{% \endgroup \noexpand\collargs@fix{% - \expandonce\collargs@next{\the\collargs@toks}% + \expandonce\collargs@next + \ifcollargsBraceCollected + {\the\collargs@toks}% + \else + \the\collargs@toks + \fi \collargs@spaces }% }% @@ -1680,7 +1711,7 @@ \collargs@CPT@def\collargs@gobbleOneB\collargs@begintag{% % Assign the collected tokens into a register. The first token in |##1| % will be |\collargs@empty|, so we expand to get rid of it. - \expandafter\toks@\expandafter{##1}% + \toks0\expandafter{##1}% % |cprotect| simply grabs the token following the |\collargs@begintag| with % a parameter. We can't do this, because we need the code to work in the % non-verbatim mode, as well, and we might stumble upon a brace there. So @@ -1692,13 +1723,13 @@ \ifcollargs@long\long\fi \collargs@CPT@def\collargs@gobbleUntilE\collargs@endtag{% % Expand |\collargs@empty| at the start of |##1|. - \expandafter\toksapp\expandafter\toks@\expandafter{##1}% + \expandafter\toksapp\expandafter0\expandafter{##1}% \collargs@gobbleUntilE@i }% % Initialize. \collargs@begins=0\relax \collargsArg{}% - \toks@{}% + \toks0{}% % We will call |\collargs@gobbleUntilE| via the caller control sequence. \collargs@letusecollector\collargs@gobbleUntilE % We insert |\collargs@empty| to avoid the potential debracing problem. @@ -1726,7 +1757,7 @@ \expandafter\collargs@gobbleOneB@v \else % Append the real begin-tag to the temporary tokens. - \etoksapp\toks@{\expandonce\collargs@begintag}% + \etoksapp0{\expandonce\collargs@begintag}% \expandafter\collargs@gobbleOneB@ii \fi }% @@ -1754,14 +1785,14 @@ \fi % Whatever the result was, we have to append the gobbled group to the % temporary toks. - \etoksapp\toks@{\collargs@grabbed@spaces\unexpanded{{#1}}}% + \etoksapp0{\collargs@grabbed@spaces\unexpanded{{#1}}}% \collargs@init@grabspaces \collargs@gobbleOneB@vi } \def\collargs@gobbleOneB@v#1{\collargs@gobbleOneB@vi} \def\collargs@gobbleOneB@vi{% % Store. - \etoksapp\collargsArg{\the\toks@}% + \etoksapp\collargsArg{\the\toks0}% % Advance the begin-tag counter. \advance\collargs@begins\collargs@begins@increment\relax % Find more begin-tags, unless this was the final one. @@ -1785,7 +1816,7 @@ }% } \def\collargs@gobbleUntilE@iii#1{% - \etoksapp\toks@{\collargs@grabbed@spaces}% + \etoksapp0{\collargs@grabbed@spaces}% \collargs@init@grabspaces \def\collargs@tempa{#1}% \ifx\collargs@tempa\collargs@tagarg @@ -1794,7 +1825,7 @@ \else % Nope, this |\end| belongs to someone else. Insert the end tag plus the % tag argument, and collect until the next |\end|. - \expandafter\toksapp\expandafter\toks@\expandafter{\collargs@endtag{#1}}% + \expandafter\toksapp\expandafter0\expandafter{\collargs@endtag{#1}}% \expandafter\collargs@letusecollector\expandafter\collargs@gobbleUntilE \fi } @@ -1802,7 +1833,7 @@ % Invoke |\collargs@gobbleOneB| with the collected material, plus a fake % begin-tag and a quark guard. \ifcollargsIgnoreNesting - \expandafter\collargsArg\expandafter{\the\toks@}% + \expandafter\collargsArg\expandafter{\the\toks0}% \expandafter\collargs@commandatend \else \expandafter\collargs@gobbleUntilE@v @@ -1812,7 +1843,7 @@ \expanded{% \noexpand\collargs@letusecollector\noexpand\collargs@gobbleOneB \noexpand\collargs@empty - \the\toks@ + \the\toks0 % Add a fake begin-tag and a quark guard. \expandonce\collargs@begintag \noexpand\collargs@qend @@ -1825,7 +1856,7 @@ \expandafter\ifx\expandafter\relax\collargs@tagarg\relax\else{% \expandonce\collargs@tagarg}\fi }% - \toks@={}% + \toks0={}% \expandafter\collargs@letusecollector\expandafter\collargs@gobbleUntilE \expandafter\collargs@empty \fi diff --git a/macros/generic/collargs/collargs.ins b/macros/generic/collargs/collargs.ins index a8673b6edc..5cde9e2b72 100644 --- a/macros/generic/collargs/collargs.ins +++ b/macros/generic/collargs/collargs.ins @@ -9,11 +9,11 @@ %% (Sa\v{s}o \v{Z}ivanovi\'{c}) %% %% This work may be distributed and/or modified under the conditions of the -%% LaTeX Project Public License, either version 1.3 of this license or (at your -%% option) any later version. The latest version of this license is in -%% http://www.latex-project.org/lppl.txt and version 1.3 or later is part of -%% all distributions of LaTeX version 2005/12/01 or later. -%% +%% LaTeX Project Public License, either version 1.3c of this license or (at +%% your option) any later version. The latest version of this license is in +%% https://www.latex-project.org/lppl.txt and version 1.3c or later is part of +%% all distributions of LaTeX version 2008 or later. +%% %% This work has the LPPL maintenance status `maintained'. %% The Current Maintainer of this work is Saso Zivanovic. %% diff --git a/macros/generic/memoize/CHANGELOG.md b/macros/generic/memoize/CHANGELOG.md new file mode 100644 index 0000000000..f0d283ee76 --- /dev/null +++ b/macros/generic/memoize/CHANGELOG.md @@ -0,0 +1,59 @@ +# Memoize changelog + +For the development history, see [Memoize's GitHub +repository](https://github.com/sasozivanovic/memoize). + +## 2024/1/02 v1.1.0 + +* Improve the extraction scripts: + * respect `TEXMF_OUTPUT_DIRECTORY`; + * respect `openin_any` and `openout_any`; + * implement `--format`; + * improve error reporting; + * drop the `Path::Class` dependency for the Perl script; + * allow for `PDF::Builder` in the Perl script; + * implement `--library` in the Perl script; + * set an appropriate exit code on exit; + * and several further minor changes. + +* Remove key `path` in favour of `prefix`. + +* `mkdir` is now initially `true`, but the directory is only created if `mkdir + command` is non-empty (and it is empty initially). The definition of `(no) + memo dir` is accordingly simpler. + +* The directory name is now appended to the value `mkdir command` when + constructing the system call. + +* A workaround for compatibility with package `morewrites`. + +* Process package options using the new LaTeX mechanism to avoids the issue of + spaces in package options. The remaining issue of `/` is addressed by + implementing option `options`. + +* Add the missing commands to `nomemoize` and `memoizable`, and implement a + generic variant of the latter (`memoizable.code.tex`). + +* Implement auto-key `to context`. + +* Write a c-memo even upon abortion. + +* Demote warning messages "memoization aborted" & "marked as unmemoizable" to + info messages. + +* Implement biblatex support. + +* Support `\DiscardShipoutBox`. + +* Advance the counter underlying `\pgfpictureid` when utilizing a tikzpicture + (`memoize tikz`). + +* Remove the `\pgfsys@getposition` hack for `tikzpicture`s. + +## 2023/10/10 v1.0.0 + +* A complete, fully documented reimplementation. + +## 2020/07/17 v0.1 + +* The proof of concept, available only at GitHub. diff --git a/macros/generic/memoize/Makefile b/macros/generic/memoize/Makefile index 7edbd7ba34..2f79a3b8ce 100644 --- a/macros/generic/memoize/Makefile +++ b/macros/generic/memoize/Makefile @@ -9,15 +9,16 @@ all: ctan/memoize.zip # Prepare the CTAN submission. PACKAGE = memoize -VERSION = 1.0.0 -YEAR = 2023 -MONTH = 10 -DAY = 10 +VERSION = 1.1.0 +YEAR = 2024 +MONTH = 01 +DAY = 02 FORMAT = generic COMMON = memoize nomemoize memoizable PLAIN = memoize-extract-one.tex +GENERIC = memoizable.code.tex SOURCE = memoize.edtx memoize.ins # $(makefiles) SCRIPTS := memoize-extract memoize-clean @@ -31,11 +32,9 @@ SCRIPTS := $(SCRIPTS:%=%.pl) $(SCRIPTS:%=%.py) %.py.1: %.1 echo .so man1/$*.1 > $@ # link to .1 man page -.PHONY: runtime -runtime: $(RUNTIME) - README = doc/README.memoize.md INSTALL = INSTALL.md +CHANGELOG = CHANGELOG.md MAKEFILE = Makefile LICENCE = LICENCE @@ -44,11 +43,11 @@ makefiles = Makefile.package Makefile.runtimes Makefile.advice Makefile.collargs codedoc-source = memoize-code.tex \ memoize-code.sty memoize-doc-common.sty -manual-source = memoize.tex \ +manual-source = memoize-doc.tex \ memoize-doc.sty memoize-doc-common.sty yadoc.sty \ - memoize.mst + memoize-doc.mst -PDF = memoize.pdf memoize-code.pdf +PDF = memoize-doc.pdf memoize-code.pdf codedoc-source := $(codedoc-source:%=doc/%) manual-source := $(manual-source:%=doc/%) @@ -80,11 +79,19 @@ ctan/$(PACKAGE).zip: ln -sr $(TDS-DOC-DIR)/examples.zip $(CTAN-DIR)/doc $(CTAN-END) +%.py.dtx: %.py + edtx2dtx -s -c '#' -B '^__version__' -E '^# Local Variables:' $< \ + | sed -e '/^% Local Variables:/Q' > $@ +%.pl.dtx: %.pl + edtx2dtx -s -c '#' -B '^my \$$PROG' -E '^# Local Variables:' $< \ + | sed -e '/^% Local Variables:/Q' > $@ -doc/memoize-code.pdf: $(SOURCE) $(codedoc-source) +doc/memoize-code.pdf: $(SOURCE) $(codedoc-source) \ + advice.edtx advice.ins collargs.edtx collargs.ins \ + $(SCRIPTS:%=%.dtx) -doc/memoize.pdf: $(manual-source) $(examples-src) +doc/memoize.pdf: $(manual-source) $(examples-src) memoize.edtx advice.edtx collargs.edtx %.pdf: %.tex latexmk -cd -lualatex -bibtex- $< && touch $@ @@ -119,11 +126,9 @@ version: $(call EDIT-VERSION-PYTHON,memoize-clean.py) $(call EDIT-VERSION-MAN,doc/memoize-extract.1.md) $(call EDIT-VERSION-MAN,doc/memoize-clean.1.md) -# Change the date of the latest release (identified by the version). - sed -Ei 's!^\\item\[\\githubrelease\{[0-9]{4}/[0-9]{2}/[0-9]{2}\}\{v$(VERSION)\}\] *$$!\\item\[\\githubrelease\{$(YEAR)/$(MONTH)/$(DAY)\}\{v$(VERSION)\}\]!' doc/memoize.tex - + $(call EDIT-DATE-CHANGELOG,CHANGELOG.md) define COLOR_VERSION -grep -E --color '[0-9]{4}[/-][0-9]{2}[/-][0-9]{2}|v?[0-9]\.[0-9]\.[0-9]|(January|February|March|April|May|June|July|August|September|October|November|December) [0-9]+, [0-9]{4}' +grep -E --color '[0-9]{4}[/-][0-9]{2}[/-][0-9]{2}|v?[0-9]\.[0-9]\.[0-9]([-a-z]*)|(January|February|March|April|May|June|July|August|September|October|November|December) [0-9]+, [0-9]{4}' endef versions-show: @@ -131,9 +136,26 @@ versions-show: @grep __version__ *.py | ${COLOR_VERSION} @grep VERSION *.pl | ${COLOR_VERSION} @grep -E '^(footer|date):' doc/memoize-*.md | ${COLOR_VERSION} - @grep -E 'githubrelease' doc/memoize.tex | ${COLOR_VERSION} + @${COLOR_VERSION} CHANGELOG.md doc/CHANGELOG.advice.md doc/CHANGELOG.collargs.md include Makefile.package include Makefile.runtimes VERSION-MAN = of Memoize v$(VERSION) + +.PHONY: all-runtimes link-all-runtimes unlink-all-runtimes test + +all-runtimes: runtimes + $(MAKE) -f Makefile.advice runtimes + $(MAKE) -f Makefile.collargs runtimes + +link-all-runtimes: link-runtimes + $(MAKE) -f Makefile.advice link-runtimes + $(MAKE) -f Makefile.collargs link-runtimes + +unlink-all-runtimes: unlink-runtimes + $(MAKE) -f Makefile.advice unlink-runtimes + $(MAKE) -f Makefile.collargs unlink-runtimes + +test: + cd testing && ./MakeTests.py diff --git a/macros/generic/memoize/doc/memoize-clean.1.md b/macros/generic/memoize/doc/memoize-clean.1.md index 138e0ee871..b296697d6c 100644 --- a/macros/generic/memoize/doc/memoize-clean.1.md +++ b/macros/generic/memoize/doc/memoize-clean.1.md @@ -2,8 +2,8 @@ title: memoize-clean section: 1 header: User Manual -footer: memoize-clean of Memoize v1.0.0 -date: October 10, 2023 +footer: memoize-clean of Memoize v1.1.0 +date: January 02, 2024 hyphenate: false --- diff --git a/macros/generic/memoize/doc/memoize-code.pdf b/macros/generic/memoize/doc/memoize-code.pdf index 7eeed4b3bf..415b401f42 100644 Binary files a/macros/generic/memoize/doc/memoize-code.pdf and b/macros/generic/memoize/doc/memoize-code.pdf differ diff --git a/macros/generic/memoize/doc/memoize-code.sty b/macros/generic/memoize/doc/memoize-code.sty index 30d48f364c..c7bddde08b 100644 --- a/macros/generic/memoize/doc/memoize-code.sty +++ b/macros/generic/memoize/doc/memoize-code.sty @@ -23,7 +23,8 @@ \RequirePackage{memoize-doc-common} % Typesetting the code & indentation -\MacroIndent=0pt +%\MacroIndent=0pt +\AddToHook{begindocument}{\MacroIndent=0pt } \MacroTopsep = 0pt \MacrocodeTopsep = 7pt plus 2pt minus 2pt \AddToHook{env/macrocode/before}{\macrocodepar}% @@ -137,22 +138,33 @@ \newcommand\hang[2]{% \setbox0=\hbox{\MacroFont \string#2\relax}% "\relax" is short enough to never trigger a hanging indent \ifdim\wd0>\dimexpr\marginparwidth\relax - \hangindent\dimexpr\wd0-\marginparwidth\relax + \hangindent\dimexpr\wd0-\marginparwidth+\hang@next@extra\relax \hangafter-#1\relax + \gdef\hang@next@extra{0pt}% \AddToHookNext{env/macrocode/before}{\par\nohang}% \AddToHookNext{para/after}{\aftergroup\nohang}% \fi \ignorespaces } -\newcommand\nohang{\hangindent 0pt\relax} -\newcommand\indentmacrocode{% - \settowidth\MacroIndent{\theCodelineNo\ }% - \addtolength\MacroIndent{\hangindent}% +\def\AdjustNextHang#1{\gdef\hang@next@extra{#1}\ignorespaces} +\gdef\hang@next@extra{0pt} +\newcommand\nohang{% + \hangindent 0pt\relax + \gdef\hang@next@extra{0pt}% +} +\newcommand\indentmacrocode[1][0pt]{% + \settowidth\@tempdima{\reset@font\scriptsize\arabic{CodelineNo}\ }% + \setlength\ExtraMacroIndent{#1}% + \addtolength\MacroIndent{\dimexpr\hangindent+\@tempdima+\ExtraMacroIndent}% } +\newdimen\ExtraMacroIndent \newcommand\noindentmacrocode{% \setlength\MacroIndent{0em}% \AddToHookNext{env/macrocode/begin}{\MacrocodeTopsep 0pt\relax}% } +\def\theCodelineNo{% + \reset@font\scriptsize\textcolor{gray}{\arabic{CodelineNo}}% + \hspace*{\dimexpr\leftskip-\MacroIndent+\ExtraMacroIndent}}% % and now ... autohang! % (the macros should be listed without spaces after commas) @@ -165,6 +177,7 @@ advice=\macro{hang}, advice=\autokey{hang}, advice=\mmzautokey{hang}, + advice=\advicekey{hang}, advice=\collargskey{hang}, advice=\handlerskey{hang}, advice=\adviceinstallkey{hang}, @@ -204,7 +217,7 @@ \def\doline{% \expandafter\futurelet\expandafter\firsttoken\expandafter\dolinei\line\fi} \def\dolinei{\ifx\firsttoken\newlabel}% -\openin0=memoize.aux +\openin0=memoize-doc.aux \loop \unless\ifeof0 \read0 to \line diff --git a/macros/generic/memoize/doc/memoize-code.tex b/macros/generic/memoize/doc/memoize-code.tex index 6daa583383..28aa812334 100644 --- a/macros/generic/memoize/doc/memoize-code.tex +++ b/macros/generic/memoize/doc/memoize-code.tex @@ -61,47 +61,23 @@ \DocInput{\docdir/collargs.dtx} -\clearpage -\restoregeometry - \section{The scripts} \label{sec:code:scripts} -\def\continuation{\llap{\textcolor{purple}{$\rightarrow$}\,}} - \subsection{The Perl extraction script \texttt{memoize-extract.pl}} - -\lstinputlisting[firstline=20, % manual (exclude the license information) - language=Perl, - breaklines, breakindent=4em, postbreak=\continuation, - commentstyle=, -]{../memoize-extract.pl} +\DocInput{\docdir/memoize-extract.pl.dtx} \subsection{The Python extraction script \texttt{memoize-extract.py}} - -\lstinputlisting[firstline=20, % manual (exclude the license information) - language=Python, - breaklines, breakindent=4em, postbreak=\continuation, - commentstyle=, -]{../memoize-extract.py} +\DocInput{\docdir/memoize-extract.py.dtx} \subsection{The Perl clean-up script \texttt{memoize-clean.pl}} - -\lstinputlisting[firstline=20, % manual (exclude the license information) - language=Perl, - breaklines, breakindent=4em, postbreak=\continuation, - commentstyle=, -]{../memoize-clean.pl} +\DocInput{\docdir/memoize-clean.pl.dtx} \subsection{The Python clean-up script \texttt{memoize-clean.py}} - -\lstinputlisting[firstline=20, % manual (exclude the license information) - language=Python, - breaklines, breakindent=4em, postbreak=\continuation, - commentstyle=, -]{../memoize-clean.py} +\DocInput{\docdir/memoize-clean.py.dtx} \clearpage +\restoregeometry \IndexPrologue{\section*{Index}Numbers written in red refer to the code line where the corresponding entry is defined; numbers in blue refer to the code diff --git a/macros/generic/memoize/doc/memoize-doc.mst b/macros/generic/memoize/doc/memoize-doc.mst new file mode 100644 index 0000000000..68a11283a5 --- /dev/null +++ b/macros/generic/memoize/doc/memoize-doc.mst @@ -0,0 +1 @@ +encap '`' diff --git a/macros/generic/memoize/doc/memoize-doc.pdf b/macros/generic/memoize/doc/memoize-doc.pdf new file mode 100644 index 0000000000..02d3caf6f0 Binary files /dev/null and b/macros/generic/memoize/doc/memoize-doc.pdf differ diff --git a/macros/generic/memoize/doc/memoize-doc.sty b/macros/generic/memoize/doc/memoize-doc.sty index 2a1d7482c4..380ab94f28 100644 --- a/macros/generic/memoize/doc/memoize-doc.sty +++ b/macros/generic/memoize/doc/memoize-doc.sty @@ -406,7 +406,7 @@ \let\mmzPrefix\gobble \let\mmzNewCMemo\gobble \let\mmzNewCCMemo\gobble - \def\mmzNewExtern##1##2##3##4##5{}% + \def\mmzNewExtern##1##2##3##4{}% \let\mmzUsedCMemo\gobble \let\mmzUsedCCMemo\gobble \let\mmzUsedExtern\gobble @@ -683,7 +683,7 @@ \ifregion{% \AddToHook{begindocument/before}{% \makeatletter - \InputIfFileExists{memoize.aux}{}{}% + \InputIfFileExists{memoize-doc.aux}{}{}% \makeatother }% \AddToHook{enddocument}{% @@ -706,6 +706,6 @@ % \captionsetup{labelformat=empty} % Local Variables: -% TeX-master: "memoize.tex" +% TeX-master: "memoize-doc.tex" % TeX-engine: luatex % End: diff --git a/macros/generic/memoize/doc/memoize-doc.tex b/macros/generic/memoize/doc/memoize-doc.tex new file mode 100644 index 0000000000..efde098ef0 --- /dev/null +++ b/macros/generic/memoize/doc/memoize-doc.tex @@ -0,0 +1,9314 @@ +\documentclass[a4paper,11pt]{article} +\usepackage[margin=2cm,top=1cm,bottom=1cm,nohead,includefoot]{geometry} + +\usepackage{nomemoize} +\usepackage{memoize-doc} +\mmzset{ + % recompile, + % readonly, + % disable, + % inverse search=false, + % include context in ccmemo, + % direct ccmemo input, + % trace, +} +\def\exampledir{examples/} + +\title{\pkg[white]{Memoize}} +\author{Sašo Živanović\\[2mm] + \emailsymbol~\url(mailto:){saso.zivanovic@guest.arnes.si}\\ + \homepagesymbol~\url(http://){spj.ff.uni-lj.si/zivanovic}\\ + \faGithub~\url(https://){github.com/sasozivanovic}} +\datefrompackageversion{memoize} +\preto\packagever{v} + +\hypersetup{ + pdftitle={Memoize}, + pdfauthor={Sašo Živanović}, + pdfsubject={externalization}, + pdfkeywords={LaTeX, externalization, memoization} +} + +\usepackage{makeidx} +\pdfsystem{makeindex -s memoize-doc.mst \jobname.idx} +\makeindex +\usepackage[hangindent=1em,justific=raggedright,font=small]{idxlayout} +\appto\indexfont{\def\pkgcolor{gray}} + +\AddToHook{begindocument}{% so that the region compilation takes these in + \docForeign{ + scope={ + after/.style={index annotation/.prefix={\hologo{LaTeX}\space}}, + cmd={name=begin}, cmd={name=end}, + cmd={name=label}, cmd={name=@currentlabel}, + cmd={name=ref}, cmd={name=pageref}, + cmd={name=index}, + cmd={name=NewDocumentCommand}, cmd={name=GetDocumentCommandArgSpec}, + env={name=verbatim}, + cmd={name=ReadonlyShipoutCounter}, + hook={name=begindocument}, + cmd={name=usepackage}, + }, + scope={ + before/.style={index annotation=\hologo{TeX} primitive}, + cmd={name=shipout}, + cmd={name=hbox}, cmd={name=vbox}, + cmd={name=ignorespaces}, cmd={name=unskip}, + cmd={name=errmessage}, + cmd={name=catcode}, + cmd={name=mag}, + }, + scope={ + before/.style={index annotation=\hologo{eTeX} primitive}, + cmd={name=scantokens}, + cmd={name=currentgrouplevel}, + }, + scope={ + before/.style={index annotation=\hologo{pdfTeX} primitive}, + cmd={name=quitvmode}, + }, + scope={ + before/.style={index annotation=\hologo{LuaTeX} primitive}, + pdfcmd={name=savepos}, + cmd={name=pdfvariable}, + }, + pdfvariable={name=compresslevel}, + pdfvariable={name=horigin}, + pdfvariable={name=vorigin}, + scope={ + after/.style={index annotation/.prefix={\hologo{ConTeXt}\space}}, + cmd={name=realpageno}, + }, + pdfcmd={name=primitive, index annotation=\hologo{LuaTeX}/\hologo{XeTeX} primitive}, + register={name=pdfmajorversion}, + register={name=pdfminorversion}, + scope={ + after/.style={index annotation/.prefix={\hologo{plainTeX}\space}}, + cmd={name=llap}, cmd={name=rlap}, + }, + scope={ + pkg=forest, + env={name=forest}, + cmd={name=Forest}, + }, + scope={ + pkg=tikz, keypath=/tikz, + cmd={name=tikz}, + env={name=tikzpicture}, + key={name=remember picture}, + key={name=overlay}, + cmd={name=tikzexternalize}, + cmd={name=pgfsys@getposition}, + }, + file={name=texmf.cnf}, + env={name=tcblisting, pkg=tcolorbox}, + cmd={name=vref, pkg=varioref}, + cmd={name=cref, pkg=cleveref}, + cmd={name=crefrange, pkg=cleveref}, + xparse type={name=l, index annotation/.append={\ (up to begin-group)}}, + xparse type={name=u, index annotation/.append={\ (until tokens)}}, + xparse type={name=r, index annotation/.append={\ (required delimited)}}, + xparse type={name=R, index annotation/.append={\ (required delimited with default)}}, + xparse type={name=d, index annotation/.append={\ (optional delimited)}}, + xparse type={name=D, index annotation/.append={\ (optional delimited with default)}}, + xparse type={name=o, index annotation/.append={\ (standard optional)}}, + xparse type={name=O, index annotation/.append={\ (standard optional with default)}}, + xparse type={name=t, index annotation/.append={\ (optional token)}}, + xparse type={name=s, index annotation/.append={\ (optional star)}}, + xparse type={name=m, index annotation/.append={\ (mandatory)}}, + xparse type={name=g, index annotation/.append={\ (optional group)}}, + xparse type={name=G, index annotation/.append={\ (optional group with default)}}, + xparse type={name=v, index annotation/.append={\ (verbatim)}}, + xparse type={name=e, index annotation/.append={\ (embellishments)}}, + xparse type={name=E, index annotation/.append={\ (embellishments with defaults)}}, + xparse modifier={name=+, index annotation/.append={\ (long)}}, + xparse modifier={name=!, index annotation/.append={\ (don't ignore spaces)}}, + xparse modifier={name=>, index annotation/.append={\ (argument processor)}}, + generic={name=shell-escape, name prefix=-, label prefix:=-, + index annotation=option of \hologo{TeX} binaries}, + generic={name=enable-write18, name prefix=--, label prefix:=--, + index annotation=option of \hologo{TeX} binaries}, + generic={name=TEXMFOUTPUT, index annotation={environment variable, + or variable in \reffile[into index=false]{texmf.cnf}}}, + generic={name=TEXMF\_OUTPUT\_DIRECTORY, label=TEXMF_OUTPUT_DIRECTORY, + index annotation={environment variable, or variable in + \reffile[into index=false]{texmf.cnf}}}, + generic={name=openout\_any, label=openout_any, + index annotation={variable in \reffile[into index=false]{texmf.cnf}}}, + generic={name=openin\_any, label=openin_any, + index annotation={variable in \reffile[into index=false]{texmf.cnf}}}, + }% + \docindex{% + cmd={name=pdfprimitive, see={\refcmd[into index=false,short]{primitive}}}, + cmd={name=pdfsavepos, see={\refcmd[into index=false,short]{savepos}}}, + cmd={name=pdfhorigin, see={\docref[into index=false]{pdfvar:horigin}}}, + cmd={name=pdfvorigin, see={\docref[into index=false]{pdfvar:vorigin}}}, + cmd={name=pdfcompresslevel, see={\docref[into index=false]{pdfvar:compresslevel}}}, + pdfvariable={name=majorversion, see={\docref[into index=false,long]{reg:pdfmajorversion}}}, + pdfvariable={name=minorversion, see={\docref[into index=false,long]{reg:pdfminorversion}}}, + }% +} + +\begin{document} + + +\maketitle +\thispagestyle{empty} + +\begin{abstract} + Memoize is a package for externalization of graphics and memoization of + compilation results in general, allowing the author to reuse the results of + compilation-intensive code. + Memoize + \begin{enumerate*}[(i)] + \item induces very little overhead, as all externalized graphics is produced + in a single compilation. It features + \item automatic recompilation upon the change of code or user-adjustable + context, and + \item automatic externalization of \TikZ pictures and Forest trees, easily + extensible to other commands and environments. Furthermore, Memoize + \item supports cross-referencing, \TikZ overlays and Beamer, + \item works with all major engines and formats, and + \item is adaptable to any workflow. + \end{enumerate*} +\end{abstract} + +\vfill + +\ExampleName{titlepage} +\makeexample{\examplename.pdf N=2} % N=2, because we show the .mmz file later +\begingroup +\tcbset{page box/.style={ + box align=center,boxsep=2mm,top=0pt,bottom=0pt,left=0pt,right=0pt, + enhanced, remember as=#1, + }}% +\begin{tcboxedraster} + [raster columns=1] + {title=The two steps of externalization of graphics in Memoize} + \tcbinputexample{ + title=\tt doc.tex, + listing only, + remember as=tex, + } + \begin{tcolorbox}[example title=doc.pdf, no attachment, remember as=pdf1] + \includeexamplepdf[page box=extern1]{page=1,trim=1in 1in 1in 1in}\hfill + \includeexamplepdf[page box=extern2]{page=2,trim=1in 1in 1in 1in}\hfill + \includeexamplepdf[page box=extern3]{page=3,trim=1in 1in 1in 1in}\hfill + \includeexamplepdf[page box=docpage]{page=4}% + \end{tcolorbox} + \vspace{10mm} + \begin{tcboxedraster}[raster columns=3, raster column skip=1cm]{blankest} + \begin{tcolorbox}[example title=.pdf]\centering + \includeexamplepdf[page box=externfile1]{page=1,trim=1in 1in 1in 1in}\hfill + \end{tcolorbox} + \begin{tcolorbox}[example title=.pdf]\centering + \includeexamplepdf[page box=externfile2]{page=2,trim=1in 1in 1in 1in}\hfill + \end{tcolorbox} + \begin{tcolorbox}[example title=.pdf]\centering + \includeexamplepdf[page box=externfile3]{page=3,trim=1in 1in 1in 1in}\hfill + \end{tcolorbox} + \end{tcboxedraster} +\end{tcboxedraster} + +\begin{tcolorbox}[title=Using the externalized graphics] + \begin{tcolorbox}[example title=doc.pdf, + halign=center, remember as=pdf2] + \tcbox[remember, enhanced]{\footnotesize + We all love Ti\emph{k}Zlings! + \def\mytikzling#1#2{% + \tcbox[blankest, baseline=1mm, remember as=used extern #1] + {\tikz[x=0.3cm,y=0.3cm]{#2;}}}% + \mytikzling1{\penguin} is a penguin, + \mytikzling2{\koala} is a koala, and + \mytikzling3{\owl} is an owl. + } + \end{tcolorbox} +\end{tcolorbox} + +\vfill + +\begin{tcolorbox}[colback=emphcolor, fontupper=\footnotesize] + This manual also documents packages Advice + ({\datefrompackageversion{advice}v\packagever}) and CollArgs + ({\datefrompackageversion{collargs}v\packagever}). These are auxiliary + packages which were developed alongside Memoize, but are distributed as + independent packages as they might be useful outside the context of Memoize, + as well. See sections~\ref{sec:advice} and \ref{sec:ref:advice} for Advice, and + sections~\ref{sec:collargs} and \ref{sec:ref:collargs} for CollArgs. + \begin{tikzpicture}[remember picture, overlay] + \draw[->, snake arrow, thick, red] (tex.west) to[out=-90-60, in=90+60] (pdf1.west); + \draw[->, snake arrow, thick, red] (tex.east) to[out=-90+19, in=90-19] (pdf2.east); + \draw[->, thick, red] (extern1.south) to[out=south, in=north] (externfile1); + \draw[->, thick, red] (extern2.south) to[out=south, out looseness=1.1, in=north, in looseness=0.9] (externfile2); + \draw[->, thick, red] (extern3.south) to[out=south, out looseness=0.7, in=north, in looseness=0.9] (externfile3); + \draw[->, thick,blue] (externfile1) to[out=south, in=north] ([yshift=0.8mm]used extern 1.north); + \draw[->, thick,blue] (externfile2) to[out=south, in=north] (used extern 2.north); + \draw[->, thick,blue] (externfile3) to[out=south, in=north] ([yshift=0.3mm]used extern 3.north); + \end{tikzpicture} +\end{tcolorbox} +\endgroup + + + +\section*{Introduction} +\label{sec:intro} + +\begin{tcolorbox}[ + title={\introboxtitle{What} is externalization and why you might want it?}, + ] + If you have ever worked on a long document full of \TikZ pictures and maybe + Forest trees, you have probably had some compilation-enforced coffee breaks + --- even on modern computers, compiling pictures, trees and such takes a lot + of time. And you might have wondered, why do I need to compile these + pictures over and over again? --- after all, I'm not changing them anymore! + Enter \emph{externalization}, a mechanism designed to deal precisely with + your issue, by saving the produced pictures into separate PDFs and including + those PDFs in subsequent compilations --- in no time at all! +\end{tcolorbox} + +\begin{tcolorbox}[ + title={\introboxtitle{Why} yet another externalization package?}, + ] + \TikZ, the popular and all-powerful graphics language for \TeX, ships + including an externalization library (described in \PGFmanual{53}). \TikZ's + library does an excellent job, but with one caveat. Assume you're using it + for the first time (or after a clean-up) on that long document full of \TikZ + pictures and Forest trees. It will take ages to produce all the externalized + graphics. Why? Because to get you up to speed, your document has to be + compiled many many times --- once for each and every externalized picture. + Even with \TikZ's advanced mechanism for skipping the parts of the document + irrelevant for the picture at hand, the first externalization can be a + daunting task. +\end{tcolorbox} + +\begin{tcolorbox}[ + title={\introboxtitle{How} does Memoize save your time?}, + ] + There is a reason why \TikZ uses an entire compilation cycle to produce a + single externalized picture: \TeX\ itself can only produce a single PDF per + compilation (at least at the moment). Memoize evades this limitation by + dumping the externalized pictures right in the middle of the document + (ouch!). More precisely, an externalized picture occurs in the PDF twice, + first on a special page of its own and then on a regular page, where you + intended it to be. The daring dump obviously necessitates a second step of + the procedure, when those special pages are \emph{extracted} into separate + PDFs, called \emph{externs}, which are then included into the document in + subsequent compilations. + + This two-step procedure, illustrated on the cover page of this manual, is + very fast. The first step, which externalizes \emph{all} the pictures into + the document itself (the squiggly red arrow), takes virtually no more time + than a regular compilation. The time needed for the second step, extraction + (the normal red arrows), depends on the system setup, but it ranges from + little to almost none. +\end{tcolorbox} + +\begin{tcolorbox}[ + title={\introboxtitle{When} should I use Memoize?}, + ] + In short, whenever you are writing a document containing lots of \TikZ + pictures, Forest trees, or other time-consuming constructs, and you are bored + waiting for the compilation to finish. + + Using Memoize on a paper containing a single picture does not make much of a + difference. But with more complex documents, the speed-up can be immense. + For example, the compilation time of my 400-page book containing about 160 + Forest trees was reduced by more than half, and the compilation time of a + 260-page Beamer presentation with a hundred complex dynamic trees went from + four minutes to a mere half minute! +\end{tcolorbox} + + +\begin{tcolorbox}[ + title={\introboxtitle{How} much extra work does Memoize require?}, + ] + In principle, none. + + For one, while allowing for manual memoization of selected document chunks + (\S\ref{sec:tut:memoize}), Memoize features a system which automatically + triggers memoization at each invocation of selected commands and + environments. Out of the box, Memoize \emph{automemoizes} \TikZ + pictures\noprint{\refenv{tikzpicture}} and Forest + trees\noprint{\refenv{forest}}, but the author can easily submit (almost) any + command or environment to automemoization (\S\ref{sec:tut:automemoization}). + Memoize also does its best to automatically prevent memoization of code that + cannot be externalized, like \TikZ pictures with \refkey{/tikz/remember + picture}, and to abort memoization in case the memoized code yields any + errors. +\end{tcolorbox} + + +\begin{tcolorbox}[ + title={\introboxtitle{Why} is Memoize not called Externalize?}, + ] + Fundamentally, Memoize is about producing and utilizing \emph{memos} --- + pieces of \hologo{TeX} code replicating the effect of the compilation of a + document chunk in a computationally less intensive manner. Typically, each + memo has an associated extern, which is where the effect of + \emph{typesetting} is stored, but conceptually, memos come first. For + example, the extern is included back into the document by the memo, and a + memo may be associated with any number of externs, including zero. + + Memos solve several externalization-related problems in a generic fashion, + allowing for a multitude of applications. For example, they store the + information about the associated externs, so that an extern can be integrated + back into the document as a box with the original orientation, width, height + and depth. They solve the problem of cross-referencing from and into the + memoized code by storing its \emph{context} (\S\ref{sec:cross-referencing}) + and replicating any \cs{label}s which occur in it. They are also crucial for + externalizing pictures in Beamer frames overlay by overlay. + + Incidentally, the term ``memoization'' is used with some programming + languages to refer to the process of remembering the result of the function, + along with the given arguments, so that on subsequent invocations of the + function with the same arguments, the result can be returned from memory + rather than recomputing it. I would say that, give or take the functions, + what Memoize does fits the bill. +\end{tcolorbox} + + +\begin{tcolorbox}[ + title={\introboxtitle{How} can I make my command (auto)memoizable?}, + ] + Any command which interacts with the rest of the document, like a command + which produces a float, must receive special treatment. Some issues can be + resolved from within Memoize. Other issues require a memoization-compatible + (re)implementation of the command. It is in the hope that package writers + will adapt their ``difficult'' commands to Memoize that this package offers a + documented interface to the memoization process, fully described in + sections~\ref{sec:tut:package}, \ref{sec:memoization-drivers} + and~\ref{sec:tut:automemoization-details}. + + An advanced user might also want to know that Memoize ships with two + auxiliary packages, which form the base of Memoize's automemoization feature. + Package Advice implements a generic framework for extending the functionality + of selected commands and environments, while package CollArgs provides a + command which can determine the argument scope of any command whose argument + structure conforms to \pkg{xparse}'s argument specification. +\end{tcolorbox} + +\begin{tcolorbox}[ + title={\introboxtitle{What else} is out there?}, + ] + Not long before submitting Memoize to CTAN, I became aware of another new + externalization package, \pkg{robust-externalize}, and it seems that the same + happened to the author of that package \Smiley, who found the + proof-of-concept version of Memoize, which was available + at \href{https://github.com/sasozivanovic/memoize}{GitHub} for a while. + + The key idea behind \pkg{robust-externalize} seems to be to extract the code + submitted to externalization into separate files, and add the necessary + preamble. While a compilation from scratch takes more time than with Memoize + (but less than with \TikZ library), the approach allows for parallel + compilation of externs --- nice! +\end{tcolorbox} + +\newpage +\tableofcontents + + +\section{Before you start} +\label{sec:setup} + +\subsection{Installing the extern extraction software} +\label{sec:install} + +Good news: using Memoize can be as easy as writing +\EmphVerbatim{\usepackage{memoize}} in the preamble. + +Bad news: Memoize won't work out of the box. The culprit is the extern +extraction --- the process which ships the externalized graphics from the main +document into separate extern files; for details, see the title page +illustration and the ``How'' box in the Introduction. For the extraction to +work, you will probably have to install some additional software, and you might +also have to allow your \hologo{TeX} to execute the extraction script. But +there's a silver lining: once Memoize is set up, it is set up for good. + +\begin{tcolorbox}[title=What do I have to do?] + + In principle, all you have to do for Memoize to work \emph{under the default + configuration} is install Perl library + \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2}; see the Perl section + below for the installation guideline. + +\end{tcolorbox} + +Consult section~\ref{sec:extraction} if you want to use an extraction method +other than the default \refmmz{extract=perl}-based method or adapt Memoize to +your particular workflow (for example, if you're compiling via a Makefile). + +If you installed Memoize through the package manager of your \hologo{TeX} +distribution, your system should be already set up to allow the execution of +Memoize's extraction scripts. If this is not the case, please contact either +me or the maintainer of your distribution; until the issue is resolved, you +have to either +\begin{enumerate}[(a)] +\item compile documents loading Memoize with command-line option + \docref{-shell-escape} (on \hologo{TeX}Live) or + \docref{--enable-write18} (on MiK\hologo{TeX}), or +\item set up the restricted shell escape mode to allow for the execution of + \refscript{memoize-extract.pl}.\footnote{\label{fn:shell-escape-commands}On + \hologo{TeX}Live, |tlmgr conf texmf shell\_escape| will tell you which shell + escape mode is currently in effect (|p| = partial = restricted, |f| = disabled + and |t| = enabled); if necessary, use |tlmgr conf texmf shell\_escape p| to + set it to restricted. Then, execute \code{tlmgr conf texmf + shell\_escape\_commands} to get the list of currently allowed commands + \meta{current}, then add the script by executing \code{tlmgr conf texmf + shell\_escape\_commands \meta{current},memoize-extract.pl}. + + On MiK\hologo{TeX}, the shell escape mode can be reported by |initexmf + --show-config-value=[Core]ShellCommandMode|, and restricted mode set by + |initexmf --set-config-value=[Core]ShellCommandMode=Restricted|, and you + get the \meta{current} list by \code{initexmf + --show-config-value=[Core]AllowedShellCommands[]}, and add to it by + \code{initexmf + --set-config-value=[Core]AllowedShellCommands[]=\meta{current};memoize-extract.pl}.} +\end{enumerate} + +Once you have set up your system, I advise you to follow the instructions in +section~\ref{sec:tut:test} to test if the setup was successful. + + +\begingroup +\setlength\intextsep{0pt} + +\vspace{-1ex}% manual + +\paragraph{Perl} If you're running GNU/Linux or macOS, Perl is most likely +already installed on your system. On Windows, you might have to install it. I +tested Memoize with Strawberry Perl, available +at \url(https://){strawberryperl.com}; see \url(https://){www.perl.org} for +other options. + +\begin{wrapfigure}[2]{r}{0.25\linewidth} + \raisebox{0.5mm}[\dimexpr\height-0.5mm][\depth]{% + \mmznext{no verbatim} + \begin{tcolorbox}[ + % example title=Installing the required library in Perl, + % attach shifted boxed title to top right, + % top=2.5mm, + fontupper=\small, + left=2mm, + force nobeforeafter, + ] + |cpan PDF::API2| + \end{tcolorbox}% + }% +\end{wrapfigure} +Once Perl is installed on your system, you will also need to install the PDF +processing library \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} (or +its fork, \hreftt{https://metacpan.org/pod/PDF::Builder}{PDF::Builder}). On +some GNU/Linux distributions, this library is included as a package --- just +use your package manager to install it. Otherwise, install it from +\hreftt{https://www.cpan.org}{CPAN} using +\hreftt{https://metacpan.org/pod/CPAN}{cpan} tool, as shown in the box. + +\vspace{-1ex}% manual + +\paragraph{Python} Installing (Python and) the required Python library is only +necessary if you decide to use the Python-based extraction script; see +section~\ref{sec:extraction}. + +If you're running GNU/Linux, install Python using your package manager. +Otherwise, download the installer from \url(https://){www.python.org}. You can +install Python even if you don't have administrator privileges: simply uncheck +the ``Install launcher for all users'' when running the installer. + +\begin{wrapfigure}[2]{r}{0.25\linewidth} + \raisebox{0.2mm}[\dimexpr\height-0.2mm][\depth]{% + \mmznext{no verbatim} + \begin{tcolorbox}[ + % example title=Installing the required library in Python, + % attach shifted boxed title to top right, + % top=2.5mm, + fontupper=\small, + left=2mm, + nobeforeafter, + ] + |pip install pdfrw2| + \end{tcolorbox}% + }% +\end{wrapfigure} + +Once Python is installed on your system, you will also need to install the PDF +processing library |pdfrw2| library (or its predecessor, |pdfrw|, which will +work just as well). Some GNU/Linux distributions offer this library as a +package; if this is not the case, and on other operating systems, install it +from \href{https://pypi.org}{The Python Package Index} +using \href{https://pip.pypa.io}{pip} tool (if you +run \href{https://pip.pypa.io}{pip} as a superuser, it will install the library +system-wide, otherwise locally), as shown in the box. + +\endgroup % \intextsep + + +\subsection{The configuration commands} +\label{sec:mmzset} + +Memoize can be configured using command \Emph{\refcmd{mmzset}\marg{keylist}} +(and friends). The \meta{keylist} argument is a comma-separated list of +configuration settings, each setting having the form \meta{key}|=|\meta{value}, +or sometimes just \meta{key}. The \meta{keylist} argument is processed by package +\pkg{pgfkeys} (see \PGFmanual{88}), so you can use all the bells and whistles +of that fantastic PGF utility (like easily defining your own styles). + +Here's some examples of \refcmd{mmzset}. For one, to whet your appetite to learn +about the various keys in the \refkeypath{/mmz} path, but more importantly now, to show +you that white-space is irrelevant in the \meta{keylist} argument, so you can +format the keylist as you wish --- as long as it does not contain an empty +line.\footnote{I like to add a comma after the final key as well, as shown in + the bottom left example, because if I don't, I often forget to insert it when + I add more keys.} + +\begin{tcbraster}[raster columns=4, raster valign=bottom] + \begin{tcboxedraster}[raster columns=1]{blankest, raster multicolumn=3, valign=bottom} + \begin{tcblisting}{listing only} +~\mmzset~{memo dir,recompile,memoize=circuitikz} + \end{tcblisting} + \begin{tcblisting}{listing only} +~\mmzset~{memo dir, recompile, memoize=circuitikz} + \end{tcblisting} + \end{tcboxedraster} + \begin{tcblisting}{listing only} +~\mmzset~{ + memo dir, + recompile, + padding=2in} + \end{tcblisting} +\end{tcbraster}\par +\begin{tcbraster}[raster columns=4, raster valign=top] + \begin{tcblisting}{listing only, raster multicolumn=2} +~\mmzset~{ + memo dir, readonly, + memoize=\qrcode{om}, + deactivate=\label, + disable, +} + \end{tcblisting} + \begin{tcblisting}{listing only} +~\mmzset~{ + memo dir, + recompile, + % comment + padding=2in +} + \end{tcblisting} + \begin{tcblisting}{listing only, bad} +~\mmzset~{ + memo dir, + recompile, + + padding=2in +} + \end{tcblisting} +\end{tcbraster} + +Command \refcmd{mmzset} can be used any time after loading the package. It is +very common in the preamble, but also useful in the document body, where its +effect is local to the \hologo{TeX} group.\footnote{Any keys with a non-local + effect are explicitly marked as such in the reference section.} For example, +you can use the idiom on the left below to force recompilation of a single +Forest tree somewhere in the middle of the document (in the code listings +below, highlighting marks the resulting scope of the \refmmz{recompile} +directive). However, as applying some setting to a single piece of +automatically memoized code is common, Memoize provides a special command for +the occasion: the keys given as the argument of \Emph{\refcmd{mmznext}} will be +applied only at the next instance of automemoization, overriding any keys set +by \refcmd{mmzset} in case of a conflict. If the command is given more than +once, only the final invocation takes effect. + +\begin{tcbraster}[raster columns=2, raster equal height=rows, raster force size=false] +\begin{tcblisting}{listing only, mark region={3}{6}} +~{~ + ~\mmzset~{recompile} + \begin{forest} + [VP[V][DP]] + \end{forest} + % ... +~}~ +\end{tcblisting} +\begin{tcblisting}{listing only, add to width=-1em, mark region={3}{5}} +% ... +~\mmznext~{recompile} +\begin{forest} + [VP[V][DP]] +\end{forest} +% ... + +\end{tcblisting} +\end{tcbraster} + +I like to follow |\usepackage{memoize}| by \refcmd{mmzset}, but if you prefer, +you can also provide the document-wide configuration as \Emph{package options.} +For example, the following are equivalent --- both will force +re-externalization of all the externs in the document. + +\begin{tcbraster}[raster columns=2, raster equal height=rows] + \begin{tcblisting}{listing only} +\usepackage{memoize} +\mmzset{recompile} + \end{tcblisting} + \begin{tcblisting}{listing only} +\usepackage[recompile]{memoize} + \end{tcblisting} +\end{tcbraster} + + +\subsection{The configuration file} +\label{sec:memoize.cfg} + +Memoize allows you to configure settings which apply to more than just one +document. It does that by attempting to load file \Emph{\reffile{memoize.cfg}} +just before executing package options. Given how \hologo{TeX} searches for +files, the location of this file determines whether it applies system-wide, +user-wide or directory-wide. The directory-wide location is clearly the +directory itself. The user-wide and system-wide location depend on the +\hologo{TeX} distribution and which format(s) you want to use +\reffile{memoize.cfg} with:\footnote{The |memoize| subfolder is not + obligatory.} + +\begin{center} + \meta{the relevant texmf tree root}|/tex/|\meta{format}|/memoize/memoize.cfg| +\end{center} + +You can say |generic| for \meta{format} if you want the configuration file to +be accessible by all formats, otherwise \meta{format} should be one of the +formats supported by Memoize: |plain|, |latex| or |context|. The texmf root +directory depends on the distribution, here's how to figure out what it is: + +\begin{tcboxedraster} + [raster columns=1, raster valign=top, raster force size=false] + {title=The roots of TEXMF trees} +\begin{tcolorbox}[title={\hologo{TeX} Live\hfill\url[white](https://){tug.org/texlive}}] + \begin{description}[itemsep=0pt] + \item[user-wide] + |tlmgr conf texmf TEXMFHOME| + or |kpsewhich -var-value TEXMFHOME|\\ + the default (on Linux): |/home/|\meta{username}|/texmf| + \item[system-wide] + |tlmgr conf texmf TEXMFLOCAL| + or |kpsewhich -var-value TEXMFLOCAL|\\ + the default (on Linux): |/usr/local/texlive/texmf-local| + \end{description} + + \smallskip + Don't forget to run |texhash| or |mktexlsr| after creating \reffile{memoize.cfg}. +\end{tcolorbox} +\begin{tcolorbox}[title={MiK\hologo{TeX}\hfill\url[white](https://){miktex.org}}, before upper={\parskip0.3\baselineskip plus 2pt\relax},] + Open the MiK\hologo{TeX} Console, select the ``Settings'' page and then the + ``Directories'' tab. + + If there is a folder marked with the ``Generic'' purpose with attribute + ``User'' (for a user-wide \reffile{memoize.cfg}) or ``Common'' (for a system-wide + \reffile{memoize.cfg}), that's the folder you are looking for. Otherwise, create a + folder following MiK\hologo{TeX}'s instructions and add it to the list. + + Don't forget to click ``Refresh file name database'' (in the ``Tools'' menu + of the MiK\hologo{TeX} Console) after creating \reffile{memoize.cfg}. +\end{tcolorbox} +\end{tcboxedraster} + +Note that a directory config file will override the user config, and the user +config will in turn override the system-wide one. This should not concern you +too much, because you will probably only want to use the user-wide config +anyway,\cprotect\footnote{If you want to have a directory-wide configuration + based on (rather than overriding) the user-wide configuration, your could + write down the real user-wide config in, say, |memoize.user.cfg| (located + user-wide), and then |\input| this file by both the user-wide and the + directory-wide \reffile{memoize.cfg}. Of course, the same logic can be used to base + a user-wide config on the system-wide one.} which might look something like +this: + +\ExampleName{memoize} +\makeexample{\examplename.cfg.c1} +\tcbinputexample[.cfg]{listing only} + +I recommend including \refmmz{memo dir} in your \reffile{memoize.cfg}, +as shown in the first line. It reduces the clutter in the document +directories; see section~\ref{sec:tut:memodir} for details. + +The \refmmz{extract} line concludes the story on ``permanently'' selecting the +extraction method, started in chapter~\ref{sec:setup}. As the final note, +observe that \refmmz{extract} and other extraction-related keys like +\refmmz{perl extraction options} only make sense as a package option or as a +\refcmd{mmzset} key in \reffile{memoize.cfg}. They will have no effect as a +\refcmd{mmzset} key in the document, because the extraction happens while the +package is loaded. + + + +\section{Your first memoized documents} +\label{sec:tut} + +\subsection{Let's see if it works!} +\label{sec:tut:test} + +\ExampleName{test} +\noprint{\refcmd{tikz}\refenv{tikzpicture}\refenv{forest}}% + +Take example file \texttt{\examplename.tex},\footnote{Where can you find + the example files? For one, they are integrated into this manual, so if your + PDF viewer supports attachments, you can simply click on the paperclip icon + on the top right of the example box (even if you're offline). Otherwise, + visit the |examples| subdirectory of wherever you found this document + \Smiley. Online, the Memoize documentation can be found at CTAN: + \url{https://ctan.org/pkg/memoize}; and if your \hologo{TeX} installation + includes the documentation, you should also find it in directory \meta{the + root of your \hologo{TeX} installation}|/doc/generic/memoize|. + + Incidentally, while we present a full example document in this section, many + code listings will only present the parts of the file relevant for the + discussion, for brevity. The example \emph{files}, however, will remain + full, compilable documents, including the document preamble etc.} or some +simple document containing a \TikZ picture or a Forest tree, add +\EmphVerbatim{\usepackage{memoize}} to the preamble,\footnote{Memoize likes to + be loaded early. If you get the warning \texttt{Cannot read file + \meta{jobname}.pdf}, move \cs{usepackage}\bracestt{memoize} up the + preamble; see section~\ref{sec:loading-order} for details.} and compile it +twice. + +\begin{itemize} +\item The first compilation of the example should produce a three-page PDF. + The first two pages are the \emph{extern pages} holding the externalized + graphics, while the final page is the (sole page of the) real document. Note + that you can see each extern twice: first on a page of its own, and then + wherever it belongs to in the real document. +\item At the second compilation, the extern pages should have disappeared from + the PDF, meaning they were successfully extracted into extern files, which + are now embedded into the main document. +\end{itemize} + +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, + comment={\centering + \includeexamplepdf[extern page,left=2mm,right=2mm]{page=1,trim=1in 1in 1in 1in}\quad + \includeexamplepdf[extern page,left=2mm,right=2mm]{page=2,trim=1in 1in 1in 1in}\\[2ex] + \includeexamplepdf[document page]{page=3} + }, +} + +You might want to play with this example a bit now. +For example, if you reverse the order of the \TikZ picture and the Forest +tree, you should notice that the externs don't get recompiled. You won't see +any extern pages again until you change the actual code of the picture or the +tree --- or until you add some other picture or tree, of course. + +If you \emph{don't} want to automatically memoize \TikZ pictures and/or Forest +trees, you can switch off their \emph{automemoization} using key +\Emph{\refmmz{deactivate}}. This key takes a list of command and environment +names. As you can see below, the command and the environment must be +deactivated separately. + +\begin{tcblisting}{listing only} +\mmzset{ + deactivate={\tikz, tikzpicture}, % deactivate automemoization of all TikZ pictures + deactivate=\tikz, % deactivate only the command + deactivate=tikzpicture, % deactivate only the environment +} +\end{tcblisting} + + +\subsection{Memoizing by hand} +\label{sec:tut:memoize} + +In the previous section, we have compiled our very first document which used +Memoize. In that document, we only had to load the package, as Memoize knows +how to externalize \TikZ pictures and Forest trees without any help from the +author. But what if you want to externalize some other code? The manual way of +doing this is by surrounding the code by a \Emph{\refenv{memoize}} environment, +or by making it the argument of the \Emph{\refcmd{mmz}} command. The only +difference between the two is that the environment, but not the command, +ignores any spaces surrounding the given code. + +\ExampleName{manual} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, lefthand ratio=0.528, + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\hfill + \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in} + }, +} + +Both the \refenv{memoize} environment and the \refcmd{mmz} command take a +configuration keylist as the optional argument, so their full syntax is +\cs{begin}\marg{\refenv{memoize}}\oarg{keylist}\meta{code to be + externalized}\cs{end}\marg{\refenv{memoize}} and and +\refcmd{mmz}\oarg{keylist}\marg{code to be externalized}. The keys given in +this optional argument take precedence over the keys set by \refcmd{mmzset}. +Note that \refcmd{mmznext} does not apply to manual memoization. + +Manual memoization is great for one-shot memoizations, but you can use it +within your own macros as well. For example, assume that you don't want to +externalize \TikZ pictures in general (so you have \refmmz{deactivate}d +automemoization of the \cs{tikz} command, as explained at the end of +section~\ref{sec:tut:test}), but that you want to easily memoize selected +pictures. You could define a memoized variant of the \cs{tikz} command, as shown +below (and similary for the environment). (Note the |%| comment characters in +the definition of |\mmztikz|. The definition was intentionally broken into +several lines to remind you that the spaces around the argument of \refcmd{mmz} +matter.) + +\ExampleName{mmztikz} +\makeexample{\examplename.pdf} +\tcbinputexample{ + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} + \quad + \includeexamplepdf[document page]{page=2} + }, +} + +\clearpage +\subsection{Memoizing automatically} +\label{sec:tut:automemoization} + +Out of the box, Memoize automatically externalizes \TikZ pictures and Forest +trees. Let us see how other commands and environments can be submitted to this +\emph{automemoization} process. + +We start with the simpler case of environments (fortunately, externalizing +environments also makes sense more often than externalizing commands). You can +submit an environment to automemoization by writing +\EmphTT{\refmmz{auto}=\marg{environment name}\braces{\refmmzauto{memoize}}} (as +a key in \refcmd{mmzset}). The natural (but not the only possible) location +for this instruction is the preamble. Below, we automemoize environment +\env{circuitikz} of package \pkg{circuitikz}, used for drawing electronic +circuits.\footnote{In section~\ref{sec:tut:test}, we learned that + automemoization can be switched off using key \refmmz{deactivate}. Memoize + also offers key \refmmz{activate}, but you probably won't have to use it, as + an \refmmz{auto} declaration automatically activates the submitted command.} + +\ExampleName{automemoize-environment} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, lefthand ratio=0.6, + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.8} + }, +} + +Commands are a bit harder to automemoize, because Memoize cannot possibly know +how far the arguments of a command extend (in contrast, the end of an +environment is clearly marked). With commands, we must inform Memoize about +their argument structure, which we achieve using key \refmmzauto{args} in the +second argument of key \refmmz{auto}: +\EmphTT{\refmmz{auto}=\meta{command}\braces{\refmmzauto{memoize}, + \refmmzauto{args}=\marg{argument specification}}}. We can only leave out +\refmmzauto{args} if the command was defined by \refcmd{NewDocumentCommand} or +similar; in this case, Memoize can retrieve the argument specification on its +own. + +And how does the \meta{argument specification} look like? It is a sequence of +letters, each letter determining an argument type. Memoize recognizes the same +argument types and their letters as package \pkg{xparse} (which defines +\refcmd{NewDocumentCommand} and friends), so you should look at (section 1 of) +the documentation of that package for details, if and when you need them. +Here, we focus on the two most commonly used types, \docref{xparse:m} and \docref{xparse:o}, and add the +optional star for good measure: +\begin{center} + \begin{tabular}{lll} + \toprule + letter & argument type & example\\ + \midrule + \docref{xparse:m} & mandatory argument \\ + & --- either surrounded with braces & |\foo{arg}|\\ + & --- or a single token & |\foo a|\\ + \docref{xparse:o} & optional argument, surrounded with brackets & |\foo[arg]|\\ + \docref{xparse:s} & optional star & |\foo*|\\ + \bottomrule + \end{tabular} +\end{center} + +Below, we write |args=om| because command \cs{qrcode} (of package \pkg{qrcode}) +takes two arguments: a bracketed optional argument, followed by a mandatory +argument (in braces). + +\ExampleName{automemoize-command} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, lefthand ratio=0.603, top=0mm, + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad + \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in} + }, +} + +This should get you started with automemoization. We'll provide some further +basic information in sections~\ref{sec:tut:working-on-a-picture} +and~\ref{sec:tut:verbatim} of the tutorial, but only package writers will +probably ever need the gory details from +section~\ref{sec:tut:automemoization-details}. It is my sincere hope that they +will support Memoize in their packages, where necessary, so that you don't even +have to write the \refmmz{auto} declarations, but there is one thing you should +know if you encounter a package supporting Memoize: you should load it +\emph{after} Memoize! + + + +\subsection{Working on a picture} +\label{sec:tut:working-on-a-picture} + + +Memoize automatically recompiles a picture when the code producing the picture +changes. However, sometimes we can modify a picture without changing its code, +like when we modify the definition of a command used in the code. In the +example below, a predefined style |emph| is applied to the node, producing a +node with a red background. Let's say we compile the document (with +memoization) and then change this style to set the yellow background. +Curiously, the node will remain red. + +\ExampleName{recompile} +\makeexample{\examplename.pdf N=3} +\begin{tcbraster}[raster columns=2, raster valign=top, raster column skip=4mm] + \tcbinputexample{ + after title pre={\ (version~1)}, + comment={\centering + \includeexamplepdf[document page]{page=2} + }, + } + \tcbinputexample[.tex][.c2]{ + no attachment, + after title pre={\ (version~2)}, + comment={\centering + \includeexamplepdf[document page]{page=1} + }, + } +\end{tcbraster} + +The curious thing happens (or rather, doesn't happen) because Memoize doesn't +keep track of how commands and styles are defined; it just uses the extern file +it created when the old style was in effect. To get a yellow node, we must ask +Memoize to reexternalize the picture. The simplest way to do that is by using +the \Emph{\refmmz{recompile}} key; below, we write +\refcmd{mmznext}\bracestt{\refmmz{recompile}} just before the picture and +compile the document again (remember from section~\ref{sec:mmzset} that +whatever keys we provide through \refcmd{mmznext} only apply to the instance of +automemoization). After the compilation, we may (and should) remove the +\refmmz{recompile} directive (otherwise, Memoize will produce the extern page +again and again). + +\tcbinputexample[.tex][.c3]{ + no attachment, + after title pre={\ (version~3)}, + sidebyside, lefthand ratio=0.6, + comment={\centering + \includeexamplepdf[document page]{page=2} + }, +} + +It is also common to put (again, for the space of a single compilation) +\refcmd{mmzset}\braces{\refmmz{recompile}} in the preamble, or to use +\refmmz{recompile} as the package option. Either will remake all the +externalized graphics in the document, so you can be sure all of them use the +latest version of your macros and styles. + +We'll revisit the issue of memoized code depending on macros and styles defined +elsewhere in section~\ref{sec:tut:redefinitions}. In this section, we will +learn how the issue can be \emph{avoided}, at least to some extent. One idea +is to turn off memoization for the picture(s) we are currently working on; +another idea is to let Memoize know which definitions the picture relies on. +The simplest way to achieve the former is by using key \Emph{\refmmz{disable}}; +by putting it into a \hologo{TeX} group, we can localize its effect to the +selected pictures. + +\ExampleName{disable} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, + lefthand ratio=0.6, + top=0mm, % manual top, bottoms and \\[...] to fit the page + comment={\centering + \includeexamplepdf[extern page,bottom=0.8mm]{page=1,trim=1in 1in 1in 1in}\\[0.5mm] + \includeexamplepdf[extern page,bottom=0.8mm]{page=2,trim=1in 1in 1in 1in}\\[0.5mm] + \includeexamplepdf[document page,bottom=0.8mm]{page=3} + }, +} + +\begin{tcolorbox}[warning] + As you can imagine, key \refmmz{disable} is complemented by \refmmz{enable}, + but it is perhaps worth mentioning a problem that can arise if you disable + memoization for a part of your document by enclosing it in a pair of + \refcmd{mmzset}\braces{\refmmz{disable}} and + \refcmd{mmzset}\braces{\refmmz{enable}}. Yes, it might work at the moment, + but say you later (e.g.\ when you are preparing the final version of the + document) decide to disable memoization for the entire document, and say you + try to do this by writing \refcmd{mmzset}\braces{\refmmz{disable}} in the + preamble. As shown below on the left, you're in for a surprise: memoization + will still be enabled in the part of the document following + \refcmd{mmzset}\braces{\refmmz{enable}}! The solution is to always disable + memoization for a part of the document by using + \refcmd{mmzset}\braces{\refmmz{disable}} in a \hologo{TeX} group (i.e.\ the + braces), as shown on the right. (In the examples below, the shaded areas + mark the parts of the document where memoization is \emph{disabled}.) +\end{tcolorbox} + +\makeexample{disable-bad.tex.c1} +\makeexample{disable-good.tex.c1} +\makeexample{disable-nomemoize.tex.c1} +\makeexample{disable-nommz.tex.c1} +\makeexample{disable-auto-cmd.tex.c1} +\makeexample{disable-auto-env.tex.c1} + +\begin{tcbraster}[raster columns=2, raster equal height=rows] + \tcbinputexample(disable-bad){ + listing and comment, bad, mark region={4}{7}, + comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}} + does not have the intended effect, + i.e.\ it doesn't apply to the whole document!}} + \tcbinputexample(disable-good){ + listing and comment, mark region={4}{10}, + comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}} applies + to the entire document, as expected.}} +\end{tcbraster} + +In fact, it might be better to disable memoization using environment +\Emph{\refenv{nomemoize}} or macro \Emph{\refcmd{nommz}}. I also like these +commands because it is easy to add and remove prefix |no| to switch manual +memoization (triggered using environment \refenv{memoize} or macro +\refcmd{mmz}) off and on. + +\begin{tcbraster}[raster columns=2, raster equal height=rows] + \tcbinputexample(disable-nomemoize){listing and comment, mark region={2}{2}, + comment=\raggedright Disable using the dedicated environment.} + \tcbinputexample(disable-nommz){listing and comment, mark region={2}{2}, + comment=\raggedright Disable using the dedicated command.} +\end{tcbraster} + +It is also possible to disable memoization for all occurrences of a selected +command or environment. In fact, we're already familiar with the procedure +from section \ref{sec:tut:automemoization}, where we used key +\refmmzauto{memoize} inside the second argument of \refmmz{auto} to +automatically memoize all instances of the command or environment given as the +first argument. All we have to do to auto-disable rather than auto-memoize, is +substitute \refmmzauto{nomemoize} for \refmmzauto{memoize}. Note that this +prevents memoization of not only the given command or environment, but also of +any (manual or automatic) memoization which would otherwise occur during its +execution; for example, if |\foo| executes \cs{tikz} under the hood, +autodisabling |\foo| prevents memoization of the inner \cs{tikz}, even though +that command is normally automemoized. + +\begin{tcbraster}[raster columns=2, raster equal height=rows] + \tcbinputexample(disable-auto-cmd){ + listing and comment, + mark region={3}{3}, mark region={5}{5}, + comment=\raggedright Autodisable within a command., + } + \tcbinputexample(disable-auto-env){ + listing and comment, + mark region={3}{5}, + comment=\raggedright Autodisable within an environment., + } +\end{tcbraster} + +All that said, Memoize actually offers a neater way to switch off the +externalization for the picture I'm currently working on. The +\Emph{\refmmz{readonly}} key instructs Memoize to use whatever externs it had +already produced (thereby reducing the document compilation time), but to +abstain from producing any new externs. In effect, the stuff we are currently +working on does not undergo memoization and therefore does not produce the +clutter which can potentially lead to trouble described in the \refmmz{recompile} +examples above. + +What I like to do is load the package using |\usepackage[readonly]{memoize}|, +work on stuff, and once I'm happy with the most recent pictures, remove +\refmmz{readonly} from the package options for one compilation. + +\ExampleName{readonly} +\makeexample{\examplename.pdf N=2} + +\begin{tcbraster}[raster columns=5, raster valign=top, raster column skip=4mm] + \tcbinputexample{ + raster multicolumn=2, + after title pre={\ (work in progress)}, + comment={\centering + \includeexamplepdf[document page]{page=1} + }, + } + \tcbinputexample[.tex][.c2]{ + raster multicolumn=3, + no attachment, + title={\tt readonly.tex} (the final version), + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad + \includeexamplepdf[document page]{page=2} + }, + } +\end{tcbraster} + +We're now ready to tackle a completely different way of avoiding the issue, by +informing Memoize which definitions the externalized picture depends on. We do +this by appending these definitions to \emph{context} --- when the context of a +picture changes, Memoize recompiles the picture, same as if the code of the +picture itself was changed. (We will talk about context in more detail in +section~\ref{sec:cross-referencing}.) + +A command can be added as a dependency using key \Emph{\refmmz{meaning to + context}}. Below, we make the following externalized picture depend on the +definition of macro \cs{answer}; changing this definition will result in the +recompilation of the extern. In general, \refmmz{meaning to context} accepts a +comma-separated list of command and environment names, e.g.\ +\code{\refmmz{meaning to context}=\bracestt{\cs{foo},bar}} (note the braces). + +Memoize offers several variants of \refmmz{meaning to context}, applicable to +various types of commands. For example, the easiest way of making the picture +depend on the definition of a \pkg{pgfkeys} style is to use handler +\Emph{\refkey{/handlers/.meaning to context}} --- note the dot in the name, and +observe that \code{emph/.meaning to context} below is executed within +\cs{tikzset}, not \refcmd{mmzset}; see \PGFmanual{87} to learn about key +handlers. + +\ExampleName{meaning-to-context} +\makeexample{\examplename.pdf N=2} +\tcbinputexample{ + after title pre={\ (version~1)}, + sidebyside, + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} + }, +} + +\tcbinputexample[.tex][.c2]{ + after title pre={\ (version~2)}, + sidebyside, + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} + }, + no attachment, +} + +All variants of \refmmz{meaning to context} (see reference +section~\ref{sec:ref:memoization:basic} for the full list) may be used within +the externalized picture itself, e.g.\ \code{\cs{node}[emph, + emph/\refkey{/handlers/.meaning to context}]\bracestt{...}} is perfectly +valid --- and also handy when you want to limit the effect of the handler to a +single picture, as \refkey{/handlers/.meaning to context} cannot be used within +\refcmd{mmznext}. + + + +\subsection{Keeping a clean house} +\label{sec:tut:memodir} + +Memoize produces lots of auxiliary files. For each piece of memoized code, it +produces two \emph{memo} files (we will learn more about these in +section~\ref{sec:memos}), which will be joined by the extern PDF upon the next +compilation.\footnote{If you're using the \hologo{TeX}-based extraction, each + extern (|.pdf|) is also accompanied by a log file (|.log|) produced during + the compilation that extracted the extern.} You can recognize these files +easily: their names start with the name of your document and include one or two +long hexadecimal numbers. + +\ExampleName{dirty-house} +\makeexample{\examplename.pdf} +{ + \relaxmmzcommands + \edef\marshal{% + \noexpand\def\noexpand\mynewextern\examplename + }\marshal.#1-#2.pdf\relax{% + \forestset{ + append={[\examplename.#1.memo\vphantom{p},pointcolor]}, + append={[\examplename.#1-#2.memo\vphantom{p},pointcolor]}, + append={[\examplename.#1-#2.pdf,pointcolor]}, + }% + } + \tcbinputexample{ + comment={% + \begin{tcolorbox}[reset, left=2mm, bottom=0mm, + my boxed title=the folder contents (after two compilations), + attach shifted boxed title to top right, + ] + \begin{forest} + before typesetting nodes={for tree={ + font=\scriptsize\tt, grow'=0, folder, s sep=0}}, + [\meta{the document folder}, inner xsep=0, + TeX={% + \def\mmzNewExtern##1##2##3##4{\mynewextern##1\relax}% + \input{\examplepath.mmz}% + }, + [\examplename.tex] + [\examplename.pdf] + [\normalsize\dots, no edge] + [\examplename.mmz, tikz={\draw[dotted](.east)--+(5em,0ex) node [anchor=south west, yshift=-1ex, align=justify, text width=14em, font=\footnotesize]{This is another auxiliary file produced by Memoize. We will mention it at the end of the section.};}] + ] + \end{forest} + \end{tcolorbox} + }, + } +} + +To top it off, changing the memoized code will produce new memo (and extern) +files, with the old files staying in place. This is all by design --- the +first hexadecimal number in these filenames is the MD5 sum of the memoized code +and that's how Memoize knows which memo belongs to which piece of +code\footnote{The second hexadecimal number in the memo and extern filenames is + the MD5 sum of the \emph{context}. The context is crucial for properly + externalizing code containing cross-references, see + section~\ref{sec:cross-referencing} for details.} --- but it has the downside +that the folder containing your document can get quite cluttered (imagine the +directory listing as above, but for a document with a hundred externalized +pictures which you have been working on for a month). + +\ExampleName{clean-house} + +I like to keep a clean house by instructing Memoize to put memos and friends +into their own directory. This can be achieved by writing +\refcmd{mmzset}\braces{\Emph{\refmmz{memo dir}}} into the preamble (anytime +after loading the package).\footnote{The \refmmz{memo dir} key is in fact + merely an abbreviation for \code{\refmmz{prefix}=\string\jobname.memo.dir}; + use this key if you need more control over the name and location of the + auxiliary files. Furthermore, there is also the \refmmz{no memo dir} key, + which reverts the configuration back to the dirty default.} This will put +the memo and the extern files into folder \meta{document name}|.memo.dir| (and +it will also omit the \meta{document name} prefix in their filenames, because +it makes no sense to repeat it there). + +\makeexample{\examplename.pdf} +{% + \relaxmmzcommands + \edef\marshal{% + \noexpand\def\noexpand\mynewextern\examplename + }\marshal.memo.dir/#1-#2.pdf\relax{% + \forestset{ + append={[#1.memo\vphantom{p},pointcolor]}, + append={[#1-#2.memo\vphantom{p},pointcolor]}, + append={[#1-#2.pdf,pointcolor]}, + }% + } + \tcbinputexample{ + comment={% + \begin{tcolorbox}[reset, left=2mm, bottom=0mm, + my boxed title=the folder contents (after two compilations), + attach shifted boxed title to top right, + ] + \begin{forest} + before typesetting nodes={for tree={ + font=\scriptsize\tt, grow'=0, folder, s sep=0}}, + [\meta{the document folder}, inner xsep=0 + [\examplename.tex] + [\examplename.pdf] + [\normalsize\dots, no edge] + [\examplename.mmz] + [\examplename.memo.dir,pointcolor, + % tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=south west, yshift=-1ex, font=\footnotesize, emphshade=black!5!white, text width=15em, align=justify]{You need to create this directory by hand, or allow Memoize to create it by adding |mkdir| to the list of restricted shell escape commands.};}, + tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=west, font=\footnotesize, emphshade=black!5!white]{This directory must be created, somehow.};}, + TeX={% + \def\mmzNewExtern##1##2##3##4{\mynewextern##1\relax}% + \input{\examplepath.mmz}% + }, + ] + ] + \end{forest} + \end{tcolorbox} + }, + } +} + +\begin{tcolorbox}[title={Why is \refmmz[link color=white]{memo dir} not the default?}] + The \code{clean-house} example most likely compiled just fine, and you are + wondering why \refmmz{memo dir} is not in effect by default. Well, out of + the box, \hologo{TeX} cannot create directories, so it is the fact that + Memoize \emph{can} create them (at least under the default settings) which + requires explanation. By default, Memoize triggers extraction by executing + the Perl extraction script \refscript{memoize-extract.pl}, and it is this + script which actually creates the memo directory. However, not everybody + will necessarily use this script \dots\ so \refmmz{memo dir} should not be + the default. + + The Python extraction script \refscript{memoize-extract.pl}, used when + \refmmz{extract}|=|\refmmz{extract=python}, works the same as the Perl + variant. With \hologo{TeX}-based extraction + \refmmz{extract}|=|\refmmz{extract=tex}, things are different. If you are + compiling the document with a full shell escape mode + (\docref{-shell-escape}), Memoize successfully creates the directory with the + system command |mkdir|.\footnote{|mkdir| is the default value of key + \refmmz{mkdir command}, but executing the extraction method + \refmmz{extract=perl} or \refmmz{extract=python} overrides this default.} + However, if you're using the restricted shell escape mode, the attempt to + create the directory won't succeed unless you include |mkdir| among the + restricted shell escape commands (see footnote~\ref{fn:shell-escape-commands} + on page~\pageref{fn:shell-escape-commands} for how to do this, but note that + it is not recommended). + + If you are using external extraction, you have to create directory + \texttt{\examplename.memo.dir} by hand, prior to the first compilation of the + document (with Memoize). This is the case even if you are performing the + extraction using one of the shipped extraction methods, and it is due to the + fact that Memoize needs the memo directory to be present even before extern + extraction, because it writes the |.memo| files into the same directory. + (When Memoize uses the extraction script to create the memo directory, it + does so completely independently of extraction, and prior to creating any + |.memo| files.) +\end{tcolorbox} + +I actually suggest adding \refcmd{mmzset}\braces{\refmmz{memo dir}} into your +user-wide \reffile{memoize.cfg} (see section~\ref{sec:memoize.cfg} for details +on this file). This will keep all your houses clean --- without work! --- as +Memoize will automatically use the memo directory for any document your create. + +It is always safe to delete memos (|.memo|) and externs (|.pdf|s residing next +to memos), in the sense that you cannot lose data this way.\footnote{The same + goes for the extern |.log| files produced by the \hologo{TeX}-based + extraction.} + +\begin{itemize} + +\item Many memos and externs are typically stale anyway, i.e.\ they reflect + some previous state of your document and are not needed anymore. These files + can be deleted without any repercussions whatsoever (unless you later revert + to a previous version of the document, of course). In fact, you might + \emph{want} to delete them periodically, or at least once you finish writing + the document. As it is hard to figure out which memos/externs are stale, + Memoize ships with a clean-up script: writing \refscript{memoize-clean.pl} + \meta{document name}\dmmz (replace |.pl| with |.py| if you use Python rather + than Perl) into the command line will delete all the \emph{stale} auxiliary + files belonging to the document. + +\item If you delete a memo or an extern currently in use, you will trigger + recompilation of their code --- so deleting a memo or an extern is actually a + perfectly legal alternative to using the \refmmz{recompile} key!\footnote{For the + users of the \hologo{TeX}-based extraction: deleting the |.log| file does + \emph{not} trigger recompilation.} + +\end{itemize} + +It is also safe to delete the \dmmz file (or any other kind of record file, see +section~\ref{sec:record-files}) residing next to your document's |.pdf|. The +\dmmz file contains the information about which externs should be extracted +from the |.pdf|. Deleting it before this is done (by default, before compiling +the document again) will prevent the extraction (same as if providing the +package option \code{\refmmz{extract}=\refmmz{extract=no}}) and ultimately +result in the recompilation of the externs produced in the previous run. +Deleting it after the extraction will have almost no effect: it will only only +prevent the clean-up script from working (the \dmmz file also lists the +currently active memos and externs, and thereby indirectly informs the clean-up +script which files are stale). For further information on the \dmmz file, see +section~\ref{sec:.mmz}. + + + +\subsection{Writing a book?} +\label{sec:tut:multifile} + +Books and other long documents are usually produced from sources which reside +in more than a single file, and to speed up the editing process, authors +usually use some system which allows them to compile each chapter separately. +Can Memoize --- designed for virtually the same task of speeding up the editing +process --- work sensibly in this kind of situation? More precisely, can the +book and the individual chapters share the memos and the externs? Yes they +can! If we instruct Memoize to use the same memo directory for both the book +job and the chapter jobs, then we can externalize graphics when compiling a +chapter and have the externs included when compiling the book (and vice +versa).\footnote{Package \pkg{docmute} makes \hologo{LaTeX} ignore the preamble of + the chapter file when including this file into the main document.} All we +need to do is use our old friend \refmmz{memo dir} from section~\ref{sec:tut:memodir} +--- we see now that this setting is good for more than just keeping a clean +house! + +\ExampleName{book} +\makeexample{\examplename.tex.c1} +\makeexample{chapters/chapter1.tex.c1} + +\begin{tcbraster}[raster columns=2, raster valign=top] + \tcbinputexample{listing only} + \tcbinputexample(chapters/chapter1){listing only} +\end{tcbraster} + +In the above example, the individual chapters reside in files stored in the +|chapters| subdirectory, and that's why the |book.tex| preamble uses +\code{\refmmz{memo dir}=chapters/book} (rather than \code{\refmmz{memo + dir}=book} or just \refmmz{memo dir}). However, Memoize has no trouble +with a situation where the main file and the chapters reside in the same +folder; the setup is even simpler, as we then say \code{\refmmz{memo dir}=book} +in both the book and the chapter preamble. The more complicated situation was +chosen to point out the following potential problem with the setup where the +chapters reside in a subdirectory. + +If you're anything like me, you would first go for having a memo directory +immediately contained in the project directory (so |examples/book.memo.dir| +above) and set up \refmmz{memo dir} as shown below. Well, this won't work, or +at least it won't work with the vanilla \hologo{TeX} Live, because \hologo{TeX} +will refuse to \emph{write} into (memo) files outside the directory where it +was executed,\cprotect\footnote{In \hologo{TeX} Live, the \reffile{texmf.cnf} + option controlling this behaviour is called \docref{openout_any}. By default, it is + set to |p| (paranoid), which ``disallow[s] opening dot files [and] + \emph{going to parent directories}, and restrict[s] absolute paths to be + under \docref{TEXMFOUTPUT}'' (emphasis mine).} and this is precisely what +the chapter compilation is asked to do below. + +\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest, bad} + \begin{tcblisting}{example title, title=the main file, listing only} +\mmzset{~memo dir=book~} +% ... +\input{chapters/chapter1.tex} + \end{tcblisting} + \begin{tcblisting}{example title, title=a chapter file, listing only} +\mmzset{~memo dir=../book~} + \end{tcblisting} +\end{tcboxedraster} + +\pagebreak % manual + +Section~\ref{sec:tut:working-on-a-picture} presented some ideas on how to work +on a single picture. Those ideas can be all easily applied to the multi-file +situation. For example, you could use \refmmz{readonly} on the chapter that you're +working on (and that chapter only). This way, the preview of the chapter will +not be tarnished by the extern pages, and if you periodically compile it +without \refmmz{readonly}, or compile the book (which does not have the \refmmz{readonly} +set), you will have a reasonably up-to-date set of externs. + +\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest} + \begin{tcblisting}{example title, title=the main file, listing only} +\mmzset{memo dir=chapters/book} +% ... +\input{chapters/chapter1.tex} + \end{tcblisting} + \begin{tcblisting}{example title, title=the current chapter file, listing only} +\mmzset{memo dir=book, ~readonly~} + \end{tcblisting} +\end{tcboxedraster} + +\ExampleName{memoize.cfg._region_} +\makeexample{\examplename.tex.c1} +\begin{tcolorbox}[title=For \Emacs users] + I often use this \refmmz{readonly} trick myself, but with a twist. As an Emacs user, + I don't use a \hologo{TeX}-based mechanism (such as the \pkg{docmute} package) to + compile a chapter, but rely on the region compilation feature of Emacs' + AUC\hologo{TeX} package. AUC\hologo{TeX} offers a way to compile the current + buffer (if you don't know what an Emacs buffer is, read ``file'') or region + (roughly speaking, the selected text). It does that by putting the buffer or + the region into a file called |_region_.tex| while dressing it up in the + preamble of the original document (when I'm working on a multi-file document, + it correctly pulls the preamble from the main document). This results in a + compilable region file. My trick is to detect whether I'm compiling a region + (this is the job of |\ifregion|), and if so, put Memoize into the \refmmz{readonly} + mode (an alternative trick would be to \refmmz{disable} it). + + This is the trick in a nutshell, but to make it really work we have to + address one further issue: the original document and the region have to share + memos and externs. This happens automatically if the original document sets + \refmmz{memo dir} explicitly (e.g.\ if a document called |doc.tex| contains + \code{\refmmz{memo dir}=doc} in the preamble), but I'm lazy and don't want to + write this in every document --- if I have to do that, what's the point of + \refmmz{memo dir} I put into my \reffile{memoize.cfg} in + section~\ref{sec:tut:memodir}? Fortunately, the region file starts with + |\message{ !name(|\meta{original document name}|.tex)}| to indicate the + origin. The complicated part of the code below (everything following + \refcmd{mmzset}\braces{\refmmz{readonly}} parses this header to extract the + \meta{original document name}, which is then fed to \refmmz{memo dir}. Now, + the trick works automatically for any document.\footnote{The assumption here + is that \refmmz{memo dir} is in effect for the original document. If not, + the trick can be adapted to use \refmmz{prefix}.} + + \tcbinputexample{ + title=\tt memoize.cfg, + attachment name=memoize-region.cfg, + listing only, + } +\end{tcolorbox} + + +\subsection{Writing a presentation?} +\label{sec:tut:beamer} + +Memoize ships with built-in support for the most widespread \hologo{LaTeX} +presentation class, Beamer, in the sense that it can externalize a picture +which changes from overlay to overlay. Before we learn how to use that +functionality, however, there's a peculiarity about loading Memoize in Beamer +to address. + +\ExampleName{beamer} +\makeexample{\examplename.pdf} + +\tcbinputexample{ + comment and listing, + listing options app={lastline=2}, + warning, no attachment, title=, + % + comment={Beamer opens the document PDF while loading the class, while + Memoize requires the PDF from the previous compilation intact in order to + extract the externs (when extraction is triggered internally, which is the + default setting). The solution is to load Memoize (a package) before + Beamer (a class), which can be done by using \cs{RequirePackage} instead of + the usual \cs{usepackage}. Easy, if hacky.}, + % +} + +To memoize a piece of code which produces different results on different +overlays --- by virtue of containing |\pause|, |\only|, and\slash or related +commands --- apply key \Emph{\refmmz{per overlay}}. Without this key, +externalization of the picture will end badly, with a single extern (the final +one) appearing on all overlays. The key may be invoked either from a prior +\refcmd{mmznext} command,\footnote{Of course, \refmmz{per overlay} may also be + invoked from \refcmd{mmzset}, but I guess this won't make sense often. For + example, if you set it for the entire presentation, and the presentation + contains static memoized pictures as well, you will compile those pictures + more times than necessary: once for each overlay, whereas once per frame + would suffice. It might occasionally make sense, however, to use \refmmz{per + overlay} as an \refmmz{auto} option --- consult + section~\ref{sec:tut:verbatim} to learn what that is.} or executed in the +memoized code itself. The example below illustrates the latter option, and +also shows that we may invoke it via its full path, +\refkeypath{/mmz}|/|\refmmz{per overlay}, when listed among options processed by +\pkg{pgfkeys}.\footnote{Read section~\ref{sec:per-overlay} to learn how the + Beamer support is implemented. The implementation only uses Memoize's public + interface, so understanding it should help if you need to support some other + presentation package.} + +\tcbinputexample{ + listing options app={firstline=3}, + middle=1mm, + comment={\def\dpwidth{0.27\linewidth}\def\epwidth{0.15\linewidth}% + \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=1,trim=1in 1in 0.4in 0.6in}}\hfill + \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=2,width=\dpwidth}}\hfill + \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=3,trim=1in 1in 0.4in 0.6in}}\hfill + \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=4,width=\dpwidth}}% + }, +} + +\begin{tcolorbox}[warning] + If the memoized code changes the value of Beamer's pause counter + |beamerpauses|, e.g.\ by issuing a |\pause|, take care that (i) \refmmz{per + overlay} is executed prior to any changes of |beamerpauses|, and that (ii) + the final value of this counter in the memoized code is the same for all + overlays. +\end{tcolorbox} + + +\subsection{When stuff sticks out} +\label{sec:tut:padding} + +Some constructs --- like plain \hologo{TeX}'s \refcmd{llap} and \refcmd{rlap}, +and, notably, \TikZ overlays --- fool \hologo{TeX} into thinking that the +``size'' of the typeset material is different than what it actually is. This +can cause trouble for externalization: a piece of your picture might disappear! +In a sentence, the solution is to manually set the \emph{padding} of the +externs, but let's slow down a bit. + +The \TikZ picture in the following example consists of node with a pin on the +right, but let's say we want to horizontally center this picture so that only +the node rather than the entire picture (including the pin) will be centered. +This can be achieved by adding key \refkey{/tikz/overlay} to the pin (actually, +we need to add it to both the pin and its edge). \TikZ normally updates the +extents (called the bounding box) of the picture every time it puts something +in it; when \refkey{/tikz/overlay} is in effect, however, these updates are +temporarily disabled. In effect, the \refkey{/tikz/overlay} key on the pin +below will fool \hologo{TeX} into thinking that the node is all there is to the +picture, so centering will work as desired. + +\ExampleName{overlay} +\makeexample{\examplename.pdf N=4} +\tcbinputexample{ + after title pre={\ (no memoization)}, + comment={\centering + % \includeexamplepdf[document page]{page=1,trim=1.8in 7.4in 1.8in 1.7in} + \includeexamplepdf[document page]{page=1} + }, +} + +What happens when we try to externalize this picture? The example below shows +what would happen if Memoize had no concept of \emph{padding} --- which we +simulate by setting \code{\refmmz{padding}=0pt}.\footnote{Unlike in the rest of + the manual, the extern pages in this section are shown without trimming the + whitespace.} Along with the rest of \hologo{TeX}, Memoize would be fooled +into thinking that the picture comprises of the node only, so the pin would +never make it into the extern. You would end up with a document missing the +pin!\footnote{On the first compilation, the document page containing the figure + without padding looks fine, as it uses the result of the compilation rather + than the extern file. But on the second compilation, when Memoize actually + uses the extern, the pin disappears.} + +\tcbinputexample[.tex][.c2]{ + after title pre={\ (memoization without padding)}, + attachment name=\examplename-no-padding.tex, + sidebyside, lefthand ratio=0.3, + comment={\centering + \includeexamplepdf[size=minimal,boxrule=0.5mm,extern page, + attach boxed title to top right={xshift=1.5mm,yshift=-1mm}, + ]{page=1} + }, +} + + +By default, Memoize puts an inch of space around (what it thinks is) the +externalized picture, and if the overlayed parts of the picture fit into this +inch of space, you will find them in the extern and therefore also in the +document. In our example, however, the default padding is not enough --- the +pin is only partially visible.\footnote{You might wonder why I didn't make the + default padding much bigger, like 10 inches. \hologo{TeX} wouldn't be + bothered (unless the resulting extern size exceeded its maximum dimension), + but you might be, because with such a large default padding, all the externs + would be huge, most often bigger than the document pages, and remember that + the externs are first dumped into the document, where they can bother you.} + +\tcbinputexample[.tex][.c3]{ + after title pre={\ (memoization with default padding)}, + attachment name=\examplename-default-padding.tex, + listing and comment, + middle=0.9mm, + comment={\centering + \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1} + }, +} + +The solution is to set the padding manually. Below, I used \refmmz{padding + right} to only increase the padding on the right side (clearly, we also have +\refmmz{padding left}, \refmmz{padding top} and \refmmz{padding bottom}), but +if you're not bothered by a large extern, you can just use +\Emph{\refmmz{padding}}, which sets all four sides at once. By the way, having +too much padding (almost) never hurts, and as you see, you can use (simple) +arithmetics in the value of these keys. + +\tcbinputexample[.tex][.c4]{ + after title pre={\ (memoization with extra padding)}, + attachment name=\examplename-extra-padding.tex, + listing and comment, + middle=0.9mm, + comment={\centering + \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1} + }, +} + +\enlargethispage{1ex} % manual +Incidentally, the \refmmz{padding} keys only change how +the externalized picture is \emph{stored}. Memoize remembers the size of the +extern as seen by \hologo{TeX} (e.g.\ the bounding box of the picture as +reported by \TikZ, with overlayed parts of the picture protruding out of it), +and it uses that size when integrating the extern into the document --- so +everything works as it should! + + + +\subsection{The verbatim mode} +\label{sec:tut:verbatim} + +Not all code will peacefully submit to memoization. In particular, this is the +case for environments which process the environment body verbatim (or perform +some other kind of \refcmd{catcode} magic). A simple environment of this kind is the +standard \hologo{LaTeX} \refenv{verbatim}, but let us illustrate the issue with +\refenv{tcblisting}, which typesets a code listing alongside its compiled effect. +(This environment is defined by the |listings| library of package \pkg{tcolorbox} +and was used extensively during the production of this manual.) To manually +memoize a \refenv{tcblisting} environment, we enclose it in a \refenv{memoize} +environment with a \Emph{\refmmz{verbatim}} key in the optional argument --- +without this key, the example below would produce nothing but +errors.\footnote{Memoize also offers a \emph{partial} verbatim mode, triggered + by key \refmmz{verb}; in this mode, the braces retain their usual category + codes. Also note that the effect of \refmmz{verbatim} can be ``undone'' by + key \refmmz{no verbatim}.} + +\ExampleName{verbatim-manual} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\\[1ex] + \footnotesize + (The document page is the same as for the \texttt{verbatim-auto} example below.) + }, +} + +Using \refmmz{verbatim} from \refcmd{mmzset} or \refcmd{mmznext} works just as +well, and the latter can be very useful with automemoization, when some +environment (say, \env{tcolorbox}) generally does not require the verbatim mode, +but a specific occurrence does (say, because it contains some verbatim +construction such as \verb!|!\meta{verbatim text}\verb!|! of the \pkg{ltxdoc} +class). + +However, for an environment such as |tcblisting|, it makes the most sense to +declare it verbatim in general, so that all instances of the environment will +be processed in the verbatim mode. This is simple to do: add \refmmz{verbatim} +to the \refmmz{auto} keylist. + +\ExampleName{verbatim-auto} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, % lefthand ratio=0.62, + comment={\centering + \footnotesize + (The extern page is the same as for the \texttt{verbatim-manual} example above.)\\[1ex] + \includeexamplepdf[document page]{page=2}\\ + }, +} + +In fact, you can add any \refkeypath{/mmz} key to the \refmmz{auto} keylist, +and the key will be applied to all occurrences of the command or the +environment. For example, adding \refmmz{recompile} to the declaration of +|tcblisting| above would recompile all and only the |tcblisting| environments; +and as an \refmmz{auto} declaration only updates (rather than completely +replaces) a previous declaration, you can also say things like +\code{\refmmz{auto}=\refcmd{tikz}\braces{\refmmz{recompile}}} to recompile all +\TikZ pictures produced by the \cs{tikz} command (handy, as you don't know how +automemoization for \cs{tikz} was declared unless you've read +section~\ref{sec:tut:automemoization-details} or looked at the Memoize's source +code). + + +\subsection{The final version of your document} +\label{sec:tut:final} + +Bluntly put, you might want to disable Memoize when compiling the final version +of your document, at least if you intend to distribute it in electronic form, +for two reasons: +\begin{itemize} +\item An externalized picture cannot contain hyperlinks. Any hyperlinks (or + hyperlink anchors) contained in the original picture will silently disappear + during the production of the extern. +\item When the document contains many externs, the size of the resulting PDF + can be several times the size of the PDF compiled without externalization. +\end{itemize} + +Below, we list several ways of fully disabling Memoize. You're of course +already familar with the first two ways, but what's this \refpkg{nomemoize} +package? The rationale behind this package is that if you want to be +absolutely sure that there is no trace of memoization in your document (for +example, see the \refmmz{disable} -- \refmmz{enable} pitfall in +section~\ref{sec:tut:working-on-a-picture}), the best thing to do is to not +load the package at all. However, you have all those \refcmd{mmzset}s etc.\ in +your source, so the document won't compile without +\cs{usepackage}\braces{\refpkg{memoize}}, right? Right, but wrong. Enter +\Emph{\refpkg{nomemoize}}, a dummy package which accepts all the commands that +Memoize does, but does nothing. In effect, your document will compile, but you +can be sure that not a single memo or extern was loaded or produced. + +\begin{tcbraster}[raster columns=3, raster force size=false, raster valign=top] + \begin{tcblisting}{listing only, add to width=2.4em, enhanced} +\usepackage[~disable~]{memoize} + \end{tcblisting} + \begin{tcblisting}{listing only, add to width=-1.6em, enhanced} +\usepackage{memoize} +\mmzset{~disable~} + \end{tcblisting} + \begin{tcblisting}{listing only, add to width=-0.8em, enhanced} +\usepackage{~no~memoize} + \end{tcblisting} +\end{tcbraster} + +There is one issue you might need to resolve manually before package +\refpkg{nomemoize} works as intended, though. If you have used any +\refkeypath{/mmz} keys outside \refcmd{mmzset}, you need to list them in +\refcmd{nommzkeys}. For example, if you used \refmmz{per overlay} in the manner +illustrated in section~\ref{sec:tut:beamer}, i.e.\ as +\refkeypath{/mmz}|/|\refmmz{per overlay} among the \TikZ keys, you need to +write \refcmd{nommzkeys}\braces{\refmmz{per overlay}} into the document +preamble. + +Another thing you might want to do once you have produced the final version of +the document (in fact, just before you disable Memoize for good) is clean up. +As we saw in sections~\ref{sec:tut:memodir} and~\ref{sec:tut:redefinitions}, +Memoize produces a lot of auxiliary files (memos and externs) and it keeps the +old versions around! Once your document is prepared, you can reduce the +clutter (and save some disk space) by deleting memos and externs belonging to +the work-in-progress versions of your document, and keep only those used in the +final version. + +You could achieve this by deleting all the memos and externs (if you're using +the \refmmz{memo dir} directive, this amounts to the entire contents of the +memo directory) and compiling your document for the final couple of times. +However, there is an easier (but \hologo{TeX}-external) way: on the command +line, change into the directory containing your (main) document and write +\EmphTT{\refscript{memoize-clean.pl}} \meta{document name}\dmmz (substitute +|.py| for |.pl| to use Python rather than Perl). The script will inspect the +contents of the \dmmz record file to see which memos and externs were used in +the final compilation, and delete all other memos and externs belonging to the +given document. + +\begin{tcolorbox}[warning] + Deleting memos and externs is never an irreversible operation, as you can + always recreate them, but it is still wise to be cautious when cleaning up. + For one, avoid cleaning up after a compilation which produced errors; a + failed compilation can lead to an incomplete \dmmz file, which can in turn + lead to over-deletion. Another bad idea is cleaning up after disabling + Memoize for a part of a document, for the same reason. + + All that said, Memoize takes some precautions itself. It will cowardly + refuse to perform the clean-up when the \dmmz file is missing the + end-of-file marker (|\endinput|), assuming that this indicates a fatal error + in the previous compilation. It will do the same in case the \dmmz file is + absent or empty. The latter is assumed to be a result of a globally + \refmmz{disable}d memoization, but note that clean-up will be performed if + memoization was disabled using package \refpkg{nomemoize}: that package does not + touch the \dmmz file, so cleaning up should work as intended. +\end{tcolorbox} + +As the final note, memos and externs (cleaned-up or not) may be copied (along +the document source) to another directory or machine, where they should be +picked up by Memoize. There is no need to copy the \dmmz file (assuming that +the document PDF contains no extern pages waiting for extraction). + + +\section{Digging deeper} +\label{sec:potential-pitfalls} + + + +\subsection{Good to know} +\label{sec:tut:good-to-know} + + +\paragraph{Line- and page-breaking} An extern can't be broken across lines or +pages. + +Externalization of a chunk of code produces a PDF, which is included into the +document at subsequent compilations as a picture --- an unbreakable object (a +horizontal box) with fixed width and height. Therefore, the original code +should produce an unbreakable object as well. For example, this means that you +cannot externalize some text and expect \hologo{TeX} to break it across lines +or pages on subsequent compilations. If you try, the compilation will succeed +--- without an error! --- but your externalized text will end up in a single +line, as shown below. + +\ExampleName{no-linebreaking} +\makeexample{\examplename.pdf N=2} +\tcbinputexample{ + bad, + comment={\centering + \includeexamplepdf[extern page,to be continued on right] + {page=1,trim=1in 1in 14in 1in}\\[1ex] + \includeexamplepdf[document page]{page=2}\\[1ex] + \includeexamplepdf[document page,title=the expected document page] + [\examplepath.c2.pdf]{page=1} + }, +} + +That said, you \emph{can} externalize a paragraph or some other vertical mode +material using \refmmz{capture}|=|\refmmz{capture=vbox}, but beware that the +vertical spacing between the memoized material and its surroundings might +change. + + +\paragraph{\env{remember picture}} +\TikZ pictures using this key cannot be externalized. + +Memoize will silently refuse to externalize any \TikZ picture using +\refkey{/tikz/remember picture} (see \PGFmanual{17.13}). Such pictures +interact with the outside world --- they either reference or are referenced by +other pictures --- and are as such unsuitable for externalization. For +example, while the colored boxes in this manual are generally externalized --- +out of principle \Smiley\ --- the title page illustration is not, and it cannot +be, because of the arrows connecting the various \TikZ pictures composing that +illustration. Some packages use the \refkey{/tikz/remember picture} mechanism +under the hood, and are thus subject to this limitation; one example is package +\pkg{todonotes}, but in general, any package dealing with absolute positions on the +page will be limited in this way. + +How does Memoize deal with this situation? Well, by cowardly refusing to +externalize any code which uses \refkey{/tikz/remember picture} or a similar +mechanism for dealing with absolute positions. Luckily, any such mechanism +eventually boils down to the \hologo{TeX} primitive \refcmd{savepos}, so +Memoize hacks --- or as we will say in this manual, \emph{advises} --- this +primitive to abort any ongoing memoization. Initializing and then aborting the +memoization takes some time, to be sure, but the overhead is negligible, +especially in the light of the fact that \emph{not} aborting wreaks real havoc. + +Memoize actually provides a user interface for aborting memoization. +Memoization can be aborted either manually, by executing \refcmd{mmzAbort}, or +automatically. The latter is a generalization of the automemoization idea: a +command such as \refcmd{savepos} can be advised to abort memoization by +\refmmz{auto}|=|\refcmd{savepos}\bracestt{\refmmzauto{abort}}. In Memoize, +commands take your advice seriously, so memoization will be aborted whenever +the advised command or environment is encountered. + + +\paragraph{Indirectly embedded environments} Such environments cannot be +memoized. + +Read this if you got an error message such as |Environment "tikzpicture" ended +as "foo"|. + +Some environments are defined so that they embed another environment using the +idiom shown on the left: the begin-code of the outer environment opens the +inner environment, and the end-code of the outer environment closes the inner +environment. While this is a fine, and common, idiom, it messes up the +memoization of the inner environment. In the example on the right, trying to +\emph{auto}memoize a \env{minipage} environment (not recommended at all!) causes +trouble with package \pkg{sectionbox}.\footnote{This is a \texttt{Package + collargs Error} because Memoize outsources the actual work of collecting + the environment body to the auxiliary package CollArgs, described in + section~\ref{sec:collargs}.}\fncomma\cprotect\footnote{Why does this happen? + As mentioned in section~\ref{sec:tut:memodir}, Memoize keeps track of memos + and externs by the MD5 sum of the memoized code. But to compute that sum for + an environment, Memoize has to \emph{grab} the environment body, meaning it + has to collect the body in advance. This presents no problem when + \refcmd{end}\meta{environment name} is already present in the input stream at the +time \refcmd{begin}\meta{environment name} is executed, like when you use the + environment normally in your document, or when some macro expands so that it + produces both \cs{begin}\meta{environment name} and \cs{end}\meta{environment + name} simultaneously --- so there would be no problem above if + |\end{minipage}| occured in the beginning code of \env{sectionbox}. The idiom +presented above is problematic for memoization because at the time \hologo{TeX} +executes |\begin{sectionbox}|, putting |\begin{minipage}| into the input + stream, |\end{sectionbox}| is not yet executed and remains as it is. The + input stream therefore contains a pair of |\begin{minipage}| and + |\end{sectionbox}|. In the normal, non-memoizing course of events this + would not be a problem, because |\end{sectionbox}| would eventually expand to +|\end{minipage}|. During memoization, however, this \emph{is} a problem, +because, as we said, Memoize needs to grab the environment body: upon +encountering |\begin{minipage}|, it looks in the input stream for + |\end{minipage}| --- but there is no |\end{minipage}| in the input stream, +there is only |\end{sectionbox}|, and this results in the |Environment +"minipage" ended as "sectionbox"| error.} + +\begingroup +\ExampleName{sectionbox} +\makeexample{\examplename.tex.c1} +\mmzset{disable} +\begin{tcbraster}[raster equal height] +\begin{tcblisting}{listing only, bad, valign=center} +\newenvironment{foo}% ... args +{% the begin-code of foo + % ... + \begin{tikzpicture} + % ... +} +{% the end-code of foo + % ... + \end{tikzpicture} + % ... +} +\end{tcblisting} +\tcbinputexample{ + bad, + comment={\small\tt + ! Package collargs Error: Environment\\"minipage" ended as "sectionbox". + }, + } +\end{tcbraster} +\endgroup + +What are your options in this kind of a situation? +\begin{enumerate} +\item The only way to perform any memoization here is to memoize the + \emph{outer} environment --- if that makes sense.\footnote{This avoids the + error because Memoize grabs and memoizes the outer environment, and while + it is memoizing it, further memoization is switched off.} You can do this + either on a case-by-case basis, by enclosing it in the \refenv{memoize} + environment, or automemoize it: \code{\refmmz{auto}=\marg{outer + environment}\braces{\refmmzauto{memoize}}}. +\item But what if memoizing the outer environment is out of the question? + Then, the only way to avoid the error is to prevent the automemoization of + the inner environment. + \begin{enumerate} + \item If you are facing a single occurrence of the problem, it is perhaps + easiest to use key \refmmz{disable} just before the start of the outer + environment. + \item Otherwise, you can automatically disable memoization for the span of + each occurrence of the outer environment: \code{\refmmz{auto}=\marg{outer + environment}% + \braces{\refmmzauto{nomemoize}}}. + \item To deactivate the automemoization of the inner environment for the span + of the outer environment, but otherwise allow for memoization inside the + outer environment: \code{\refmmz{auto}=\marg{outer environment}% + \braces{\refmmzauto{noop}, \refmmz{deactivate}=\meta{inner + environment}}}. Key \refmmzauto{noop} does nothing but apply the + given options (in this case, \refmmz{deactivate}=\meta{inner environment}) + to the advised command or environment. + \end{enumerate} +\end{enumerate} + +\subsection{Extraction methods and modes} +\label{sec:extraction} + +Remember that in Memoize, externalization is a two-step process. First, +externs are typeset on separate pages of the main document, called extern +pages. Then, these extern pages are extracted out of the main document PDF +into extern files. The process is illustrated on the title page. + +Memoize is flexible in terms of which piece of software is used to perform +extern extraction. It ships with three \emph{extraction methods}: + +\begin{description} +\item[\refmmz{extract=perl}] A Perl script, \refscript{memoize-extract.pl}. + This method is the default because it is fast and because Perl is usually + already installed on a system running \hologo{TeX}. However, you will most + likely still need to install the PDF processing library + \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2}, or its fork + \hreftt{https://metacpan.org/pod/PDF::Builder}{PDF::Builder}; the + installation guidelines can be found in section~\ref{sec:install}. +\item[\refmmz{extract=python}] A Python script, \refscript{memoize-extract.py}. + This method is even faster than the Perl script, though not by much. Try it + out if you have problems installing Perl or the required libraries, or if the + Perl script chokes on your document (see section~\ref{sec:troubleshooting} + for the list of known issues). Besides Python ($\geq 3.8$), you will also + need the Python library \hreftt{https://pypi.org/project/pdfrw/}{pdfrw} or + \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2}. For the installation + guidelines, see section~\ref{sec:install}. +\item[\refmmz{extract=tex}] \hologo{TeX}-based extraction requires no + additional software, but it is much slower than the scripts. As \hologo{TeX} + can only produce a single PDF per compilation, an instance of \hologo{TeX} + (loading the entire document PDF) has to be invoked for each extern, and this + takes time (although the entire process is still much faster than the + venerable \TikZ externalization library). +\end{description} + +Memoize is also flexible in terms of how extern extraction is triggered, +providing two \emph{extraction modes}: + +\begin{description} +\item[internal]\label{item:setup:who} By default, extern extraction is + triggered internally, i.e.\ by Memoize during the compilation of the + document; more precisely, any externs produced in a compilation are extracted + during the next compilation. To choose an extraction method other than the + default Perl script, load Memoize with the package option + \EmphTT{\refmmz{extract}=\meta{extraction method}}. + +\item[external] Loading Memoize with with package option + \EmphTT{\refmmz{extract}=\refmmz{extract=no}} instructs Memoize to \emph{not} + trigger the (internal) extraction. When instructed to use extraction + ``method'' \refmmz{extract=no}, Memoize expects you to trigger the extraction + yourself, in any way that is convenient to you: manually from the command + line, or automatically through your editor, a Makefile, etc. --- all Memoize + cares about is that the extraction takes place before the next compilation of + the document. +\end{description} + +Summing up, the extraction mode and method are selected by providing the +appropriate value to package option key \refmmz{extract}; the possible values +are listed in the table below. Note that this key can only be used as a +package option, or in \refcmd{mmzset} within \reffile{memoize.cfg}. In +particular, it is disabled in the document preamble, because Memoize performs +extraction while it is loaded. + +\medskip +\begin{center} + \begin{tabular}{lll} + \toprule + extraction method&external program&Memoize invocation\\ + \midrule + \refmmz{extract=perl}&\refscript{memoize-extract.pl}&\refcmd{usepackage}\braces{\refpkg{memoize}}\footnotemark\\ + \refmmz{extract=python}&\refscript{memoize-extract.py}&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=python}}\braces{\refpkg{memoize}}\\ + \refmmz{extract=tex}&|pdftex|&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=tex}}\braces{\refpkg{memoize}}\\ + \refmmz{extract=no}&none (external extraction)&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=no}}\braces{\refpkg{memoize}}\\ + \bottomrule + \end{tabular}% + \footnotetext{Or + \refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=perl}}% + \braces{\refpkg{memoize}}. This is useful if you have changed the default + using \reffile{memoize.cfg}.}% +\end{center} +\smallskip + +For internal extraction, \hologo{TeX} must be allowed to execute the external +program implementing the chosen extraction method. Both |memoize-extract| +scripts should be listed among restricted shell escape mode commands in your +\hologo{TeX} distribution; their execution should therefore be allowed under +the default, restricted shell escape mode. However, the |pdftex| program, +executed by extraction method \refmmz{extract=tex}, is not listed there, nor +should it be. If you are forced to use this fallback method, I suggest you +compile documents loading Memoize under the full shell escape mode, by adding +command-line option \docref{-shell-escape} (on \hologo{TeX}Live) or +\docref{--enable-write18} (on MiK\hologo{TeX}) to the invocation of the +\hologo{TeX} program. The answers linked from question +``\href{https://tex.stackexchange.com/q/598818/16819}{How can I enable + shell-escape?}'' on \href{https://tex.stackexchange.com}{\hologo{TeX} + StackExchange} will tell you how you can ask your editor to do this for you. + +You may use any extraction method to perform external extraction. The simplest +option is to use the Perl or the Python script. Supposing you are doing this +manually from the command line, change into the directory containing your +document, which should contain the auxiliary \dmmz file produced by Memoize, +and execute: + +\begin{enumerate}[(a)] +\item \refscript{memoize-extract.pl} \meta{document name}\dmmz \hfill (for + the Perl script) +\item \refscript{memoize-extract.py} \meta{document name}\dmmz \hfill (for + the Python script) +\end{enumerate} + +See sections~\ref{sec:.mmz} and \ref{sec:ref:extraction-perl-python} for +further details on the \dmmz file and the extraction scripts. + +Things are a bit more complicated if you want to use the \hologo{TeX}-based +extraction method externally, because an instance of |pdftex| needs to be +invoked for each extern (and these have unwieldy names and can be many in +number), but Memoize can help you here by producing a shell script or a +makefile, executing which will extract all the externs at once. To have Memoize +produce a shell script, use package option +\code{\refmmz{record}=\refmmz{record=sh}} (or +\code{\refmmz{record}=\refmmz{record=bat}} on Windows); package option +\code{\refmmz{record}=\refmmz{record=makefile}} will make a makefile. By +default, these files are named |memoize-extract.|\meta{document name} plus the +|.sh|, |.bat| or |.makefile| suffix. If neither a shell script nor a makefile +works for you, you can also define your own kind of \emph{record file}, to be +processed by the external tool of your choice (and implementation) in order to +extract the externs; see section~\ref{sec:new-record-file} to learn how to do +this. + + +\subsection{From cross-references to the context} +\label{sec:cross-referencing} + +Cross-referencing presents a challenge to externalization, because without +special provisions, the ``communication channel'' between the \cs{label} and the +\cs{ref} is broken once we start utilizing the extern. + +One direction of the issue occurs when a \refcmd{label} within the memoized +code is referenced by a \cs{ref} on the outside. Without the (built-in) +workaround, the \cs{label} command would only be executed when the extern is +being produced, but not on subsequent compilations of the document, when it is +merely included. Memoize addresses this problem by generalizing +externalization (which can only produce a picture, the extern) to memoization +(which can additionally produce arbitrary code). When Memoize is externalizing +code which contains a \cs{label}, it automatically replicates it into the +\emph{memo}, which is input into the document on subsequent compilations. In +effect, the memo--extern team will continue to produce the label even when it +is utilized rather than compiled. As far as the author is concerned, \cs{label}s +in memoized code ``just work,'' without any observable differences to the +situation without memoization. This is why we will not discuss this direction +of the issue here; a reader interested in how precisely the system works is +invited to read section~\ref{sec:memos}. + +The other direction of the issue occurs when a \refcmd{ref} within the memoized +code references \cs{label} on the outside. In this situation, the extern +should be recompiled when the value of the label it refers to changes. Again, +Memoize addresses this problem in full generality, by associating with each +extern a \emph{context}, and recompiling the extern whenever the value of the +context changes.\footnote{The dependency of an extern upon prior definitions + and such can also be addressed in a more \emph{ad hoc} manner, by recompiling + manually; we have already touched upon this subject in + section~\ref{sec:tut:working-on-a-picture}, and will revisit it in + section~\ref{sec:tut:redefinitions}.} All that needs to be done for \cs{ref} +and friends, specifically, is to advise them to add their reference keys to the +context. + +As we shall see presently, for the author, the only difference between a +non-memoized and a memoized \cs{ref} is that the latter will take one more +compilation cycle to ``stabilize'' the resulting document. (More precisely, +the memoized situation will take one more cycle if the reference is undefined +on the first compilation.) Then, we will show how we can teach Memoize about +cross-referencing commands other than \cs{ref} and \cs{pageref}. Finally, we will +learn about key \refmmz{context}, the backbone of the cross-referencing support +in Memoize. (The inner workings of the context are further explained in +section~\ref{sec:c-memos}.) + +\ExampleName{ref} +\makeexample{\examplename.pdf N=7} + +When the memoized code contains a \cs{ref} referring to a label given in another +part of the document, the code is recompiled when (and only when) the reference +changes. Let us look at the following example, jumping in at the point where +it was already compiled enough times that the resulting PDF had stabilized into +a single (document) page with correct references. (Environment +\refenv{nomemoize} disables memoization +of \href{https://ctan.org/pkg/tikzlings}{Ti\emph{k}Zlings}, so that their +externs don't disturb us, and we can focus on the \cs{tikz} command, which does +get externalized and contains a \cs{ref}.) + +\tcbinputexample[.tex][.c3]{ + after title pre={\ (with stable output after three compilations)}, + middle=1mm, + comment={\centering + \includeexamplepdf[document page]{page=1} + } +} + +Let us add an owl in front of the penguin. In the next compilation, neither +the ``normal'' nor the memoized reference is yet updated, as expected --- in +this compilation, the new value of the penguin label only makes it into the +|.aux| file. + +\tcbinputexample[.tex][.c4]{ + after title pre={\ (after the first compilation with the added owl)}, + no attachment, + comment={\centering + \includeexamplepdf[document page]{page=1} + } +} + +During the following compilation, the \refcmd{ref}s pick up the new value of the +penguin label, and the \cs{ref} inside the automemoized \cs{tikz} command +forces recompilation of the extern (\emph{how} this is done will be explained +later). + +\tcbinputexample[.tex][.c5]{ + after title pre={\ (after the second compilation with the added owl)}, + comment only, no attachment, + comment={\centering + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad + \includeexamplepdf[document page]{page=2} + } +} + +In the next compilation, the resulting PDF is finally stabilized, as the +updated extern is (extracted and) included into the document. + +\tcbinputexample[.tex][.c6]{ + after title pre={\ (after the third compilation with the added owl)}, + comment only, no attachment, + comment={\centering + \includeexamplepdf[document page]{page=1} + } +} + +The message to take home? When some memoized code contains a reference and +that reference changes, it will take three compilation cycles (so, one more +cycle than without memoization) for the resulting document to ``stabilize.'' + +Out of the box, Memoize supports the standard \hologo{LaTeX} cross-referencing +commands \cs{ref} and \refcmd{pageref}. To automatically recompile code +containing some other cross-referencing command, like \refcmd{vref} of package +\pkg{varioref}, we use the advising framework implemented by package Advice. +This framework is a generalization of automemoization: we use the familiar +\refmmz{auto}, but with advice offered by \Emph{\refmmzauto{ref}} rather than +\refmmzauto{memoize}. + +{\ExampleName{vref} +\makeexample{\examplename.tex.c1} +\tcbinputexample{listing only}} + +Key \refmmzauto{ref} only works for commands which operate on a single +reference key. However, that single key (which must be enclosed in braces) may +be preceded by optional argumen(s) of any kind. Extensions to \refcmd{ref}, +e.g.\ the \pkg{hyperref}'s variant, which accepts an optional |*|, work out of +the box. Furthermore, Memoize offers support for cross-referencing commands +which work on multireferences and reference ranges, such as \pkg{cleveref}'s +\refcmd{cref} and \refcmd{crefrange}. Those commands should be advised by +\refmmz{auto} keys \refmmzauto{multiref} and \refmmzauto{refrange}, +respectively. + +We have jumped into first example of this section with the assumption that it +had already been compiled several times, allowing the resulting PDF to +stabilize. Let us now take a look at what happens at the very first, fresh +compilation of our original example (the one without the owl). (Removing the +|.aux| file before compiling the example again will start afresh.) The curious +thing is that we don't get the extern page containing +\tikz[baseline]\node[draw=red,thick,fill=yellow,anchor=base,font=\bf]{??};. +This is so because by default, Memoize aborts a memoization containing an +undefined reference. + +\tcbinputexample[.tex][.c1]{ + after title pre={\ (after the fresh compilation of the original example)}, + comment only, no attachment, + comment={\centering + \includeexamplepdf[document page]{page=2} + } +} + +Now sometimes you might want to produce an extern even if it contains an +undefined reference --- for example, you might intend to write the code +containing the \cs{label} much later but enjoy the speed-up offered by Memoize +until then. In that case, apply the \refmmz{auto} key \Emph{\refmmzauto{force + ref}} to \cs{ref}. + +\tcbinputexample[.tex][.c7]{ + after title pre={\ (after the fresh compilation with \refmmzauto{force ref})}, + attachment name=\examplename-force.tex, + comment={\centering + \includeexamplepdf[extern page][\examplepath.c1.pdf]{page=1,trim=1in 1in 1in 1in}\quad + \includeexamplepdf[document page][\examplepath.c1.pdf]{page=2} + } +} + +However, note that when you use \refmmzauto{force ref}, \hologo{LaTeX} will +\emph{not} complain about the undefined reference once the extern containing it +is included (unless that reference also occurs in some non-memoized piece of +code). Using \refmmzauto{force ref} is therefore a tiny bit dangerous, and +this is why \refmmzauto{ref}, with the abortion mechanism, is the default +handler for \cs{ref} and \refcmd{pageref}. + +As already noted in the previous section, \refcmd{ref} works by appending the +cross-reference to the \emph{context}, the change of which triggers +recompilation. Memoize initializes the context to contain the four +\refmmz{padding}s --- as a result, an extern recompiles if we change the +padding --- but we can append stuff to the context by ourselves, as well. +Below, we use key \Emph{\refmmz{context}} to append the font size (we'll talk +about the value given to this key a bit later); as a result, the picture is +recompiled whenever the font size changes. Below, we change the font size +using command |\small|; changing the default size with a class option such as +|12pt| works as well. + +\ExampleName{fontsize} +\makeexample{\examplename.pdf N=2} +\tcbinputexample{ + sidebyside, lefthand ratio=0.7, after title pre={\ (the first version)}, + comment={ + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} + } +} + +\tcbinputexample[.tex][.c2]{ + sidebyside, lefthand ratio=0.7, after title pre={\ (the second version)}, + no attachment, + comment={ + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} + } +} + +How does this work? Key \refmmz{context} appends the given tokens to the +\emph{context expression}. When creating an extern or trying to use it, +Memoize (fully) expands this expression and computes the MD5 sum of the +expansion. This \emph{context MD5 sum} then serves as a part of the extern's +filename (see sections~\ref{sec:tut:memodir} and~\ref{sec:memos}). In effect, +Memoize will only find (and utilize) the extern if \emph{the context MD5 sum + computed during (attempted) utilization matches the one computed during + memoization}. + +As revealed by looking at the \hologo{LaTeX} source code, \hologo{LaTeX} holds +the current font size in macro |\f@size|, and above, we have effectively added +the contents of this macro to the context. Now, why didn't we simply write +\refmmz{context}|=\f@size|? First, we used |\csname ... \endcsname| because we +were under the normal \hologo{LaTeX} catcode regime, where |@| cannot be a part +of the command name. Of course, we could have temporarily changed the catcode +of |@| using |\makeatletter| and |\makeatother|, but I would advise against +that, because the approach does not work in general: it fails when key +\refmmz{context} is used \emph{within} memoized code (we will explain why in +section~\ref{sec:memos}). Another reason why I recommend the |\csname +... \endcsname| approach is that it does not result in an error when the +control sequence is not defined (|\csname ... \endcsname| will expand to +|\relax| then); this behaviour is handy for undefined cross-references, for +example. Second, why did I write |fsize={...}| around the control sequence? +Well, because I'm being paranoid, really. Writing \refmmz{context}|={\csname + f@size\endcsname}| would work just as well, but I like to explicitly +``announce'' the value to prevent any possibility of a conflict with an +alternative context. Imagine that we don't use the ``announcements'' and we +decide to add some other dimension instead of the font size to the context. +Now if that dimension happened to have the same value as the font size, Memoize +would incorrectly pick up the ``font size extern'' instead of producing a new +one. + +It bears emphasizing that whatever you add to the context expression must be +fully expandable, and also not merely declared as robust. So writing +\code{\refmmz{context}=\cs{ref}\marg{key}}, for example, would be unwise, since +it would not work as intended when package \pkg{hyperref} is loaded. (This +package declares \refcmd{ref} as robust, so it won't expand to the +cross-reference value.) You have to look up where the cross-references are +stored internally; the cross-reference for \meta{key} turns out to be stored in +the internal control sequence |\r@|\meta{key}, so it is |\csname +r@|\meta{key}|\endcsname| that the \refmmzauto{ref} handler actually appends to +the context. + +The padding and font-size contexts are useful quite generally. However, the +context can be pretty command-specific, as well. Consider the \pkg{skak} +package used for typesetting chess games. The board is drawn using command +|\showboard|, but this command has no arguments, because it draws the state of +the board that is reached by the moves given by command |\mainline|. Memoizing +|\showboard| as such will therefore yield the wrong result --- all the boards +will be one and the same board! The solution is to provide the correct +context: we dig into the \pkg{skak} sources and realize that the current +board is stored in macro |\csname chessgame.skak.mainline\endcsname|. + +\ExampleName{skak} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, lefthand ratio=0.48, + comment={ + \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.45}\quad + \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in,scale=0.45} + } +} + +If you remove \code{\refmmz{context}=\bracestt{...}} from the code above, you +will end up with a document where the final board drawn takes place of all the +boards. This is so because in that case, all externs are written into the same +file, so the final extern overwrites the previous ones, but note that you will +only observe this after the second compilation, when the externs are actually +used. + + + +\subsection{More on redefinitions and stale externs} +\label{sec:tut:redefinitions} + +In this subsection, we elaborate on an issue touched upon at the beginning of +section~\ref{sec:tut:working-on-a-picture}: what happens if the memoized code +depends on some macro or style which gets redefined? The answer was +``nothing,'' and one solution was to \refmmz{recompile} the code. Let us take +the example from that section a bit further. We will propose no new solution +or workaround, but deepen our understanding of the issue. + +\ExampleName{redefinitions} +\makeexample{\examplename.pdf N=6} +\tcbset{ + recompile step/.style={ + sidebyside, lefthand ratio=0.74, title=, + comment={\includeexamplepdf[blankest][\examplepath.c\theenumi.pdf]{#1}}, + /utils/exec={\addtocounter{enumi}{1}}, + enhanced, overlay app={\node[at=(frame.north west), font=\scriptsize, circle, fill=black!80!white, text=white, inner sep=1pt]{\theenumi};}, + }, +} +\begin{tcboxedraster}[raster columns=1]{% + title=Working on \texttt{\examplename.tex}, enhanced, breakable} + \setcounter{enumi}{0}% + \tcbox[blankest]{I like red. My emphasized nodes will have red background.} + \tcbinputexample[.tex][.c1]{recompile step={page=2}} + \tcbox[blankest]{Hmm, this particular node is really important, + let me put the text in italics as well!} + \tcbinputexample[.tex][.c2]{recompile step={page=2}, no attachment} + \tcbox[blankest]{You know what? Perhaps yellow background would work better + --- in general.} + \tcbinputexample[.tex][.c3]{recompile step={page=1}, bad, no attachment} + \tcbox[blankest]{How come my node is still red?! Oh yes, I changed the + style, so I have to recompile the extern!} + \tcbinputexample[.tex][.c4]{recompile step={page=2}, no attachment} + \tcbox[blankest]{Ahh, yellow background, that's much better. But you know + what, this double emphasis won't do after all, let me go back to the + upright shape.} + \tcbinputexample[.tex][.c5]{recompile step={page=1}, bad, no attachment} + \tcbox[blankest]{Red???!???!? Ok, I know that recompiling will help, but what + happened here?} + \tcbinputexample[.tex][.c6]{recompile step={page=2}, no attachment} +\end{tcboxedraster} + +What happened is that the externs from steps~1 and~5 share the very same code. +In step~1, this code was compiled when the red |emph| style was in effect, and +that extern lingered and was eventually picked up again in step~5, Memoize +having no idea that it is including an extern produced with the obsolete +definition of the style. + +There are two points to this story. First (and forgetting for a moment about +the context, which we started discussing in +section~\ref{sec:cross-referencing}), Memoize identifies externs (and memos) by +the code that produced them --- or more precisely, by the MD5 sum of the code, +as each piece of code has a unique (well, unique enough) MD5 sum. Each extern +is saved in a file whose name contains this MD5 sum; see +section~\ref{sec:tut:memodir} for illustration. Generally, this is a very +useful feature. You can move your picture to another location in the document, +insert some other (externalized) picture in front of it, and so on, all without +triggering recompilation of the extern(s). (None of this is possible with +\TikZ's externalization library, which identifies the externs by the order in +which they appear in the document.) + +The downside of the MD5 sum approach is the potential pitfall illustrated +above, and the downside comes about because of the second point of the story: +Memoize does not attempt to delete the ``old'' externs. (However, as described +in section~\ref{sec:tut:memodir}, stale memos and externs can be easily removed +using the \refscript{memoize-clean.pl} script.) That would be not only +dangerous (as any deletion inherently is) but also potentially wasteful: what +if you have only temporarily removed some code, or compiled only a portion of +the document --- you surely wouldn't want your hard-won externs to disappear in +such a situation! + +The pitfall described above applies to any command which depends on parameters +which can be set prior to the invocation of the command, like \TikZ pictures, +which depend on the settings given in \cs{tikzset}. After customizing the +settings, you will have to recompile the externs: +\refcmd{mmznext}\braces{\refmmz{recompile}} is useful when you only have to +recompile a single extern; use \refcmd{mmzset}\braces{\refmmz{recompile}}, or +the package option \refmmz{recompile}, to recompile all externs in the +document; and there is also the middle road: if you have changed only Forest's +settings, you can write +\code{\refmmz{auto}=\braces{forest}\braces{\refmmz{recompile}}} to recompile +all and only the Forest trees. + +Above, we have seen the ``same code, same extern'' issue manifested ``through +time,'' i.e.\ Memoize was (incorrectly) reusing externs produced in previous +compilations, but the issue can also manifest ``through space.'' This can +happen if the same code appears twice in the same document --- but, crucially, +with some parameters which it depends on changed from one occurrence to the +next. Observe what happens in the following example, where the settings for +|\progressbar| are changed by |\progressbarchange|. After the first +compilation, everything looks fine. But as both extern pages were produced by +the same code, they will be stored into the same file, the second one +overwriting the first one. The second compilation pulls in the externs --- or +rather, the single extern --- resulting in the document containing the second +progressbar in the place of the first one as well. + +\ExampleName{progressbar} +\makeexample{\examplename.pdf N=2} +\tcbinputexample{ + bad, float, + comment={ + \begin{tcbitemize}[raster columns=3,raster equal height,halign=center,valign=center] + \tcbitem[title=After 1\textsuperscript{st} compilation] + \includeexamplepdf[extern page,remember as=extern1]{page=1,trim=1in 1in 1in 1in}\\ + \includeexamplepdf[extern page,remember as=extern2]{page=2,trim=1in 1in 1in 1in}\\ + \includeexamplepdf[document page]{page=3} + \tcbitem[title=After extern extraction\vphantom{p\textsuperscript{nd}}] + \includeexamplepdf[my boxed title=the extern file,attach shifted boxed title to top right,remember as=externfile]{page=2,trim=1in 1in 1in 1in} + \tcbitem[title=After 2\textsuperscript{nd} compilation] + \includeexamplepdf[document page,remember as=doc][\examplepath.c2.pdf]{page=1} + \end{tcbitemize} + \begin{tikzpicture}[remember picture,overlay] + \draw[->, thick, red] (extern1.east) to[out=east, in=west] ([yshift=0.5ex]externfile.west); + \draw[->, thick, red] (extern2.east) to[out=east, in=west] ([yshift=-0.5ex]externfile.west); + \draw[->, thick, blue] (externfile.east) to[out=east, in=west] (doc.west); + \end{tikzpicture} + }, +} + +The same ``extern duplication'' can arise due to how a particular command is +implemented. Say we deactivate automemoization of Forest trees +(\code{\refmmz{deactivate}=forest}), but keep on automemoizing \TikZ pictures. +Forest uses \refenv{tikzpicture}\noprint{\refenv{forest}} under the hood (a +lot); in particular, the tree itself is typeset as a \env{tikzpicture} +environment. But the code that typesets it is the same for all trees, +regardless of their content (the actual content of the tree is hidden in +various macros and boxes, rather than ``pasted'' into the \env{tikzpicture}). +Consequently, the final tree of the document will overwrite all other trees in +the document, just as the second (and thus final) progress bar overwrote the +first one above.\footnote{That is assuming that \hologo{TeX} doesn't simply + spew a bunch of errors. This can happen as well. In the interest of full + disclosure, compiling a Forest tree in the situation described above would + actually also produce --- but only in the first compilation --- a number of + small empty extern pages, one for each node of the tree. A promise: Forest + will soon fully support Memoize and (among other things) avoid this pitfall. + But the principle will remain.} Ouch! + +Generally speaking, this final sort of extern duplication issue can arise +whenever we have an ``outer'' command that we don't want to (auto)memoize which +uses an ``inner'' command that we \emph{do} want to automemoize. The solution +is to use the \refmmz{auto} key \refmmzauto{nomemoize} on the outer command; +remember that this key disables memoization for the space of the command or +environment. For example, the correct way to ``deactivate'' automemoization of +\env{forest} environments (but keep automemoizing \TikZ pictures) is +\code{\refmmz{auto}=\bracestt{forest}\braces{\refmmzauto{nomemoize}}}. + + + + +\subsection{Supporting Memoize in your package} +\label{sec:tut:package} + +\subsubsection{Loading Memoize?} +\label{sec:loading-memoize} + +So you want to support Memoize in your package? That's great! + +What form precisely this support will take of course depends on your package. +For some commands, a simple \refmmz{auto} declaration will suffice; for other +commands, you might need to write a dedicated \emph{memoization driver}, as +explained in section~\ref{sec:memoization-drivers}. However, one thing is +clear: you \emph{don't} want to require Memoize's presence by +\cs{RequirePackage}\braces{\refpkg{memoize}} in your package. That would +trigger memoization, but triggering memoization should be left at the sole +discretion of the author. The question is, if you're not allowed to load +Memoize, how can you even issue the \refmmz{auto} declaration? + +Well, it's not that you really want to \emph{memoize} anything; you want to +make the commands of your package \emph{memoizable}. So: +\cs{RequirePackage}\braces{\Emph{\refpkg{memoizable}}} --- and note that in +`memoizable', the final `e' of `memoize' is dropped, apparently this is the +correct way to spell it. + +Loading \refpkg{memoizable} does nothing if Memoize is already loaded, and +behaves like package \refpkg{nomemoize} otherwise --- remember from +section~\ref{sec:tut:final} that \refpkg{nomemoize} is a dummy package which +accepts all the commands that Memoize does, but does nothing. + +I have decided to require that Memoize must be loaded before any package that +supports it. Allowing for an arbitrary loading order would complicate the +implementation (and possibly even turn out to be problematic), and furthermore, +Memoize likes to be loaded early anyaway, because it needs to be loaded before +the document PDF is opened if it is to perform the embedded extern extraction. +I don't think the ordering requirement will cause any problems --- let me know +if it does! --- but perhaps it is wise to inform the author about it in the +documentation of your package (I did so at the end of +section~\ref{sec:tut:automemoization}). Anyway, I have enforced the +requirement by raising an error and refusing to load the package in case +Memoize detects \refpkg{memoizable} to be loaded. + +Note that the loading order requirement implies that you can use +|\@ifpackageloaded{memoize}| to specifically react to the presence Memoize, if +necessary. + + +\subsubsection{Memoizable design} +\label{sec:memoizable-design} + +Many commands and environments can be submitted to externalization with a +single-line \refmmz{auto} declaration, as illustrated in +section~\ref{sec:tut:automemoization}, perhaps requiring an addition to the +context (section~\ref{sec:cross-referencing}), or some pre- or post-memoization +code (section~\ref{sec:tut:beamer}). In some situations, however, these simple +approaches won't work. Most often, this will happen when the extern must be +integrated into the document in some special way. For example, a command might +internally create floating material, or surround the core typeset material with +some stretchable space.\footnote{Commands and environments of package + \pkg{tcolorbox} exhibit both these issues (see \pkg{tcolorbox} options |float|, + |before| and |after|), and were in fact the inspiration for several technical + details of Memoize.} None of these behaviours can be replicated by merely +including the extern; with respect to the stretchable space, remember that an +extern, being a picture, has fixed size, so if our extra space ended up in the +extern, it would lose the stretchability. + +The key to successful memoization of problematic commands is their design. In +a nutshell, the idea is to \emph{break up the command's definition into two + parts, the outer command and the inner command, and only submit the inner + command to automemoization}. We will illustrate this with a simple +environment --- |poormansbox| --- which produces a potentially framed box of +the given width, and surrounds this box with some configurable material --- by +default, this material will be stretchable vertical space, and this will be the +source of our memoization problem. (In terms of user experience, the solution +in this section will leave something to be desired, but we will revisit the +example in section~\ref{sec:memoization-complex-single-driver} and make things +right.) + +Let us first take a look at a document using our to-be-developed box +environment. The |poormansbox| environment takes one optional argument, a +keylist of options, which can also be set with the |\pmbset| command. This +being a \textbf{p}oor \textbf{m}an's \textbf{b}ox, it doesn't recognize many +options. One can set the |width| of the box, or request that it occurs in a +|frame|, and the surrounding material can be configured using keys |before| and +|after|. As we will see later in the listing of the package, the box is +|\linewidth| wide by default, has no frame, and is surrounded by vertical glue +|\vskip 2ex plus 1ex minus 1ex| (|2ex| of natural vertical space which may both +stretch and shrink for |1ex|); furthermore, the default value of |before| +contains |\centering| to center the box horizontally (centering is of course +only observable when we change the width of the box). + +\ExampleName{poormansbox} +\makeexample{\examplename.sty} +\makeexample{\examplename.pdf} +\tcbinputexample{ + sidebyside, lefthand ratio=0.482, + comment={\centering + \includeexamplepdf[document page,left=0pt,right=0pt] + {page=1,scale=0.5,trim=0mm 3mm 0mm 3mm} + }, +} + +You might want to play with the example to see that the surrounding vertical +space is indeed stretchable. The example is set up so that the surrounding +space is shrunk a bit to fit all the material onto one page. But if you remove +the final |\lipsum[144]|, the natural amount of all vertical space can be +accommodated on the page, so you should observe an increase of vertical +spacing. + +You might have noticed that the example contains nested |poormansbox|es: the +second box (the one which contains |\lipsum[66]|) is nested within the first +one (between |\lipsum[101]| and |\lipsum[75]|). This is intentional: when we +will revisit the |poormansbox| example in +section~\ref{sec:memoization-complex-single-driver}, the implementation will +have to pay special attention to nesting (which presents no problem to the +implementation in this section). + +As you can see in the package listing below (\texttt{\examplename.sty}), the +implementation of our environment is straightforward. We first define the +configuration command |\pmbset| and the option keys (we're using +\pkg{pgfkeys}), and set the option defaults. Then, we move on to the +environment itself: we apply the given options, execute the pre-code, typeset +the box (which is a |minipage| of the given width, potentially wrapped in a +|\fbox|), and execute the post-code. + +\tcbinputexample[.sty]{listing only, float} + +\ExampleName{poormansbox-memoizable} +Now let's make our |poormansbox| externalizable (\texttt{\examplename.sty}). As +announced above, the idea is to split the definition of the environment into +the outer part (below, the user-level environment |poormansbox|), which +(applies the options and) executes the pre- and the post-code, and the inner +part (below, the macro |\@poormansbox|), which typesets the actual box. If we +then then submit |\@poormansbox|, rather than |poormansbox|, to +automemoization, the outer part will be executed at every compilation (giving +us stretchable space if we request it), while the inner command will be +executed (and memoized) at the first compilation, and substituted for the +extern (the fixed-size box) at subsequent compilations.\cprotect\footnote{You + might have wondered why our definition of the |poormansbox| environment grabs + the body into an argument (|+b|, yielding |#2|), necessitating the use of + |\NewDocumentEnvironment| over the venerable |\newenvironment|. One reason + was that having the environment body as an argument simplifies wrapping the + |\fbox| around the |minipage|, but there is a more important reason. If we + did not grab the environment body, we would have to implement the internal + part of the definition as an environment (|@poormansbox|) as well, and embed + it into the user-level environment using the following idiom: + |\newenvironment{poormansbox}[2][]{...\begin{@poormansbox}}{\end{@poormansbox}...}|. + However, as illustrated in section~\ref{sec:tut:good-to-know}, automemoizing + an environment indirectly embedded in such a way produces an error, because + Memoize is prevented from collecting the environment body.} + +\makeexample{\examplename.sty} +\makeexample{\examplename.pdf}% just to know if it compiles fine +\tcbinputexample[.sty]{listing only, float} + +Looking at the definition of the internal |\@poormansbox| command, it might +strike you as weird that we have equipped this command with an optional +argument (|#1|) it never uses. However, this optional argument is crucial for +memoization. It will become a part of the memoized code (note +\code{\refmmzauto{args}=om} in the \refmmz{auto} declaration) and thereby +ensure that Memoize will produce separate externs for invocations of +|\@poormansbox| with the same environment body but different options; or in +other words, it will ensure that changing the options recompiles the +extern.\cprotect\footnote{Of course, this only holds for options given in the + optional arguments; if the user changes an option value using a prior + \cs{pmbset} (and that option does not occur in the optional argument), + Memoize won't detect the change. But the end-user knows about this issue, as + it was addressed in sections~\ref{sec:tut:working-on-a-picture} + and~\ref{sec:tut:redefinitions}, and she is also aware of two workarounds: + manual recompilation, or setting the context + (section~\ref{sec:cross-referencing}). + + While we're on the subject of the context, note that it is also possible to + deploy context to trigger recompilation of the inner command upon change of + parameters it depends on. We could simply omit the optional argument of + \cs{@poormansbox} and add + \refmmz{context}|={width=\pmb@width,frame=\ifpmb@frame true\else false\fi}|, + to the \refmmz{auto} declaration. The advantage of such an approach is that + Memoize reacts to the change of parameters regardless of whether they are set + using the optional argument or \cs{pmbset}. However, the approach is + unfeasible for commands depending on many parameters: can you imagine listing + all the \TikZ options in the context? Not to mention that a particular + picture usually only depends on a small subset of these options --- by and + large, \TikZ externs would get recompiled too often if the context contained + all \TikZ options.}\footnote{I have toyed with the idea of + splitting (using \pkg{pgfkeys} key filtering) the given options into outer + options, relevant for the outer command, and inner options, relevant for the + inner command, and only passing the inner options to the inner command. The + thought was that would (i) avoid recompiling the extern when only outer + options change, as these options don't affect the inner command, and (ii) + avoid applying the inner options when utilizing the extern, as these options + don't affect the outer command. However, it then hit me that the end-user + might define a style which incorporated both inner and outer options --- I + know I do this with my |tcolorbox|es.} + +The downside to automemoizing an internal command is that this might be +counter-intuitive for the author. For example, to deactivate automemoization +of |poormansbox|, the author will have to write +\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{@poormansbox}} (note the +|\@|), but they will have no clue they have to do this unless they have +carefully read |poormansbox|'s documentation. Even worse, the above +\refcmd{mmzset} command will not work unless surrounded by |\makeatletter| and +|\makeatother|, as it refers to an internal control sequence containing |@|. +Well, Memoize offers \refmmz{auto csname}, \refmmz{activate csname} and +\refmmz{deactivate csname}, so that |@| catagory code manipulations can be +omitted by writing \refcmd{mmzset}\bracestt{\refmmz{deactivate + csname}|=@poormansbox|}, but still. + +Another downside could occur when you use the same (automemoized) internal +command in service of several user interface commands. For the sake of +illustration, assume we have also defined an UI-macro |\pmb| which again relies +on |\@poormansbox|. How is the author to deactivate automemoization of |\pmb| +but leave the |poormansbox| environment intact? This is how: +\refcmd{mmzset}\bracestt{\refmmz{auto}=\cs{pmb}\bracestt{\refmmzauto{args}=m, + \refmmzauto{nomemoize}}}. Again, counter-intuitive; the author expects +\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{pmb}} to work. + +One other consequence of this approach is that the code included in the c-memo +(if \refmmz{include source in cmemo} is in effect) will not faithfully reflect +the source: as shown in the c-memo listing below, it will contain +|\@poormansbox{...}| instead of |\begin{poormansbox}...\end{poormansbox}| --- +even if this might actually count as an advantage, as the discrepancy will at +least inform the author who refuses to read the fine material accompanying our +|poormansbox| that something funky is going on. + +\begingroup +\relaxmmzcommands +\def\mmzNewCMemo#1{% fetch the last c-memo filename + \def\mycmemo{#1}% +} +\input{\examplepath.mmz.c1} +\sed{% + s/[]]\lbrace/]\n\space\space\space\space\space\space\space + \space\space\space\space\space\space\space\lbrace/; +}{\exampledir\mycmemo} +\tcbinputlisting{ + listing only, + listing file=\exampledir\mycmemo, + example title, + title=the c-memo of the last \texttt{poormansbox} environment, + left=0.5em, right=0em, +} +\endgroup + +In a nutshell, automemoizing an internal command might be counter-intuitive for +the author. But the core idea --- to support memoization of a resistant +command by splitting its definition into the outer and the inner command --- is +sound, and we will elaborate on this idea in +section~\ref{sec:memoization-complex-single-driver}, where we will revisit our +|poormansbox| example and develop a variant of this environment which is both +memoizable and user-friendly. + + +\section{Under the hood} +\label{sec:under-the-hood} + +This chapter is written for three audiences: a curious user who wants to +know how Memoize does what it does; a package writer who wants to support +Memoize in a tricky situation; and myself, lest I forget why I made the design +decisions that I made. + +\subsection{The entry point} +\label{sec:Memoize} + + +From the author's perspective, the functionality of this package is entered +either through the manual memoization commands (macro \refcmd{mmz} and +environment \refenv{memoize}), or via automemoization. And while that is +correct, those user interface entry points merely determine what code is +submitted to memoization, and set any options specific to the upcoming +memoization. The real fun starts with command \Emph{\refcmd{Memoize}}, which +is eventually executed by both manual and automatic memoization. + +Not every call to \refcmd{Memoize} results in memoization. Calling this macro +has three possible outcomes. It can result in \emph{memoization}, which +produces the memos and externs; in \emph{utilization} of the result of an +earlier memoization (which boils down to inputting the memos); or in +\emph{regular compilation}, whereby the code is compiled +(almost)\footnote{\label{fn:regular-compilation-almost}This is absolutely true + for memoized code which is ``contained'' in the sense of not peeking into the + input stream following the memoized code. In general, code which fails to + satisfy this containment requirement is most likely simply not memoizable; + but there are borderline cases. For example, \refcmd{ignorespaces} at the + end of some code will have the expected effect in the absence of Memoize, but + no effect when executed either during memoization or regular compilation + under Memoize, simply because it will hit some code belonging to Memoize + rather than the continuation of the document. Memoize offers the + \refmmz{ignore spaces} provision to work around this specific problem.} as if +Memoize was not there. Which outcome obtains depends on several factors. The +decision logic is depicted below, and note that you can \refmmz{trace} the +action on the terminal. + +\begin{center} + \begin{forest} + yes/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,left,#1]{yes}}}, + no/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,right,#1]{no}}}, + use/.style={content=utilization\\(of the cc-memo)\vphantom{dj},fill=orange,align=center}, + memoize/.style={content=memoization\vphantom{dj},fill=green}, + compile/.style={content=regular compilation\vphantom{dj},fill=emphcolor}, + for tree={ + l sep*=1.5, + if n children={0}{}{align=center}, + child anchor=north, + }, + [Is Memoize enabled?\\(\refcmd{ifmemoize}), + [Are we currently undergoing memoization?\\(\refcmd{ifmemoizing}), yes + [,compile, yes] + [Is the\\\refmmz{recompile}\\mode in effect?, no + [,memoize, yes] + [Do the relevant\\memos and externs exist?, no + [,use, yes] + [Is the\\\refmmz{readonly}\\mode in effect?,no + [,compile, yes] + [,memoize, no] + ] + ] + ] + ] + [,compile, no] + ] + \end{forest} +\end{center} + +As the memoization options were already set by the user interface entry points, +you might expect, quite reasonably, that \refcmd{Memoize} takes a single +argument, the code submitted to memoization. After all, what more does it +need? Clearly, executing this code is what produces the typeset material, and +to detect whether the code has ``changed'' (in order to recompile the memos and +externs), we compute the MD5 sum of this very code, don't we? Well, the +reality is a bit more complicated. When it comes to automemoized commands, the +code which the MD5 sum is computed off of (and which is displayed in the c-memo +if \refmmz{include source in cmemo} is in effect) is not exactly the same as +the code we compile (during either memoization or regular compilation). We'll +see what the difference is in section~\ref{sec:tut:automemoization-details}; +what matters here is that we must provide \refcmd{Memoize} with both and that +this macro therefore takes two arguments: the \emph{identification code}, which +the MD5 sum is computed off of, and the \emph{executable code}, which, well, is +the code that gets executed during memoization (or regular compilation). + +Let's illustrate this with an example which is probably entirely useless (but +don't worry, we'll get to a realistic example in +section~\ref{sec:tut:automemoization-details}). We first memoize some text +manually, using command \refcmd{mmz}, and then do something very stupid: we use +this very text as the identification code for the following \refcmd{Memoize}, +even if the executable code of that command is completely different. The +second line of the typeset output should convince you that the first argument +to that command was really used to produce the extern; and one further +compilation should convince you that the first argument was indeed used to +identify the extern: the extern produced by \refcmd{mmz} was overwritten by the +extern produced by \refcmd{Memoize}, in the fashion of the |progressbar| +example from section~\ref{sec:tut:redefinitions}. + +\ExampleName{memoize-internal} +\makeexample{\examplename.pdf N=2} +\tcbinputexample{ + comment={% + \includeexamplepdf[document page, + after title pre={\ (after the first compilation)} + ]{page=3}\hfill + \includeexamplepdf[document page, + after title pre={\ (after the second compilation)} + ][\examplepath.c2.pdf]{page=1} + }, +} + +The example above also illustrates a(nother) peculiar feature of +\refcmd{Memoize}. \refcmd{Memoize} does not open a new \hologo{TeX} group, but +it \emph{expects a group to be opened prior to calling it}, as it will issue an +|\endgroup| at some point. Specifically, the memoization group will be closed +before regular compilation or utilization, but after memoization. If you want +to know why, read the boxed text below. + +\begin{tcolorbox}[title=\string\Memoize\ and grouping, enhanced, breakable] + One important desideratum behind the design of Memoize was that using the + package should disrupt the original, Memoize-less compilation as little as + possible. In particular, if the memoized code contains local assignments + whose effect (in the original compilation) persists into the rest of the + document (until the end of the surrounding \hologo{TeX} group, of course), + wouldn't one want these local effects to persist when Memoize is around, as + well? Fortunately, most memoized code does not have persistent local effects + (at least for me, it is usually environments, like \env{tikzpicture}s, that I + want to memoize, and environments introduce a group anyway) --- fortunately, + because there are design reasons for enclosing memoization in a \hologo{TeX} + group (or two), and this enclosure will of course cancel the effect of local + assignments in the memoized code. + + For one, the user interface memoization commands, such as \refcmd{mmz} and + automemoized commands, allow for options specific to a particular piece of + memoized code (the options given as the optional argument to manual + memoization, the next-options and the auto-options), and to delimit their + effect, it makes most sense to apply them in a group. I have toyed with the + idea of working around the introduction of a group by manually saving and + restoring all the options, but I quickly gave up on this line of thought. + For one, manually saving and restoring the options would be cumbersome and + error-prone, and probably also slower than using the group. But even worse, + all that work would not really solve the problem of the persistence of local + effects, because memoization itself introduces a group, as well: during + memoization, the typeset material is collected into a box, and opening a box + introduces a group. In some particular situations, this could be avoided by + typesetting the memoized code as-is and collecting the resulting material + using |\lastbox|, but this approach cannot work in general. In general, + memoization will take place in a group, so the issue of local effects must be + addressed in some other way. Memoize offers the following workaround: during + memoization, the memoized code can (globally) add code to the \refmmz{after + memoization} hook, which gets executed immediately after closing the + memoization group. + + Does this mean it would be best if the user interface memoization commands + straightforwardly surrounded \refcmd{Memoize} by |\begingroup| and + |\endgroup|? For example, \refcmd{mmz} would open the memoization group, let + \refcmd{Memoize} do its work, and then close the group. Not really. + Remember that memoization is not the only possible outcome of calling + \refcmd{Memoize}. Perhaps we can at least retain the local effects of a + regular compilation, and of utilization? + + We can, by finely tuning the timing of the memoization group closure within + \refcmd{Memoize}. This command is designed to close the memoization group + after memoization, but before regular compilation and utilization. Closing + the group after memoization makes sure that the given options are in effect + during this process. By closing the group prior to regular compilation, + regular compilation of the memoized code (which takes place when Memoize is + disabled, for example) is guaranteed to have (almost, see + footnote~\ref{fn:regular-compilation-almost}) exactly the same effect as the + compilation of that code in absence of Memoize; in particular, the effect of + any local assignments will persist into the rest of the document. Finally, + closing the group before utilization simplifies the construction of the memo + in the cases where we need to replicate local effects of the memoized code + --- the group closed, there is no need to smuggle local assignments out of a + memo. +\end{tcolorbox} + +\subsection{Memos} +\label{sec:memos} + +Up until now, we have pretended that there is a single kind of a memo file. In +truth, there's two kinds: \emph{code memos}, or \emph{c-memos} for short; and +\emph{code--context memos}, or \emph{cc-memos} for short. In this section, we +will learn what they are for, and how they look like --- and also a bit on how +they are produced, even if the details on that will have to wait until +section~\ref{sec:memoization-drivers}. + +We will see that when Memoize utilizes memos, c-memos are processed first. But +conceptually, cc-memos are more important, so we will start the discussion with +these. + +\subsubsection{Cc-memos (and extern inclusion)} +\label{sec:cc-memos} + +When it is input, a cc-memo replicates the effect of the memoized code. This +includes the reproduction of its visual output, which takes the form of +inclusion of any externs produced by memoization. And yes, you got the +implication right: a cc-memo can have any number of associated externs, +including zero, even if the most common case is that of exactly one extern per +cc-memo. The number of externs mostly depends on the memoization driver (see +section~\ref{sec:memoization-drivers}); the default driver always produces +exactly one extern. + +A cc-memo is located in the directory determined by key \refmmz{prefix} +(everything up to the final |/| in the value of this key, or the current +directory if this value contains no |/|). You can recognize it by its +filename, which has the following form (\meta{prefix name} is everything +following the final |/| in the value of \refmmz{prefix}): +\begin{center} + \code{\meta{prefix name}\meta{code md5sum}-\meta{context md5sum}.memo} +\end{center} + +In fact, this is how Memoize recognizes --- or rather, searches for --- a +cc-memo as well: Memoize will utilize a cc-memo when the code and the context +MD5 sum computed during an attempted utilization match the code and the context +MD5 sum computed during some previous memoization (for details on the context +MD5 sum, see section~\ref{sec:c-memos}). In detail, a cc-memo is created at +the end of memoization, at which point Memoize computes the MD5 sum of the +memoized code and the MD5 sum of the context, and writes the results of +memoization into the cc-memo identified by (the prefix and) these two MD5 sums. +And when Memoize, on a subsequent compilation, encounters a piece of memoized +code, it again computes the MD5 sum of that code and the MD5 sum of the +context, and tries to input the cc-memo identified by (the prefix and) these +two MD5 sums. If the inputting is successful, we have utilized the cc-memo +(which in the typical case amounts to including the one associated extern); if +the cc-memo cannot be found, Memoize starts the memoization process, which +creates the memos and the externs. + +Let us take a look at the contents of a cc-memo in detail. Here's a typical +cc-memo (it belongs to the titlepage penguin): + +\begingroup +\relaxmmzcommands +\def\mmzNewCCMemo#1{% fetch the first cc-memo filename + \def\myccmemo{#1}% + \endinput +} +\input{\exampledir titlepage.mmz.c1} +\sed{% + s/\cmd{quitvmode} \cmd{mmzIncludeExtern}/\cmd{quitvmode}\n\cmd{mmzIncludeExtern}/; + s/\(\marg\)\cmd{global}/\1\n\cmd{global}/; +}{\exampledir\myccmemo} +\tcbinputlisting{ + listing only, + listing file=\exampledir\myccmemo, + example title=\myccmemo, +} +\endgroup + +A cc-memo begins by listing the externs which the memo will (actually, might) +attempt to include into the document. When the cc-memo is input, each +\refcmd{mmzResource} command checks if the given extern exists. If some +existence check fails, Memoize enters the memoization mode, same as if the +cc-memo itself did not exist. If all the resources pass the existence check, +Memoize inputs the core of the cc-memo, i.e.\ everything following the +\refcmd{mmzMemo} marker. + +The core might contain arbitrary code, but most often, it will consist of only +two commands. The first one is \refcmd{quitvmode} and it is included if the +extern was \refmmz{capture}d into a horizontal box (which is the usual +situation). The second one is \refcmd{mmzIncludeExtern}, and it is this +command which actually includes the extern into the document upon inputting the +cc-memo. The core code is executed without introducing any groups, i.e.\ the +effect of any local assignments in the cc-memo will persist into the code +following the memoized code. + +Command \refcmd{mmzIncludeExtern} takes nine parameters. The first is the +sequential number of the extern associated with the cc-memo, starting with 0; +usually, this is simply 0 as most memos are associated with a single extern. +The second one is a \refcmd{hbox} or \refcmd{vbox}, noting the type of the box +the memoized code was externalized into. The next three numbers are the +expected width, height and the depth of the extern. Finally, we have the four +\refmmz{padding} amounts (left, bottom, right and top). We should arrive at +the expected size after trimming the extern PDF by the +padding amounts; Memoize will complain if we don't. + +Let's look at a more interesting cc-memo. Using the advising framework, +described in section~\ref{sec:tut:automemoization-details}, Memoize hacks +\refcmd{label} to support \cs{label}s inside memoized code --- the following +code ``just works.'' + +\ExampleName{label} +\makeexample{\examplename.pdf N=3} +\tcbinputexample{ + comment={\centering + \includeexamplepdf[document page, after title pre={\ (compilation 1)}, + left=1.5mm, right=1mm]{page=2}\quad + \includeexamplepdf[document page, after title pre={\ (compilation 2)}, + left=1.5mm, right=1mm][\examplepath.c2.pdf]{page=1}\quad + \includeexamplepdf[document page, after title pre={\ (compilation 3)}, + left=1.5mm, right=1mm][\examplepath.c3.pdf]{page=1} + }, +} + +Everything seems normal --- after the first compilation, we get ``\textbf{??}'' +because the label has not made it into the |.aux| file yet, but in subsequent +compilations, we learn where the penguin lives --- but it is far from normal +under the hood. If we de-hacked \cs{label} by writing +\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{label}}, the third compilation +(and subsequent compilations) would revert to ``\textbf{??}''. Why would that +happen? The memoized code containing the \cs{label}s is only executed in the +first compilation; in the subsequent compilations, we're simply inputting the +cc-memo, so the memoized code, including any \cs{label}s in contains, is not +compiled, and the labels don't get into the |.aux| file anymore. + +The \cs{label} hack deploys Memoize's ability to put arbitrary code into the +cc-memo. During memoization, the memoized code may add arbitrary code to +register \refcmd{mmzCCMemo}, and the contents of this register at the end of +the memoization form the free-form part of the cc-memo.\footnote{This is also + how the above-described code containing \refcmd{mmzIncludeExtern} gets into + the cc-memo. The code is produced by \refcmd{mmzExternalizeBox} and appended + to \refcmd{mmzCCMemo} by the default memoization driver + \refcmd{mmzSingleExternDriver}; see section~\ref{sec:memoization-drivers} for + details.} When the hacked \cs{label} is encountered during memoization, it +appends \refcmd{mmzLabel}\marg{label name}\marg{current label value} to +\refcmd{mmzCCMemo}, so this command winds up in the cc-memo. It is then a +simple job for \refcmd{mmzLabel}, executed when the cc-memo is input at +subsequent compilations, to temporarily store \meta{current label value} (i.e.\ +the contents of \refcmd{@currentlabel} at the time the \cs{label} was invoked) back +into \cs{@currentlabel} and to execute \cs{label}\marg{label name}. In effect, any +\cs{label} command contained within the memoized code is executed at every +compilation, even if the memoized code itself is not compiled. + +\begingroup +\relaxmmzcommands +\def\mmzNewCCMemo#1{% fetch the first cc-memo filename + \def\myccmemo{#1}% + \endinput +} +\input{\examplepath.mmz.c1} +\sed{% + s/~//g; + s/\cmd{quitvmode} \cmd{mmzLabel}/\cmd{quitvmode}\n\cmd{mmzLabel}/; + s/\(\marg\)\cmd{mmzIncludeExtern}/\1\n\cmd{mmzIncludeExtern}/; + s/\(\cmd{mmzLabel}\) *\(\marg\marg\) */~\1\2~\space/g; + s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; +}{\exampledir\myccmemo} +\tcbinputlisting{float, + listing only, right=1.5mm, + listing file=\exampledir\myccmemo, + example title=\myccmemo, +} +\endgroup + +We will continue the discussion of \refcmd{label} in section~\ref{sec:label+} using +a funkier example. + + +\subsubsection{C-memos (and context)} +\label{sec:c-memos} + +As explained in the previous section, a cc-memo belonging to a piece of +memoized code is identified by two MD5 sums: the MD5 sum of the memoized code, +and the MD5 sum of the associated context. However, when Memoize encounters +some code submitted to memoization, the context expression is not yet fully +known, as it may be adjusted by the memoized code itself during memoization --- +and this potential adjustment is crucial for \cs{ref} and friends to work as +advertised (see section~\ref{sec:cross-referencing}). Upon being invoked, +Memoize therefore cannot immediately attempt to input the cc-memo; it needs to +first learn about the context adjustments. Here's where c-memos enter the +picture: \emph{the primary job of a c-memo is to store the context adjustments + made by the memoized code}. Let's see how this works in detail. + +Same as cc-memos, c-memos are located in the directory determined by key +\refmmz{prefix}, and their filenames start by \meta{prefix name} determined by +the same key. However, a c-memo belonging to some memoized code is identified +by the MD5 sum of that code alone: +\begin{center} + \code{\meta{prefix name}\meta{code md5sum}.memo} +\end{center} + +The c-memo is created at the end of the memoization process. At that time, the +context expression is fully known, as the memoized code was already processed. +Even more, Memoize keeps track of both the state of the context expression +prior to memoization, stored in token register \refcmd{mmzContext}, and of the +\emph{additions} to the context expression made by the memoized code, which are +stored in token register \refcmd{mmzContextExtra}. (Incidentally, key +\refmmz{context} automatically adapts to the situation by appending to +\refcmd{mmzContext} outside memoization and to \refcmd{mmzContextExtra} during +memoization.) The complete context expression is the concatenation of the +contents of these two registers, but it is only the context expression +additions, i.e.\ the contents of \refcmd{mmzContextExtra}, which Memoize stores +into the c-memo, with the idea that during subsequent compilations, the initial +context (\refcmd{mmzContext}) will be set up again via ``normal'' compilation, +while inputting the c-memo will restore the additions, jointly reconstructing +the complete context expression associated with a piece of memoized code to +what it was at the end of memoization. + +We can now complete the picture of a utilization attempt started in +section~\ref{sec:cc-memos}. Memoize begins by trying to input the c-memo; this +can be done as the c-memo can be identified based solely on (the MD5 sum of) +the memoized code. If the c-memo does not exist, Memoize starts the +memoization process, which will produce the memos and the externs. But if it +does exist, inputting it reconstructs the context expression to the state at +the end of memoization. Therefore, as the MD5 sum of the \emph{expansion} of +the context expression \emph{at the end of memoization} is baked into the +cc-memo filename, trying to load the cc-memo identified by (the prefix, the +code MD5 sum and) the MD5 sum of the \emph{expansion} of the context expression +\emph{at attempted utilization} will succeed precisely when the context +remained unchanged from memoization to attempted utilization. + +All this might have sounded very complicated, but in the end, most c-memos are +quite boring, the titlepage penguin's c-memo shown below being no exception. A +c-memo starts with the \refcmd{mmzMemo} marker, which is always followed by a +(global) assignment to token register \refcmd{mmzContextExtra}, holding the +context expression additions. As promised, the c-memo below is boring: it +assigns an empty token list to this register, leaving the context expression +as-is. Next comes the free-form part of the memo. Below, it is boringly empty +as well (just the percent sign), but in principle, it will contain any code +gathered in register \refcmd{mmzCMemo} during memoization; see +\ref{sec:per-overlay} for an example. A c-memo is concluded by an optional +part consisting of the \refcmd{mmzSource} marker, followed by the memoized +code. The source code section is not used by Memoize in any way and can be +switched off by \code{\refmmz{include source in cmemo}=false}; it is included +by default so that an interested user can know which code produced which memo, +which can be useful if one wants to trigger recompilation of an extern by +deleting the corresponding memo. Incidentally, any newlines in the source code +are lost in the c-memo replica (unless \refmmz{verbatim} is in effect), but we +will only see this once we arrive at the \pkg{beamer} example below. + +\begingroup +\relaxmmzcommands +\def\mmzNewCMemo#1{% fetch the first c-memo filename + \def\mycmemo{#1}% + \endinput +} +\input{\exampledir titlepage.mmz.c1} +\tcbinputlisting{ + listing only, + listing file=\exampledir\mycmemo, + example title=\mycmemo, +} +\endgroup + +A c-memo of code containing a cross-reference will prove more interesting. The +following c-memo was produced by the |ref| example from section +\ref{sec:cross-referencing}. As we know from that section, when a \refcmd{ref} (as +hacked by Memoize's \refmmzauto{ref} key) occurs in some memoized code, it +appends the cross-reference to the context. In the c-memo below, this is +reflected by the (global) assignment of an expression containing the +cross-reference macro to token register \refcmd{mmzContextExtra}, holding +the context expression additions. + +\begingroup +\relaxmmzcommands +\def\mmzNewCMemo#1{% fetch the first c-memo filename + \def\mycmemo{#1}% + \endinput +} +\input{\exampledir ref.mmz.c1} +\sed{% + s/~//g; + s/\cmd{global}.*\marg/~\0~/; +}{\exampledir\mycmemo} +\tcbinputlisting{ + listing only, + listing file=\exampledir\mycmemo, + example title=\mycmemo, +} +\endgroup + + +\subsubsection{More on \texorpdfstring{\cs{label}}{\textbackslash label}} +\label{sec:label+} + +A \refcmd{label} inside memoized code works out of the box in the usual situation +when label value is fully determined by the memoized code, as in the example in +section~\ref{sec:cc-memos}, where the memoized code contained the outermost +(and only) \env{enumerate} environment. However, the out of the box approach does +not work if the label value is (fully or partially) determined outside the +memoized code. To illustrate the problem, and some potential solutions, we +define two very simple enumeration environments, |listi| and |listii|, which +use counters |counti| and |countii|, and which are intended as the outer and +the inner environment, respectively. Our interest here is in the inner +environment, |listii|. While it prefixes each item by an indented +|\thecountii)|, the label is a composite of both counters: +|\thecounti\thecountii|. The label is stored into \refcmd{@currentlabel}, so +referencing works as usual. However, problems arise when we automemoize the +inner environment. + +\ExampleName{label+} +\makeexample{\examplename.tex.c1 N=6} +\makeexample{\examplename.pdf N=2} +\tcbinputexample{ + sidebyside, lefthand ratio=0.53, + comment={\centering\includeexamplepdf[document page][\examplepath.c2.pdf]{page=1}}, +} + +While the result looks fine at first, changing the order of |listii| +environments, for example by moving ``pets'' below ``domestic'', will result in a +problem: the reference at the bottom will remain unchanged. This is so because +the reference text is baked into the cc-memo, as shown below. + +\begingroup +\relaxmmzcommands +\def\mmzNewCCMemo#1{% fetch the first cc-memo filename + \def\myccmemo{#1}% + \endinput +} +\input{\examplepath.mmz.c1} +\sed{% + s/~//g; + s/\cmd{mmzLabel} \marg\marg/~\0~/; + s/\(\marg\)\(\cmd{mmzIncludeExtern}\)/\1 \2/; + s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; +}{\exampledir\myccmemo} +\tcbinputlisting{ + listing only, + listing file=\exampledir\myccmemo, + example title=\myccmemo, +} +\endgroup + +How can we remedy this? The manual option is to force the recompilation of the +extern by putting an (invisible) reference to the outer item into the inner +item: add |\label{item:pets}| to item ``pets'' and refer to it at ``dog'' by +\refcmd{mmzNoRef}|{item:pets}|.% +\attachexample[\examplename mmzNoRef.tex][\examplepath.tex.c3.attachment] + +An automatic variant of the recompilation solution is to add \cs{@currentlabel} +to the context upon memoizing |listii|. This can be achieved by adding +\refmmz{context}|={@currentlabel={\csuse{@currentlabel}}}| to the \refmmz{auto} +declaration for |listii|. The downside of this approach is that every |listii| +will get reexternalized upon movement, whether it actually contains a label or +not.\attachexample[\examplename context.tex][\examplepath.tex.c4.attachment] + +In fact, given that the externs produced by the inner environment do not +contain the value of the outer counter, it seems wasteful to recompile any +extern just to change the reference. And indeed, it is possible to avoid this, +but the approach unfortunately requires adapting the inner environment code +(and this is why I have not illustrated the problem using an environment of an +elaborate package like \pkg{enumitem}). The idea is to ``unbake'' the +reference to the outer item in the cc-memo. We can achieve this by changing +|listii| to define \cs{@currentlabel} to be |\unexpanded{\thecounti}\thecountii|. +Under this definition, the cc-memo will contain |\mmzLabel +{item:dog}{\thecounti a}|, and rearranging |listii| environments will produce +(upon two compilations, of course) the correct reference without recompiling +the extern. Note again, however, that this solution can only work when the +value of the outer counter does not appear in the extern, i.e.\ it would not +work the ``dog'' item was prefixed by |1a)| rather than simply |a)|. In those +cases, one should deploy one of the other solutions.% +\attachexample[\examplename listii.tex][\examplepath.tex.c5.attachment] + +The final solution, presented below, is an elaboration on the second one. +Rather than append \cs{@currentlabel} to the context immediately upon beginning +to memoize environment |listii|, we will at that point redefine \cs{label} to do +that. In effect, changing the location of |listii| will only recompile it if +it contains a \cs{label}. + +At announced, we redefine \cs{label} once the memoization of |listii| begins, +so within \refmmz{at begin memoization}.\footnote{Another generally good + location for such redefinitions is among the auto-options of |listii|. We + could include an \refmmz{auto}\cs{label}\marg{...} there, or a |/utils/exec| + with \refcmd{AdviceSetup}. However, in this particular case this would be + wasteful, as it would be applied regardless of whether memoization will take + place or not, whereas we only need the redefined \cs{label} when memoizing.} +However, we do not redefine \cs{label} directly, as Memoize advises this +control sequence out of the box (see +section~\ref{sec:tut:automemoization-details} for details). What we redefine +is the command which the advising framework executes instead of \cs{label} --- +its so-called \refmmzauto{outer handler} --- and we do this by calling +\refcmd{AdviceSetup}, the low-level variant of the familiar key +\refmmz{auto}.\footnote{We could have also used \refmmz{auto}, but we don't, + because (a) \refcmd{AdviceSetup} is faster, (b) it is easier to prepend + material to a handler using the low-level interface, and (c) I wanted to + showcase \refcmd{AdviceSetup}.} The first argument of \refcmd{AdviceSetup} +is the installation path (|/mmz|), the second one the command or environment we +are submitting to the framework (\cs{label}), and the third one the setup +\emph{code} --- here lies the biggest difference between \refmmz{auto} and +\refcmd{AdviceSetup}: the former expects a keylist, and the latter \hologo{TeX} +code which directly manipulates settings macros like +\refcmd{AdviceOuterHandler} (for the full list, see +section~\ref{sec:tut:automemoization-details} or~\ref{sec:ref:advice}). + +\tcbinputexample[.tex][.c6]{listing only, float, + attachment name=\examplename auto.tex, +} + +Within \refcmd{AdviceSetup}, we prefix (using macro \cs{preto} of package +\pkg{etoolbox}) the original outer handler \refcmd{AdviceOuterHandler} by code +which causes \cs{outerlabeltocontext} to be executed at the end of memoization +(by globally appending this macro to \refcmd{mmzAtEndMemoizationExtra}; +``Extra'' because we're appending during memoization). It is +\cs{outerlabeltocontext} which then appends \refcmd{@currentlabel} to the +context (\refcmd{mmzContextExtra}; again, ``Extra'' because we're appending +during memoization),\footnote{As a courtesy, we clear out macro + \cs{outerlabeltocontext} once it did it's job, so that multiple \cs{label}s + do not include multiple \cs{@currentlabel}s into the context. But the code + would work even without this addendum, can you see why?} and it is crucial +that this happens at the end of memoization rather than when \cs{label} is +executed. When \cs{label} is executed, we're inside an inner item, and +\cs{@currentlabel} refers to that item, while at the end of memoization, the +value of this macro equals the value at the beginning of memoization, namely +the label of the outer item. While the outer list remains unshuffled, the +value of \refcmd{@currentlabel} that contributes to the context MD5 sum during +the utilization attempt will therefore match the value which contributed to the +context MD5 sum during memoization, resulting in matching MD5 sums and +therefore in actual utilization of the extern; once the outer list is shuffled, +this will cease to be the case and the extern will be recompiled. + + +\subsubsection{The Beamer support explained} +\label{sec:per-overlay} + +The implementation of \refmmz{per overlay}, which makes memoization sensitive +to Beamer overlays, provides an example of a complex interaction between +various components of memoization. At the core, the Beamer support works by +adjusting the context, but we will also have the occasion to observe the +free-form part of the c-memo, add a bit of extra code to the cc-memo, and +deploy several memoization hooks. We will show the complete Beamer support +code later on; let us build our understanding of that code step by step. +(Before you read on, you might want to refresh your memory about the \pkg{beamer} +example from section~\ref{sec:tut:beamer}, as we will refer to it in the +present section.) + +The core idea behind \refmmz{per overlay} is to append the current beamer +overlay number to the context:\cprotect\footnote{As we saw in + section~\ref{sec:tut:beamer}, it is convenient to execute \refmmz{per + overlay} inside memoized code. But remember, from + section~\ref{sec:cross-referencing}, that when \refmmz{context} is executed + from within the memoized code, its argument winds up in the c-memo. As the + c-memo is processed under the normal category code regime, where |@| is not a + letter, we have to access |\beamer@overlaynumber| using the |\csname + ... \endcsname| construct.} \refmmz{context}|={overlay=\csname + beamer@overlaynumber\endcsname}|. This makes Memoize produce a separate +extern for each overlay. However, only the first of these externs will get +utilized on subsequent compilations, in general at least. Even worse, we will +lose each frame whose creation is driven solely by the memoized code. We will +lose the second overlay in our example (i.e.\ in the \pkg{beamer} example from +section~\ref{sec:tut:beamer}) as the second overlay was only created because +Beamer encountered |only={2}{...}| (resolving to |\only<2>{...}| under the +hood) inside the picture code; once we utilize the extern instead of compiling +the picture on the first overlay, the |\only| command is not executed anymore, +so Beamer thinks it is done with the frame. + +As the compilation of our picture is substituted by utilization of its +cc-memos, we have to somehow drive the creation of the necessary overlays from +these files. An easy way to achieve this is to furnish them with a dummy +|\only<|\meta{final overlay number}|>{}|,\footnote{`The final overlay' here + should be understood as relative to our memoized picture, i.e.\ as the final + overlay containing the memoized picture.}\footnote{Actually, putting this + \cs{only} command only into the first cc-memo would suffice, but would be + harder to implement.} but there is a problem: the final overlay number is +unknown when we're memoizing our picture --- it is unknown even when we're +memoizing the picture on final overlay itself (we simply don't know yet that +this overlay will end up being the final one), let alone during the memoization +on the first overlay. + +The solution exploits the fact that \emph{the c-memo is rewritten at each + memoization}: at each memoization of our picture, we store the the +\emph{current} overlay number to the c-memo; after all memoizations, the c-memo +will thus contain the number of the \emph{final} overlay containing our +memoized picture. To access this number from the cc-memo, we store it as a +macro definition, and then use the defined macro, |\mmzBeamerOverlays|, in the +overlay specification of the dummy |\only|. Below, you can see all this in +code, as the argument to \refmmz{at begin memoization}. + +\ExampleName{per-overlay-v1} +\makeexample{\examplename.sty} +\tcbinputexample[.sty]{ + listing only, + title={The implementation of \refmmz{per overlay} (first attempt)}, +} + +A couple of remarks are in order here. First, the definition of +|\mmzBeamerOverlays| in the c-memo is global, because it will be accessed from +the cc-memo, but the cc-memo is input after closing the memoize group (which +the c-memo \emph{is} processed in). Second, using \refmmz{at begin + memoization} makes it possible to use \refmmz{per overlay} both outside and +during memoization: if \refmmz{at begin memoization} is executed outside +memoization, its argument is (locally) stored into hook +\refcmd{mmzAtBeginMemoization}, to be called at the beginning of each +memoization; if the key executed outside memoization, the argument is executed +immediately. Third, the Memoize keys in the definition of \refmmz{per overlay} +are prefixed with \code{\refkeypath{/mmz}/}, so that this key can be called from +\pkg{pgfkeys} option lists of other packages, for example the option list of +the \env{tikzpicture} environment, as shown in the example in +section~\ref{sec:tut:beamer}. + +I used the above version of \refmmz{per overlay} for quite a while. In +general, it worked as I expected, but there were glitches. Occasionally, the +picture would appear on the wrong overlay, or I would get an extra overlay, or +perhaps lose an overlay. Eventually, I figured out this happens when I play +with the overlay structure of the frame: when I add or remove a \cs{pause} or +similar. In hindsight, it is easy to see what was happening. Once the picture +is memoized, it is fixed, forever, which extern will appear on which overlay. +I cannot expect the extern--overlay correlation to change just because I added +a \cs{pause} in front of the picture. Furthermore, the number of overlays the +memos will drive to be created is fixed as well. If I memoize the picture +while it follows a \cs{pause}, and the picture creates 10 overlays, the c-memo +will define |\mmzBeamerOverlays| to 11. So what, if I then remove that +\cs{pause}! The c-memo will still define |\mmzBeamerOverlays| to 11, and drive +the creation of 11 overlays --- one too many. + +By now, the road ahead is probably clear --- we put the |beamerpauses| counter +into the context --- but we will see there are still obstacles on the way. The +issue is that the context is evaluated at the \emph{end} of memoization (so +that those cross-references from section~\ref{sec:cross-referencing} actually +get into it). However, the memoized code might contain a \cs{pause} or similar +itself, and change the value of |beamerpauses|. For one, this means that we +have to write down the changed value of |beamerpauses| into the cc-memo; below, +we do this using key \refmmz{at end memoization} (the code given code to this +key is executed after the driver but before Memoize writes down the memos and +ships out the extern pages; the key itself may be executed either before or +during memoization). Furthermore, if the memoized code changes the value of +|beamerpauses|, the value of |beamerpauses| at the attempted utilization, which +would nicely match the value from the start of memoization, will never match +the changed, final value from the end of memoization, in effect preventing our +hard-won externs from ever getting utilized. + +We therefore have to invent a way to get the memoization-\emph{initial} value +of |beamerpauses| into the context.\footnote{This imposes a requirement on the + in-code usage of \refmmz{per overlay}, namely, that it should be executed + prior to any changes of |beamerpauses|.} Fine, we store it in a +macro,\footnote{We must define \cs{mmzBeamerPauses} globally, because + \refmmz{per overlay} can be arbitrarily deeply embedded in the memoized + code.} |\mmzBeamerPauses|, when we start the memoization (\refmmz{at begin + memoization}), and put |\mmzBeamerPauses| rather than |beamerpauses| into the +context. Will this work? Not yet, because |\mmzBeamerPauses| is undefined at +utilization. We need to set up the context expression so that it will expand +to the value of |\mmzBeamerPauses| at (the end of) memoization, and to the +value of |beamerpauses| at utilization. This leads to the +|pauses=\ifmemoizing...| part of the context expression in (the final version + of) the Beamer support code below. + +\medskip % manual, to avoid an orphan + +\ExampleName{per-overlay} +\makeexcerpt{per-overlay} +\tcbinputexample[.tex][.excerpt]{% + listing only, one file, + left=0pt, % manual indentation gobble + title={The implementation of \refmmz{per overlay}}, +} + +Are we done? Almost. The final issue is that once we have introduced support +for pauses, we have to relativize |\mmzBeamerOverlays| (the final overlay +number) to |beamerpause|. So instead of a simple |\gdef\mmzBeamerOverlays| in +the first version, we define |\mmzSetBeamerOverlays|\nohyphen\marg{beamer + pauses}\marg{final overlay number}, which sets |\mmzBeamerOverlays| only if +\meta{beamer pauses} argument matches the value of |beamerpauses| (at its +invocation in the c-memo). Well, the macro has some other housekeeping to do +as well: it is self-replicating, so that during potential memoization, the +|\mmzBeamerOverlays| values belonging to non-current |beamerpauses| values get +rewritten into the c-memo.\footnote{As replication should only occur during + memoization (actually, it \emph{can} only occur then, anyway), the + instruction to append to the c-memo is appended to the + \cs{mmzAtBeginMemoization} hook (the low-level interface to \refmmz{at begin + memoization}). Note that the assignment to this hook must be local (once + local, always local), and it \emph{can} be local because of a little + implementation detail: while the c-memo is processed in the memoize + \hologo{TeX} group, we don't open an additional group to process it; so the + local effects from c-memo will persist into memoization (but not into + utilization, because remember that the memoize group is closed before + inputting the cc-memo).} (Fine, there is another fine detail, regarding +anti-pollution: the macro also ensures that, relative to \marg{beamer pauses}, +only the instance with the greatest \meta{final overlay number} is replicated.) + +We are now truly done, and we can look at the final result, the c-memo and the +cc-memo belonging to the extern on the first overlay of the example from +section~\ref{sec:tut:beamer}. Specifically, look at the |\mmzSetBeamerOverlays +{1}{2}|, which says that the extern chain started when |beamerpauses| equals 1 +should continue up to overlay 2, and at the (expanded) context included at the +end of the cc-memo, courtesy of \refmmz{include context in ccmemo}, where you +can see that the cc-memo will be used when on the first overlay (|overlay=1|) +when preceded by no \cs{pause} command (|pauses=1|). + +\begingroup +\relaxmmzcommands +\def\mmzNewCMemo#1{% fetch the first c-memo filename + \def\mycmemo{#1}% +} +\def\mmzNewCCMemo#1{% fetch the first cc-memo filename + \def\myccmemo{#1}% + \endinput +} +\input{\exampledir beamer.mmz.c1} +\sed{% + s/~//g; + s/\cmd{mmzSetBeamerOverlays} \marg\marg/~\0~/; +}{\exampledir\mycmemo} +\sed{% + s/~//g; + s/overlay=[0-9]*/~\0~/; + s/pauses=[0-9]*/~\0~/; + s/\(\cmd{quitvmode}\) \(\cmd{only}\)/\1\n\2/; + s/\rbrace\(\cmd{mmzIncludeExtern}\)/\rbrace\percentchar\n\1/; + s/\rbrace\(\cmd{setcounter}\)/\rbrace\percentchar\n\1/; + s/\(\marg\)\cmd{global}/\1\n\cmd{global}/; +}{\exampledir\myccmemo} + +\tcbinputlisting{ + listing only, + listing file=\exampledir\mycmemo, + example title=\mycmemo, +} + +\tcbinputlisting{ + listing only, + listing file=\exampledir\myccmemo, + example title=\myccmemo, +} +\endgroup + + +\subsection{Record files} +\label{sec:record-files} + +We have seen that externalization is a two-step process in Memoize: as it is +impossible for \hologo{TeX} to create multiple PDFs during a single +compilation, the externs are first dumped into the document PDF as special +extern pages, and only later extracted from the main document into separate PDF +files. But extraction requires a complete PDF, which is unavailable even at +the very end of the compilation which produces the externs. The externs can +therefore only be extracted \emph{after} that compilation (either before or at +the beginning of the next one), and this necessitates some form of +communication whereby the memoization step informs the extraction step which +pages should be extracted from the document PDF and into which (PDF) files they +should be stored. This communication is implemented through auxiliary files +called \emph{record files}. + +\subsubsection{The \texttt{.mmz} file} +\label{sec:.mmz} + +By default, Memoize records the information needed for the extraction in a file +named \meta{document name}\dmmz,\footnote{For \hologo{TeX}perts: the + \meta{document name} is of course the expansion of \cs{jobname}.} henceforth +a \dmmz file. In fact, this file contains more than information on externs +created during the last compilation: it records which memos and externs were +either used or created during the compilation. The full information contained +in the \dmmz file is used by the clean-up script \refscript{memoize-clean.pl} +to safely remove stale memos and externs. Let us take a look at the \dmmz file +produced by the titlepage illustration. In fact, we have two versions of this +file, as it changes upon the second compilation. + +\ExampleName{titlepage} +\makeexample{\examplename.pdf N=2} +\sed{% + s/~//g; + s/\cmd{mmzNewExtern}/~\0~/; +}{\examplepath.mmz.c1} +\tcbinputexample[.mmz][.c1]{ + listing only, one file, no attachment, float, + after title pre={\ (after the first compilation)}, + listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar}, +} +\tcbinputexample[.mmz][.c2]{ + listing only, one file, no attachment, float, + after title pre={\ (after subsequent compilations)}, + listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar}, +} + +As you can see, the \dmmz file takes the form of a \hologo{TeX} script (the +format was chosen because it facilitated the implementation of the internally +triggered \hologo{TeX}-based extraction). The crucial lines in this file, and +the only lines used by the extraction script, occur in the first version of the +file: they contain command \refcmd{mmzNewExtern}, which informs the extraction +script that it should extract the document page given by the second argument +into the extern file given by the first argument.\footnote{If you look at the + \dmmz file after extracting the externs using \refscript{memoize-extract.pl} + without the \refscript{memoize-extract.pl--keep} option, you will find that + the \refcmd{mmzNewExtern} commands are commented out; this is to prevent + multiple extractions (even if they are harmless).} (The following two +arguments provide the expected width and height of the extern; the +extraction script may check whether the extern size conforms to these +expectations, but this is not crucial, as the extern size is checked every time +it is included anyway.) + +A \dmmz file also contains a record of the memos (both c-memos and cc-memos) +created in the last compilation; this information is provided by the sole +argument of commands \refcmd{mmzNewCMemo} and \refcmd{mmzNewCCMemo}. And once +memos and externs get used in subsequent compilations, the \dmmz file will +reflect this with \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and +\refcmd{mmzUsedExtern}, as shown in the second version of the file above. + +Finally, both versions illustrate that a \dmmz file always begins with command +\refcmd{mmzPrefix} and ends with the |\endinput| marker. The argument of +\refcmd{mmzPrefix} is the prefix to the memo and extern files, as +determined by the invocation of key \refmmz{prefix}. The initial +\refcmd{mmzPrefix} line is written to the \dmmz file at the beginning of the +document, but an additional \refcmd{mmzPrefix} line will occur for every +invocation of \refmmz{prefix} in the document body. Finally, the |\endinput| +marker signals that the \dmmz file is complete. + +As mentioned above, the full contingent of \dmmz file commands is only used by +the clean-up script \refscript{memoize-clean.pl}. By default, this script +removes all memos and externs with the prefix given by \refcmd{mmzPrefix} +(relative to the directory hosting the \dmmz file) but those listed by any of +\refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo}, \refcmd{mmzNewExtern}, +\refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and \refcmd{mmzUsedExtern}. +Furthermore, the clean-up script will cowardly refuse to delete anything if the +\dmmz file does not end with |\endinput|, as this means that the compilation +ended prematurely and that the \dmmz file might not mention all memos and +externs actually used in the document. If given option +\refscript{memoize-clean.pl--all}, the clean-up script removes even the memos +and externs mentioned in the \dmmz file, and as this option is intended to +bring Memoize to a clean slate after any ``disasters,'' the +\refscript{memoize-clean.pl--all} mode also ignores the potential absence of +the |\endinput| marker. Incidentally, the \refscript{memoize-clean.pl--all} +mode is also the raison d'être for \refcmd{mmzPrefix}: while the prefix is +usually recognizable from \refcmd{mmzNewCMemo} and friends, these commands +might not make it into the \dmmz file in a fatally failed compilation, but it +is precisely such compilations that could occasionally require the full +clean-up. + +\subsubsection{Defining a new record type} +\label{sec:new-record-file} + +The \dmmz file is not the only kind of a record file that can be produced by +Memoize. Out of the box, it can also write down the extraction instructions +into a makefile or a shell script. These are useful on systems which have to +employ the \hologo{TeX}-based extraction but cannot trigger it internally. +Running the \hologo{TeX}-based extraction manually would be painful, as it must +be done on extern-by-extern basis, so Memoize offers to automate the extraction +by a makefile or a shell script; here, the record file is named +\code{memoize-extract.\meta{document name}.\meta{record type}} by default, +where \meta{record type} is either \refmmz{record=makefile}, +\refmmz{record=sh} (for shell scripts on Linux), or \refmmz{record=bat} (for +shell scripts on Windows). + +To turn on recording of an alternate record type, use key +\refmmz{record}|=|\meta{record type}. Memoize can record any number of files +simultaneously, so saying \code{\refmmz{record}=\refmmz{record=sh}} will +produce the shell script alongside \dmmz (Memoize internally executes +\code{\refmmz{record}=\refmmz{record=mmz}} to start recording the \dmmz file); +this should not be a problem, but if you really want to disable the \dmmz file +production, you can say \refmmz{no record}. + +The predefined record types are defined through a generic system open to the +user. To define an additional record type, one needs to define, using +\pkg{pgfkeys}, the relevant hooks of the form \refkeypath{/mmz/record/record + type}|/|\meta{hook}. The following \meta{hook}s can be defined (the hooks +not needed for the record file type may be left undefined): +\begin{itemize} +\item Key \refmmz{record/record type/begin} will be executed at the beginning + of the document; it will receive no argument. Use it to open the record + file. +\item Key \refmmz{record/record type/end} will be executed at the end of the + document; it will receive no argument. Use it to close the record file. +\item Key \refmmz{record/record type/prefix} will be executed at the end of the + document and at every invocation of key \refmmz{prefix} in the document body; + it will receive a single argument, the prefix determined by key + \refmmz{prefix}. +\item Keys \refmmz{record/record type/new cmemo}, \refmmz{record/record + type/used cmemo}, \refmmz{record/record type/new ccmemo} and + \refmmz{record/record type/used ccmemo} will be executed after creating or + inputting a memo; they will receive a single argument, the full path to the + memo. +\item Key \refmmz{record/record type/used extern} will be executed after an + extern was included into the document; it will receive a single argument, the + full path to the extern. +\item Key \refmmz{record/record type/new extern} will be executed after + creating creating an extern, more precisely at the end of memoization, right + after shipping out the extern page. It will receive a single argument, the + full path to the extern, but additionally, Memoize prepares the following + macros: + \begin{itemize} + \item \refcmd{ne:externbasepath} holds the full path to the extern, but (unlike |#1|) + without the |.pdf| suffix; + \item \refcmd{ne:pagenumber} holds the ``physical'' page number of the extern page in + the document (the numbering starts by $1$); + \item \refcmd{ne:expectedwidth} and \refcmd{ne:expectedheight} hold the width and + the height (total height, i.e.\ the sum of \hologo{TeX}'s height and depth) + of the extern page. + \end{itemize} +\end{itemize} + +Below, we present two simple examples of a record file. The first type simply +records the names of all memos and externs used or created by Memoize; the +resulting file could be included by |.gitignore| to have +\hreftt{https://git-scm.com}{git} automatically ignore all files produced by +Memoize. The second type lists the new externs, each preceded by its page +number in the |.pdf|; this file could be fed to a custom extern extraction +tool. + +\ExampleName{record-files} +\makeexample{\examplename.tex.c1} +\tcbinputexample{listing only} + +\ExampleName{record-extern-pages} +\makeexample{\examplename.tex.c1} +\tcbinputexample{listing only} + +Finally, note that (unlike memos and externs) record files are auxiliary files +and may be deleted at any time after the extraction of the externs produced in +the final compilation --- actually, even if these externs were not yet +extracted, deleting the record file(s) will merely force their recompilation. + + +\subsection{The memoization process} +\label{sec:memoization-drivers} + +We now turn to the memoization process itself. The job of memoization is to, +while compiling the given code in a regular fashion, prepare the cc-memo +(which, when it is input, will replicate the effect of the given code), +alongside any externs that the cc-memo will include (these hold the typeset +material to be replicated). Clearly, merely compiling the code cannot have +this effect (unless that code was written specifically to support memoization; +more on this later), and this is why the memoized code is typically wrapped by +a \emph{memoization driver}, which can be set using key \Emph{\refmmz{driver}}. +We'll inspect the default memoization driver, \refcmd{mmzSingleExternDriver}, +in the first subsection, and we will learn how to write specialized drivers in +the remaining subsections. But first, let us say some words about a +grouping-related \hologo{TeX}nical detail we need to take care about during +memoization. + +During memoization, we have to collect certain information, like build the +contents of the cc-memo. Some of that information might be contributed by the +memoized code itself. For example, a \cs{label} ``adds itself'' to the cc-memo +(by appending to token register \refcmd{mmzCCMemo}); a +\refkey{/tikz/remember picture} aborts memoization (by issuing +\refcmd{mmzAbort}); etc. The issue is that the memoized code might open any +number of \hologo{TeX} groups; we have no idea how deeply embedded the +\cs{label} or \refkey{/tikz/remember picture} might be. Therefore, we have +to collect all the information about the ongoing memoization \emph{globally}: +all assignments to \refcmd{mmzCCMemo} must be global; \refcmd{mmzAbort} sets +the underlying conditional globally; etc. (Clearly, all these global variables +are initialized at the start of memoization.) + +This was the easy part. An additional complication arises with some options +which may be set either outside memoization, or during this process. For +example, you can append the font size to the context expression in the preamble +(see section~\ref{sec:cross-referencing}), so that the externs will be +automatically recompiled when the font size changes, and clearly, this context +adjustment should respect \hologo{TeX} grouping; but a \cs{ref} or some other +cross-referencing command in the memoized code needs to append to the context +as well, and as this \cs{ref} occurs \emph{within} the memoized code, the +assignment must be global, as explained above. + +Mixing the local and global assignments to the token register +\refcmd{mmzContext}, which holds the (in the actual implementation, local) +context expression, will not do. For one, we \emph{do} want to restore the +pre-memoization context expression after we have memoized the code, and +furthermore, mixing local and global assignments to the same variable is not +recommended for save stack reasons anyway. + +Memoize addresses this issue by having \emph{two} context registers, +\refcmd{mmzContext} and \refcmd{mmzContextExtra} --- when computing the context +MD5 sum (which happens at the end of memoization), the two registers are +concatenated (the local one comes first). A package writer should know when to +use which register, and how. Outside memoization, one should assign to +\refcmd{mmzContext} --- \emph{locally}. During memoization, one should assign +to \refcmd{mmzContextExtra} --- \emph{globally}. The user interface key +\refmmz{context} respects this requirement automatically: it locally appends to +\refcmd{mmzContext} outside memoization, and it globally appends to +\refcmd{mmzContextExtra} during memoization. (The same idea is applied to the +post-memoization hooks \refmmz{at end memoization} and \refmmz{after + memoization}.) + + +\subsubsection{The default memoization driver} +\label{sec:memoization-driver-default} + +The default memoization driver, \refcmd{mmzSingleExternDriver}, produces +exactly one extern, which contains whatever is typeset by the code submitted to +memoization. The driver compiles the code into a horizontal or vertical box +depending on the value of key \refmmz{capture}. Let us look at the definition +of the driver line by line: + +\makeexcerpt{single-extern-driver} +\tcbinputexample[.tex][.excerpt]{% + listing only, one file, + listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em}, + title=The default memoization driver, + float, +} + +\begin{enumerate} +\item Macro \refcmd{mmzSingleExternDriver} (and in fact any memoization + driver) takes a single argument, the code to compile. Memoize will call the + driver with the code given as the second argument to \refcmd{Memoize}, but + wrapped in a macro which re-reads it using \refcmd{scantokens} when + \refmmz{verbatim} is in effect. +\item If we're capturing into a horizontal box + (\refmmz{capture}|=|\refmmz{capture=hbox}), we put \refcmd{quitvmode} into + the cc-memo --- putting it to the very beginning should make sure that any + replicated \cs{label} and \cs{index} commands refer to the correct page. +\item We compile the given code, storing the typeset material into a box + (above, a temporary box called |\mmz@box|). |\mmz@capture| resolves into a + box construction command, depending on the value \refmmz{capture}. +\item Macro \refcmd{mmzExternalizeBox} instructs Memoize to externalize the box + given as its first argument. However, this macro does not directly produce + an extern page or write any instructions into the cc-memo; the road to this + final destination is indirect. \refcmd{mmzExternalizeBox} has two effects. + First, it adds the contents of the given box (above, |\mmz@box|) to an + internal box dedicated to holding all the externs produced in this + memoization (the contents of |\mmz@box| remain as they are) --- it is only at + the end of memoization that the contents of this internal box are shipped off + to extern pages. Second, \refcmd{mmzExternalizeBox} produces the code which + will include the extern into the document on subsequent compilations (this + will be a call to \refcmd{mmzIncludeExtern}, potentially prefixed by + \refcmd{quitvmode}; see section~\ref{sec:cc-memos} for details). This code + is stored into the token register gives as the second argument (above, + |\mmz@temptoks|), and it is the responsibility of the driver to include it + into the cc-memo. (In the interest of full disclosure, + \refcmd{mmzExternalizeBox} also updates the list of externs produced in this + memoization. At the end of memoization, this list is written to the + beginning of the cc-memo, resulting in the \refcmd{mmzResource} lines + preceding the \refcmd{mmzMemo} marker.) +\item The construction of the cc-memo is indirect as well. In the third line + of the definition, we globally append the extern-inclusion code residing in + |\mmz@temptoks| to token register \refcmd{mmzCCMemo}. At the end of + memoization, the contents of \refcmd{mmzCCMemo} are written into the cc-memo, + preceded by the \refcmd{mmzMemo} marker. +\item We put the typeset material into the document, again preceded by + \refcmd{quitvmode} when capturing in a horizontal box. +\end{enumerate} + +You might wonder why the construction of the extern pages and the cc-memo (and +actually, of the c-memo as well) is indirect, as described above. +\begin{itemize} +\item For one, the indirect construction facilitates potential abortion of + memoization (see section~\ref{sec:tut:good-to-know}). With the indirect route, + aborting is easy --- as nothing was permanently written anywhere yet, Memoize + simply skips the final part of the process, where extern boxes are shipped + into extern pages and the memo registers written into memo files --- and also + clean: if \refcmd{mmzExternalizeBox} immediately shipped out the extern + pages, these pages would remain in the document even in the case of abortion. +\item Even more importantly, the cc-memo filename contains the \meta{context + md5sum} (see section~\ref{sec:cc-memos}), but the context expression is not + yet fully known when memoization starts --- remember (from + section~\ref{sec:cross-referencing}) that a \cs{ref} in the memoized code will + update the context! The cc-memo can therefore only be opened at the end of + memoization, which necessitates a buffer (i.e.\ the \refcmd{mmzCCMemo} + register) for storing its contents during memoization. +\end{itemize} + + +\subsubsection{Pure memoization} +\label{sec:pure-memoization} + +The default memoization driver discussed above is really an externalization +driver: it produces a single extern. We now move to examples of drivers with +other functions, starting with a pure memoization driver, which does not +externalize any typeset output --- simply because it does not call +\refcmd{mmzExternalizeBox} at any point --- but rather remembers the result +of a (pgfmath) computation (let's pretend that the computation is +time-consuming). + +\ExampleName{pgfmathparse} +\makeexample{\examplename.pdf} +\begingroup +\relaxmmzcommands +\def\mmzNewCCMemo#1{% fetch the first cc-memo filename + \def\myccmemo{#1}% + \endinput +} +\input{\examplepath.mmz.c1} +\tcbinputexample{% + comment={% + \tcbinputlisting{ + listing only, width=.4\linewidth, nobeforeafter, + listing file=\exampledir\myccmemo, + example title, title=the cc-memo, + }\quad + \includeexamplepdf[document page]{page=1} + }, +} +\endgroup + +Command \refcmd{mmz} above memoizes its mandatory argument with the memoization +\refmmz{driver} set to the previously defined macro \cs{mmzPgfmathDriver}. +Just as the default driver above, \cs{mmzPgfmathDriver} first executes the +given code. However, there is no need to do this in the context of a +|\setbox|, as the memoized code, which is obviously expected to consist of a +single \cs{pgfmathparse} call, does not typeset anything: \cs{pgfmathparse} +evaluates the given expression and stores the result into macro +\cs{pgfmathresult}. The driver has two jobs: first, it must store this result +into the cc-memo, to be utilized in subsequent compilations; second, because +the assignment to \cs{pgfmathresult} (within \cs{pgfmathparse}) is local, the +driver also needs to somehow smuggle the result out of the \cs{endgroup} issued +by \refcmd{Memoize} and thereby make it into the following code (the final line +of the example, which typesets the equation). Both jobs are easy enough: the +expansion of |\def\noexpand\pgfmathresult{\pgfmathresult}| (in this case, +|\def\pgfmathresult{42.0}|) is (globally) appended both to the token register +\refcmd{mmzCCMemo}, which Memoize later writes into the cc-memo, and to the +macro underlying the \refmmz{after memoization} hook, whose contents are +executed \emph{after} closing the memoization group. + +Let us consider an alternative implementation of the same goal of memoizing the +result of a pgfmath computation, showcasing a couple of useful tricks. + +\ExampleName{pgfmathparse-embellished} +\makeexample{\examplename.pdf} +\tcbinputexample{listing only} + +For one, this ``embellished'' example reminds us that we can list the +\refmmz{driver} key among the auto-options (even if I don't really recommend +automemoizing \cs{pgfmathparse}). But even more importantly, the example shows +that the driver consist of more than a single control sequence; the only +requirement is that the given driver code will consume the memoized code. In +this example, we have developed a generic smuggling driver and applied it to +\cs{pgfmathresult} in particular --- \cs{pgfmathresult} will become the first +argument of \cs{mmzSmuggleOneDriver}, and the memoized code will become its +second argument. + +In the first version of the example, we have appended the same code to macro +\refcmd{mmzCCMemo} and to macro \refcmd{mmzAfterMemoizationExtra} --- no +surprise here, as we want the effect of memoization and utilization to be the +same. In the embellished version, we advertise another way to achieve the same +effect, a way which might be useful for complicated drivers: we simply smuggle +out the entire cc-memo. The idea works even when memoization procudes externs; +in that case, however, the driver also has to say \cs{mmzkeepexternstrue} --- +conditional \refcmd{ifmmzkeepexterns} decides whether Memoize keeps the externs +around, in memory, even after shipping them out (but they are always gone at +the start of the next memoization). + +Finally, remember that the default context expression contains the +\refmmz{padding} values. However, these really have no place in the context +expression of some purely memoized code. We have therefore emptied out the +context expression using \refmmz{clear context}. + + +\subsubsection{Multiple externs per memo} +\label{sec:memoization-multiple-externs} + +In the next example, we show how to produce multiple externs for a single piece +of memoized code. The usage case I find most appealing is breaking the typeset +material, like a table, across pages --- but of course, table-breaking is too +complicated an example, so we illustrate the idea by defining command +|\countdown|, which counts down from the given number, typesetting each number +into its own line. Clearly, if we were to externalize a call to this command +using the default memoization driver, page breaking would stop working, as the +entire countdown would be seen as a single, unbreakable box. To externalize it +properly, the chunks of the countdown that should appear on separate pages must +be externalized into separate externs, as shown below. + +\ExampleName{countdown} +\makeexample{\examplename.sty N=2} +\makeexample{\examplename.pdf} +\tcbinputexample{% + listing options app={ + basicstyle=\ttfamily\footnotesize, + }, + middle=1mm, bottom=0mm, + comment={\centering + \relaxmmzcommands + \def\mmzUsedExtern##1{% + \saveexpandmode\fullexpandarg + \StrGobbleRight{##1}{4}[\mytemp]% + \StrRight{\mytemp}{5}[\mytemp]% + \restoreexpandmode + \relaxmmzcommands + }% + \def\mmzNewExtern##1##2##3##4{\mmzUsedExtern{##1}} + \def\myexternname##1{extern \dots\texttt{\mytemp##1.pdf}}% + \input{\examplepath.mmz}% + \begin{tabular}{ccc} + \includeexamplepdf[extern page,title/.expand once=\myexternname{}] + {page=2,trim=1in 1in 1in 1in,scale=0.333}& + \includeexamplepdf[extern page,title/.expand once=\myexternname{-1}] + {page=3,trim=1in 1in 1in 1in,scale=0.333}& + \includeexamplepdf[extern page,title/.expand once=\myexternname{-2}] + {page=4,trim=1in 1in 1in 1in,scale=0.333}\\ + \includeexamplepdf[document page]{page=1,scale=0.333}& + \includeexamplepdf[document page]{page=5,scale=0.333}& + \includeexamplepdf[document page]{page=6,scale=0.333} + \end{tabular} + }, +} + +To achieve this, we will have to integrate the memoization driver into the very +code of |\countdown|. This approach contrasts sharply with the standard +memoization driver, which is simply wrapped around the memoized code. Let's +say we have implemented a non-memoization-aware variant of |\countdown| as a +loop which gathers the countdown lines into a vertical box, and periodically, +when this box holds all the material that will fit onto the page, places it +into the main vertical list (i.e.\ on the page).\footnote{Of course, + implementing \cs{countdown} this way would be idiotic; a sane implementation + would simply spit out the countdown lines one by one, and let \hologo{TeX} + deal with page-breaking. However, remember that we are pretending that we + are typesetting (and page-breaking) some complex material, like a table; in + such a case, the loop outlined in the main text would make perfect sense.} +To have this command support memoization, we have to externalize our box every +time we're placing it into the main vertical list. This is precisely what we +do in the definition of |\countdowntypeset| below:\footnote{We omit the + definition of the core algorithm of \cs{countdown} in the listing, because it + is mostly irrelevant for our discussion, and only show the + memoization-related code.} the final line of this macro adds the material to +the main vertical list, and the preceding lines externalize it (the two lines +inside \cs{ifmemoizingcountdown} should be familiar from the definition of the +standard memoization driver; we'll explain about the conditional below); note +that Memoize automatically deals with the fact that our box is vertical. As a +result of having our memoization driver integrated into the loop of the core +command, we can create as many externs as necessary, complete with the code in +the cc-memo for including each and every one of them on subsequent +compilations. Each extern eventually makes it into its own extern file, and +note that the filenames of the non-first externs have their sequential number +(we start counting at $0$) appended to the basename, as shown in the +example.\cprotect\footnote{The \refmmz{auto} declaration of |\countdown| adds + some relevant parameters to the context (see + section~\ref{sec:cross-referencing}). The countdown will be recompiled upon + change of either the font size (|f@size|), the text height (|\textheight|), + or the height of the material in the main vertical list collected so far + (|\pagetotal|). The |\pagetotal| parameter is especially important; + including it makes sure that the countdown will be recompiled when it is + pushed up or down the page. Also note that we want the context to record the + value of |\pagetotal| when the (automemoized) |\countdown| is encountered + (rather than at the end of memoization), so we expand it when applying the + auto-options.} + +\tcbinputexample[.sty]{listing only, after title pre={\ (version 1)}} + +Of course, the chunks of the countdown should only be externalized when the +code is actually being memoized, and not, say, when Memoize is disabled or +performing regular compilation. (Note that this is a problem that only affects +integrated drivers and not wrapped drivers such as the default driver.) The +first thought is to detect whether we're undergoing memoization using +conditional \refcmd{ifmemoizing}, which Memoize sets to true at the start of +every memoization. This conditional is used in the run conditions of advice +for \cs{ref} and \cs{label}, the idea being that they should add stuff to the +context (\cs{ref}) and the cc-memo (\cs{label}) only when undergoing +memoization. However, deploying \refcmd{ifmemoizing} in the current example +would not be exactly right. It would work well with the main document as it +is, but it would fail if |\countdown| was called from a piece of code that was +independently submitted to memoization, e.g.\ +\refcmd{mmz}|{\countdown{30}}|.\footnote{Such embedding occurs more often than + you might think. For example, \env{forest} calls \env{tikzpicture} under the + hood, and both environments are automemoized.} In that case, both the +\refcmd{mmz} driver and the |\countdown| integrated driver would get executed, +resulting in the creation (and in subsequent compilations, utilization) of four +externs: first, the |\countdown| driver would externalize each countdown chunk +separately, and then, the \refcmd{mmz} driver would externalize them, all +together, yet again. You can try this out by replacing +\cs{ifmemoizingcountdown} in \cs{countdowntypeset} by \refcmd{ifmemoizing} (and +wrapping the \cs{countdown} call in \cs{mmz}). + +The solution to the \refcmd{ifmemoizing} problem deployed in the example is to +declare a new, \cs{countdown}-specific memoization conditional, and set it to +true in \cs{countdown}'s formal driver, i.e.\ the macro set as the +\refmmz{driver} in the \refmmz{auto} declaration for \cs{countdown}. In fact, +Memoize can do most of this for you: when we write \refmmzauto{integrated + driver}|=countdown|, Memoize creates the countdown-specific memoization +conditional and declares the formal driver which sets this conditional to true; +you only have to access this conditional in your code, and you should do this +using the \hologo{LaTeX}-style conditional \refcmd{IfMemoizing}, as shown +below.\footnote{You shouldn't directly use the plain \hologo{TeX} + countdown-specific conditional created by \refmmzauto{integrated driver} --- + to prevent accidental access, Memoize doesn't actually name it + \cs{ifmemoizingcountdown} --- because this conditional is undefined when + Memoize is not loaded, i.e.\ when only package \refpkg{memoizable} is in + effect. Furthermore, \refcmd{IfMemoizing} addresses a problem faced by + integrated drivers of potentially recursive commands; we will talk about this + in section~\ref{sec:memoization-complex-single-driver}.} + +\tcbinputexample[.sty][.c2]{listing only, after title pre={\ (version 2)}, + attachment name=\examplename-integrated-driver.sty, +} + + +\subsubsection{Driver-based memoizable design} +\label{sec:memoization-complex-single-driver} + +In the previous section, we used the integrated driver approach to produce +memos including multiple externs, but the approach can be useful for one-extern +memos as well, when the extern must be integrated into the document in some +special way. We already discussed such situations in +section~\ref{sec:memoizable-design}, where we suggested to split a +``difficult'' command into the outer command and the inner command, and only +submit the inner command to automemoization. However, the vanilla flavour of +this approach had a negative impact on the user interface to automemoization. +In this section, we will deploy the memoization driver to overcome the issue. + +\ExampleName{poormansbox-driver} +\makeexample{\examplename.sty} +\makeexample{\examplename.pdf N=2} + +Let us revisit the |poormansbox| example from +section~\ref{sec:memoizable-design}. Remember that that environment produced +a potentially framed box of a certain width, surrounded by some pre- and +post-code, and that the issue was that the pre- and the post-code should not be +memoized, but rather executed at every invocation of the command, as it was +primarily intended to put some stretchable vertical space around the box. + +The document source\attachexample\ and the resulting PDF of the example are the +same as in section~\ref{sec:memoizable-design}, so we will not repeat them +here, but jump directly into a revised definition of the environment. We will +retain the core idea from the original implementation: the outer command will +execute the pre- and the post-code, and the inner command will typeset the box. +But unlike in the original implementation, we will not automemoize the inner, +internal command (this was the source of the author's discomfort) but the outer, +user-level command --- and we will equip it with a custom memoization driver. +The major idea here is to have the driver compose a cc-memo which not only +includes the extern, but also executes the outer command. + +\tcbinputexample[.sty]{listing only, float} + +In detail, the implementation (partially shown in the |.sty| listing) is as +follows. The outer command (|\poormansbox@outer|) first applies the options +(|#1|) and then wraps the pre-code (|\pmb@before|) and the post-code +(|\pmb@after|) around some arbitrary code (|#2|). During memoization or +regular compilation, the outer command is invoked through the |poormansbox| +environment, and you can see that in the definition of that environment, the +second argument to |\poormansbox@outer| is a call to the inner command +(|\poormansbox@inner|; this command takes two arguments, the options and the +environment body). During utilization, the outer command is invoked from the +cc-memo,\cprotect\footnote{As you can see, in the cc-memo the outer command is invoked + by |\csuse{poormansbox@outer}|. A straight |\poormansbox@outer| would not + work because we're in the middle of the document where |@| is not a letter, + and including a |\makeatletter| in front of it (and in a group) only works if + \refmmz{direct ccmemo input} was in effect. Under the default, indirect + cc-memo input regime, the core cc-memo is tokenized before |\makeatletter| can + take effect.} and as you can see in the cc-memo listing below, the second +argument to |\poormansbox@outer| there is a call to \refcmd{mmzIncludeExtern}. + +\begingroup +\relaxmmzcommands +\def\mmzNewCCMemo#1{% + \def\mmzNewCCMemo##1{% fetch the second cc-memo filename + \def\myccmemo{##1}% + \endinput + }% +} +\input{\examplepath.mmz.c1} +\sed{% + s/~//g; + s/\(\cmd{csuse} *{poormansbox@outer}\) *\({\nobrace*\marg\nobrace*}\)\({.*}\rbrace\)/~\1~\2~\3~/; + s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; +}{\exampledir\myccmemo} +\tcbinputlisting{ + listing only, left=1.5mm, right=1mm, + listing options app={basicstyle=\tt\small}, + listing file=\exampledir\myccmemo, + example title, title=the second poor man's box's cc-memo, +}% +\endgroup + +And how does |\poormansbox@outer| get into the cc-memo, which normally only +includes a call to \refcmd{mmzIncludeExtern}, you ask? This is the job of the +memoization driver, which is in this case integrated into the inner command. +The overall shape of the driver is the same as the shape of the standard +driver, discussed in section~\ref{sec:memoization-driver-default}: typeset the +extern material into a box, externalize this box, append the extern-inclusion +code to the cc-memo, and put the extern box into the document. It is the +cc-memo part which interests us right now: unlike the standard driver, we don't +simply append the contents of |\mmz@temptoks|, i.e.\ a +\refcmd{mmzIncludeExtern} call; we rather append a call to +|\poormansbox@outer|, which gets the \refcmd{mmzIncludeExtern} call as its +second argument (and the options as its first argument). + +The core part of the driver, which externalizes the box and appends to the +cc-memo, is embedded inside the true branch of conditional +\refcmd{IfMemoizing}|[1]{pmb}|. We already used this conditional in +section~\ref{sec:memoization-multiple-externs}, but without the optional +argument. Such usage will not work here, because it is not recursion-safe. +Unlike in the \cs{countdown} situation, one |poormansbox| environment can be +embedded in another one (and, in our example, it is). If we deployed +\refcmd{IfMemoizing}|{pmb}| in the inner command, the driver would be executed for +\emph{both} the outer and the inner instance of the environment, whereas it +should really be executed only for the outer instance. + +When used in a recursion-safe way, i.e.\ with the optional argument, +\refcmd{IfMemoizing} first tests whether the auxiliary command-specific +conditional from the previous section is true, and then proceeds to compare the +current group level (\hologo{eTeX}'s \refcmd{currentgrouplevel}) to the group +level at the start of memoization (which Memoize stored in +\refcmd{memoizinggrouplevel}). Only if these group levels match do we know +that we're working on the outer instance of the environment, and that we should +therefore execute the memoization driver. Importantly, though, the two group +levels are compared modulo the offset, given as the optional parameter to +\refcmd{IfMemoizing}: in our example, the offset is 1, because the driver is +located inside the |poormansbox| environment, which opens a group --- note that +0 zero (no offset) is not the default optional parameter; the absence of the +optional parameter indicates that the non-recursion safe method should be +used.\footnote{Even this approach is not completely bullet-proof. It will only + work when the inner instance of the command is guaranteed to occur in an + additional group, i.e.\ when our command opens up a group for any free-form + code. I will assume that situations which require externalization of a + potentially recursive command which, for some reason, cannot open the group + before processing a free-form argument, are rare enough to not warrant a + generic solution here.} + + +\FloatBarrier + + +\subsubsection{Shipout} + +Memoize is a hypocrite: when it is creating extern pages, it uses +\refcmd{primitive}\refcmd{shipout} to bypass the regular shipout routine of the +format, but it is offended if anyone else does that. + +Memoize bypasses the regular shipout because the extern pages should really not +be modified or discarded by a foreign package. But using the primitive +\refcmd{shipout} means that extern shipouts can't be detected by another +package, at all. To facilitate peaceful coexistence with a potential package +which needs to know about our extern pages, we offer public counter +\refcmd{mmzExternPages} holding the number of externs shipped out so far. And +if anyone really needs to \emph{do} something at every extern shipout, they can +always (ab)use \refmmz[show keypath]{record/record type/new extern} as a +post-extern-shipout hook. + +The other side of the story is about Memoize needing to know the ``physical'' +page numbers of its externs in the document PDF --- how else are we to extract +them? Memoize computes these page numbers by adding the values of several +counters: \refcmd{mmzRegularPages}, which holds the number of regular shipouts; +the above-mentioned \refcmd{mmzExternPages}, which holds the number of extern +shipouts; and \refcmd{mmzExtraPages}, which holds the number of other shipouts. +The latter counter should be advanced by a package which, like Memoize, +bypasses the regular shipout routine. + +\hologo{LaTeX} and \hologo{ConTeXt} kindly provide the number of regular +shipouts as publicly a accessible counter, so we define +\refcmd{mmzRegularPages} as synonymous with their +\refcmd{ReadonlyShipoutCounter} and \refcmd{realpageno}. In \hologo{plainTeX}, we +have to hijack the \refcmd{shipout} control sequence and count regular shipouts +ourselves; as we have to hijack it while it still refers to the +\refcmd{shipout} primitive, this format provides another reason for preferring +Memoize to be loaded early. + + +\subsection{Automemoization} +\label{sec:tut:automemoization-details} + +Automemoization is a mechanism that automatically memoizes the result of the +compilation of certain commands and environments. Writing Memoize, I went to +great lengths to make it flexible, yet easy to use. This resulted in +automemoization deploying two specifically developed auxiliary packages: +package Advice, which provides a generic framework for extending the +functionality of selected commands and environments, and package CollArgs, +which provides a command for collection of the arguments conforming to the +given (slightly extended) \pkg{xparse} argument specification. + +This section lists the considerations which went into designing the system, +followed by short tutorials on both auxiliary packages, which include several +examples of how Memoize uses the underlying advising framework. + +\paragraph{(De)activation} +Ideally, all commands and environments where memoization makes sense would +support Memoize (or Memoize would support them) and nothing would ever go +wrong. In this dream world, memoization would be completely transparent to the +author. However, things will go wrong, so at the very least, we need to +offer the author a \emph{simple} way to \emph{selectively} switch automemoization +on and off. This is achieved by keys \refmmz{activate} and +\refmmz{deactivate}. + +\paragraph{Submission} +Of course, there will be commands without official support by either Memoize or +the package which defines them; clearly, at the moment when I write this, all +commands but \cs{tikz}, \env{tikzpicture} and \env{forest} are such. Or, the +author might want to automemoize his or her own command. Ideally, submitting a +new command to automemoization would be as simple as +|memoize=|{\meta{command}}, and for environments, this is in fact achievable, +although the actual interface is +\refmmz{auto}|=|\braces{\meta{environment}}\braces{\refmmzauto{memoize}}. But +simply submitting the name cannot work for commands, because commands are where +we encounter the major \hologo{TeX}nical problem with automemoization: we need +to somehow collect the arguments of the command --- without executing the +command itself. + +\paragraph{Argument collection using CollArgs} +\hologo{TeX} being \hologo{TeX}, automatically determining the scope of a +command in general is just plain impossible. Note that inspecting the +|\meaning| is not enough in general, because the ``real'' and the formal +arguments of a command can, and quite often do, differ wildly. The author (or the +package writer) will need to tell Memoize about the argument structure of the +command. And as there is already a nice and general argument specification on +the market --- I'm obviously referring to the argument specification of package +\pkg{xparse}, which was recently even integrated into the core \hologo{LaTeX} +--- why not use that? Memoize comes with an auxiliary package CollArgs, which +(given the slightly extended \pkg{xparse}-style argument specification) +collects the arguments of a command into a single entity. All the user needs +to write to enable automemoization for a command is thus +\refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize}, + \refmmzauto{args}|=|\braces{\meta{argument specification}}}. Even simpler, +when it comes to commands defined by \pkg{xparse}'s \refcmd{NewDocumentCommand} +or friends, writing \refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize}} +will suffice, as the argument specification of these commands can be retrieved +by \refcmd{GetDocumentCommandArgSpec}. + +\paragraph{Weird commands} +Not every argument structure can be described using \pkg{xparse}'s argument +specification, a case in point being \refcmd{tikz} with its totally +idiosyncratic syntax --- and if Memoize won't support \cs{tikz}, why have it at +all? The interface to automemoization must be flexible enough to cover even +the craziest commands, and this is why Memoize allows for arbitrary argument +collectors. These are defined by the advanced user or package writer and then +declared to be used for parsing the argument structure of a command by writing +\code{\refmmz{auto}=\marg{command}\bracestt{...,\refmmzauto{collector}=\meta{argument + collector}}}. + +\paragraph{Over and above automemoization: handlers} +The framework facilitating automemoization must cover more than just that. For +one, it sometimes makes sense to automatically \emph{prevent} memoization +during the execution of certain commands (as in the \refmmzauto{nomemoize} +example in section~\ref{sec:tut:redefinitions}). It follows that the action +performed to an invocation of a command should not be fixed. In the advising +framework, implemented by the auxiliary package Advice, we assign each +advised command a handler --- a command which does the real work of +memoizing or whatever. Crucially, the handler and the collector are +independent of each other, allowing a single memoization handler to handle +commands with both standard and non-standard argument structure, and allowing a +single collector to serve either the memoization or the no-memoization handler. + +\paragraph{Over and above automemoization: the outer and the inner handler} +Second, unlike the memoization handler set by \refmmzauto{memoize}, not all +handlers work with the entire argument list of the advised command. Some +handlers don't care about the arguments of the advised command at all: +\refmmzauto{abort} simply aborts memoization whenever the advised command is +executed. Other handlers are only intended to advise a single command or a +small family of commands, and need to inspect specific arguments of the advised +command: for example, \refmmzauto{ref} needs to append the internal +cross-reference macro to the context, and it of course constructs the name of +this macro from the reference key. For all such handlers, it would be plain +wasteful to first collect the arguments and then tear them apart to inspect +them (or not). The advising framework therefore recognizes two kinds of +handlers. The abortion and the cross-reference handler are examples of an +\emph{outer} handler, which is simply placed in front of the arguments of the +handled command as they are, without invoking the collector. The memoization +handler, on the other hand, is an example of an \emph{inner} handler, which +receives the entire argument list from the collector (as a single argument) --- +or more precisely, even \refmmzauto{memoize} sets up an outer handler, but this +outer handler doesn't do much more than invoke the collector, which in turn +invokes the inner handler. + +\paragraph*{Run conditions} are another, if minor, lego piece of the +advising framework. Using key \refmmzauto{run conditions}, the user can set the +conditions under which the (outer) handler is executed; for example, +cross-reference commands are only advised when memoization is underway. And +the same goes for \refcmd{label}, and for the \refmmzauto{replicate}d +\refcmd{index}, and for \refcmd{savepos}, upon which memoization must be +aborted. The bottom-line is that run conditions repeat across handlers, so +it makes sense to separate them out as an independent component of the +framework, with the added bonus that the system can make sure that an +invocation of a advised command which does not satisfy the run conditions +will incur as litte overhead as possible. + +\paragraph{Bailout handler} +An automemoized command applies the next-options (set by \refcmd{mmznext}), but +what happens when the run conditions are not satisfied? If nothing happened, +the existing next-options might apply to the next instance of +(auto)memoization, which would not be what the author intended. This is why +Advice introduces the bailout handler, a piece of code executed before the +original command when the run conditions are not met. Obviously, the bailout +handler for memoization clears out the next-options (and does not process +them). + +\paragraph{The structure of advice} +Together, the components mentioned above form a piece of \emph{advice}: + +\begin{center} + \begin{forest} + /tikz/edge node/.style={midway, font=\scriptsize, yshift=0.5ex}, + for tree={l sep=5ex, % manual, to avoid a page break + child anchor=north, s sep+=2em}, + [activated? + [\refmmzauto{run conditions}?, + edge=dashed, edge label={node[edge node, left]{yes}} + [\refmmzauto{outer handler}\meta{args}, + edge=->, edge label={node[edge node, left]{yes}}, + [default:, + edge=dashed, + [\refmmzauto{collector}\meta{args}, edge=->, + [\refmmzauto{inner handler} + \textcolor{red}{\bracestt{\textcolor{black}{\meta{args}}}}, + edge=-> + ] + ] + ] + [other:\\whatever\rlap\footnotemark, + align=center, edge=dashed] + ] + [\refmmzauto{bailout handler}, + edge=->, edge label={node[edge node, right]{no}}, + [\meta{original command}\meta{args}, edge=->] + ] + ] + [\meta{original command}\meta{args}, + edge=dashed, edge label={node[edge node, right]{no}}, + ] + ] + \end{forest}% + \footnotetext{The handler may do whatever as long as it consumes all and + only the arguments of the original command.} +\end{center} + + + +\paragraph{Deferred activation} +Memoize needs to be loaded early, but activation should take place late, so +that it can surely override the submitted commands; a case in point, +\pkg{hyperref} redefines \refcmd{ref} very late. To address the issue, the +advising framework implements the deferred \refmmz{activation} regime, under +which (de)activation commands are not executed but collected in a special +style, \refmmz{activate deferred}. Memoize deploys the deferred activation +regime throughout the preamble, and executes \refmmz{activate deferred} at the +latest possible \docref{hook:begindocument} hook; as a bonus, it also offers +the author a way to avoid automatic activation completely by invoking key +\refmmz{manual}. + + +\paragraph{Install anywhere} +Once all this machinery is developed, why not offer it to others as well? + +Once I decided to offer Advice (at the time, still called Auto) as a standalone +package (and I freely admit that the framework got much cleaner once I +separated its code out of Memoize) it became immediately clear that if it is +to serve as a generic framework, it should be possible for multiple packages to +use it without interfering with each other. The package thus allows any number +of installations into different namespaces, each namespace a \pkg{pgfkeys} +keypath. The installation is a breeze: +\cs{pgfkeys}\bracestt{\meta{namespace}/\refkey{/handlers/.install advice}}. + + +\subsubsection{Using package Advice} +\label{sec:advice} + +In this section, we will provide some examples of handler declarations, mainly +based on how Memoize deploys the advising framework. + +\paragraph{\refmmzauto[show keypath]{memoize}} + +In section~\ref{sec:tut:automemoization}, the author was instructed to submit a +command to automemoization by writing +\code{\refmmz{auto}=\meta{command}\bracestt{\refmmzauto{memoize},...}}. The +auto-key \refmmzauto{memoize} is a style (defined by Memoize rather than +Advice) which sets the appropriate components of the automemoization advice. +Residing in keypath \refkeypath{/mmz/auto}, it is effectively defined as +follows: + +\begin{tcblisting}{listing only} +\mmzset{ + auto/memoize/.style={ + run if memoization is possible, + bailout handler=\mmz@auto@bailout, + outer handler=\mmz@auto@outer, + inner handler=\mmz@auto@memoize + } +} +\end{tcblisting} + +The heart of this advice is its inner handler, which actually triggers +memoization by executing \refcmd{Memoize}. Remember that the first argument of +\refcmd{Memoize} is the code which the md5sum is computed off of. This +argument must therefore be identical to the author's invocation of the +automemoized command or environment. Given what Advice offers, this is easy to +construct: \refcmd{AdviceReplaced} holds the code replaced by the advice, and +the \meta{arguments} of the automemoized command are waiting for us in |#1|. +The second argument of \refcmd{Memoize} will be similar, but as this is the +code which will get compiled, we have to execute the original definition of the +command, followed by the (unbraced!) \meta{arguments} as |#1|; this is a job +for \refcmd{AdviceOriginal}. (Note that \emph{executing} +\refcmd{AdviceReplaced} would run the auto-handler again, resulting in an +infinite loop! Or at least a pile of errors.) + +However, the overly simplistic approach shown below won't necessarily work. +The issue is that the arguments of \refcmd{Memoize} contain +\refcmd{AdviceReplaced} and \refcmd{AdviceOriginal} themselves, instead of their +contents, i.e.\ (first) expansions. + +\begin{tcblisting}{listing only, bad} +\long\def\mmz@auto@memoize#1{% + \Memoize{\AdviceReplaced#1}{\AdviceOriginal#1}% +} +\end{tcblisting} + +Regarding the first argument, the problem is that the code md5sum will be +computed off of the token list \refcmd{AdviceReplaced}\meta{arguments} --- +exactly as you see it.\footnote{If you inspected a c-memo, you would find + \refcmd{AdviceReplaced}\meta{arguments} in the \refcmd{mmzSource} section.} +This implies that two commands sharing exactly the same \meta{arguments} will +receive the same (c-)memo. For example, if you automemoized first +|\textit{foo}| and then |\textbf{foo}|, both would come out as a bold ``foo'' +upon utilization. + +The second argument illustrates a general issue about the lifespan of +\refcmd{AdviceOriginal} and other |\Advice...| commands.\footnote{The list of + all commands available only within the handler can be found in the + documentation of key \refmmzauto{outer handler} in + section~\ref{sec:ref:advice}.} By executing \refcmd{Memoize}, we leave the +advice and thereby cannot be sure that when expanded, \refcmd{AdviceOriginal} +will mean what it means at the moment. In general, another piece of advice might be +triggered until its expansion, or the group might be closed, etc. For example, +the author may have issued \refcmd{mmznext}\bracestt{\refmmz{at begin + memoization}=\cs{label}\marg{key}}, and as the pre-memoization code is +executed before the memoization driver, the \cs{label}, which is submitted to +Advice so that any \cs{label}s inside the memoized code ``just work'', would +execute another piece of advice, redefining \refcmd{AdviceOriginal} and friends. +Effectively, you'd end up memoizing an invocation of the \cs{label} command. + +The bottom line is that while the code following the above template +\emph{might} sometimes work, Advice offers no guarantees that it will, so I +advise against using it. The actual definition of the memoization inner +handler is shown below. In this definition, we expand \refcmd{AdviceReplaced} +and \refcmd{AdviceOriginal} --- exactly once! --- into the respective arguments +of \refcmd{Memoize}; of course, as the entire invocation of \refcmd{Memoize} is +expanded, we have to guard against expanding the collected arguments (|#1|) and +\refcmd{Memoize} itself.\footnote{As the final touch, the handler also contains + \refcmd{ignorespaces} after the invocation of \refcmd{Memoize}, if this was + requested using \refmmz{ignore spaces}. Note that this could not be done + without pre-expanding the \cs{ifmmz@ignorespaces} conditional, as + \refcmd{Memoize} closes the group in which the auto- and the next-options are + applied.} The result of the expansion is shown under the code: the first and +the second line assume we are automemoizing command |\foo| and environment +|bar|, respectively. Note that \refcmd{AdviceOriginal} expands into an +invocation of \refcmd{AdviceGetOriginal}, a command which may be safely used +outside the advice; the first argument of this command is the auto-namespace +(in our case, \refkeypath{/mmz}), and the second argument is the advised +command. For a \hologo{LaTeX} environment, the advised command is actually +\refcmd{begin}, and this is why the call of \refcmd{AdviceGetOriginal} is of +course followed by the environment name. + +\makeexcerpt{_auto-memoize-inner} +\tcbinputexample[.tex][.excerpt]{% + listing and comment, one file, no attachment, + title={The implementation of the \refmmzauto{inner handler} + for automemoization}, + comment={% + $\rightarrow$ \refcmd{Memoize}\braces{\cs{foo}\Arg1}% + \braces{\refcmd{AdviceGetOriginal}\bracestt{/mmz}\bracestt{\cs{foo}}\Arg1}\\ + $\rightarrow$ \refcmd{Memoize}\braces{\cs{begin}\bracestt{bar}\Arg1}% + \braces{\refcmd{AdviceGetOriginal}% + \bracestt{/mmz}\bracestt{\cs{begin}}\bracestt{bar}\Arg1} + } +} + +Let us now move backwards in time and look at the outer handler installed by +\refmmzauto{memoize}. It is very simple, but performs an important function of +applying the auto- and the next-options (in this order), which also +necessitates opening a group (closed by \refcmd{Memoize}). The final line +invokes the argument collector, which then calls the inner handler; remember +that the invocation of \refcmd{AdviceCollector} is the sole function of the +default outer handler. + +\makeexcerpt{_auto-memoize-outer} +\tcbinputexample[.tex][.excerpt]{% + listing only, one file, no attachment, + title={The implementation of the \refmmzauto{outer handler} + for automemoization}, +} + +Moving even further back in time, we arrive at the run conditions. The +\refmmzauto{memoize} style invokes \refmmzauto{run if memoization is possible}, +defined as \code{run conditions=\cs{mmz@auto@rc@if@memoization@possible}}, with +the installed macro as shown below. Indeed, memoization only makes when +Memoize is \refmmz{enable}d (which we test using \refcmd{ifmemoize}), but we're +not already ``inside \refcmd{Memoize}'' (which we test using +\refcmd{ifinmemoize}). The latter condition is true when we're either +memoizing or regularly compiling some code submitted to memoization (see the +diagram in section~\ref{sec:Memoize}). Note that it is not necessary to invoke +\refcmd{AdviceRunfalse} in branches where the run conditions are not satisfied. + +\makeexcerpt{_auto-run-if-memoization-is-possible} +\tcbinputexample[.tex][.excerpt]{% + listing only, one file, no attachment, + title={The implementation of + \refmmzauto{run if memoization is possible}}, +} + +While it is clear that double memoization is a no-no, why should we avoid +memoizing inside a regular compilation? Imagine that Memoize decides not to +memoize a Forest tree, perhaps because \refmmz{readonly} is in effect. Under +the hood, Forest creates many \env{tikzpicture}s. Should all of them be +(auto)memoized now? Certainly not. + +Finally, what happens when the run conditions are not met? Not much, but +something important nevertheless: by consuming the next-options, the +\refmmzauto{bailout handler} makes sure they will not erroneously apply to the +next instance of (auto)memoization. + +\makeexcerpt{_auto-memoize-bailout} +\tcbinputexample[.tex][.excerpt]{% + listing only, one file, no attachment, + title={The implementation of the \refmmzauto{bailout handler} + for automemoization}, +} + +The only component of the automemoization advice not determined by style +\refmmzauto{memoize} is the argument collector, which allows the user to submit +a command with a weird argument structure to automemoization simply by setting +key \refmmzauto{collector} in addition to executing \refmmzauto{memoize}. For +example, Memoize submits \refcmd{tikz} to automemoization by loading +\reffile{advice-tikz.code.tex}, which contains Advice's definition of the +\cs{tikz} collector \refcmd{AdviceCollectTikZArguments}, and issuing the +following declaration.\footnote{This is a simplification, see + \refmmzauto{memoize tikz} for the full story.} + +\begin{tcblisting}{ + example title={Declaring automemoization of command \cs{tikz}}, + listing only, + } + auto=\tikz{memoize, collector=\AdviceCollectTikZArguments}, +\end{tcblisting} + +\paragraph{\refmmzauto[show keypath]{ref}} + +The cross-reference advice presents an example of an outer handler radically +different from the default outer handler. This outer handler does not invoke +the collector at all. As shown below, it grabs the argument of \refcmd{ref} (or +whichever cross-referencing command) on its own --- remember that the outer +handler receives the arguments of the handled command ``as they are,'' i.e.\ +uncollected. It then asks \refcmd{mmzNoRef} to do the real job of getting the +reference key into the context, and finally executes the original \cs{ref}. + +\ExampleName{_auto-ref} +\makeexample{\examplename.tex.c1} +\tcbinputexample{% + listing only, no attachment, + title={\hypercolor{link}{white}A simplified\footnotemark{} definition + of \refmmzauto{ref}}, + % This only works if there is a single footnote in the memoized code. + /mmz/context/.expanded={footnote=\numexpr\thefootnote-1}, +}\footnotetext{The real outer handler allows for arbitrary optional arguments of + the cross-referencing command, and shares code with \refmmzauto{force + ref}.} + +The run conditions of this style are agonizingly simple: \refmmzauto{run if + memoizing} sets \refmmzauto{run conditions} to a macro defined as +\refcmd{ifmemoizing}\hyp\refcmd{AdviceRuntrue}\hyp\cs{fi}. + +\paragraph{\refmmzauto[show keypath]{abort}} + +The advice for aborting memoization is very simple --- it merely executes +\refcmd{mmzAbort} --- but also very sneaky. Here, the \refmmzauto{run + conditions} do the real work of aborting memoization, while the ``real,'' +i.e.\ outer handler, never even gets executed; note the absence of +\refcmd{AdviceRuntrue}, which implies \refcmd{AdviceRunfalse}, which triggers +the execution of the original command after the run conditions are ``checked.'' + +\makeexcerpt{_auto-abort} +\tcbinputexample[.tex][.excerpt]{% + listing only, one file, no attachment, + title={The definition of \refmmzauto{abort}}, +} + +The point here is that executing \refcmd{mmzAbort} (itself a single-liner +setting an internal conditional) is cheaper than testing for the real run +conditions (\refmmzauto{run if memoizing}) and aborting only if they are +satisfied. Of course, the trick only works because (i) the advice doesn't +need to inspect any arguments of advised command, and because (ii) setting the +internal abortion conditional outside memoization does no damage. + +\paragraph{Advice in chains} + +A command may be submitted to several instances of the advising framework, +i.e.\ instances installed under different keypaths. In the example below, we +submit |\foo| both to the instance of Advice installed in keypath |/one| and to +the one installed in keypath |/two|. Under |/one|, the result of |\foo{...}| +will be boxed (|\fboxWrap|); under |/two|, in will be parenthesized +(|\parenWrap|). The order in which this happens depends on the order in which +|\foo| was activated under different keypaths. If we first activate it under +|/one| with the boxing effect and then under two with the parenthesizing +effect, the box will appear within parenthesis; if we reverse the activation +order, the parenthesis will appear inside the box. + +\ExampleName{chained-advice} +\makeexample{\examplename.tex.c1 N=2} +\tcbinputexample{comment=\input{\examplepath.tex.c2}} + +First of all, looking at the code above, you have probably noticed the absence +of key \refmmz{auto}. This is because by default, \refkey{/handlers/.install + advice} defines the \emph{setup key} to be |advice| --- Memoize overrides +this default by installing the framework with \refkey{/handlers/.install + advice}|=|\bracestt{\refkey{/advice/install/setup key}=auto, ...}. + +Next, |advice'| is a variant of |advice| which prevents automatic activation +upon setup (and the same holds for |auto'| vs.\ |auto| in Memoize). We have +used the bar variant above to make it clear that it is the order of activation, +rather than declaration by |advice|\slash \refmmz{auto}, which matters in +determining which handler is applied first. + +Finally, note that the deactivation order must be the reverse of the activation +order. So if we activate |\foo| first in |/one| and then in |/two|, we should +deactivate it in |/two| first and in |/one| next, otherwise Advice will +complain. + + +\paragraph{A simple collector} + +Let us implement a collector for a command which accepts one (standard +\hologo{LaTeX}) optional argument and one mandatory argument; in \pkg{xparse} +terms, a command with argument specification |om|. + +Using \cs{NewDocumentCommand}, such a collector is very easy to implement. We +simply define a command with signature |om| and distinguish two possibilities +regarding the presence of the optional argument, which we test using +\cs{IfValueTF}. If the true branch, we pass a \emph{braced} |[#1]{#2}| to the +inner handler, which we invoke by \refcmd{AdviceInnerHandler}; in the false +branch, we omit the optional argument, passing it an (additionally) braced +|{#2}|. + +\ExampleName{om-collector-NewDocumentCommand} +\makeexample{\examplename.tex.c1} +\tcbinputexample{listing only} + +Defining a functionally equivalent collector using \cs{newcommand} would be a +bit more involved, as \hologo{LaTeX2e} does not offer a standardized way to +test for the \emph{presence} of the optional argument. Consider the following +collector, whose optional argument has the same default value as the advised +command. Is it functionally equivalent to the one above? + +\ExampleName{om-collector-newcommand} +\makeexample{\examplename.tex.c1} +\tcbinputexample{listing only} + +While there will be no visual difference, there is a difference under the hood. +If you compile both documents, you will see that the first one creates three +memos\slash externs, while the second one only creates two: |\foo{...}| does +not have its own memo anymore, but creates and uses the same memo as +|\foo[green]{...}|. + +While the second version might sometimes be preferred, perhaps even in the +context of memoization, the initial collector, which deploys command +\refcmd{CollectArguments}, behaves like the \cs{NewDocumentCommand}-defined +collector above,\footnote{In fact, there is a slight difference after all. + While the above-defined collector won't distinguish between the single-token + mandatory argument given with or without braces, \refcmd{CollectArguments} + will again faithfully replicate the original argument tokens.} as it attempts +to perfectly replicate the command invocation. Furthermore, this behaviour +makes it unnecessary for the author to provide the default values of optional +arguments (and even allows them to replace |O{default}| in the argument +specification by \docref{xparse:o}). + +We now turn to the package CollArgs, which implements the actual argument +collection; we'll revisit the initial collector of Advice at the end of the +following subsection. + + + +\subsubsection{Using package CollArgs} +\label{sec:collargs} + +Automemoization is implemented on top of the framework offered by package +Advice, and that package in turn couldn't really work as intended without +package CollArgs. A regular user of Memoize shouldn't need to know anything +about CollArgs, but a package writer wanting to support Memoize might have to. + +The package provides two public commands, \refcmd{CollectArguments} and +\refcmd{CollectArgumentsRaw}; we'll focus on the former first. +\refcmd{CollectArguments} takes three arguments: optional \meta{options} in the +form of a \pkg{pgfkeys} keylist; a mandatory \meta{argument specification} in a +(slightly extended) \pkg{xparse} format; and the \meta{next-code}: +\begin{center} + \refcmd{CollectArguments}\oarg{options}\marg{argument specification}\marg{next-code}% + \textcolor{gray}{\meta{tokens}} +\end{center} +Following the three formal arguments of \refcmd{CollectArguments} are some +\meta{tokens} --- the rest of the document, really --- and the job of +\refcmd{CollectArguments} is to figure out the extent to which these +\meta{tokens} conform to the given \meta{argument specification}. In other +words, \refcmd{CollectArguments} will consume as many of the \meta{tokens} as a +\meta{command} defined by \refcmd{NewDocumentCommand}\meta{command}\marg{argument + specification}\bracestt{...} would. Once these \meta{argument tokens} are +collected, \refcmd{CollectArguments} executes the \meta{next-code} with the +\meta{argument tokens} given as a single, braced argument (clearly, the +\meta{rest} of the \meta{tokens}, i.e.\ the non-consumed tokens, will follow): +\begin{center} + \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}} +\end{center} + +In the example below, we define macro |\PrintAndDo|, which takes two arguments, +a command and the collected arguments of that command, prints out which command +we're about to execute and with what arguments, and then executes that command +with those arguments --- |#1#2| at the end of the definition. Note that |#2| +immediately following |#1| is not braced, so +|\PrintAndDo\makebox{[5em][r]{text}}| executes |\makebox[5em][r]{text}|. + +{\tolerance 1000 Executing |\PrintAndDo\makebox{[5em][r]{text}}| directly would + thus yield the first line of the result below --- and in fact, this is + precisely what gets executed to yield that line, but in a roundabout fashion. + Given the argument specification |oom| (two optional arguments followed by a + mandatory argument), \refcmd{CollectArguments} figures out how many tokens + following its formal arguments conform to this argument specification --- + below, these would be |[5em][r]{text}| following + \refcmd{CollectArguments}\hyp|{oom}{\PrintAndDo\makebox}|--- and puts them, + braced, behind its \meta{next-code} argument, |\PrintAndDo\makebox|, yielding + |\PrintAndDo|\hyp|\makebox{[5em][r]{text}}|.\par} + +\ExampleName{collargs-makebox} +\makeexample{\examplename.tex.c1} +\tcbinputexample{listing and compile} + +Seeing the arguments of |\makebox| without the immediately preceding |\makebox| +might seem strange, but remember that \refcmd{CollectArguments} is about the +arguments of a command, not about the command's control sequence. It doesn't +know or care which command the argument tokens ``belong'' to, as long as they +conform to the given specification. In the example above, it is only in |#1#2| +of |\PrintAndDo| that |\makebox| is ``reunited'' with its arguments, but note +that the reunion is far from obligatory. + +CollArgs supports all the argument types (and modifiers) that \pkg{xparse} +does, including the environment type \docref{xparse:b}, as exemplified below. +Again, the code below might seem strange, as it features an |\end{minipage}| +without the matching |\begin{minipage}|, but the logic is similar as for + commands: just as \refcmd{CollectArguments} occurs in front of the command + arguments, without the command itself, so it occurs in front of the + environment body, without the opening of that body. However, while + \refcmd{CollectArguments} never needs to know the command name, we need to + inform it of the environment name, so that it can find the end of the + environment. This can be achieved as shown below, using key + \refcollargs{environment} in the optional argument of the command, or by our + extension to the \pkg{xparse} argument specification, where the environment + argument type \docref{xparse:b} may be followed by a braced environment name. + In the example below, we could therefore also invoke argument collection by + \refcmd{CollectArguments}\braces + {\docref{xparse:+}\docref{xparse:b}\braces{\env{minipage}}} (we have preceded + \docref{xparse:b} with a \docref{xparse:+} to allow for an environment body + containing paragraph tokens).\footnote{CollArgs automatically adapts to the + format, i.e.\ it knows that environments are tagged by \cs{}\meta{name} and + \cs{end}\meta{name} in \hologo{plainTeX} and by \cs{start}\meta{name} and + \cs{stop}\meta{name} in \hologo{ConTeXt}.} + +\ExampleName{collargs-minipage} +\makeexample{\examplename.tex.c1} +\tcbinputexample{comment=\input{\examplepath.tex.c1}} + +You might wonder why didn't we provide \refcmd{CollectArguments} in the +previous example with argument specification |omb| --- after all, the +\env{minipage} environment takes an optional and a mandatory argument. While that +would work, and produce the same result,\footnote{Unless argument processing + was in effect; see section~\ref{sec:ref:collargs} for details.} note that +\refcmd{CollectArguments} is only interested in finding the scope of the +arguments, and grabbing everything until |\end{minipage}| is the same as first +grabbing the optional argument, maybe, then the mandatory argument, and finally +the argument body. + +\refcmd{CollectArguments} not only supports \pkg{xparse}'s verbatim argument +type \docref{xparse:v}, it can grab an argument of \emph{any} type in the +verbatim mode, triggered by option \refcollargs{verbatim}.\footnote{We refer to + the verbatim mode triggered by \refcollargs{verbatim} as the full verbatim + mode, where all characters are of category ``other''. There is also the + partial verbatim mode, triggered by \refcollargs{verb}, where braces retain + their normal category codes.} We illustrate this key below, where we also use +option \refcollargs{tags}, which makes CollArgs automatically surround the +grabbed environment body with the begin tag \refcmd{begin}\marg{environment + name} and the end tag \cs{end}\marg{environment name}, and use +\refcmd{scantokens} to execute the grabbed environment. Consult +section~\ref{sec:ref:collargs} for the full reference on the verbatim mode and +its limitations. + +% A harmless warning: +% Missing character: There is no ^M (U+000D) in font [lmmono10-regular]:! +% because \texttt in the example is fed a (verbatim) carriage return +\ExampleName{collargs-verbatim} +\makeexample{\examplename.tex.c1} +\tcbinputexample{listing and compile} + +Finally, CollArgs extends the \pkg{xparse} specification by modifier +\docref{xparse:amp}, which allows the user to specify options which apply only +to the following argument, as opposed to the options given as the optional +argument of \refcmd{CollectArguments}, which apply to all the arguments. A +third way to invoke the environment body collection in the above example is +thus \refcmd{CollectArguments}% +\bracestt{\docref{xparse:amp}\bracestt{\refcollargs{environment}=minipage}+b}. + +Both the single-argument and the common options can be given not only as +\pkg{pgfkeys} keys, but also in the raw, ``programmer's interface'' format. +Every option key has a corresponding macro; for example, key +\refcollargs{environment} is matched by macro \refcmd{collargsEnvironment}. +The macros are listed alongside their corresponding keys in the reference +section~\ref{sec:ref:collargs}; here, we merely learn how to use them. + +To use raw options for a single argument, double the ampersand in the argument +specification. Therefore, the fourth way to specify the environment name is +\code{\docref{xparse:amp}\docref{xparse:amp}\bracestt{% + \refcmd{collargsEnvironment}\bracestt{minipage}}\docref{xparse:+}\docref{xparse:b}}. + +To set the raw options for all arguments, use \refcmd{CollectArgumentsRaw}, the +second public command of the package. This command is exactly like +\refcmd{CollectArguments}, excepts that it expects the options in the raw +format and as a \emph{mandatory} argument: + +\begin{center} + \refcmd{CollectArgumentsRaw}\marg{raw options}\marg{argument specification}% + \marg{next-code}\textcolor{gray}{\meta{tokens}} +\end{center} + +This leads us to the fifth way to set the environment name (an overkill, I +know): \refcmd{CollectArgumentsRaw}\hyp +\bracestt{\refcmd{collargsEnvironment}\bracestt{minipage}}\bracestt{+b}\marg{next-code}. +Furthermore, you can use a mixture of raw and key--value options: the raw +option commands include \refcmd{collargsSet}, which applies the given option +keylist. The idea here (incarnated by both Auto and Memoize) is that the +package will provide CollArgs with the raw options, for speed, while the +author can supplement them in the friendly keylist format --- and this leads +us to the sixth, and thankfully final way to set the environment name: +\refcmd{CollectArgumentsRaw}\bracestt{\refcmd{collargsSet}% + \bracestt{\refcollargs{environment}=minipage}}\bracestt{+b}\marg{next-code}. + + +\paragraph{The initial collector} + +As the final example, let us study Advice's initial collector; this is a macro +which is used as the collector when key \refmmzauto{collector} is not given. +This macro is not really \refcmd{CollectArguments}, as we sometimes state to +simplify matters, but a macro which acts as the ``bridge'' between Advice and +CollArgs, by compiling an invocation of \refcmd{CollectArgumentsRaw} from the +given advice setup, and executing it. + +The bridge macro is shown below in its full glory, but it is really less +complicated than it might appear at first sight. In line~2, we use +\refcmd{AdviceIfArgs} to see whether the argument structure of the handled +command was given by the user. If it wasn't, we assume that the handled +command was defined using \cs{NewDocumentCommand} or similar, and use +\refcmd{GetDocumentCommandArgSpec} to retrieve it (line~3; note that +\refcmd{AdviceName} holds the handled control sequence) and store it into +\refcmd{AdviceArgs} (line~4), which also receives the argument specification +when given by the user via key \refmmzauto{args}. + +\makeexcerpt{_advice-CollectArgumentsRaw}(../../advice.edtx) +\tcbinputexample[.tex][.excerpt]{% + listing only, one file, no attachment, + listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em}, + title={The definition of the initial collector}, +} + +Lines~6--17 are somewhat of an expansion mess, because we have to construct the +invocation of the CollArgs' collector from the advice setup stored in various +macros. But once we think away all the (non-)expansion commands, we're left +with \refcmd{CollectArgumentsRaw} plus the following three arguments: +\begin{enumerate} +\item The raw options (lines 8--11): + \begin{enumerate} + \item In line 8, the advised command's control sequence is designated as the + \refcollargs{caller}. The effect is that if the given arguments don't + conform to the specification, the error thrown seems to come from the + advised command rather than some internal CollArgs macro. The author will + be grateful for this little detail. + \item In line 9, we add any \refmmzauto{raw collector options} set by Advice + (plus the package deploying Advice, like Memoize); user-given options are of + course possible, but not really expected here, because: + \item In lines 10--11, we add the user-given \refmmzauto{collector options}, + if there are any, embedded under \refcmd{collargsSet}. + \end{enumerate} +\item The argument specification (line 14). +\item The \refmmzauto{inner handler} (line 15). +\end{enumerate} + + + +\section{Reference} +\label{sec:reference} + +In this section, the extra information about keys, offered at the right of a +key in parenthesis, may contain the initial value of a key, and also a default +value of a key. In this context, the terms ``initial'' and ``default'' have +the meaning employed by the \pkg{pgfkeys} utility (\PGFmanual{87}). The term +``initial value'' applies to the setting underlying a key (when there is no +such setting, the key is marked as a ``style''), and refers to the value of +this setting that is set by the package. In other contexts, we would call this +the default or the package-default value, but in the \pkg{pgfkeys} parlance, +the term ``default value'' applies to a key taking an argument, and refers to +the value that is passed to the key in the absence of that argument. +(Honestly, I only keep to this convention in the reference section; elsewhere, +I often say ``default'' or ``package-default'' and mean the initial value.) + +Another convention I keep to in this section is the color-coding of the keys +and commands. Green background indicates a basic key or command, which any +user might want to know about. Red background indicates other, more or less +advanced keys and commands. + + +\subsection{Loading and initialization} +\label{sec:ref:loading} +\paragraph{\hologo{LaTeX}} + +Load Memoize by \cs{usepackage}\braces{\docaux{package}{memoize}} or +\cs{usepackage}\oarg{options}\bracestt{memoize}. The latter form functions +almost identically to the former followed by +\refcmd{mmzset}\bracestt{\meta{options}}, with two exceptions. + +First, when used as package options, \meta{options} may not contain the slash +character (|/|), which is necessary to invoke \pkg{pgfkeys} handlers, because +is it misinterpreted by \hologo{LaTeX}.\footnote{The historic \hologo{LaTeX} + constraint prohibiting spaces in package options does not apply anymore.} In +the rare situations requiring key handling in the package options, use option +\refmmz{options}. + +Second, key \refmmz{extract} and other extraction-related keys such as +\refmmz{perl extraction options} should normally be used as a package option, +or within \reffile{memoize.cfg}. Because Memoize extracts the externs while it +is being loaded, executing these keys after the package is loaded will have a +different effect; see the documentation of \refmmz{extract} for details. + +Memoize extracts the externs while it is being loaded (and not, say, at the +beginning of the document) because extern extraction can only be performed +before the output PDF is opened, and some packages cause it to be opened when +they are loaded. This also implies that Memoize must be loaded before any such +package. In particular, it must be loaded before \PGF library |fadings| (see +section~\ref{sec:fadings-problem}) and before the \pkg{beamer} class (see +section~\ref{sec:tut:beamer}). + +If you're familiar with the \TikZ externalization library, you might wonder +whether Memoize has an equivalent of the \refcmd{tikzexternalize} command. It +doesn't. Memoize assumes that if you loaded it, you want to use it --- but you +can always disable it using key \refmmz{disable}, or even by loading package +\refpkg{nomemoize} in its stead. + +In \hologo{LaTeX}, initialization and finalization are completely automatic. +Memoize defines several initialization and finalization styles --- +\docaux{key}{begindocument/before}, \docaux{key}{begindocument}, +\docaux{key}{begindocument/end} and \docaux{key}{enddocument/afterlastpage} +---and executes them at the cognominal \hologo{LaTeX} hooks. + +\paragraph{\hologo{plainTeX}}{} + +Load Memoize by |\input memoize|. As package options cannot be provided in +\hologo{plainTeX}, the author must trigger extraction from \refcmd{mmzset} +using key \refmmz{extract}; I recommend doing this immediately after loading +the package. This key may be invoked with or without a value. In the latter +case, Memoize will extract using the package default method +\refmmz{extract=perl}, unless its has been overriden from +\reffile{memoize.cfg}. + +Furthermore, as \hologo{plainTeX} has no concept of a document body, the text +must be manually enclosed in \refcmd{mmzset}\bracestt{\docaux{key}{begin + document}} and \refcmd{mmzset}\bracestt{\docaux{key}{end document}}; this +is where the initialization and finalization hooks described above will be +executed. Note that \refmmz{extract}, when used, must precede this enclosure. + +Finally, \hologo{plainTeX} has another reason for preferring the early loading +of the package. In this format, Memoize must redefine \refcmd{shipout}, at a +time the meaning of this control sequence is still primitive. In particular, +this means that Memoize must be loaded before \pkg{atbegshi}. + +\begin{tcboxedraster}[raster column skip=1cm]{blankest} + \ExampleName{plainTeX}% + \makeexample{\examplename.tex.c1}% + \tcbinputexample{listing only, title=A minimal \hologo{plainTeX} example} + % + \ExampleName{ConTeXt}% + \makeexample{\examplename.tex.c1}% + \tcbinputexample{listing only, title=A minimal \hologo{ConTeXt} example} +\end{tcboxedraster} + +\paragraph{\hologo{ConTeXt}}{} + +Load Memoize by \cs{usemodule}\bracketstt{memoize} or +\cs{usemodule}\bracketstt{memoize}\oarg{options}. Unlike in \hologo{LaTeX}, +there are no restrictions on characters allowed within \meta{options}; the +remarks on key \refmmz{extract} and loading order are the same as for +\hologo{LaTeX}. + +In \hologo{LaTeX}, Memoize automatically executes its initialization and +finalization code when at the beginning and the end of the document body. Due +to my very limited experience with \hologo{ConTeXt}, and its project structure +in particular, I don't know what the appropriate place for initialization and +finalization would be in \hologo{ConTeXt}. I therefore provisionally leave it +to the author to execute \refcmd{mmzset}\bracestt{\refmmz{begin document}} and +\refcmd{mmzset}\bracestt{\refmmz{end document}} manually, and hope for advice +on how to handle this properly. + + +\paragraph{Auxiliary packages} + +\begin{doc}{package={name=nomemoize}} + Loading this package instead of Memoize completely disables memoization, but + does not require removing any Memoize commands from the document (they all + become no-ops). This package accepts any package options (and ignores them). +\end{doc} + +\begin{doc}{package={name=memoizable}} + This package is a programmer's stub: if Memoize is loaded, it does nothing; + otherwise, it provides the no-op variants Memoize commands. This package + accepts no package options. Generic packages, e.g.\ \TikZ libraries, can + also load this package via |\input memoizable.code.tex|. See + section~\ref{sec:loading-memoize} for details. +\end{doc} + + +\subsection{Configuration} +\label{sec:ref:configuration} + + +\begin{doc}{easy,cmd={name=mmzset, par=\marg{options}}} + + Update the Memoize configuration. + + The \meta{options} are a comma-separated list of \meta{key}|=|\meta{value} + pairs. They are processed using the \pkg{pgfkeys} utility of PGF/\TikZ (see + \PGFmanual{87}), with the default path set to \docaux{key path}{mmz}. + + The changes are local to the current \hologo{TeX} group, except for keys + where explicitly noted otherwise. +\end{doc} + +\begin{doc}{easy, + cmd={name=mmznext, par=\marg{options}}, + } + + This command accepts the same \meta{options} as \refcmd{mmzset}, but + interprets them as \emph{next-options} --- options which will be applied to + the next, and only the next, \emph{auto}memoized command or environment. + (Remember that a command or environment is submitted to automemoization by + \refmmz{auto}|=|\marg{command or + environment}\bracestt{\refmmzauto{memoize},...}; see + sections~\ref{sec:tut:automemoization}, \ref{sec:tut:working-on-a-picture} + and \ref{sec:ref:advicememoization} for details.) + + Remarks for the author: + + \begin{itemize} + \item Key \refmmz{enable} has no effect inside \refcmd{mmznext}. + \item If \refcmd{mmznext} is used more than once preceding an automemoized + command, only the final invocation takes effect. + \item The next-options also apply to commands and environments for which + memoization is autodisabled via \refmmz{auto}|=|\marg{command or + environment}\bracestt{\refmmzauto{nomemoize},...}. + \item It is safe to set the next-options in front of a command submitted to + automemoization which does not actually undergo memoization in this + particular instance. In other words, the absence of memoization will not + cause the next-options to ``leak'' to the next automemoized command. + \item Only the linear (execution) order of \refcmd{mmznext} and the + automemoized command matters. Key \refcmd{mmznext} will correctly apply to + a single following automemoized command even if it occurs outside the group + which that command is executed from; and it will apply to the following + automemoized command even if it is called within a group closed before that + command is executed. + \end{itemize} + + Remarks for the programmer: + \begin{itemize} + \item The next-options are set globally. + \item The effect of \refcmd{mmznext} is \emph{not} cumulative. Consequently, + \refcmd{mmznext}\braces{} clears the next-options. + \item The next-options are applied by executing \refcmd{mmzAutoInit} within + the advice. Any piece of advice applying the next options should also clear them + when the \refmmzauto{run conditions} are not met. This is streamlined by + style \refmmzauto{apply options}, intended for use within \refmmz{auto} + declarations. Out of the box, this style is deployed by + \refmmzauto{memoize}, \refmmzauto{nomemoize} and \refmmzauto{noop}, but it + may be used by any piece of advice. Note that the \refmmzauto{outer handler} + declared by this style opens a group (to apply the options in) but leaves + it to the (undeclared) \refmmzauto{inner handler} to close that group. + \item Key \refmmz{enable} has no effect inside \refcmd{mmznext} because when + Memoize is disabled upon encountering an automemoized command, the advice + bails out without ever applying the next-options. More generally, this + applies to any advised command whose run conditions require + \refcmd{ifmemoize} to be true. Key \refmmz{disable}, on the other hand, + takes effect, because \refcmd{ifmemoize} is checked within \refcmd{Memoize} + as well. + \end{itemize} +\end{doc} + +\begin{doc}{easy,file={name=memoize.cfg, desc=file}} + The configuration file, loaded just before processing the package options. + It will typically contain a \refcmd{mmzset} command, but it may contain any + \hologo{TeX} code. + + As for any other file loaded by \hologo{TeX}, the location of the file + determines whether it applies system-wide, user-wide or directory-wide. + + This file is also loaded by package \refpkg{nomemoize}, on the off chance it + defines some commands other than \refkeypath{/mmz} keys. (It is not loaded + by package \refpkg{memoizable}, though). +\end{doc} + +\begin{doc}{key={name=options, par=\marg{options}}} + Execute \meta{options} as if they were given an an argument to \refcmd{mmzset}. + + This option primarily exists to allow any key--value pair accepted by + \refcmd{mmzset} to be used as a package option. In particular, this applies + key handlers (see \PGFmanual{87}), because their invocation includes a slash + (|/|). For example, one cannot directly use + \begin{center} + \code{\cs{usepackage}[\refmmz{perl extraction + options}/.prefix=\braces{\braces{\refscript{memoize-extract.pl--quiet}}} ]\braces{memoize}} + \end{center} + to add (prepend) option \refscript{memoize-extract.pl--quiet} to the + invocation of the perl extraction script \refscript{memoize-extract.pl}, but + this will work: + \begin{center} + \code{\cs{usepackage}[\refmmz{options}=\braces{\refmmz{perl extraction + options}/.prefix=\braces{\braces{\refscript{memoize-extract.pl--quiet}}} ]\braces{memoize}}} + \end{center} +\end{doc} + +\begin{doc}{cmd={name=nommzkeys, par=\marg{key}}} + In package \refpkg{memoize}, this command is a no-op; in packages + \refpkg{nomemoize} and \refpkg{memoizable}, it defines key + \refkeypath{/mmz}|/|\meta{key} as a no-op. + + As explained in section~\ref{sec:tut:final}, use this command to declare any + \refkeypath{/mmz} keys you have used outside \refcmd{mmzset} when switching + to package \refpkg{nomemoize}. +\end{doc} + + +\subsection{Memoization} +\label{sec:ref:memoization} + +\subsubsection{Manual memoization commands} +\label{sec:ref:memoization:manual} + +\begin{doc}{easy, + cmd={name=mmz, par=\oarg{options}\marg{code}}, + env={name=memoize, par=\oarg{options}}, + } + Submit \meta{code} or \meta{environment body} to memoization. + + Prior to memoization, the configuration is locally updated by executing + \meta{options} given as the optional argument to this command, i.e.\ the + given options take precedence to options previously set via \refcmd{mmzset}. + Note that next-options, set by \refcmd{mmznext}, are not applied. + + The effect of the macro and of the environment version of this command is the + same, except that the command version memoizes \meta{code} exactly as-is, + while the environment version trims away any spaces at the beginning and the + end of the code.\footnote{What actually happens is that at the beginning of + the environment body, all space tokens will be discarded. At the end of + the body, no spaces are actually discarded; Memoize simply issues an + \refcmd{unskip}. This should not matter to a regular user who simply + writes down the environment.} The space-trimming feature of the environment + ensures that you can write |\begin{memoize}| and |\end{memoize}| in separate + lines (as shown above), but no extra space will creep into the extern. + + The space-trimming feature of the environment, which trims spaces at the + beginning and at the end of the \meta{environment body}, should not be + confused with the effect of \refmmz{ignore spaces}, which ignores spaces + following the environment end-tag (in \hologo{LaTeX}, |\end{...}|) --- and + which does not apply to manual memoization at all! + + The argument of \refcmd{mmz} \emph{must} be enclosed in braces. +\end{doc} + +\begin{doc}{easy, + cmd={name=nommz, par=\oarg{options}\marg{code}}, + env={name=nomemoize, par=\oarg{options}}, + } + Disable Memoize for the span of the compilation of \meta{code} or + \meta{environment body}. + + This command consumes the \meta{options} in the same way as + \refcmd{mmz}\slash \refenv{memoize} described above. The macro and the + environment version of the command exhibit the same space-trimming behaviour + as their \refcmd{mmz}\slash \refenv{memoize} counterparts, and the argument + of \refcmd{nommz} \emph{must} be enclosed in braces. +\end{doc} + + +\subsubsection{Basic configuration} +\label{sec:ref:memoization:basic} + +\begin{doc}{easy, + key={name=enable}, + key={name=disable}, + } + Enable or disable the functionality of the package. + + What happens when Memoize is enabled depends on the memoization mode + (\refmmz{normal}, \refmmz{readonly}, \refmmz{recompile}) and many other + factors. When the package is disabled, it neither creates new memos and + externs, nor uses the existing ones; this applies to both manual and + automatic memoization. The effect is close to not having Memoize loaded, or + to loading NoMemoize, but it is not completely the same; for example, the + record file (\dmmz) \emph{is} updated while Memoize is disabled, reflecting + the fact that nothing was memoized (or utilized) in the disabled + state.\footnote{Cleaning the folder (\S\ref{sec:cleanup-scripts}) after + disabling the package for the entire document is thus a bad idea.} + + If these keys are used in the preamble, their effect is delayed until the + beginning of the document, to ensure that Memoize is never enabled in the + preamble. Other than that, all these keys do is set the \hologo{TeX} + conditional \refcmd{ifmemoize}, which you may use in your code to test + whether Memoize is enabled. You may also use \refcmd{memoizetrue} and + \refcmd{memoizefalse}, as long as you never enable the package in the + preamble. + + Key \refmmz{enable} cannot be applied to automemoized commands via + \refcmd{mmznext}. It will take effect for manual memoization, though, and + key \refmmz{disable} will work for both, as expected. +\end{doc} + +\begin{doc}{easy, + key={name=normal, desc=the initial mode}, + key={name=readonly}, + key={name=recompile}, + } + + Select the memoization mode. + + Each piece of code submitted to (either manual or automatic) memoization is + associated to several files: one c-memo, one cc-memo, and some externs (zero + or more, typically one). When Memoize encounters a piece of code submitted + to memoization, it takes one of the following actions: + + \begin{description} + \item[memoization] The code is compiled in a special way which produces the + associated memos and externs. + \item[utilization] The code is not compiled. Its effect is replicated by + processing the cc-memo; typically, this includes the single extern into the + document. + \item[regular compilation] The code is compiled as if Memoize was absent or + disabled (the memos and externs are neither utilized nor produced). + \end{description} + + \begin{minipage}[t]{0.48\linewidth} + The action taken depends on the memoization mode and on whether all the + memos and externs associated with the code exist, as shown in the table on + the right. Note that a single missing memo or extern implies the ``no'' + column of the table, and that memoization will create \emph{all} the + associated memos and externs, even those which already exist. + \end{minipage} + \hfill + \begin{tabular}[t]{@{}lll@{}} + \toprule + &\multicolumn{2}{c}{Do the memos and externs exist?}\\ + \cmidrule{2-3} + mode&yes&no\\ + \midrule + normal&utilization&memoization\\ + readonly&utilization®ular compilation\\ + recompile&memoization&memoization\\ + \bottomrule + \end{tabular} + + The memoization mode only has effect when Memoize is \refmmz{enable}d. Mode + selection is orthogonal to enabling\slash disabling the package; for example, + if you switch to a new mode while the package is disabled, the new mode will + be in effect once the package is enabled. +\end{doc} + + +\begin{doc}{easy, + desc={style}, + key={name=verbatim}, + key={name=verb}, + key={name=no verbatim, desc={style, the initial mode}}, + } + When \refmmz{verbatim} or \refmmz{verb} is in effect, the code submitted to + memoization is read verbatim; \refmmz{no verbatim} reverts to the normal, + non-verbatim collection of the code. This applies to both manual and + automatic memoization. + + The long version, \refmmz{verbatim}, switches to the full verbatim mode, + where all characters are assigned category code 12 (other). With the short + version, \refmmz{verb}, the braces, |{| and |}|, retain category codes 1 and + 2, which can be useful for verbatim collection of optional arguments. For + details, see the documentation of CollArgs' \refcollargs{verbatim} in + section~\ref{sec:ref:collargs}. + + Under the hood, these keys have two effects. First, they are passed on to the + argument collector (typically, \refcmd{CollectArguments} of the auxiliary + package CollArgs; for details, see section~\ref{sec:ref:advicememoization}), + instructing it to collect the code in the specified fashion, as described + above. Second, if the collected verbatim code is eventually compiled (either + regularly, or memoized), Memoize first rescans it using \refcmd{scantokens}. +\end{doc} + +\begin{doc}{easy, + key={name=padding left, par=\meta{dimension}, + desc={no default, initially |1| |in|}}, + key={name=padding right, par=\meta{dimension}, + desc={no default, initially |1| |in|}}, + key={name=padding top, par=\meta{dimension}, + desc={no default, initially |1| |in|}}, + key={name=padding bottom, par=\meta{dimension}, + desc={no default, initially |1| |in|}}, + } + Set the left/right/top/bottom padding of the extern in the extern PDF. + + Without padding, the (PDF) page holding the extern would tightly fit the + bounding box of the extern. These keys enlarge the extern page by the given + amounts, so that any parts of the extern lying outside the bounding box will + be correctly included when using the extern. See + section~\ref{sec:tut:padding} for details. + + \meta{dimension} is evaluated with \hologo{eTeX}'s |\dimexpr|, and may + contain control sequences \docaux{cmd}{width}, \docaux{cmd}{height} and + \docaux{cmd}{depth}, which will refer to the dimensions of the extern. + \refcmd{width} and friends behave like dimension registers, so it is ok to write + e.g.\ \refmmz{padding right}|=0.5\width|. + + The default padding is what \hologo{pdfTeX} puts around the page anyway, 1 + inch, but we use |1 in| rather than |1 true in|, which is the true default + value of PDF registers \docref{pdfvar:horigin} and \docref{pdfvar:vorigin}, + as we want the padding to adjust with \refcmd{mag}nification. +\end{doc} + +\begin{doc}{easy,key={name=padding, par=\meta{dimension}, desc={style, no default}}} + Set all the above |padding| keys to the given value. +\end{doc} + +\begin{doc}{ + easy, + key={name=context, par=\meta{tokens}, desc={cumulative, no default, + initially set by \refmmz{padding to context}}}, + key={name=clear context}, + } + These keys append the given \meta{tokens} to the \emph{context expression}, + or clear this expression. Memoized code gets recompiled whenever the + expansion of the context expression changes. + + The \meta{tokens} must be fully expandable (modulo protection); in + \hologo{LaTeX}, they will be expanded by |\protected@edef| when calculating + the md5sum of the context. + + The context expression is evaluated at the end of the memoization, and at + utilization attempts after inputting the c-memo (note that the c-memo + contains any additions to the context expression accumulated during + memoization). At evaluation, the given context expression is fully expanded, + yielding the \emph{context value}, whose md5 sum forms the \meta{context md5 + sum} part of the filename of the cc-memo and the extern. + + These keys may be used both prior to the memoization process, or during + memoization. In the former case, their effect is local to the current group; + in the latter case, the effect is global, so that the changes surely survive + until the end of memoization, when the c-memo, where the context expression + is stored, is written into a file. + + Under the hood, these keys manipulate token registers + \docaux{cmd}{mmzContext} and \docaux{cmd}{mmzContextExtra}, changing the + contents of the former while not memoizing, and the contents of the latter + during memoization. These token registers may also be manipulated directly + by the user, as long as one keeps to the convention of adjusting + \refcmd{mmzContext} locally and only while not memoizing, and adjusting + \refcmd{mmzContextExtra} globally and only during memoization. +\end{doc} + +\begin{doc}{ + key={easy, name=meaning to context, + par=\marg{comma-separated list of commands and environment names}}, + key={name=csname meaning to context, par=\marg{control sequence name}}, + key={name=key meaning to context, + par=\marg{full path to a pgfkeys command key}}, + key={name=key value to context, + par=\marg{full path to a pgfkeys value key}}, + } + These keys append the definition of the given construct to the context. + + Essentially, the ``meaning'' keys append \cs{meaning}\meta{control sequence} + to the context, for the appropriate \meta{control sequence}. For example, + \refmmz{meaning to context} appends \cs{meaning}\cs{foo} when given \cs{foo} + as in item in the list, and it appends the internal environment macros + appropriate to the format when given an environment name. Similarly, + \refmmz{key meaning to context} resolves \meta{control sequence} to the + internal macro holding the key command. + + Key \refmmz{key value to context} should be used on keys which store values, + e.g.\ keys initialized by \pkg{pgfkeys} handler \code{.initial}; see + \PGFmanual{87.3.4 and \S87.4.5}. + + All the keys prefix the meaning\slash value by the name of the command\slash + environment\slash key, in order to prevent ambiguous contexts, see + section~\ref{sec:cross-referencing} for details. Furthermore, they all + operate through \code{\cs{csname}...\cs{endcsname}} construct, allowing one + to safely add internal commands to the context. +\end{doc} + +\begin{doc}{ + keypath={/handlers}, desc=handler, + key={easy, name=.meaning to context}, + key={name=.value to context}, + } + These are the handler variants of \refmmz{key meaning to context} and + \refmmz{key value to context}. +\end{doc} + + +\begin{doc}{key={name=padding to context,desc=style}} + This key appends the values of the \refmmz{padding} keys to the context, + causing the memoized code to be recompiled whenever the padding values are + changed. This key is used to initialize \refmmz{context}, so the author + normally shouldn't have to use this key. +\end{doc} + +\begin{doc}{ + cmd={name=mmzNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only}, + cmd={name=mmzForceNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only}, + } + These macros append the current value of the \meta{reference key} to the + context,\footnote{More precisely, it is \meta{reference key}|=|\marg{current + value} which is appended.} causing the memoized code to be recompiled + when the reference changes. + + If the reference key is undefined, \refcmd{mmzNoRef} aborts memoization, + while \refcmd{mmzForceNoRef} uses \cs{relax} as the reference string. + + These commands are deployed in the implementation of \refmmzauto[show + keypath]{ref}, \refmmzauto{force ref} and friends. If the + cross-referencing commands you are using are advised by these keys, you most + likely have no need of these macros. +\end{doc} + +\begin{doc}{easy, key={name=per overlay,desc=style}} + This style is only defined in the Beamer class. When applied, the memoized + code will produce a cc-memo (and extern) for each overlay of the frame. For + implementation, see section~\ref{sec:per-overlay}. +\end{doc} + +\begin{doc}{ + key={name=capture, par=\docAux{of=key:/mmz/capture, value={name=hbox}, text=\alt, + value={name=vbox,of=key:/mmz/capture}}, + desc={no default, initially \refmmz{capture=hbox}}} + } + Select the capture mode. This setting only applies to the default + memoization \refmmz{driver}. + + By default, it is assumed that the memoized code should be executed in the + horizontal mode, so the default memoization driver captures the output of the + memoized code in a \refcmd{hbox} --- and also issues a \refcmd{quitvmode} + (both in the document and in the cc-memo), just in case the memoized code + occurs at the start of the paragraph. + + Use \refmmz{capture}|=|\refmmz{capture=vbox} to execute the memoized code in + the vertical mode: Memoize will capture the output of the memoized code in a + \refcmd{vbox}, and avoid issuing \refcmd{quitvmode}. For example, this + capture mode is necessary to memoize a \refenv{verbatim} environment. +\end{doc} + + +\subsubsection{Inside the memoization process} +\label{sec:ref:memoization:process} + +\begin{doc}{easy, cmd={name=mmzAbort}} + This command aborts the ongoing memoization. + + The memoization will proceed as usual (i.e.\ the extern boxes and the cc-memo + code will be produced), but at the end of this process, no memos will be + produced, no externs shipped out to extern pages, and no record files + updated. +\end{doc} + +\begin{doc}{easy, cmd={name=mmzUnmemoizable}} + This command aborts the ongoing memoization, and marks the submitted code as + unmemoizable. + + The ongoing memoization produces a c-memo setting conditional + \docaux{cmd}{ifmmzUnmemoizable} to true. Upon utilizing this c-memo, the + system switches to regular compilation. + + For example, if you are automemoizing |tcolorbox|es of package + \pkg{tcolorbox}, you will want to refrain from memoizing boxes marked as + |breakable| or |float|ing. Simply aborting the memoization cannot do the + trick here, as memoization compiles the submitted code in a \hologo{TeX} box. + Marking a |breakable| or |float|ing |tcolorbox| as unmemoizable (either + manually using this macro, or automatically using \refmmz{auto key}) makes + sure that after the first compilation when memoization is attempted, the box + will be compiled regularly, and will have the intended ability to break + across pages, or float. +\end{doc} + + +\begin{doc}{ + cmd={name=Memoize, par=\marg{key}\marg{code}}, + } + Depending on various factors, this command either memoizes \meta{code} under + key \meta{key}, utilizes the results of a previous memoization, or performs a + regular compilation of \meta{code}. + + The outcome of executing this command --- memoization, utilization or regular + compilation --- depends upon the Memoize's state (\refcmd{ifmemoize}, + \refcmd{ifmemoizing}) and mode (\refmmz{normal}, \refmmz{readonly}, + \refmmz{recompile}), and the existence of the relevant memos and externs. + The decision process is depicted in section~\ref{sec:Memoize}. + + This command expects to be executed in a dedicated group, which it will close + itself. + + Invoking memoization through \refcmd{Memoize} might be useful for packages + which want to save the results of intensive computations, regardless of + whether the author loads (and enables) memoization or not. However, this + usage is not yet officially allowed, because there is currently no way to + load the core memoization routines without loading the entire package, + thereby forcing the author to use Memoize. +\end{doc} + +\begin{doc}{ + key={ + name=driver, par=\marg{code}, + desc={no default, initially \refcmd{mmzSingleExternDriver}} + }, + } + This key sets \meta{code} as the memoization driver. + + Given some code submitted to memoization, the memoization driver should + produce the memos and externs which will replicate the effect of that code + (while retaining its regular effect). For details, see + section~\ref{sec:memoization-drivers}. + + Typically, the \meta{code} argument of this key will consist of a single + control sequence (the driver control sequence), but any amount of tokens is + allowed. Memoize executes the driver followed by the code which it is + supposed to memoize, in braces, and only cares that the driver consumes that + code. +\end{doc} + + +\begin{doc}{ + key={name=at begin memoization, par=\meta{code}, desc={cumulative, initially empty}}, + key={name=at end memoization, par=\meta{code}, desc={cumulative, initially empty}}, + key={name=after memoization, par=\meta{code}, desc={cumulative, initially empty}}, + } + Use these keys to set up memoization hooks. + + These keys may be used both prior to the memoization process, or during + memoization. In the former case, their effect is local to the current group; + in the latter case, the code given to \refmmz{at begin memoization} is + executed immediately, while the assignment performed by the other two keys is + global, so that the changes surely survive until the end of memoization. + + The code given to hook \refmmz{at begin memoization} is kept in macro + \docaux{cmd}{mmzAtBeginMemoization}, while the content of the other two hooks + resides in two macros per hook: \docaux{cmd}{mmzAtEndMemoization} and + \docaux{cmd}{mmzAtEndMemoizationExtra}, and \docaux{cmd}{mmzAfterMemoization} + and \docaux{cmd}{mmzAfterMemoizationExtra}. All these macros may be + manipulated directly by the user,\footnote{Use \cs{appto}, \cs{eappto}, + \cs{gappto} and \cs{xappto} of package \pkg{etoolbox} (loaded by Memoize) + to easily append code to these macros.} as long as one keeps to the + convention of adjusting the macros without ``\texttt{Extra}'' locally and + only while not memoizing, and adjusting the macros with ``\texttt{Extra}'' + globally and only during memoization. The ``\texttt{Extra}'' macros require + global assignments as they might be manipulated by code residing within a + \hologo{TeX} group of any depth. + + These complications explained, let us take a look at how memoization proceeds + to learn when the hooks are used: + \begin{enumerate} + \item Initialize various conditionals, macros and token registers. (Here is + where the ``\texttt{Extra}'' hooks are cleared.) Remember that at this + point, we're inside a group opened by \refcmd{Memoize}. + \item Execute \refmmz{at begin memoization} hook, i.e.\ the contents of + macro \refcmd{mmzAtBeginMemoization}. + \item Execute the memoization \refmmz{driver}. + \item Execute \refmmz{at end memoization} hook, i.e.\ the contents of macros + \refcmd{mmzAtEndMemoization} and \refcmd{mmzAtEndMemoizationExtra} (in this + order). + \item Write out the memos and ship out the externs to extern pages (unless + memoization was aborted). + \item Close the memoization group. + \item Execute \refmmz{after memoization} hook, i.e.\ the contents of + macros \refcmd{mmzAfterMemoization} and \refcmd{mmzAfterMemoizationExtra} + (in this order). + \end{enumerate} +\end{doc} + +\begin{doc}{ + cmd={name=mmzCMemo, desc={token register, global, empty at the start of memoization}}, + } + This token register mediates the construction of the c-memo. During + memoization (and only during memoization), arbitrary code may be added to + this register; at the end of memoization, Memoize writes out its contents to + the free-form part of the c-memo. + + All assignments to this register should be global. Use \refcmd{gtoksapp} and + \refcmd{xtoksapp} to easily append tokens to the register. +\end{doc} + + +\begin{doc}{ + cmd={name=mmzCCMemo, desc={token register, global, empty at the start of memoization}}, + } + This token register mediates the construction of the cc-memo. During + memoization, the memoization driver should append cc-memo code to + \refcmd{mmzCCMemo}; at the end of memoization, Memoize writes out its + contents to the cc-memo (preceded by the list of produced externs). + + All assignments to this register should be global. Local assignments would + not work, because the memoized code may contain commands, like \cs{label} and + \cs{ref}, which contribute content to cc-memo as well, but these commands may + appear within a \hologo{TeX} group of any depth. + + Use \refcmd{gtoksapp} and \refcmd{xtoksapp} to easily append tokens to the + register. +\end{doc} + +\begin{doc}{ + par=\meta{token register}\marg{tokens}, + cmd={name=toksapp}, + cmd={name=gtoksapp}, + cmd={name=etoksapp}, + cmd={name=xtoksapp}, + } + These commands append the given \meta{tokens} to the \meta{token register}. + \refcmd{etoksapp} and \refcmd{xtoksapp} expand the \meta{tokens} before + appending them; \refcmd{gtoksapp} and \refcmd{xtoksapp} perform a global + assignment. + + These commands are actually provided by CollArgs, and they are defined only + if they don't already exist; in particular, note that \hologo{LuaTeX} + provides them as primitives. + + Unlike the \hologo{LuaTeX} primitive variant, these commands require the + \meta{token register} to be given by a (|\toksdef|fed) control sequence; it + cannot be given as |\toks|\meta{number}. +\end{doc} + +\begin{doc}{ + cmd={name=ifmemoize}, + cmd={name=memoizetrue}, + cmd={name=memoizefalse}, + } + Use the \hologo{TeX}-style conditional \refcmd{ifmemoize} to test whether + Memoize is currently enabled. Within the document body, the conditional may + be set using \refcmd{memoizetrue} and \refcmd{memoizefalse}, which are then + functionally equivalent to \refmmz{enable} and \refmmz{disable}. Do not set + the conditional in the preamble of the document (unless you really know what + you are doing). +\end{doc} + +\begin{doc}{ + cmd={name=ifmemoizing,desc={readonly}}, + } + Use this \hologo{TeX}-style conditional to test whether Memoize is currently + memoizing. It may be only inspected; you should \emph{never} set this + conditional yourself. +\end{doc} + +\begin{doc}{ + cmd={name=ifinmemoize,desc={readonly}}, + } + Use this \hologo{TeX}-style conditional to test whether Memoize is currently + active, in the sense of either memoizing or regularly compiling some code --- + so inside a call to \refcmd{Memoize}. The conditional may be only inspected; + you should \emph{never} set it yourself. +\end{doc} + +\begin{doc}{ + cmd={name=mmzSingleExternDriver, par=\marg{code}}, + } + This is the default memoization \refmmz{driver}, producing exactly one extern + containing whatever is typeset by the submitted \meta{code}. + + The \meta{code} is compiled either within a horizontal or vertical box, + depending on the value of key \refmmz{capture}. In the case of a horizontal + capture, the driver makes sure that the horizontal mode is entered prior to + both typesetting the resulting box in the document, or utilizing the extern. + + For the implementational details, see + section~\ref{sec:memoization-driver-default}. +\end{doc} + +\begin{doc}{ + cmd={name=mmzExternalizeBox, par=\marg{box}\marg{token register}}, + } + This macro is indended to be called by memoization drivers to produce an + extern page. The given \meta{box} is dumped into the document as a separate + extern page, while the \meta{token register} receives the cc-memo extern + inclusion code. + + The \meta{box} may be given either as a control sequence (declared via + |\newbox|), or as box number. The resulting extern page will contain a + \emph{copy} of the given box, padded by the \refmmz{padding} values in effect + at the time of invocation of \refcmd{mmzExternalizeBox}. + + An implementation detail is that \refcmd{mmzExternalizeBox} does not ship out + the extern page immediately. This action is delayed until the end of the + memoization process; more precisely, it is carried out (in tandem with + writing out the c-memo and the cc-memo) between execution of hooks \refmmz{at + end memoization} and \refmmz{after memoization}. This delay guarantees + that no extern pages are produced in the event of aborting memoization, even + if the abortion is triggered after executing \refcmd{mmzExternalizeBox}. + + The \meta{token register} may be given either as a control sequence (declared + via |\newtoks|) or as control sequence |\toks| followed by the register + number. The register will receive the code, which, when executed from the + cc-memo, includes the extern file into the main document. This code consists + of a single invocation of \refcmd{mmzIncludeExtern}. It is the + responsibility of the driver to include the code received by \meta{token + register} in the register \refcmd{mmzMemo}, whose contents are, unless + memoization is aborted, written into the cc-memo. (See + \refcmd{ifmmzkeepexterns} and \refmmz{after memoization} to learn about + another way to use the code received by \meta{token register}.) + + The invocation of \refcmd{mmzIncludeExtern} in the produced extern-inclusion + code is adapted to the type of the box (horizontal or vertical), which is + detected automatically --- the memoization driver does not need to inform + \refcmd{mmzExternalizeBox} about this type explicitly.\footnote{This does not + negate the need for key \refmmz{capture}, which applies to the default --- + and therefore generic --- memoization driver. This driver cannot know + whether the memoized code would prefer to be compiled in a horizontal or + vertical box. It is precisely key \refmmz{capture} which gives the user an + opportunity to inform Memoize about this preference. Only once the + memoized code is compiled into a box of the appropriate type, it is trivial + to detect the type of that box.} +\end{doc} + +\begin{doc}{ + cmd={name=ifmmzkeepexterns, desc={initially \cs{iffalse}}}, + cmd={head prefix=\texttt{\string\global}, name=mmzkeepexternstrue}, + cmd={head prefix=\texttt{\string\global}, name=mmzkeepexternsfalse}, + } + Setting this conditional to true makes Memoize keep the extern boxes in the + global temporary storage even after shipping them out as extern pages. (The + temporary storage is emptied at the start of the next memoization.) + + The extern inclusion code received by the \refcmd{mmzCCMemo} when executing + \refcmd{mmzExternalizeBox} is primarily meant to be executed by inputting the + cc-memo file; i.e.\ when the cc-memo is input, \refcmd{mmzIncludeExtern} is + defined to include the extern file into the document. However, it sometimes + makes sense to execute the cc-memo contents immediately after memoization; + for example, if memoization produces several externs, intricately integrated + into the surrouding environment, it might be cumbersome to replicate their + typesetting both in the memoizing compilation and in the cc-memo code --- + easier to build up the cc-memo code and execute it right after memoization. + This is why Memoize, just before executing the contents of \refmmz{after + memoization} hook, redefines \refcmd{mmzIncludeExtern} to include externs + from the temporary storage rather than from (at that point still + non-existing) extern files. However, as this mechanism requires Memoize to + keep the externs around even after memoization, it is not enabled by default: + it must be enabled by (globally) setting conditional + \refcmd{ifmmzkeepexterns} to true. +\end{doc} + +\begin{doc}{ + key={keypath=/mmz/auto, name=integrated driver, par=\marg{name}, desc=style} + } + Use this key to easily setup a memoization driver which is integrated into + the command itself. + + \begin{tcolorbox}[warning] + This is an auto-key residing in keypath \refkeypath{/mmz/auto}. + \end{tcolorbox} + + An integrated driver must have a way of telling whether it is memoizing or + regularly compiling the code. This key declares a driver-specific + conditional which may be inspected, using \refcmd{IfMemoizing}, to determine + this. The conditional is set to true by the formal \refmmz{driver} of the + command (set up by the invocation of this key), executed at the start of + memoization; it should never be set elsewhere. See + section~\ref{sec:memoization-multiple-externs} for details and an example. +\end{doc} + +\begin{doc}{ + cmd={name=IfMemoizing, par=\oarg{offset}\marg{name}\marg{true code}\marg{false code}} + } + This \hologo{LaTeX}-style conditional is meant to be used by the + \refmmzauto{integrated driver} with the given \meta{name}. It tests whether + this particular driver is currently memoizing some code. + + Potentially recursive commands are supported via the optional argument + \meta{offset}. If given, the conditional will only execute the \meta{true + code} when the current \hologo{TeX} group level matches the \hologo{TeX} + group level at the time of the invocation of the formal driver (held in + \refcmd{memoizinggrouplevel}), plus the \meta{offset}. In effect, the inner + invocation of the integrated driver will perform a regular compilation. For + details, see section~\ref{sec:memoization-complex-single-driver}. +\end{doc} + +\begin{doc}{ + cmd={name=memoizinggrouplevel,desc={readonly}}, + } + During memoization, this macro holds the \hologo{TeX} group level in effect + at the start of the memoization. +\end{doc} + +\begin{doc}{ + cmd={name=mmzRegularPages, desc=readonly counter}, + } + This counter holds the number of pages shipped out (so far) by the format's + regular shipout routine. \emph{Do not change its value!} + + In \hologo{LaTeX}, this counter is synonymous with + \refcmd{ReadonlyShipoutCounter}, and in \hologo{ConTeXt}, it is synonymous + with \refcmd{realpageno}. Memoize does not touch its value. + + In \hologo{plainTeX}, Memoize hijacks the \refcmd{shipout} control sequence + to count (and only to count) regular shipouts. In order for its value to be + realistic, Memoize should be loaded before other packages which hack + \cs{shipout} --- in particular, before \pkg{atbegshi}. +\end{doc} + +\begin{doc}{ + cmd={name=mmzExternPages, desc=readonly counter}, + } + This counter holds the number of extern pages Memoize has shipped out (so + far). \emph{Do not change its value!} + + A third-party tool may inspect this counter to have a realistic count of + shipped-out pages. +\end{doc} + +\begin{doc}{ + cmd={name=mmzExtraPages, desc=counter}, + } + This counter holds the number of pages shipped out (so far) in a way not + tracked by either \refcmd{mmzRegularPages} or \refcmd{mmzExternPages}. It + \emph{should} be advanced by any code which performs such shipouts, or + Memoize won't work correctly. +\end{doc} + + +\subsubsection{Tracing} +\label{sec:ref:memoization:tracing} + +\begin{doc}{ + key={name=trace, conditional=false}, + } + When tracing is on, Memoize shows information about its decision processes on + the terminal. You can learn whether the memoized code is being memoized, + utilized or regularly compiled; find out the md5sum of the code and which + input line it comes from; etc. + + This key has the syntax of a conditional, but there is no underlying + \hologo{TeX} conditional. The low-level interface for switching the tracing + on and off consists of macros \docaux{cmd}{mmzTracingOn} and + \docaux{cmd}{mmzTracingOff}. + + To learn about tracing the ``auto'' part of the automemoization process, also + \refcmd{AdviceTracingOn}. +\end{doc} + +\begin{doc}{ + key={name=include source in cmemo, conditional=true}, + } + + As a courtesy towards a curious user and a debugging aid, Memoize can include + a copy of the memo source in the c-memo. This feature is switched on by + default, but as the package itself never uses that information, it can be + safely switched off at any time. +\end{doc} + +\begin{doc}{ + key={name=include context in ccmemo, conditional=false}, + } + When this key is in effect, the expanded context expression is appended to + the cc-memo, behind the \docaux{cmd}{mmzThisContext} marker. + + Memoize never uses the context information from the cc-memo; this information + is only for tracing purposes. +\end{doc} + +\begin{doc}{ + key={name=direct ccmemo input, conditional=false}, + } + When this key is set to false, a cc-memo is processed indirectly: it is first + read into a token register, and it is the contents of this register which are + executed. When the key is set to true, the cc-memo is simply |\input|ted. + + The indirect execution is implemented to facilitate inverse search. Under + the direct cc-memo input, inverse search pointed at an included extern will + visit the cc-memo, which is not practical; under the indirect regime, the + inverse search will work as expected, and this is why the indirect cc-memo + input is the default. + + The overhead produced by the default indirect input method seems negligible, + but there are other factors which might make the user switch to the direct + input. For one, a cc-memo changing some category codes will require direct + input (no such cc-memos are ever produced out of the box). Less crucially, + sometimes one would like to use the inverse search to figure whether a part + of the document was produced by regular compilation or utilization, and which + memos\slash externs were utilized if the latter. Figuring this out under the + indirect input regime is harder: (i) reading the tracing information shown by + \refmmz{trace} is the surest way to learn what's going on, although (ii) + visual inspection of the externs and (iii) grepping through the |.memo.dir| + folder for particular code often help, as well. + + Both input methods use the same cc-memos; there is no need to + \refmmz{recompile} the memos when switching the cc-memo input method. Note + that the default indirect input method crucially relies on cc-memos ending + with \refcmd{mmzEndMemo}; this macro should not appear in the cc-memo itself. +\end{doc} + + + +\subsubsection{Internal memo commands} +\label{sec:ref:memoization:internal} + +The end-user should never have to use these commands. They are not formally +marked as internal by a |@| in their name only because doing so would +complicate |\input|ting the memos due to the category code changes it would +require. + +\begin{doc}{ + cmd={name=mmzMemo}, + } + This macro marks the beginning of a c-memo and a cc-memo core. Without it, + utilization of a memo will not work. +\end{doc} + +\begin{doc}{cmd={name=mmzSource}} + This macro marks the beginning of the memoized source in the c-memo. That + source is not used by Memoize in any way. It's inclusion into the c-memo may + be switched off by \refmmz{include source in cmemo}|=false|. +\end{doc} + +\begin{doc}{ + cmd={name=mmzResource, par=\marg{filename}}, + } + This is an internal command, which only occurs in a cc-memo. It checks + whether file \meta{filename} exists and is non-empty, and triggers + recompilation of the memoized code if the check fails. +\end{doc} + +\begin{doc}{ + cmd={ + name=mmzIncludeExtern, + par=\marg{seq}\braces{\refcmd{hbox}\alt\refcmd{vbox}}% + \marg{expected width}\marg{expected height}\marg{expected depth}% + \marg{padding left}\marg{padding bottom}\marg{padding right}\marg{padding top}, + } + } + + This is an internal command, which only occurs in a cc-memo. It includes the + extern identified by the sequential number \meta{seq} into the document as a + box of the specified type (horizontal or vertical). The extern is trimmed by + the given padding values. After trimming, the command checks whether the + size of the resulting box matches the given expectations; if it doesn't, a + warning is yielded. + + Before this command is executed, the externs should be listed by a sequence + of \refcmd{mmzResource} commands; \meta{seq} refers to the sequential number + of an extern in this sequence. + + This command may also be executed by executing the entire contents of + \refcmd{mmzCCMemo} after memoization. +\end{doc} + +\begin{doc}{ + cmd={name=mmzLabel, par=\marg{label key}\marg{label value}} + } + This is an internal command written into the cc-memo by the auto-handler of + \refcmd{label}. It temporarily stores \meta{label value} into + \refcmd{@currentlabel} and then executes \cs{label}\marg{label name}. +\end{doc} + +\begin{doc}{ + cmd={name=mmzEndMemo}, + } + This macro marks the end of a cc-memo. It is used to grab the cc-memo core + (everything between \refcmd{mmzMemo} and \refcmd{mmzEndMemo}) under the + indirect cc-memo input regime, i.e.\ when \refmmz{direct ccmemo input} is not + in effect. +\end{doc} + + +\subsection{Location of memos and externs} +\label{sec:ref:dirs} + +\begin{doc}{easy, + key={name=memo dir,par=\meta{name}, desc={style, default \texttt{\string\jobname}}}, + } + A convenient way to store memos and externs in a dedicated directory (see + sections~\ref{sec:tut:memodir} and~\ref{sec:tut:multifile} for the tutorial). + + Without an argument, this key places these files in subdirectory + \meta{document name}|.memo.dir| of the current directory. With an argument, + these files are stored in subdirectory \meta{name}|.memo.dir|. In both + cases, memo and extern filenames contain no prefix, as \meta{name} already + occurs in the directory name. + + The definition of this style is very simple: |memo + dir/.style={|\refmmz{prefix}|={#1.memo.dir/}}| (note the slash). Feel free + to redefine it, or to define another style to suit your needs. +\end{doc} + +\begin{doc}{easy,key={name=no memo dir,desc=style}} + A convenient way to undo the effect of \refmmz{memo dir} and revert to the + initial \refmmz{prefix} setting which locates memos and externs in the + current directory, with filenames prefixed by \meta{document name}|.| (mind + the dot). +\end{doc} + +\begin{doc}{ + key={name=prefix, par=\meta{prefix}, + desc={no default, initially \texttt{\string\jobname.}}}, + } + This key determines the location of memo and extern files, and the initial + part of their filenames. + + If \meta{prefix} contains no |/|, memos and externs are stored in the current + directory, alongside the output PDF, and their filenames begin with + \meta{prefix} (and continue with the identifier, see below). For example, + this is what happens with the initial value of this key, where memo and + extern filenames are prefixed by \meta{jobname} (i.e.\ the name of the + document) plus |.| (a dot). Such a situation is shown in the |dirty-house| + example in section~\ref{sec:tut:memodir}. + + If \meta{prefix} contains slashes, everything up to the final slash + determines the directory which memos and externs will be stored into, and the + part after the final slash determines the starting part of their filenames; a + slash (|/|) as a directory separator should be used even on Windows, where + the system directory separator is a backslash (|\|). For example, + \refmmz{memo dir} sets \refmmz{prefix} to |\jobname.memo.dir/|. As shown in + the |clean-house| example in section~\ref{sec:tut:memodir}, this results in + memos and externs stored in directory \meta{jobname}|.memo.dir|, with their + filenames consisting solely of the identifier. + + In detail, the paths to memos and externs are constructed as shown below, + where \meta{code md5 sum} and \meta{context md5 sum} identify the memoized + code and the \refmmz{context} in effect at its memoization, and $N$ is the + sequential number of the extern with respect to that code and context ($N$ is + usually $0$, as memoization normally produces a single + extern).\footnote{Normally, these paths are relative to the current directory + (i.e.\ the directory where \hologo{TeX} is executed from; usually, this + will be the directory where the compiled |.tex| file resides). However, + when \hologo{TeX} is invoked with option |-output-directory|, these paths + are relative to the specified output directory. Furthermore, when a memo + or extern cannot be written into the current\slash output directory, it + will be stored into the temporary directory \docref{TEXMFOUTPUT}, if + specified.} + \begin{center} + \begin{tabular}{r@{ }l} + c-memo:&\meta{prefix}\meta{code md5 sum}|.memo|\\ + cc-memo:&\meta{prefix}\meta{code md5 sum}|-|\meta{context md5 sum}|.memo|\\ + extern:&\meta{prefix}\meta{code md5 sum}|-|\meta{context md5 sum}-$N$|.pdf|\\ + &(where ``-$N$'' is only present when $N\neq 0$, i.e.\ for non-first externs)\\ + \end{tabular} + \end{center} + + \begin{tcolorbox}[warning,title=The final slash matters] + To reiterate, the presence vs.\ absence of a slash (|/|) determines whether + memos and externs are stored in a dedicated directory or not. For example, + if you want to store memos and externs in directory |memos|, you should set + \code{\refmmz{prefix}=memos/}, with the final slash. Without the final + slash, these files would end up in the current directory. + \end{tcolorbox} + + In principle, the directory specified by this key must already exist. + However, Memoize does it best to create it for you, and should succeed at + least when extraction method is set to \refmmz{extract=perl} or + \refmmz{extract=python}. See also \refmmz{mkdir} and \refmmz{mkdir command}. + + When invoked from the document body, each invocation of this key records the + new prefix by invoking \refmmz{record/record type/prefix} (this typically + results in a \refcmd{mmzPrefix} entry in the \dmmz file) and attempts to + create memo directory if \refmmz{mkdir} is in effect. When \refmmz{prefix} + is executed in the document preamble, these actions are only carried out at + the beginning of the document, for the final value of the key. + + +\end{doc} + +\begin{doc}{key={name=mkdir, conditional=true}} + When this key is set to true and the value of \refmmz{mkdir command} is + non-empty, Memoize will attempt to create the memo directory set by + \refmmz{prefix} if it does not yet exist. + + The directory is created using the system command specified by \refmmz{mkdir + command}. The directory creation takes place at the beginning of the + document and at every subsequent invocation of key \refmmz{prefix}. +\end{doc} + +\begin{doc}{ + key={name=mkdir command, par=\meta{system command}, + desc={no default, initially empty}}, + } + This key sets the command used to create the memo directory specified + by \refmmz{prefix} when \refmmz{mkdir} is in effect. + + Memoize attempts to create the directory by executing \meta{system command} + followed by (a space and) \meta{directory}, where \meta{directory} is the + directory part of the \meta{prefix} set by \refmmz{prefix}. The directory + creation takes place at the beginning of the document and at every subsequent + invocation of key \refmmz{prefix}. + + This key could be set to |mkdir| (on Linux, |mkdir -p| would be an even + better choice), however, this is not advisable as |mkdir| does not respect + \hologo{TeX}'s output restrictions, set by \docref{openout_any} in + \hologo{TeX}Live and |[Core]AllowUnsafeInputFiles| in MiK\hologo{TeX}. + Further note that as the value of this key is a system command, an + appropriate shell escape mode must be in effect to execute it successfully; + again, not something to be taken lightly. + + The extractions scripts shipped with Memoize accept option + \refscript{memoize-extract.pl--mkdir}, which makes them behave as a safe + variant of |mkdir|, i.e.\ a |mkdir| which respects \hologo{TeX}'s output + restrictions. Therefore, extraction methods \refmmz{extract=perl} and + \refmmz{extract=python} set \refmmz{mkdir command} to + \refscript{memoize-extract.pl} \refscript{memoize-extract.pl--mkdir} and + \refscript{memoize-extract.py} \refscript{memoize-extract.pl--mkdir}, + respectively. In effect, as the initial extraction method is + \refmmz{extract=perl}, the memo directory (if set via \refmmz{prefix} or + \refmmz{memo dir}) should be created under the initial settings without any + user intervention. +\end{doc} + + +\subsection{Extern extraction} +\label{sec:ref:externs} + +\begin{doc}{easy, + key={name=extract,par=\marg{extraction method}, + desc={preamble-only, initially \refmmz{extract=perl}, default: see below}}, + } + This key selects or executes the extern \meta{extraction method}, i.e. the + method which Memoize will use to extract the extern pages out of the document + PDF. + + Out of the box, Memoize recognizes the following \meta{extraction method} + keywords: \docAux{easy, of=key:/mmz/extract, value={name=perl}, comma, + value={name=python}, comma, value={name=tex}, and, value={name=no}}. The + first three keywords trigger extern extraction using the methods documented + in sections~\ref{sec:ref:extraction-perl-python} + and~\ref{sec:ref:extraction-tex}. The final keyword (not available in + \hologo{plainTeX}) instructs Memoize to \emph{not} perform the extraction at + the end of loading the package; it should be used when extraction is + performed externally (for details, see section~\ref{sec:setup}). Additional + methods may be installed by defining key \docaux{key + path}{mmz/extract}|/|\meta{extraction method}. + + When invoked from \reffile{memoize.cfg} or used as a package option, this key + \emph{selects} the extraction method. In this case, the key has no default + value, i.e.\ it is illegal to use it without an argument. The method + selected by the package option overrides the method selected in + \reffile{memoize.cfg}, which in turn overrides the package-initial value + \refmmz{extract=perl}. + + In \hologo{LaTeX} and \hologo{ConTeXt}, the selected method is automatically + executed at the end of loading the package. This does not happen in + \hologo{plainTeX}, because we want to allow the author to override the + initial extraction method (\refmmz{extract=perl}) or the extraction method + specified in \reffile{memoize.cfg}, even if \hologo{plainTeX} has no package + options. In \hologo{plainTeX}, internal extraction must be triggered by an + explicit invocation of \refmmz{extract} in the ``document preamble'' --- + i.e.\ between |\input memoize| and \refcmd{mmzset}\braces{\refmmz{begin + document}}. If given an \meta{extraction method} argument, the key will + execute the given extraction method; otherwise, it will execute either the + initial method, \refmmz{extract=perl}, or the overwide specified in + \reffile{memoize.cfg}. + + In general, Memoize guards against triggering the extraction more than once. + In formats other than \hologo{plainTeX}, invoking key \refmmz{extract} from + the document preamble is thus only allowed when + \refmmz{extract}|=|\refmmz{extract=no} was previously selected. In this + case, \refmmz{extract} in the document preamble behaves as in + \hologo{plainTeX}, i.e.\ it triggers the given extraction method; if no + extraction method is given, Memoize executes either the initial extraction + method \refmmz{extract=perl}, or the extraction method specified in + \reffile{memoize.cfg}. The possibility of invoking key \refmmz{extract} from + the document preamble makes it possible to bake Memoize into a user format + (with \refmmz{extract}|=|\refmmz{extract=no}), and trigger the internal + extraction manually. + + Executing extraction method \refmmz{extract=perl} or \refmmz{extract=python} + has an additional effect of setting the \refmmz{mkdir command} to the + extraction script with option \refscript{memoize-extract.pl--mkdir}. This + obviates the need to include |mkdir| among the restricted shell commands if + one is using the restricted shell mode. +\end{doc} + +\subsubsection{Perl- and Python-based extraction} +\label{sec:ref:extraction-perl-python} + +Perl- and Python-based extraction is triggered by +\refmmz{extract}|=|\refmmz{extract=perl} and +\refmmz{extract}|=|\refmmz{extract=python}, respectively. + +\begin{doc}{ + key={name=perl extraction command, par=\meta{system command}, + desc={no default, initially \refscript{memoize-extract.pl}}}, + key={name=perl extraction options, par=\meta{options}, + desc={no default, initially: \code{\refscript[short]{memoize-extract.pl--format} + \meta{format} \string\jobname}}}, + key={name=python extraction command, par=\meta{system command}, desc={no + default, initially \refscript{memoize-extract.py}}}, key={name=python + extraction options, par=\meta{options}, desc={no default, initially: + \code{\refscript[short]{memoize-extract.pl--format} \meta{format} + \string\jobname}}} + } + + These keys determine the system calls used for invoking the extraction + scripts \refscript{memoize-extract.pl} and \refscript{memoize-extract.py}. + All the details below apply both to the Perl and the Python version. + + Use |perl|\slash|python extraction command| to set the name of the extraction + script. If necessary, include the full path to the script, or |perl|\slash + |python| plus the path to the script. Whatever you set here must be allowed + by the shell escape mode. + + Use |perl|\slash|python extraction options| to set the options that the + script will receive; consult the documentation of + \refscript{memoize-extract.pl} for their meaning. The initial value asks the + script to process \meta{document name}\dmmz, and informs it that it is + executed from within a \hologo{TeX} compilation of a document in the given + \meta{format} (|latex| for \hologo{LaTeX}, |plain| for \hologo{plainTeX}, + |context| for \hologo{ConTeXt}); the latter makes the script yield any + warnings or errors in a form expected by the format. + + During the execution of a system call, the values of these settings are fully + expanded. Furthermore, these keys were initialized using \pkg{pgfkeys} + handler |.initial|, so their values may be modified by handlers |.prefix|, + |.append|, etc. The initial value of the extraction options contains a space + on both sides, so that these handlers are easy to use. For example, write + \code{\refmmz{perl extraction + options}/.append=\refscript{memoize-extract.pl--quiet}} to ask + for less output. +\end{doc} + +\begin{doc}{ + script={name=memoize-extract.pl, par=[\meta{options}] \meta{name}\texttt{.mmz}}, + script={name=memoize-extract.py, par=[\meta{options}] \meta{name}\texttt{.mmz}}, + } + These scripts extract the new externs recorded in \meta{name}\dmmz from + \meta{name}|.pdf|. Memoize invokes them when loaded with package option + \refmmz{extract}|=|\refmmz{extract=perl} (the default) or + \refmmz{extract}|=|\refmmz{extract=python}. + + Argument \meta{name}\dmmz may be given either in full (e.g.\ |doc.mmz|), or + merely as the stem (|doc|). In the latter case, |.mmz| is appended to the + given argument even if it already contains a suffix (e.g.\ |my.doc| will + result in |my.doc.mmz|); the exception is suffix |.tex|, which is replaced by + |.mmz|. + + The script inspects the given record file, \meta{name}\dmmz, for lines of + form \refcmd{mmzNewExtern}\marg{extern filename}\marg{extern page + number}\braces{\meta{expected width}\texttt{pt}}\braces{\meta{expected + height}\texttt{pt}}. For each such line, page number \meta{extern page + number} is extracted from \meta{name}|.pdf| into \meta{extern filename}. + (The script also creates directories specified by \refcmd{mmzPrefix} lines. + Other lines are ignored, and so are commented invocations of + \refcmd{mmzNewExtern} and \refcmd{mmzPrefix}.) + + The \meta{extern filename} may contain a (relative or absolute) path to the + new extern file. The relative paths are relative to the location of the + \dmmz file, even when the script is invoked from some other directory. + + To guard against extracting a wrong page, the script checks whether the size + of each extracted page matches the \meta{expected width} and \meta{expected + height}.\footnote{\label{fn:tolerance}To avoid false positives, the match + need not be exact, a difference up to |0.01pt| is tolerated. Some PDF + tools, notably the |PDF::API2| library deployed by the Perl version of the + script, round the dimensions of a PDF page, recorded in |/MediaBox|, to two + digits.} If it does not, the script refuses to extract the page, yields a + warning and even removes the extern file if it exist. + + The extraction script's paranoia extends further. It will refuse to extract + the page, yielding a warning, if a (c)c-memo associated to the extern does + not exist. And it will respect the |openin| and |openout| settings of + |texmf.cnf|; under the default configuration, it will therefore refuse + (yielding an error) to write to any file whose absolute path does not occur + under the current directory, the temporary directory set by + \docref{TEXMFOUTPUT} (in \reffile{texmf.cnf} or as an environment variable), + or the output directory (\docref{TEXMF_OUTPUT_DIRECTORY}). Furthermore, it + will refuse to write into the root directory (except on Windows, where + writing into a drive root might potentially make sense). + + Exit codes: 0 = success, 1 = Perl/Python error, 2 = usage error, 10 = + extraction warning, 11 = extraction error. + + \yadocset{ of=script:memoize-extract.pl, override/.style={ index + annotation={option of {% + \hypercolor{link}{gray}% + \hyperref[script:memoize-extract.pl]{\texttt{memoize-extract}}% + }}, }, } + \begin{doc}{ + option={name=pdf, short name=P, par=\meta{pdf}} + } + Extract the externs from the given \meta{pdf} instead of the default + \meta{name}\texttt{.mmz}. Note that file \meta{pdf}, despite the different + name, should be produced by the same compilation that produced + \meta{name}\dmmz, otherwise wrong pages might be extracted. + \end{doc} + \begin{doc}{ + option={name=prune, short name=p} + } + After extraction, remove the extracted extern pages from the document PDF. + \end{doc} + \begin{doc}{ + option={name=keep, short name=k} + } + By default, the script comments out the \refcmd{mmzNewExtern} lines in the + \dmmz file, to prevent multiple extractions. Specifying this option + prevents this behaviour. + \end{doc} + \begin{doc}{ + option={name=format, short name=F, + par=\code{latex}\Alt\code{plain}\Alt\code{context}} + } + When this option is given, the script assumes that it was called from + within a \hologo{TeX} compilation of a document in the given format + (|latex| for \hologo{LaTeX}, |plain| for \hologo{plainTeX}, |context| for + \hologo{ConTeXt}). + + For one, the script will prefix all its output by the script name, to be + easily identifiable in the terminal output, but more importantly, it will + create a log file (\code{\string\jobname.mmz.log}) which will receive any + warnings and errors yielded by the script (in absence of this option, the + warnings and errors are printed to the standard error). The log contains + messages in a form recognized by the format (e.g.\ \cs{PackageError} for + \hologo{LaTeX}), so Memoize can then \cs{input}s this log to reproduce the + warnings and errors in the compilation. + + There should be no need to use this option when executing the script form + the command line. + \end{doc} + \begin{doc}{ + option={name=force, short name=f} + } + Force extern extraction even if the size check or (c)c-memo existence check + fails. The failure will still yield a warning. + \end{doc} + \begin{doc}{ + option={name=quiet, short name=q} + } + Normally, the script prints what it is doing to the standard output; in + particular, it prints out the page number and the filename of each extern + it is extracting. This option disables this behaviour. + \end{doc} + \begin{doc}{ + option={name=library, short name=L, + par=\texttt{PDF::API2\Alt PDF::Builder}, desc=Perl-only} + } + Use the specified library for PDF processing. If this option is not given, + the script uses the first available library (in the order given above). + \end{doc} + \begin{doc}{ + option={name=mkdir, short name=m} + } + When given this option, the extraction script transforms into a paranoid + |mkdir| (|-p|). Argument \meta{name}|.mmz| is interpreted as a path to the + directory to create; all other options are ignored. + + The ancestors of the directory are created as needed. The script will + refuse to create any directory whose absolute path does not occur under the + current directory or a directory listed in \docref{TEXMFOUTPUT} (set in + \reffile{texmf.cnf} or as an environment variable). + + This option exists so that the author using the restricted shell mode does + not have to list |mkdir| among the restricted shell mode commands (and it + is also safer than a plain |mkdir|). + + Memoize automatically uses this script to create the memo directory when + option \refmmz{extract} is given value \refmmz{extract=perl} or + \refmmz{extract=python}. + \end{doc} + \begin{doc}{ + option={name=version, short name=V} + } + Show Memoize version. + \end{doc} + \begin{doc}{ + option={name=help, short name=h} + } + Show help. + \end{doc} + + Functionally, the Perl (|.pl|) and the Python (|.py|) version of the script + are almost equivalent. The minor differences, listed below, are mostly due + to the underlying PDF-processing library: + \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} in Perl, and + \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2} in Python. + \begin{itemize} + \item The Python script is about twice as fast as the Perl script. However, + both scripts are very fast compared to the \hologo{TeX}-based extraction. + On my computer, extracting 160 externs out of a 400-page book takes 1.4s + with Python, 3.7s with Perl, and 65s with \hologo{TeX}. But even when + using \hologo{TeX}-based extraction, externalization using Memoize is + extremely fast compared to the \TikZ's externalization library. Adding + the regular compilation time of one minute to the above numbers, we arrive + at the maximum externalization time of about two minues, whereas my + estimate for the production of all 160 externs using \TikZ's + externalization would be an \emph{hour} or more. + \item The Python script cannot extract externs out of PDFs created without + stream compression, i.e.\ with \refcmd{pdfvariable} + \docref{pdfvar:compresslevel} set to |0|. + \item Occasionally, the Perl script crashes during extraction externs; see + section~\ref{sec:troubleshooting} for details. + \end{itemize} +\end{doc} + +\subsubsection{\texorpdfstring{\hologo{TeX}}{TeX}-based extraction} +\label{sec:ref:extraction-tex} + +\hologo{TeX}-based extraction is triggered by +\refmmz{extract}|=|\refmmz{extract=tex}. + +\begin{doc}{ + key={name=tex extraction command, par=\meta{system command}, + desc={no default, initially |pdftex|}}, + key={name=tex extraction options, par=\meta{options}, + desc={no default, initially: see below}}, + key={name=tex extraction script, par=\meta{\hologo{TeX} code}, + desc={no default, initially: see below}}, + } + \yadocset{ + before/.style={label prefix=cmd:te}, + index annotation={defined at \hologo{TeX} extraction}, + } + + Together, these keys determine the system call used for invoking the + \hologo{TeX}-based extraction script, \refscript{memoize-extract-one.tex}. + (They were initialized using \pkg{pgfkeys} handler |.initial|, so their + values may be modified by handlers |.prefix|, |.append|, etc.) + + Memoize uses the resulting system call string at the following occasions. + First, it executes it, once for each new extern in the \dmmz file, during the + internal extraction, i.e.\ when it is loaded with package option + \refmmz{extract}|=|\refmmz{extract=tex}. Second, it uses it to construct + record files of types \refmmz{record=sh}, \refmmz{record=bat} and + \refmmz{record=makefile} when recording of these types is requested. + + Key \refmmz{tex extraction command} sets the \hologo{TeX} binary used for + \hologo{TeX}-based internal extraction. By default, the \hologo{pdfTeX} + engine |pdftex| is used; other sensible values for this key are |luatex| and + |xetex|, or the program name including the path. Note that shell escape mode + must be configured appropriately, and that + \refscript{memoize-extract-one.tex} must be compiled with the + \hologo{plainTeX} format. + + The value of key \refmmz{tex extraction options} is passed as options to the + \hologo{plainTeX} binary. As shown in the initial value in the frame below, + it may use the temporary macro \docaux{cmd}{externbasepath}, which expands to + the path to the extern without the |.pdf| suffix. This macro is available + during internal \hologo{TeX}-based extraction and during the execution of + \refmmz[show keypath]{record/record type/new extern} key for and \meta{record + type}. + + \makeexcerpt{tex-extraction-options} + \tcbinputexample[.tex][.excerpt]{% + listing only, one file, no attachment, title={The initial value of + \refmmz{tex extraction options}}, + } + + Key \refmmz{tex extraction script} defines the command-line script executed + to extract the extern. The default value, shown in the frame below, invokes + \refscript{memoize-extract-one.tex} after setting its parameter macros. + Temporary macros \docAux{cmd={name=pagenumber}, comma, cmd={name=expectedwidth}, + and, cmd={name={expectedheight}}} are defined at the same occasions as + \refcmd{te:externbasepath} above, i.e.\ during internal \hologo{TeX}-based + extraction and during the execution of a \refmmz{record/record type/new + extern} key. The initial value requests the extern to be of the same version as + the main document, if possible;\footnote{As far as I know, it is impossible + to access the version of the PDF being produced in \hologo{XeTeX}, i.e.\ + there are no registers \docref{reg:pdfmajorversion} and + \docref{reg:pdfminorversion}. To request production of a specific version of + PDF, \hologo{XeTeX} must be invoked by with command-line option + \code{-output-driver 'xdvipdfmx -V $N$'}.} note that Memoize defines + \docref{reg:pdfmajorversion} and \docref{reg:pdfminorversion} in \hologo{LuaTeX}. + + \makeexcerpt{tex-extraction-script} + \tcbinputexample[.tex][.excerpt]{% + listing only, one file, title=The initial value of \refmmz{tex extraction + script}, no attachment} + + As the value of \refmmz{tex extraction script} is fully expanded when used, + the initial value shown above must prevent the expansion of much code. + Furthermore, the initial value varies with the \hologo{TeX} format, as + indicated by the |.dtx| guards in the definition of + \refcmd{meo:warningtemplate}. +\end{doc} + +\begin{doc}{ + script={ + head prefix={% + (\texttt{pdf}\alt\texttt{lua}\alt\texttt{xe})\texttt{tex -jobname} + \meta{extern filename} + \texttt{"}\meta{parameters} + \cs{input}\texttt{\textbraceleft} + }, + name=memoize-extract-one.tex, + head infix=\texttt{\textbraceright"}, + }, + } + Compiling \refscript{memoize-extract-one.tex} with \hologo{plainTeX} + produces an extern file containing a single (extern) page extracted from the + document PDF. + + Memoize invokes this script, once for each new extern appearing in the \dmmz + file, when loaded with package option + \refmmz{extract}|=|\refmmz{extract=tex}. + + The desired \meta{extern filename} is given as the value of option |-jobname| + of the \hologo{TeX} binary. To set the extraction \meta{parameters}, define + the following macros before |\input|ting the file: + \yadocset{ + index annotation=option of \texttt{memoize-extract-one.tex}, + before/.style={head prefix=\cs{def}, label prefix=cmd:meo}, + } + \begin{doc}{cmd={name=fromdocument, parameters=\marg{document pdf filename}}} + Defining this macro sets the filename of the PDF which the externs will be + extracted from. The filename is relative to the working directory. + \end{doc} + \begin{doc}{cmd={name=pagenumber, parameters=\marg{number}}} + Defining this macro sets the number of the page to extract. The first page + has number 1. + \end{doc} + \begin{doc}{ + cmd={name=expectedwidth, parameters=\marg{dimension}, desc=optional}, + cmd={name=expectedheight, parameters=\marg{dimension}, desc=optional}, + } + Defining these macros sets the expected width and height of the extracted + page. + + To guard against extracting a wrong page, the dimensions of the extracted + page are compared against the expected width and height. If the size check + fails,\footnote{The match need not be exact, see + footnote~\ref{fn:tolerance}.} the resulting extern PDF is empty (which + counts as non-existent when Memoize checks for its presence when it + attempts to utilize it), and a warning message (formatted via + \refcmd{meo:warningtemplate}) is printed to the log file, if logging was + requested via \refcmd{meo:logfile}. + + If any of these macros is undefined, the size check will be skipped. + \end{doc} + \begin{doc}{cmd={name=logfile, parameters=\marg{filename}, desc=optional}} + Defining this macro sets the name of the log file. If not defined, no log + file will be produced. + + The log file is intended to be used when the script is invoked from an + outer \hologo{TeX} compilation. In particular, it is intended to be + |\input| by that compilation to see whether the extraction was successful. + Upon a failed size check, it will contain a warning (formatted by + \refcmd{meo:warningtemplate}, if that macro is defined). The log file ends + with |\endinput| to signal that extraction actually took place. + \end{doc} + \begin{doc}{cmd={name=warningtemplate, parameters=\marg{code}, desc=optional}} + Defining this macro determines how to log the warning message in the case + of a failed size check. The macro should expand to a \hologo{TeX} + format-specific warning message code containing the warning text given in + \docAux{cmd={name=warningtext, label prefix=cmd:meo, into + index=false}}.\footnote{Macro \refcmd{meo:warningtemplate} is passed + the warning text by a macro rather than a formal parameter to avoid + category code problems with the parameter character when setting key + \refmmz{tex extraction script}.} + + While the script formats the warning message \emph{text} on its own (``I + refuse to extract page \dots''), the warning message is not written into + the log unadorned. The log file is intended to be \cs{input} by the outer + \hologo{TeX} compilation, and the idea is that inputting it should yield a + warning in that compilation (in the case of a failed size + check). Therefore, the content of the log file must contain an invocation + of the command used to produce warning messages in the \hologo{TeX} format + used by the outer compilation. + + For example, when this script is invoked from within a \hologo{LaTeX} + compilation, it makes sense to define something like + |\def\warningtemplate{\PackageWarning{memoize}{\warningtext}}|. + \end{doc} + \begin{doc}{cmd={name=force, + parameters=\bracestt{\texttt{true}\Alt\texttt{false}}, desc=optional}} + If this macro is defined to |true|, extern extraction will be carried out + even if the size-check fails. The failure will still be logged. + \end{doc} + \begin{doc}{ + cmd={name=mmzpdfmajorversion, parameters=\marg{number}, desc=optional}, + cmd={name=mmzpdfminorversion, parameters=\marg{number}, desc=optional}, + } + Defining (one or both of) these macros requests that the extern PDF be + produced with the given major\slash minor PDF version, i.e.\ the extraction + script will set registers \docref{reg:pdfmajorversion} and + \docref{reg:pdfminorversion}. + \end{doc} + + After extracting the extern, the script will end the compilation, i.e.\ + intentionally, only one page documents can be produced. +\end{doc} + +\subsubsection{The clean-up scripts} +\label{sec:cleanup-scripts} + +\begin{doc}{ + script={name=memoize-clean.pl, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]}, + script={name=memoize-clean.py, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]}, + } + This script removes memo and extern files whose filenames start with + \meta{prefix}es mentioned in the given \dmmz files or by the + \refscript{memoize-clean.pl--prefix} option. Unless option + \refscript{memoize-clean.pl--all} is given, the script only deletes the + \emph{stale} files, i.e.\ the files not mentioned in any of the given \dmmz + files. + + A \meta{prefix} of a memo or an extern is what was set by key + \refmmz{prefix}, or more commonly, one of the shortcut keys \refmmz{memo dir} + and \refmmz{no memo dir}; see section~\ref{sec:ref:dirs} for details on the + form of a memo/extern filename. + + In detail, the script scans the given \dmmz files for occurrences of + \refcmd{mmzPrefix}, and adds their \meta{prefix} arguments to the list of + prefixes given on the command line by option + \refscript{memoize-clean.pl--prefix}; a \meta{prefix} occurring in some \dmmz + file is interpreted relatively to the location of the \dmmz file. The script + removes all files whose full pathname (relative to the current directory) + matches pattern + \meta{prefix}\meta{md5sum}(|-|\meta{md5sum})(|.memo|\alt(|-|$N$)|.pdf|\alt|.log|),% + \footnote{The |.log| files are produced by the \hologo{TeX}-based extraction + script.} except those which occur as the \meta{filename} argument to one + of \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo}, \refcmd{mmzUsedExtern}, + \refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo} and \refcmd{mmzNewExtern} in one + of the \dmmz files. + + The script is fairly paranoid. It refuses to delete anything if a \dmmz file + is malformed in any way (but not if it doesn't exist or is completely empty, + which facilitates its usage in clean-up scripts), or if it would remove a + file not residing under the current directory. Before removing the files, it + lists the files to be removed and asks for confirmation. + + Functionally, the Perl (|.pl|) and the Python (|.py|) version are completely + equivalent. + + \yadocset{ + of=script:memoize-clean.pl, + override/.style={ + index annotation={option of {% + \hypercolor{link}{gray}% + \hyperref[script:memoize-clean.pl]{\texttt{memoize-clean}}% + }}, + }, + } + \begin{doc}{ + option={name=prefix, short name=p, par=\meta{prefix}} + } + Add \meta{prefix} to the list of prefixes; the given prefix is + relative to the current directory. This option may be given multiple + times. + \end{doc} + \begin{doc}{ + option={name=all, short name=a} + } + When given this option, the script removes \emph{all} memos and externs + belonging to the document, not just the stale ones, i.e.\ it effectively + ignores the occurences of \refcmd{mmzUsedCMemo} and friends in the \dmmz + file. + \end{doc} + \begin{doc}{ + option={name=yes, short name=y} + } + When given this option, the script does not ask for confirmation before + removing the files. + \end{doc} + \begin{doc}{ + option={name=quiet, short name=q} + } + Normally, the script prints what it is doing to the standard output; in + particular, it prints out the filename of each file as it is deleting it. + This option disables this behaviour. + \end{doc} + \begin{doc}{ + option={name=help, short name=h} + } + Show help. + \end{doc} + \begin{doc}{ + option={name=version, short name=V} + } + Show Memoize version. + \end{doc} +\end{doc} + +\subsubsection{Record files} +\label{sec:ref:record-files} + +\begin{doc}{ + key={name=record, par=\marg{record type}, + desc={cumulative, initially \refmmz{record=mmz}, no default}}, + key={name=no record}, + } + Memoize records which externs were produced and used in the compilation, + producing a record file of every type found in the record-type list. These + keys add \meta{record type} to the record-type list, or clear this list. See + section~\ref{sec:record-files} for details. + + Note that passing an undefined \meta{record type} to this key will not yield + an error. + + Out of the box, the following \meta{record type}s are recognized: + \yadocset{of=key:/mmz/record} + \begin{doc}{value={name=mmz}} + This record type produces a \dmmz file recording new/used + externs/c-memos/cc-memos and changes in the \refmmz{prefix} to these + files; see section~\ref{sec:.mmz} for details. + + The produced file is named \docAux{file={name=mmz, + name prefix=\meta{jobname}., + ref prefix=., + index prefix=., + }}. This name cannot be changed. + + The \dmmz file is a \hologo{TeX} file, but uses only a simple subset of + the \hologo{TeX} syntax, to be easily parsable by the external scripts such + as \refscript{memoize-extract.pl}. Each line of the file consists of a + (possibly commented) invocation of one of the commands listed below; the + final line is |\endinput|. The \meta{prefix} below consists of the + path to memos/externs and the immutable \refmmz{prefix} of their filename. + \begin{doc}{ + cmd={name=mmzUsedCMemo, par=\marg{filename}}, + cmd={name=mmzUsedCCMemo, par=\marg{filename}}, + cmd={name=mmzUsedExtern, par=\marg{filename}}, + } + Record that the (c)c-memo or extern residing in file \meta{filename} was + utilized. + \end{doc} + \begin{doc}{ + cmd={name=mmzNewCMemo, par=\marg{filename}}, + cmd={name=mmzNewCCMemo, par=\marg{filename}}, + } + Record that a new (c)c-memo residing in file \meta{filename} was + produced. + \end{doc} + \begin{doc}{cmd={name=mmzNewExtern, par=\marg{filename}\marg{page number}% + \marg{expected width}\marg{expected height}}} + Record that a new extern was produced and dumped as page \meta{page + number} into the document, that it should be extracted into file + \meta{filename}, and that it should be \meta{expected width} wide and + \meta{expected height} high (modulo tolerance of |0.01pt|, see + footnote~\ref{fn:tolerance}), where the height is the total height + comprising both \hologo{TeX} height and depth. + \end{doc} + \begin{doc}{cmd={name=mmzPrefix, par=\marg{prefix}}} + Record that the \refmmz{prefix} of memo and extern files was + changed. + \end{doc} + \end{doc} + \begin{doc}{value={name=makefile}} + This record type produces a makefile which, when processed by the |make| + utility, triggers \hologo{TeX}-based extraction of the new externs. + \begin{doc}{ + key={name=makefile, par=\marg{filename}, desc={no default, + initially \texttt{memoize-extract.\string\jobname.makefile}}} + } + Use this key to change the filename of the produced makefile. + \end{doc} + \end{doc} + \begin{doc}{value={name=sh}, value={name=bat}} + These record types produce a shell script which, when executed, triggers + \hologo{TeX}-based extraction of the new externs. + + Use \refmmz{record=sh} on Unix-like systems, and \refmmz{record=bat} on + Windows. + + \begin{doc}{ + key={name=sh, par=\marg{filename}, desc={no default, + initially \texttt{memoize-extract.\string\jobname.sh}}}, + key={name=bat, par=\marg{filename}, desc={no default, + initially \texttt{memoize-extract.\string\jobname.bat}}} + } + Use these keys to change the filename of the produced shell script. + \end{doc} + \end{doc} +\end{doc} + +\begin{doc}{ + desc=definable, + keypath=/mmz/record/\meta{record type}, + keypath label=/mmz/record/record type, + name prefix={\meta{record type}/}, ref prefix=, + index annotation={key in \texttt{/mmz/\meta{record type}}}, + key={name=begin}, + key={name=prefix, par=\marg{prefix}}, + key={name=new extern, par=\marg{filename}}, + key={name=new cmemo, par=\marg{filename}}, + key={name=new ccmemo, par=\marg{filename}}, + key={name=used extern, par=\marg{filename}}, + key={name=used cmemo, par=\marg{filename}}, + key={name=used ccmemo, par=\marg{filename}}, + key={name=end}, + } + A new record type can be implemented by defining these keys in keypath + \docAux{key path={name=mmz/record/\meta{record type}, label=mmz/record/record + type}} (using the standard \pkg{pgfkeys} handlers such as |.code| and + |.style|). The keys are invoked by Memoize where appropriate if recording + for the defined type is activated by \refmmz{record}|=|\meta{record type}, + just as for the predefined types. Only those keys which are required for + implementing the desired functionality need to be defined. + + The following macros are available during the execution of key + \refmmz{record/record type/new extern}: + \yadocset{before/.style={label prefix=cmd:ne}, + index annotation=defined at \refmmz{record/record type/new extern}} + \begin{doc}{cmd={name=pagenumber}} + This macro holds the number of the extern page. + \end{doc} + \begin{doc}{ + cmd={name=expectedwidth}, + cmd={name=expectedheight}, + } + These macros hold the width and the height of the extern page. + \end{doc} + \begin{doc}{cmd={name=externbasepath}} + This macro holds the filename of the extern, minus the |.pdf| suffix (but + including the path leading to the extern). + \end{doc} +\end{doc} + + + +\subsection{Automemoization} +\label{sec:ref:advicememoization} + + +\subsubsection{Package Advice} +\label{sec:ref:advice} + +Package Advice is a namesake of \Emacs's Advice. As such, it implements a +generic framework for extending the functionality of selected commands and +environments. Each \emph{advised} command and environment is assigned a piece +of \emph{advice} --- a command which is executed instead of the advised command +and environment, and which may, or may not, invoke the original command or +environment during its execution. The package offers an elegant way of +declaring advice, setting up the conditions upon which the advised command will +actually be replaced by the advice, collecting the arguments of the advised +command and invoking it, and (de)activating the advice. + +Before the advising framework can be used, it must be installed into a selected +\pkg{pgfkeys} keypath (multiple installations into different keypaths are +allowed, even if they handle the same commands). Memoize installs the +framework into keypath \refkeypath{/mmz} and (primarily) uses it to +automatically memoize the results of compilation of selected commands and +environments. + +\begin{doc}{ + keypath={/handlers}, desc=handler, + key={name=.install advice, + sort index=install advice, + par=\marg{configuration keylist}, + }, + } + This key is a \pkg{pgfkeys} key handler (see \PGFmanual{87.3.5}) which + installs the advising framework into the keypath which it was invoked from + --- henceforth, the \meta{namespace}. + + For example, \code{\cs{pgfkeys}\bracestt{/my/\refkey{/handlers/.install + advice}}} installs the framework into keypath |/my|. + + Argument \meta{configuration keylist} may contain the following keys: + \yadocset{keypath=/advice/install} + \begin{doc}{key={ + name=setup key, + parameters=\marg{name}, + description={no default, initially \texttt{advice}}}, + } + This key determines the names of the user-interface keys used to setup + advice for commands and environments in \meta{namespace}. + + The keys whose names are determined by this key are the following: + \meta{name}, \code{\meta{name} csname}, \code{\meta{name} key}, + \code{\meta{name}'}, \code{\meta{name} csname'} and \code{\meta{name} + key'}. Memoize sets \refkey{/advice/install/setup key}|=auto|, and + thereby defines \refmmz{auto}, \refmmz{auto csname}, \refmmz{auto key}, + \refmmz{auto'}, \refmmz{auto csname'} and \refmmz{auto key'}. + \end{doc} + \begin{doc}{key={ + name=activation, + par=\meta{initial activation type}, + desc={no default, initially \texttt{immediate}} + }, + } + This key sets the \meta{initial activation type} for \meta{namespace}. + + At the end of the installation, the system will execute + \meta{namespace}|/|\refmmz{activation}|=|\meta{initial activation type}; + consequently, \meta{initial activation type} must be one of + \refmmz{activation=immediate} and \refmmz{activation=deferred}. In + Memoize, the \meta{initial activation type} is + \refmmz{activation=deferred}. + + Setting the \refmmz{activation} type during the installation only matters + in \hologo{LaTeX}, where the installation ends by advising \refcmd{begin} + to implement advising of environments. + \end{doc} + + \begin{tcolorbox}[warning] + Writing the documentation for Advice, I was faced with a dilemma. Should + the documentation reflect the fact that the full names of keys defined by + the package depend on the installed instance of the framework, in + particular on \meta{namespace} and \meta{setup key}? For example, should + the reference headers contain things like \meta{namespace}|/activate| and + \meta{namespace}|/|\meta{setup key} |csname|? In my opinion, this would + make the reference hard to read, so I decided to have the reference headers + refer to the Advice keys of the Memoize installation, where + \meta{namespace}=|/mmz| and \meta{setup key}=|auto|, resulting in + friendlier headers such as \refkeypath{/mmz}|/|\refmmz{activate} and + \refkeypath{/mmz}|/|\refmmz{auto csname}. (Consequently, it also made + sense to document Advice within the Memoize documentation.) + + The bottomline: if you're reading this section with a non-Memoize + installation in mind, you have to mentally replace any \refkeypath{/mmz} + and \refmmz{auto} in the reference headers with the \meta{namespace} and + the \meta{setup key} selected by that installation. + (Section~\ref{sec:ref:advice:memoization} is another matter. Keys + described there are only available in Memoize.) + \end{tcolorbox} + + In more detail, key handler \refkey{/handlers/.install advice} performs the + following actions (as explained in the box above, we assume that the + advising framework was installed into keypath \refkeypath{/mmz} with the + setup key named |auto|): + \begin{itemize} + \item \edef\origtolerance{\the\tolerance}\tolerance=1000 + It defines the following keys in keypath \refkeypath{/mmz}: + \refmmz{auto}, + \refmmz{auto csname}, + \refmmz{auto key}, + \refmmz{auto'}, + \refmmz{auto csname'}, + \refmmz{auto key'}, + \refmmz{activation}, + \refmmz{activate deferred}, + \refmmz{activate}, + \refmmz{deactivate}, + \refmmz{activate csname}, + \refmmz{deactivate csname}. + \refmmz{activate key}, + \refmmz{deactivate key}. + \refmmz{force activate}, + \refmmz{try activate}. + \item + It defines the following keys in keypath \refkeypath{/mmz/auto}: + \refmmzauto{run conditions}, + \refmmzauto{outer handler}, + \refmmzauto{bailout handler}, + \refmmzauto{collector}, + \refmmzauto{args}, + \refmmzauto{collector options}, + \refmmzauto{clear collector options}, + \refmmzauto{raw collector options}, + \refmmzauto{clear raw collector options}, + \refmmzauto{inner handler}, + \refmmzauto{options}, + \refmmzauto{clear options}, + \refmmzauto{reset}. + \item \tolerance\origtolerance\relax + It defines the |.unknown| key handler for \refkeypath{/mmz/auto}. + This handler appends any unknown keys (and their values) to + \refmmzauto{options}. + \item It executes \refmmz[show keypath]{activation}|=|\meta{initial + activation type}. + \item In \hologo{LaTeX}, it submits \refcmd{begin} to advising, thereby + enabling environment support in this format. Consequently, advising of + environments can switched off by writing \refmmz{deactivate}|=\begin|. + \end{itemize} +\end{doc} + +\paragraph*{The keys installed into keypath \metabf{namespace}} are used to +declare and (de)activate advice. In the documentation in this subsection, +we assume that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup + key}=\refmmz{auto}. In particular, this also applies to the reference +headers. + +\begin{doc}{ + key={name=activation, par=\docAux{of=key:/mmz/activation, value={name=immediate}, + text=\Alt, value={name=deferred}}, desc={no default, initially set by \refkey{/handlers/.install advice}}}, + key={name=activate deferred, desc={style}}, + } + Key \refmmz{activation} selects the activation regime. Under the + \refmmz{activation=immediate} regime, keys \refmmz{activate}, + \refmmz{deactivate}, \refmmz{force activate} and \refmmz{try activate} behave + as described in their documentation below. Under the + \refmmz{activation=deferred} regime, however, those keys are not executed; + rather, their invocations are appended to style \refmmz{activate deferred}. + For example, writing \refmmz{activate}|=\foo| in the deferred activation + regime appends \refmmz{activate}|=\foo| to \refmmz{activate deferred}. It is + up to the user if and when to execute the keys collected in \refmmz{activate + deferred}; see the documentation of \refmmz{manual} to learn what Memoize + does with the contents of this style. +\end{doc} + + +\begin{doc}{easy, + par=\marg{list of commands and/or environments}, + desc={style}, + key={name=activate}, + key={name=deactivate}, + } + These keys activate or deactivate the advice for the given commands and + environments. When the advice is activated, it replaces the advised command; + when it is deactivated, the command is reverted to its original definition. + + In Memoize, these keys are most commonly used to activate or deactivate + automemoization for the given commands or environments. For example, write + \refmmz{deactivate}|={\tikz,tikzpicture}| to deactivate automemoization of + \TikZ pictures (which is declared and active by default). The curly braces + may be omitted if the list contains a single command or environment, e.g.\ + \refmmz{deactivate}|=\tikz| or \refmmz{deactivate}|=tikzpicture|. + + (De)activation of a piece of advice is completely orthogonal to its declaration with + \refmmz{auto}. For example, there is no need to deactivate a command before + redeclaring its advice, and reactivate it afterwards. A command may be + activated even before declaring its advice --- however, the command itself + must be defined at the time of activation. + + As the advice is normally automatically activated upon declaration with + \refmmz{auto}, explicit activation is rarely needed, but see \refmmz{auto'}. + The effect of these keys under the deffered activation regime is described in + \refmmz{activation}. + + Note that I sometimes speak of (de)activating a command, and sometimes of + (de)activating its advice. I mean the same thing. +\end{doc} + +\begin{doc}{ + par=\marg{control sequence name}, + desc={style}, + key={name=activate csname}, + key={name=deactivate csname}, + } + These keys activate and deactivate a command given by its \meta{control + sequence name}; for example, \refmmz{activate csname}|=foo| is equivalent + to \refmmz{activate}|=\foo|. Note that unlike the regular \refmmz{activate} + and \refmmz{deactivate}, their |csname| variants only accept a single command + at a time (otherwise, including a comma in the command name would be + impossible). +\end{doc} + +\begin{doc}{ + par=\marg{list of full key names}, + desc={style}, + key={name=activate key}, + key={name=deactivate key}, + } + These keys activate and deactivate |pgfkeys| keys. Note that \emph{full} key + names must be given, i.e.\ the names must include the keypath. + + Under the hood, these keys merely execute \refmmz{activate} and + \refmmz{deactivate} on the internal macros corresponding to the given keys. +\end{doc} + +\begin{doc}{ + key={name=try activate, conditional=false}, + } + When this conditional is set to true, \refmmz{activate} will not yield an + error if the advice is already activated, and \refmmz{deactivate} will not + yield an error if the advice is not yet activated. + + This key applies to the next, and only to the next, invocation of key + \refmmz{activate} or \refmmz{deactivate}, i.e.\ it is reset back to |false| + after invoking \refmmz{activate} or \refmmz{deactivate}. +\end{doc} + +\begin{doc}{ + key={name=force activate, conditional=false}, + } + When this conditional is set to true, \refmmz{activate} will activate even a + previously activated command, provided that, additionally, the command has + been redefined since the prior activation. + + In more detail, the original definition of the advised command is saved upon + activation (to provide the possibility of both deactivation and the usage of + the original command by the handler). Consequently, activation of an already + activated command would result in the saved original definition being + overwritten by the redefinition made during the first activation. However, + if the handled command was meanwhile redefined by a third party, reactivation + makes sense, under the assumption that the former original definition is + obsolete and should be replaced by the (third party) redefinition. As a + safeguard, however, \refmmz{activate} requires such reactivation to be + explicitly requested using conditional \refmmz{force activate}.\footnote{A + potential problem, not (yet) addressed, is that the third party might be + another incarnation of the advising framework. In this case, forced + reactivation will result in the loss of the original command and a circular + dependency between the two pieces of advice.} + + This key applies to the next, and only to the next, invocation of key + \refmmz{activate}, i.e.\ it is reset back to |false| after invoking + \refmmz{activate}. This key does not apply to \refmmz{deactivate}. +\end{doc} + +\begin{doc}{easy, + key={name=auto, par=\marg{command or environment}{\marg{keylist}}, desc=style}} + + This key sets up the advice for the given command or environment, or updates + the configuration of an existing piece of advice. + + In Memoize, this key is most commonly used to submit a command or environment + to automemoization. For an environment (say, |bar|), it suffices to write + \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{memoize}}; for a command, + we usually need to include its argument specification: + \refmmz{auto}|=\foo|\bracestt{\refmmzauto{memoize}, + \refmmzauto{args}=\bracestt{...}}. Another common usage is to prevent + memoization during the execution of a command or environment: + \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{nomemoize}}. For + details, see sections~\ref{sec:tut:automemoization}, + \ref{sec:tut:working-on-a-picture} and~\ref{sec:tut:verbatim}. + + Note that in \hologo{LaTeX}, advising an environment (say, |bar|) is + different than advising the internal command (|\bar|) containing the + \meta{begin-code} of the environment, created by \cs{newenvironment}. + Advising environment |bar| effectively replaces the entire + |\begin{bar}...\end{bar}| construction, so that \hologo{LaTeX}'s |\begin| + code is never executed. The advice of command |\bar|, on the other hand, + is executed after |\begin| initializes the environment; in particular, it + is executed within the group introduced by the environment. + + The advice is configured by the given \meta{keylist}, which is executed with + the default keypath set to \docaux{key path}{mmz/auto}. Any unknown keys in + \meta{keylist} are passed on to key \refmmzauto{options}; for example, a + plain \refmmz{verbatim} or \refmmz{padding}|=2in| have the same effect as + \refmmzauto{options}|=|\refmmz{verbatim} or + \code{\refmmzauto{options}=\braces{\refmmz{padding}=2in}}. + + This key automatically activates the declared advice, unless it is already + activated; under the deferred activation regime, the automatic activation is + deferred as well. Use variant \refmmz{auto'} when you don't want to + automatically activate the advice. + + When this key is used on a command or environment with an existing piece of advice, + the advice is merely updated. This makes it easy to, for example, + temporarily switch to verbatim collection of an environment in Memoize: + \refmmz{auto}|=|\bracestt{tcolorbox}\braces{\refmmz{verbatim}}. Use key + \refmmzauto{reset} to setup the advice from scratch: + \refmmz{auto}|=|\bracestt{...}\bracestt{\refmmzauto{reset}, ...}. + + A piece of advice consists of several interlocked components, declared by keys + residing in path \refkeypath{/mmz/auto}: \refmmzauto{run conditions}, + \refmmzauto{bailout handler}, \refmmzauto{outer handler}, + \refmmzauto{collector} and \refmmzauto{inner handler}. During the execution + of the advice, these components are available through the following macros: + \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler}, + \refcmd{AdviceOuterHandler}, \refcmd{AdviceCollector} and + \refcmd{AdviceInnerHandler}. These macros are also defined during setup, and + it is possible to change the configuration by modifying them directly; in + that case, it likely also makes sense to use the low-level variant of this + key, macro \refcmd{AdviceSetup}. + + Control sequences used in the advice components do not need to be defined at + the time of invoking key \refmmz{auto}, or activating the advice; they must + only be defined at the time the advice is actually executed. It is thus + perfectly fine to declare \refmmzauto{inner handler}|=\myinnerhandler| before + defining |\myinnerhandler|, or to redefine |\myinnerhandler| between the + invocations of the advised command. + + This key configures not only the components, but also the options of the + advice. These options are set by keys \refmmzauto{args}, + \refmmzauto{collector options}, \refmmzauto{raw collector options} and + \refmmzauto{options}. Whether these options are used or not depends on the + advice components. (Same as the components, the options have their + corresponding low-level macros: \refcmd{AdviceArgs}, + \refcmd{AdviceCollectorOptions}, \refcmd{AdviceRawCollectorOptions} and + \refcmd{AdviceOptions}.) + + Parameter symbols (i.e.\ |#|) are not allowed in advice settings. + + A command or environment may be submitted to several instances of the + advising framework, i.e.\ instances installed under different keypaths. The + effect of such chained advice depends on the order of activation. If advice + $A$ is activated before advice $B$, it will also be applied before $B$. + + The advice setup takes place in a group. Use key \refmmzauto{after setup} to + execute code outside this group. + + In general, the name of this key equals whatever was submitted to + \refkey{/advice/install/setup key} during the installation of the advising + framework via \refkey{/handlers/.install advice}; the initial value is + |advice| (and Memoize sets the value to |auto|). +\end{doc} + +\begin{doc}{ + key={name=auto csname, par=\marg{control sequence name}{\marg{keylist}}, desc=style} + } + This key is a variant of \refmmz{auto}, but with the command of the first + argument given as a control sequence name, i.e.\ \refmmz{auto + csname}|={foo}{...}| is equivalent to \refmmz{auto}|=\foo{...}|. +\end{doc} + +\begin{doc}{ + key={name=auto key, par=\marg{full key}{\marg{keylist}}, desc=style}, + } + This key is a variant of \refmmz{auto}, but it works with \pkg{pgfkeys} keys. + The first argument should be a \meta{full key} like |/tcb/float|, i.e.\ it + must consist of both the keypath and the keyname. + + This key sets up advice for the internal command corresponding to the given + \meta{full key}, and also properly initializes the collector, so that + \refmmzauto{inner handler} will ``just work.'' +\end{doc} + +\begin{doc}{ + key={name=auto', par=\marg{command or environment}{\marg{keylist}}, desc=style}, + key={name=auto csname', par=\marg{control sequence name}{\marg{keylist}}, desc=style}, + key={name=auto key', par=\marg{full key}{\marg{keylist}}, desc=style}, + } + These keys are variants of \refmmz{auto}, \refmmz{auto csname} and + \refmmz{auto key} which do not attempt to activate the command after setting + it up. +\end{doc} + +\begin{doc}{ + cmd={ + name=AdviceSetup, + par=\marg{namespace}\marg{command or environment}{\marg{setup code}} + }} + This macro is the low-level variant of key \refmmz{auto}. The differences + between the two are the following: + \begin{itemize} + \item An invocation of the macro must provide the namespace (i.e.\ the + installation keypath) as the first argument. + \item There is no automatic activation at the end of the setup. + \item The final argument should not be a keylist (of keys belonging to + \refkeypath{/mmz/auto}) but \hologo{TeX} code adjusting the contents of the + settings macros \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler}, + \refcmd{AdviceOuterHandler}, etc. For the full list of available macros, see + the documentation of their corresponding keys below; the setting macros are + mentioned at the end of each entry. + \end{itemize} +\end{doc} + +\begin{doc}{ + cmd={name=AdviceTracingOn}, + cmd={name=AdviceTracingOff}, + } + Advice tracing is initially off. When it is on, Advice will show (on the + terminal and in the |.log| file) which advice components are executed, and + what arguments and options they have received. +\end{doc} + +\paragraph*{The keys installed into keypath \metabf{namespace}\texttt{/}\metabf{setup key}} +are used to configure advice. They may only occur within the second +argument of the setup key. In the documentation in this subsection, we assume +that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup + key}=\refmmz{auto}. In particular, this also applies to the reference +headers. + +\begingroup +\yadocset{keypath=/mmz/auto} + +\begin{doc}{% + key={name=run conditions, par=\meta{\hologo{TeX} code}, + desc=initially and default: \refcmd{AdviceRuntrue}}, + } + + This key declares the \meta{control sequence} as the run conditions component + of the advice. + + The run conditions macro is executed at the very start of the advice. Its + function is to decide whether we should proceed to advise the command by + executing the outer handler, or execute the original command (after invoking + the bailout handler). + + The run conditions macro should take no arguments. If it determines that the + run conditions are satisfied, it should set the \hologo{TeX} conditional + \docaux{cmd}{ifAdviceRun} to true by executing \docaux{cmd}{AdviceRuntrue}. + There is no need to execute \docaux{cmd}{AdviceRunfalse} when the run + conditions are not satisfied. + + Initially, the run conditions are set to \refcmd{AdviceRuntrue}, translating to + ``always run.'' For two non-trivial examples, see \refmmzauto{run if + memoization is possible} and \refmmzauto{run if memoizing}. Executing this + key without a value restores it to the initial value. + + During advising and advice setup, the run conditions of the advised command + are accessible through \docaux{cmd}{AdviceRunConditions}, a parameterless + macro expanding to the given \meta{\hologo{TeX} code}. +\end{doc} + +\begin{doc}{% + key={name=bailout handler, par=\meta{\hologo{TeX} code}, + desc=initially and default: \cs{relax}}, + } + + This key declares the \meta{\hologo{TeX} code} as the bailout handler + component of the advice. + + The bailout handler is executed when the run conditions are not met, just prior + to executing the original definition of the advised command. The bailout + handler should take no arguments. + + The initial bailout handler, |\relax|, does nothing. Memoize defines and + uses a bailout handler which clears the next-options. Executing this key + without a value restores it to the initial value. + + During advising and advice setup, the bailout handler of the handled command is + accessible through \docaux{cmd}{AdviceBailoutHandler}, a parameterless macro + expanding to the given \meta{\hologo{TeX} code}. +\end{doc} + +\begin{doc}{ + key={name=outer handler,par=\meta{\hologo{TeX} code}, + desc=initially and default: see below}, + } + + This key declares the \meta{\hologo{TeX} code} as the outer handler component + of the advice. + + The outer handler can be safely imagined as the command which replaces the + handled command. This also holds for handled environments, but with a + caveat: for a \hologo{plainTeX} or \hologo{ConTeXt} environment |foo|, the + outer handler replaces |\foo| and |\startfoo|, respectively; in the case of a + \hologo{LaTeX} environment, it replaces |\begin{foo}|.%] + + The outer handler is the first component which has the opportunity to inspect + the arguments given to the handled command. It is invoked just in front of + these arguments (which are, in case \hologo{TeX} hasn't seen them yet, + untokenized), and while it is expected that the advice will consume + the same arguments as the advised command itself would, how precisely that + happens may vary from situation to situation. In particular, the argument + structure of the outer handler is not prescribed. + + In fact, the outer handler has complete control over the remainder of the + advising process. In situations where advising requires knowledge of the + advised command's arguments as a whole, the outer handler executes the + collector, which in turn invokes the inner handler, which does the real work; + see \refmmzauto{memoize} for the usage case which inspired this design. + Sometimes, however, it is the outer handler which does the real work (and + there is thus no inner handler). This is the case in situations when the + arguments of the handled command are irrelevant for the functioning of the + advice, or when the advice needs to inspect some individual argument of the + handled command; for examples of such situations, see \refmmzauto{abort} and + \refmmzauto{ref}. + + To reiterate the argument situation of the outer handler, it sees the + arguments of the handled command as they were given. The arguments are + \emph{not} collected before invoking the outer handler --- in fact, avoiding + the argument collection is the raison d'être of the outer handler! (In the case + of an advised environment, the environment body can be seen as an argument of + \pkg{xparse} type \docref{xparse:+}\docref{xparse:b}.) + + The outer handler (and any other component of the advice it invokes) has + access to the following auxiliary macros, defined by the framework: + \begin{itemize} + \item \tolerance 1000 the macros holding the configuration of the advised + command, as set up by \refmmz{auto}: \refcmd{AdviceRunConditions}, + \refcmd{AdviceBailoutHandler} and \refcmd{AdviceOuterHandler} are probably + useless, as they refer to components already invoked, but the remaining + components (\refcmd{AdviceCollector} and \refcmd{AdviceInnerHandler}) and + their options (\refcmd{AdviceArgs}, \refcmd{AdviceCollectorOptions}, + \refcmd{AdviceRawCollectorOptions} and \refcmd{AdviceOptions}) should be + commonly used. + \item the macros holding information about the namespace and the advised + command or environment: \refcmd{AdviceNamespace}, \refcmd{AdviceName}, + \refcmd{AdviceCsname}, \refcmd{AdviceReplaced} and \refcmd{AdviceOriginal}. + (Commands \refcmd{AdviceGetOriginal} and \refcmd{AdviceCsnameGetOriginal} + might also be useful, although using \refcmd{AdviceOriginal} will likely be + more practical.) + \end{itemize} + + This key is initially set to an internal control sequence which merely + invokes the collector by executing \refcmd{AdviceCollector}; in other words, + the initial outer handler leaves all the work to the collector and the inner + handler. There is no need to specifically set up the outer handler when + using the inner handler. Executing this key without a value restores it to + the initial value. + + During advising and advice setup, the outer handler of the advised command is + accessible through \docaux{cmd}{AdviceOuterHandler}, a parameterless macro + expanding to the given \meta{\hologo{TeX} code}. +\end{doc} + +\begin{doc}{ + key={name=collector,par=\meta{\hologo{TeX} code}, + desc=initially and default: see below}, + } + + This key declares the \meta{\hologo{TeX} code} as the collector component of + the advice. + + The collector, if used, is invoked by the outer handler. It is invoked + immediately in front of the advised command's arguments (which are, in case + \hologo{TeX} hasn't seen them yet, untokenized), and its function is to + collect these arguments and pass them on, as a single argument, to the inner + handler. + + While this manual occasionally states that the initial argument collector is + \refcmd{CollectArguments} of package CollArgs, this is, if we're precise, + incorrect on two counts. For one, the initial collector is not a CollArgs + command, but a macro which acts as the ``bridge'' between Advice + and CollArgs. Second, the initial collector does not really invoke + \refcmd{CollectArguments}, but its cousin, \refcmd{CollectArgumentsRaw}, + which allows Advice (and Memoize) to fine tune its behaviour + using the fast low-level (``programmer interface'') commands rather than the + slower \pkg{pgfkeys} interface; clearly, the latter point also provides + raison d'être for \refmmzauto{raw collector options}. Summing up, this key + is initially set to an internal control sequence which compiles the settings + provided by \refmmzauto{args}, \refmmzauto{collector options} and + \refmmzauto{raw collector options} into an invocation of + \refcmd{CollectArgumentsRaw} of package CollArgs.\footnote{The initial + collector also sets the CollArgs' option \refcollargs{caller} to the + name of the advised command or environment.} Executing this key without a + value restores it to the initial value. + + The above-mentioned collector settings were clearly tailored to suit + \refcmd{CollectArgumentsRaw}. In general, a collector might or might not use + them, and if it does, it may interpret them in any way. For example, Advice + ships with a \refcmd{tikz} collector, \docaux{cmd}{mmzCollectTikZArguments}, which + ignores them completely, as it knows everything about the idiosyncrasies of + that command anyway. Incidentally, \refcmd{mmzCollectTikZArguments} becomes + available upon loading \reffile{advice-tikz.code.tex} (which Memoize does + automatically in the presence of \TikZ). + + The collector has access to the same auxiliary macros as the outer handler. + In particular, it will \emph{have} to use \refcmd{AdviceInnerHandler} (followed + by the braced collected arguments) to invoke the inner handler. + + During advising and advice setup, the collector of the advised command is + accessible through \docaux{cmd}{AdviceCollector}, a parameterless macro + expanding to the given \meta{\hologo{TeX} code}. +\end{doc} + +\begin{doc}{easy, + key={name=args,par=\meta{argument specification}, desc=initially and default: unset}, + } + + This key describes the \meta{argument specification} of the advised command. + + Assuming that the initial value of \refmmzauto{collector} has not been + modified, the given \meta{argument specification} is eventually interpreted + by command \refcmd{CollectArguments} of package CollArgs, which expects an + argument specification in the format specified by package \pkg{xparse}; the + format is summarized in the frame below for convenience, for details, see the + \pkg{xparse} manual. If the specification is not given, the initial collector + assumes that the advised command was defined using \refcmd{NewDocumentCommand} + (or similar) of package \pkg{xparse}, and will attempt to retrieve the argument + specification automatically via \refcmd{GetDocumentCommandArgSpec}. + + \begin{tcolorbox}[float, before float=\hfill, + title={The {\pkg[white]{xparse}} argument specification + (as understood by \refcmd[link color=white]{CollectArguments})}] + \begin{tabularx}{\linewidth}{>{\tt}lX} + \multicolumn{2}{l}{\rm\textbf{Mandatory argument types}}\\ + m&standard (a single token or multiple tokens in braces)\\ + r\meta{token$_1$}\meta{token$_2$}&delimited by + \meta{token$_1$} and \meta{token$_2$} \\ + v&verbatim, in the style of \cs{verb}\\ + b&the body of an environment\\ + [1ex]\multicolumn{2}{l}{\rm\textbf{Optional argument types}}\\ + o&in square brackets\\ + d\meta{token$_1$}\meta{token$_2$}&delimited by + \meta{token$_1$} and \meta{token$_2$}\\ + s&an optional star\\ + t\meta{token}&an optional \meta{token}\\ + e\marg{tokens}&a set of embellishments\\ + [1ex]\multicolumn{2}{l}{\rm\textbf{Weird argument types}}\\ + l&a mandatory argument until the first begin-group token\\ + u\marg{tokens}&\hologo{TeX}'s delimited argument\\ + g&an optional argument inside braces\\ + [1ex]\multicolumn{2}{l}{\rm\textbf{Modifiers}}\\ + +&allow the next argument to be long\\ + !&disallow spaces before arguments of type \docref{xparse:d} and \docref{xparse:t}\\ + >\marg{processor}&process the next argument\\ + [1ex]\multicolumn{2}{l}{\rm\textbf{CollArgs extensions}}\\ + \docref{xparse:b}\marg{name}&set the environment name for this environment\\ + \docref{xparse:amp}\marg{options}&apply CollArgs options to the next argument\\ + \docref{xparse:amp}\docref{xparse:amp}\marg{raw options}&apply raw CollArgs options to the next argument\\ + \end{tabularx} + + \smallskip + + \refcmd{CollectArguments} can grab an argument of any type in the + \refcollargs{verbatim} mode. + + As \refcmd{CollectArguments} does not use the arguments but only collects + them, it does not care about the default values of optional arguments. + Therefore, argument types with defaults (\docref{xparse:O}, + \docref{xparse:D} and \docref{xparse:R}) may be substituted by their + \texttt{-NoValue-} counterparts (\docref{xparse:o}, \docref{xparse:d} and + \docref{xparse:r}) and are therefore not included in the above table. + \end{tcolorbox} + + In general, however, an argument collector may this interpret this setting it + in any way it sees fit --- or not at all. For example, in Memoize the value + of \refmmzauto{args} is ignored for command \refcmd{tikz}, which requires a special + collector (\refcmd{mmzCollectTikZArguments}). + + When setting up advice for a \emph{command}, this key is initially + ``unset,'' i.e.\ it holds a special value indicating that the argument + specification is not provided. Note that this special value is not an empty + string --- \refmmzauto{args}|={}|, or simply \refmmzauto{args}|=|, indicates + a command which takes no arguments. During the execution of the advice, one + may use the \hologo{LaTeX}-style conditional + \docaux{cmd}{AdviceIfArgs}\marg{true branch}\marg{false branch} to test + whether the argument specification was provided. Executing this key without + a value restores it to the initial, unset value. + + When setting up the advice of an \emph{environment}, this key is + initialized to \docref{xparse:+}\docref{xparse:b} (a long environment body), + making it unnecessary to specify this value manually. Note that this holds + even for environments with arguments other that the environment body + ``argument'': those arguments will be caught as the start of the body even if + not explicitly specified. + + During advising and advice setup, the argument specification of the advised + command is accessible through \docaux{cmd}{AdviceArgs}, a parameterless macro + expanding to the given \meta{argument specification}. +\end{doc} + + +\begin{doc}{ + key={name=collector options, par=\marg{keylist}, + desc={cumulative, initially empty, value required}}, + key={name=raw collector options, par=\marg{code}, + desc={cumulative, initially empty, value required}}, + } + These keys append the given value to the list of user-friendly and raw + collector options, respectively. A comma is prefixed to the user-friendly + \meta{keylist} before appending it. + + Both kinds of collector options are intended to be used by the collector, + which may interpret them in any way it sees fit --- or not at all. The + initial collector, which invokes \refcmd{CollectArgumentsRaw} of package + CollArgs, passes both lists to this command, which interprets + \refmmzauto{collector options} as a user-friendly \pkg{pgfkeys} keylist + (which therefore requires a bit of processing) and \refmmzauto{raw collector + options} as plain \hologo{TeX} code (expecting it to contain only the + allowed, ``programmer's interface'' macros).\footnote{Clearly, + \refmmzauto{raw collector options} are why Advice deploys + \refcmd{CollectArgumentsRaw} rather than \refcmd{CollectArguments}. But + how does it then pass the user-friendly \refmmzauto{collector options} to + that command? It embeds them in \refcmd{collargsSet}.} The raw variant is + used internally by both Advice and Memoize, and may be used by a package + deploying the advising framework which wants to save a few processing cycles. + In CollArgs, the two kinds of options are functionally equivalent; both are + documented in section~\ref{sec:ref:collargs}. + + Initially, the list of collector options is empty, and for commands, so is + the list of raw collector options. For environments, however, the latter + list is initialized to set (the raw equivalent of) \refcollargs{environment} + to the environment name, and \refcollargs{end tag} to true. The rationale + for the latter is that the environment body containing the end tag (e.g.\ + |\end{foo}|) is nicely compatible with \refcmd{AdviceReplaced} (which equals +the begin tag, e.g.\ |\begin{foo}|) and \refcmd{AdviceOriginal} (which executes + the original definition of e.g.\ |\begin{foo}|). For example, thanks to + \refcollargs{end tag}, writing \refcmd{AdviceOriginal}|#1| in the inner + handler executes the original environment. Importantly, the original + environment can be executed without explicitly referring to the + environment's name, and with code that works not only for environments of + any \hologo{TeX} format, but is actually the same as the code which invokes + an original \emph{command}. Consequently, the same inner handler works for + both commands and environments, and in all \hologo{TeX} formats. + + Furthermore, the initial collector also sets option \refcollargs{caller} + to the name of the advised command or environment (however, + \refcollargs{caller} never appears in any of the collector options + lists; it is simply prefixed to them while constructing the invocation of + \refcmd{CollectArgumentsRaw}). And in Memoize, using keys + \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} triggers the + addition of the cognominal \refcollargs{verbatim}, \refcollargs{verb} + or \refcollargs{no verbatim} among the collector options. + + Precious few CollArgs' options thus remain to be set by the author. For + memoization, the most likely candidates are \refcollargs{ignore nesting} + and \refcollargs{ignore other tags}, which could help deal with unusual + environments. Overriding the initial \refcollargs{end tag} by + \refcollargs{begin tag}, \refcollargs{end tag} and/or \refcollargs{tags} + might also be useful on occasion. + + During advising and advice setup, the \pkg{pgfkeys} and the raw collector + options of the advised command are accessible through + \docaux{cmd}{AdviceCollectorOptions} and + \docaux{cmd}{AdviceRawCollectorOptions}, both a parameterless macro expanding + to the given \meta{keylist} and \meta{code}, respectively. +\end{doc} + +\begin{doc}{ + key={name=clear collector options}, + key={name=clear raw collector options}, + } + These keys empty the list of user-friendly and raw collector options, + respectively. +\end{doc} + +\begin{doc}{ + key={name=inner handler, par=\meta{\hologo{TeX} code}, long description=0.5, + desc=initially and default: see below}, + } + + This key declares the \meta{\hologo{TeX} code} as the inner handler component + of the advice. + + The inner handler is intended to be used in situations which require + knowledge of the advised command's arguments as a whole. In such situations, + the outer handler will normally invoke the collector, which will in turn + execute the inner handler and provide it with a single (braced) argument, + containing the collected arguments of the advised command. See + \refmmzauto{memoize} for the usage case which inspired this design. + + The simplest example of an inner handler is a (single-parameter) macro which + does nothing. Surprisingly enough, such an inner handler could be useful. + Defining |\def\Gobble#1{}| and setting + \code{\refmmz{auto}=\cs{foo}\bracestt{\refmmzauto{inner handler}=\cs{Gobble}, + \refmmzauto{args}=\bracestt{...}}} with the argument structure + appropriate for |\foo| could be used to eradicate all invocations of |\foo| + from the document. + + The inner handler has access to all the macros available to the outer + handler, but given that most of them have already fulfilled their function, + only the following will likely be useful in the inner handler: + \refcmd{AdviceNamespace}, \refcmd{AdviceName}, \refcmd{AdviceCsname}, + \refcmd{AdviceReplaced}, \refcmd{AdviceOriginal}, and \refcmd{AdviceOptions}. + + Because there is clearly no reasonable default for the inner handler, this key + is initially set to an internal control sequence producing an ``undefined + inner handler'' error. Note that it is not necessary to define a dummy inner + handler when handling is entirely performed by the outer handler, i.e.\ in + cases when the inner handler is not invoked. Executing this key without a + value restores it to the initial value. + + During advising and advice setup, the inner handler of the advised command is + accessible through \docaux{cmd}{AdviceInnerHandler}, a parameterless macro + expanding to the given \meta{\hologo{TeX} code}. +\end{doc} + +\begin{doc}{easy, + key={name=options,par=\marg{keylist}, + desc={cumulative, initially empty, value required}}, + key={name=clear options}, + } + + The first key appends the given \meta{keylist} to the list of advice options + (after prefixing it by a comma), and the second one empties this list. For a + \meta{key} undefined in keypath \refkeypath{/mmz/auto}, + \meta{key}|=|\meta{value} has the same effect as + \refmmzauto{options}|=|\bracestt{\meta{key}=\meta{value}}. + + In Memoize, the options set by this key are known as + \emph{auto-options} --- options which are applied (using \refcmd{mmzset}) at + every invocation of the advised command or environment. For example, the + \env{tcolorbox} environment of package \pkg{tcolorbox} is used extensively + for typesetting this manual, and I have submitted this environment to + automemoization. However, the \env{tcolorbox}es in this manual often include + code listings. To memoize such environments successfully, their bodies must + be grabbed verbatim. I have therefore submitted the \env{tcolorbox} + environment to automemoization like this: + \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize}, + \refmmzauto{options}=\refmmz{verbatim}}; the simpler + \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize}, \refmmz{verbatim}} + would work as well. + + In general, whether to use the options set by this key, and how, remains at + the sole discretion of the advice. Note that they might be used by either + the outer or the inner handler, or perhaps even the collector. + + During advising and advice setup, the options of the advised command are + accessible through \docaux{cmd}{AdviceOptions}, a parameterless macro expanding + to the given \meta{keylist}. +\end{doc} + +\begin{doc}{key={name=reset, desc=style}} + Executing this key restores all \refmmz{auto} keys to their initial values. + + Invoking \refmmz{auto} on the same command or environment again + \emph{updates} the advice configuration. Use this key to start from scratch. +\end{doc} + +\begin{doc}{key={name=after setup, desc={initially empty, cumulative}}} + The code given to this key will be executed after exiting the group opened by + \refmmz{auto}. The same effect may be achieved by appending to macro + \docaux{cmd}{AdviceAfterSetup}. + + For example, \refmmzauto{integrated driver} uses this key to declare a new + conditional. +\end{doc} + +\paragraph{Commands available during the execution of advice} + +With the exception of \refcmd{AdviceGetOriginal} and +\refcmd{AdviceCsnameGetOriginal}, the commands listed below only become +available in the outer handler, and if that handler does nothing funky, they +should be available in the collector and the inner handler, as well. However, +once the advice yields control to foreign code, these macros are not guaranteed +to hold the expected values anymore, because the foreign code might trigger +another piece of advice. Consequently, these macros should be expanded, once, +before integrating them into arbitrary (non-advice) code; in particular, this +applies to \refcmd{AdviceOriginal}. + +\begin{doc}{cmd={name=AdviceNamespace}} + This macro holds the \meta{namespace}, i.e.\ the keypath which this instance + of the advising framework was installed into. +\end{doc} + +\begin{doc}{cmd={name=AdviceName}} + This macro holds the name of the advised command or environment, i.e.\ the + name which was used as the first argument to \refmmz{auto}. For a command, + this will be a control sequence, e.g.\ |\foo|; for environments (in any + \hologo{TeX} format), their name, e.g.\ |foo|. +\end{doc} + +\begin{doc}{cmd={name=AdviceCsname}} + This macro holds the control sequence name of the advised command; it is + undefined for environments. For example, when \refcmd{AdviceName} contains + |\foo@bar|, this macro will hold |foo@bar|. +\end{doc} + +\begin{doc}{cmd={name=AdviceReplaced}} + This macro holds the code which was replaced by the outer handler. For + commands, this will be the command itself, e.g.\ |\foo|, so + \refcmd{AdviceReplaced} will equal \refcmd{AdviceName}. For an environment + |foo|, \refcmd{AdviceReplaced} is set to |\begin{foo}| in \hologo{LaTeX}, + |\foo| in \hologo{plainTeX} and |\startfoo| in \hologo{ConTeXt}. +\end{doc} + +\begin{doc}{cmd={name=AdviceOriginal}} + This macro executes the original code of the advised command. + + This macro is defined as \refcmd{AdviceGetOriginal}\marg{namespace}\marg{name}, + and therefore acts as a shortcut for an explicit invocation of + \refcmd{AdviceGetOriginal}. When executing the original command directly from + the advice, one may safely write \refcmd{AdviceOriginal}. However, whenever + \refcmd{AdviceOriginal} is embedded in code which might contain other advised + commands, it should be pre-expanded, exactly once. +\end{doc} + +\begin{doc}{cmd={name=AdviceGetOriginal, par=\marg{namespace}\marg{control sequence}}} + This command invokes the original definition of the \meta{control sequence} + advised by the \meta{namespace} instantiation of Advice; more precisely, the + full expansion of this macro produces the (internal) control sequence holding + the definition of \meta{control sequence} in effect when this control + sequence was activated in \meta{namespace}. This macro may be safely used + outside the advice, even if the advised command is not activated. + + For example, upon executing key |/ns/|\refmmz{auto}|=\foo{...}|, + |\AdviceGetOriginal{/ns}{\foo}| will recall the original definition of |\foo| + if |\foo| is activated, and simply execute |\foo| otherwise. + + The second argument of this command should \emph{not} be an environment name. + To execute the original environment |foo| in \hologo{plainTeX} or + \hologo{ConTeXt}, use \refcmd{AdviceGetOriginal} with the appropriate macro: + |\AdviceGetOriginal{/ns}{\foo}| or |\AdviceGetOriginal{/ns}{\startfoo}|. In + \hologo{LaTeX}, one should use |\AdviceGetOriginal{/ns}{\begin}{foo}|, which + executes the original \refcmd{begin} and provides it with the environment + name. + + Within the advice, you will probably never have to use this command directly, + but will rather rely on the (plain or pre-expanded) \refcmd{AdviceOriginal}. + However, outside the advice, this command provides the only official means to + access the original definition of an advised command. (Unlike the + commands described above, this command is available throughout the + document.) + + \begin{tcolorbox}[warning] + A typo in the invocation of this command may result in an infinite loop. + Assume that the advice for |\foo|, declared in namespace \refkeypath{/mmz}, + executes \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which incorrectly refers + to the non-existing namespace |/zzm|, and that command |\foo| is activated. + Executing |\foo| will eventually execute + \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which won't find the original + definition of |\foo| in the non-existing namespace |/zzm| and will thus + execute macro |\foo| (again), which, being advised, will lead to another + \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, etc. My advice is to define an + abbreviation like |\def\mmzAdviceGetOriginal{\AdviceGetOriginal{/mmz}}|. + And note that the name\-space is a full keypath, which begins with a slash + (|/|), but has no slash at the end. + \end{tcolorbox} +\end{doc} + +\begin{doc}{cmd={name=AdviceCsnameGetOriginal, + par=\marg{namespace}\marg{control sequence name}}} + This is a version of \refcmd{AdviceGetOriginal} which accepts a control + sequence name as the second argument. + + This macro is used by auto-key \refmmzauto{to context} to include the meaning + of any command (even internal commands containing |@|, or \pkg{expl3} + commands) into the context. +\end{doc} + +\paragraph{Support for specific packages} +At the moment, Advice only implements specific support for \TikZ, by defining a +\refmmzauto{collector} for command \refcmd{tikz}. + +\begin{doc}{ + cmd={name=AdviceCollectTikZArguments}, + } + This command collects the arguments in the format expected by \refcmd{tikz}, and + executes macro \refcmd{AdviceInnerHandler} with the collected arguments given + as a single braced argument. The collector supports both the group and the + semicolor invocation of |\tikz|, i.e.\ both |\tikz{...}| and |\tikz...;|. + + This command is only available upon |\input|ting file + \docaux{file}{advice-tikz.code.tex}. +\end{doc} + +\endgroup % keypath = /mmz/auto + +\subsubsection{Memoization-related additions to the advising framework} +\label{sec:ref:advice:memoization} + +In section~\ref{sec:ref:advice}, we have seen that Memoize installs the +advising framework into keypath \refkeypath{/mmz}, with setup key name +\refmmz{auto}. This populates keypaths \refkeypath{/mmz} and +\refkeypath{/mmz/auto} with various generic advice keys. However, Memoize +installs further advice\slash automemoization-related keys into these keypaths. +It is these keys which are described in this section.\footnote{One such key, + \refmmzauto{integrated driver}, is actually documented in + section~\ref{sec:ref:memoization}.} + +Therefore, in contrast to section~\ref{sec:ref:advice}, \refkeypath{/mmz} and +\refmmz{auto} have no secret generic meaning here, i.e.\ they should \emph{not} +be generalized to \meta{namespace} and \meta{setup key} of +\refkey{/handlers/.install advice}. + +\paragraph{Keys residing in \refkeypath{/mmz}} + +\begingroup +\setlength\textfloatsep{20pt plus 2pt minus 4pt}% back to the default + +\begin{doc}{easy, + key={name=manual, description={preamble-only,{ }}, conditional=false}, + } + When this conditional is set to true, no commands are \refmmz{activate}d at + the beginning of the document. The list of commands and environments advised + and activated out of the box can be found in Table~\ref{tab:advised-commands}. + + The auto-framework allows the activation to be deferred (see + \refmmz{activation} and \refkey{/handlers/.install advice}) but leaves it + open to the specific instance of the framework to use the deferred activation + commands as it sees fit. Normally, Memoize switches to immediate activation + at the end of the preamble (hook \refmmz{begindocument/before}) and issues + \refmmz{activate deferred} at the beginning of the document, more precisely + in hook \refmmz{begindocument/end} (afterwards, \refmmz{activate deferred} is + emptied). However, when \refmmz{manual} is in effect, the deferred + activation is suppressed (though it may be still carried out by the user by + executing \refmmz{activate deferred}). + + In \hologo{LaTeX}, \refmmz{manual} affects the (internal) activation of + \refcmd{begin} as well, which effectively deactivates handling of all environments. +\end{doc} + +\begin{table} + \centering + \begin{tabularx}{\linewidth}{llX} + \toprule + command/environment&handler¬es\\ + \midrule + \refcmd{begin}&custom&Only in \hologo{LaTeX}; declared by Advice.\\ + \refcmd{errmessage}&\refmmzauto{abort}&Not available in \hologo{LuaTeX}, + where better error-detection is implemented.\\ + \refenv{forest}&\refmmzauto{memoize}\\ + \refcmd{Forest}&\refmmzauto{memoize}\\ + \refcmd{index}&\refmmzauto{replicate}&The argument is expanded prior to replication.\\ + \refcmd{label}&custom&\refmmzauto{run if memoizing}; globally appends + \refcmd{mmzLabel}\marg{label key}\marg{current label} + to register \refcmd{mmzCCMemo}.\\ + \refcmd{pageref}&\refmmzauto{ref}\\ + \refcmd[into index=false]{pdfsavepos}&\refmmzauto{abort}&Not available in \hologo{LuaTeX}.\\ + \refcmd{pgfsys@getposition}&\refmmzauto{abort}&Available only in \TikZ is + loaded, it aborts memoization of a picture which gets accidentally marked + as + ``remembered''.\\ + \refcmd{ref}&\refmmzauto{ref}\\ + \refcmd[short]{savepos}&\refmmzauto{abort}&Available only in \hologo{LuaTeX}.\\ + \refcmd{tikz}&\refmmzauto{memoize tikz}\\ + \refenv{tikzpicture}&\refmmzauto{memoize tikz}\\ + \bottomrule + \end{tabularx} + \caption{Commands advised by Memoize} + \label{tab:advised-commands} +\end{table} + +\begin{doc}{easy, + key={name=ignore spaces, conditional=false}, + } + Ignore any spaces following \emph{auto}memoized code. This key has no effect + for manual memoization, i.e.\ command \refcmd{mmz} and environment + \refenv{memoize}. + + It is common practice to conclude the definition of a command by \hologo{TeX} + primitive \refcmd{ignorespaces}, which consumes any following spaces, to + prevent unintended blank space after the command's invocation. Automemoizing + such a command disrupts this behaviour.\footnote{It is clear that + \refcmd{ignorespaces} is disrupted during utilization; in this case, the + original command, including the concluding \cs{ignorespaces}, is never even + executed. However, the disruption also occurs during memoization, and even + during regular compilation. In both cases, the memoized code is embedded + in some internal Memoize code. Therefore, the original \cs{ignorespaces} + does not occur directly in front of the rest of the document.} The + workaround is to use this key, normally as an option in the \refmmz{auto} + declaration; it will work both for automemoized macros and environments. +\end{doc} + +\paragraph{Keys residing in \refkeypath{/mmz/auto}} + +\begingroup +\yadocset{keypath=/mmz/auto} + +\begin{doc}{easy,key={name=memoize, desc=style}} + This key sets up advice which triggers memoization of the command or + environment whenever it is encountered; we often refer to such a command as + ``automemoized,'' or say that it was ``submitted to automemoization.'' + + An automemoized command will consume the next-options, whether memoization + actually occurs or not. + + Under the hood, this key declares both an \refmmzauto{outer handler} and an + \refmmzauto{inner handler}. The outer handler opens the memoization group + (so that options can be applied locally), applies the auto-options (given by + \refmmzauto{options} within \refmmz{auto}) and the next-options (given by + \refcmd{mmznext}) by executing \refmmzauto{apply options}, and appends the + verbatim keys to \refmmzauto{collector options} if necessary. The inner + handler invokes \refcmd{Memoize} (which closes the group opened by the outer + handler): the first argument is \refcmd{AdviceReplaced}, expanded once and + followed by the arguments of the handled command; the second argument is + \refcmd{AdviceOriginal}, also expanded once and followed by the arguments of + the handled command. The inner handler also makes sure that \refmmz{ignore + spaces} is respected. +\end{doc} + +\begin{doc}{easy,key={name=nomemoize,desc=style}} + This key installs advice which disables memoization for the space of the + command or environment; we sometimes refer to such commands as + ``autodisabled.'' + + This key is merely an abbreviation for \refmmzauto{noop}|,| + \refmmzauto{options}|=|\refmmz{disable}. See the documentation of + \refmmzauto{noop} for further details. +\end{doc} + +\begin{doc}{key={name=noop,desc=style}} + This key sets up advice which does nothing. + + Ok, not nothing at all. The installed handler applies the auto-options and + the next-options by executing \refmmzauto{apply options}, and makes sure that + \refmmz{verbatim} and \refmmz{ignore spaces} are respected. + + For commands and non-\hologo{LaTeX} environments, this key declares the same + outer handler as \refmmzauto{memoize}, while the inner handler merely + executes the original command (respecting the potential verbatim mode), + closes the group opened by the outer handler, and makes sure that + \refmmz{ignore spaces} is respected. + + For \hologo{LaTeX} environments, which open the group necessary for the local + application of options themselves, this key declares an outer handler which + adds the relevant code into the next hook |env/|\meta{environment + name}|/begin|. There is no need to open a group, collect the environment + body, or make special provisions for the verbatim mode. +\end{doc} + + +\begin{doc}{key={name=apply options,desc=style}} + This style, used by \refmmzauto{memoize}, \refmmzauto{nomemoize} and + \refmmzauto{noop} described above, installs two handlers: + \begin{itemize} + \item an outer handler which opens a group, applies auto-options and + next-options by executing \refcmd{mmzAutoInit}, and executes the collector; + and + \item a bailout handler which clears the next-options. + \end{itemize} +\end{doc} + + +\begin{doc}{cmd={name=mmzAutoInit}} + This macro applies the auto-options and the next-options. + + Additionally, if \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} was + previously executed, this style appends the corresponding CollArgs key + (\refcollargs{verbatim}, \refcollargs{verb} or \refcollargs{no + verbatim}) to \refcmd{AdviceRawCollectorOptions}. In case several of the + verbatim keys were executed, the final one takes effect. +\end{doc} + + +\begin{doc}{easy, key={name=abort,desc=style}} + This key sets up advice which aborts any ongoing memoization. + + Under the hood, the advice merely executes \refcmd{mmzAbort} followed by + \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the + next-options. Out of the box, we submit two control sequences to this + handler: + \begin{itemize} + \item \refcmd{errmessage}: this allows us to detect and abort upon at least some + errors. + \item \refcmd[into index=false]{pdfsavepos} (in \hologo{LuaTeX}, + \refcmd[short]{savepos}): one common effect is that memoization of any + \TikZ picture with \refkey{/tikz/remember picture} set is aborted. + \end{itemize} +\end{doc} + + +\begin{doc}{key={name=unmemoizable,desc=style}} + This key sets up advice which aborts the ongoing memoization and marks the + automemoized code as unmemoizable, so that it will be henceforth compiled + regularly. + + Under the hood, the advice merely executes \refcmd{mmzUnmemoizable} followed + by \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the + next-options. + + Out of the box, we submit no control sequences to this advice, but it might + make sense to submit \refcmd[into index=false]{pdfsavepos}\slash + \refcmd[short]{savepos}. Keys \refmmzauto{abort}|=|\refcmd[short]{savepos} + and \refmmzauto{unmemoizable}|=|\refcmd[short]{savepos} will most often have + the same effect, as far as the author is concerned; the former was chosen as + the default because it does not produce a c-memo; see + \refcmd{mmzUnmemoizable} for a situation where \refmmzauto{unmemoizable} is + preferred. +\end{doc} + +\begin{doc}{key={name=memoize tikz,desc=style}} + This key is used to declare memoization of \TikZ pictures; it may also be + used for PGF pictures. Besides executing \refmmzauto{memoize}, it uses + \refmmz{at begin memoization} and \refmmz{at end memoization} to add the code + which measures the increase of the PGF picture ID during memoization of a + picture, and increases this ID for the measured amount upon the utilization + of the extern, thereby making sure that a utilized \TikZ\slash\PGF extern + advances the PGF picture ID as if the picture was compiled. +\end{doc} + +\begin{doc}{easy, + key={name=ref,desc=style}, + key={name=force ref,desc=style}, + } + + These keys set up advice which adds the reference key to the context + expression. They are intended to be used with cross-referencing commands + such as \refcmd{ref} and \refcmd{pageref}. + + Indeed, \refcmd{ref} and \refcmd{pageref} are submitted to this advice by + Memoize, with the effect that standard cross-referencing inside memoized code + ``just works.'' Note that the stabilization of the document after changing + the reference takes three compilation cycles, i.e.\ one cycle more than + without memoization. + + The advice set up by \refmmzauto{ref} aborts memoization if the reference + key is undefined, the rationale being that the produced memo and extern would + most often be useless, and could even obscure an undefined reference. The + \refmmzauto{force ref} handler produces the memo and the extern even when the + reference is undefined. + + The reference produced by the advised command should be fully expandable + (because it will be expanded as a part of the context expression). + + Typically, a \refcmd{ref} command takes a single argument, the reference key. + However, some packages may define a reference command which takes optional + arguments, as well; in particular, the \pkg{hyperref}'s incarnation of \refcmd{ref} + takes an optional star. This advice does not care: it will accept any number + of any kind of optional arguments, as long as the reference key is the first + braced argument following the advised command; for example, |\ref*{key}|, + |\ref[opt]{key}|, |\ref*[opt]{key}| etc.\ will all be handled correctly, + while |\ref{mand}{key}| will not work. Effectively, it is as if we had set + \refmmzauto{args}|=lm| --- and with the same downside, namely that an + unlikely unbraced single-token reference key, like |\ref k|, will not work. + + Under the hood, these two pieces of advice pass the reference key to macros + \refcmd{mmzNoRef} and \refcmd{mmzForceNoRef}, and it is these commands --- + which may also be used in user-defined advice or the document itself --- + which actually add the reference key to the context expression. +\end{doc} + +\begin{doc}{easy, + key={name=refrange,desc=style}, + key={name=force refrange,desc=style}, + } + These keys have the same function as \refmmzauto{ref} and \refmmzauto{force + ref}, but they operate on reference-range commands, such as \pkg{cleveref}'s + \refcmd{crefrange}, which take two arguments (the starting and the ending reference + key). +\end{doc} + +\begin{doc}{easy, + key={name=multiref,desc=style}, + key={name=force multiref,desc=style}, + } + These keys have the same function as \refmmzauto{ref} and \refmmzauto{force + ref}, but they operate on ``multireference'' commands, such as + \pkg{cleveref}'s \refcmd{cref}, which allow the author to list several + comma-separated reference keys in a single argument. +\end{doc} + +\begin{doc}{ + key={name=to context, desc=style} + } + This key sets up advice which appends the original meaning of the advised + command to the context. It invokes \refmmzauto{run if memoizing}, so that + the command is only advised during memoization. + + It is safe to apply this key to internal commands such as commands whose name + contains |@|, or \pkg{expl3} commands. It is not necessary to provide the + argument structure of the advised command (using \refmmzauto{args}). + +\end{doc} + +\begin{doc}{ + key={name=replicate, desc=style} + } + This key sets up advice which replicates the invocation of the command in + the cc-memo during memoization. + + When using this key, it is necessary to set \refmmzauto{args} as well. For + \refcmd{index}, Memoize executes + \refmmz{auto}|=\index|\bracestt{\refmmzauto{args}=m, \refmmzauto{replicate}}. + + This key takes an auto-option, \docAux{keypath=/mmz/auto/replicate, + key={name=expanded}}.\footnote{This option is unrelated to Memoize's options, + settable by \refcmd{mmzset}.} If given, the collected arguments will be + expanded before replicating them in the cc-memo; in \hologo{LaTeX}, this + expansion is |\protect|ed. + + In \hologo{LaTeX}, Memoize submits \refcmd{index} to this handler (with + expansion). Therefore, any |\index{key}| in the memoized code gets copied + into the cc-memo. Effectively, indexing from within the memoized code ``just + works.'' + + Note that \refcmd{label}, despite essentially requiring replication, cannot + use this advice, because it needs to replicate not only the label key but + \refcmd{@currentlabel} as well. +\end{doc} + +\begin{doc}{key={name=run if memoization is possible, desc=style}} + Under the run conditions installed by this key, a command is only advised if + Memoize is enabled but we're not already ``within Memoize,'' i.e.\ memoizing + or normally compiling some code submitted to memoization. In code: + \refcmd{ifmemoize}\refcmd{ifinmemoize}|\else|\refcmd{AdviceRuntrue}|\fi\fi|. + + Internally, this key is used by \refmmzauto{memoize} and \refmmzauto{noop}. +\end{doc} + +\begin{doc}{key={name=run if memoizing, desc=style}} + Under the run conditions installed by this key, a command is only advised + during memoization. In code: + \refcmd{ifmemoize}\refcmd{ifmemoizing}\refcmd{AdviceRuntrue}|\fi\fi|. + + Internally, this key is used by \refmmzauto{abort}, \refmmzauto{replicate}, + and \refmmzauto{ref} and friends. +\end{doc} + +\endgroup % keypath = /mmz/auto +\endgroup + + + +\subsubsection{Package CollArgs} +\label{sec:ref:collargs} + +\begingroup +\yadocset{keypath=/collargs} + +\begin{doc}{cmd={ + name=CollectArguments, + par=\oarg{options}\marg{argument specification}\marg{next-code}% + \textcolor{gray}{\meta{tokens}}}} + + This command determines the extent to which the \meta{tokens} following the + the three formal arguments of the command conform to the given \meta{argument + specification}, effectively splitting \meta{tokens} into \meta{argument + tokens} and the \meta{rest} of the tokens, and then executes + \meta{next-code} with the \meta{argument tokens} provided as a single, braced + argument: + \begin{center} + \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}} + \end{center} + If the initial part of \meta{tokens} does not conform to \meta{argument + specification}, \refcmd{CollectArguments} throws an error. (In this case, + \meta{next-code} is not executed, and the \meta{tokens} collected until the + error are thrown away.) + + The optional \meta{options} are processed using the \pkg{pgfkeys} utility of + PGF/\TikZ (see \PGFmanual{87}), with the default path set to + \docaux{key path}{collargs}. The given options apply to all the arguments in + \meta{argument specification}. The recognized keys are listed in the rest of + the section. + + The \meta{argument specification} should be given in the \pkg{xparse} format + (we summarize this format in the documentation for \refmmzauto{args} in + section~\ref{sec:ref:advice}), with several extensions:\footnote{Collargs + internally uses a dot (|.|) to delimit the argument specification from the + following argument tokens. Therefore, the dot really counts as an extra + argument type, in the sense that Collargs will stop working if the dot + becomes an argument type or a modifier in some future release of + \pkg{xparse}.} + \begin{itemize} + \item We introduce modifier \docAux{xparse modifier={name=\&,label=amp,index + annotation/.prefix={additional\ }, index annotation/.append={\ + (options)} }} taking a mandatory argument specifying the options to + apply to the following argument in the specification. Options given here + override the \meta{options} given as the optional argument. + \item The environment body type \docAux{xparse type={name=b,index + annotation/.append={\ (environment body)}}} may be followed by an optional + \emph{braced} argument providing the name of the environment to collect. + The name given here overrides the name given by the + \refcollargs{environment} option. + \item The number of collected ``arguments'' is unlimited. + \end{itemize} + + Also note that the effect of \docref{xparse:O}\marg{default} is the same as + the effect of \docref{xparse:o}, and similarly for other pairs of types with + and without defaults (\docref{xparse:R} and \docref{xparse:r}, + \docref{xparse:D} and \docref{xparse:d}, and \docref{xparse:E} and + \docref{xparse:e}). CollArgs is dedicated to collecting the argument tokens + precisely as they are given: if an optional argument is missing, its default + value is \emph{not} inserted among the collected arguments --- consequently, + \refcmd{CollectArguments} is utterly uninterested in the default value. + + Collection of environments automatically adapts to the format, i.e.\ given + environment body name |foo|, \refcmd{CollectArguments} knows to search for + \refcmd{begin}\bracestt{foo} |...| \refcmd{end}\bracestt{foo} in + \hologo{LaTeX}, \cs{foo} |...| \cs{endfoo} in \hologo{plainTeX}, and + \cs{startfoo} |...| \cs{stopfoo} in \hologo{ConTeXt}. For further + information on environment collection, see keys \refcollargs{ignore + nesting} and \refcollargs{tags}. +\end{doc} + +\begin{doc}{cmd={ + name=CollectArgumentsRaw, + par=\marg{option-setting code}\marg{argument specification}\marg{next-code}% + \textcolor{gray}{\meta{tokens}}}} + + This command is the programmer's interface to CollArgs, intended to be used + instead of \refcmd{CollectArguments} when compilation speed is an issue. The + two commands only differ in how they deal with options. + + One difference is that for \refcmd{CollectArgumentsRaw}, the options form a + mandatory rather than an optional argument. More importantly, however, they + do not take the form of a keylist, but should be composed out of low-level + option-setting commands. Each key documented in this section has a + corresponding low-level macro; these macros are listed in footnotes alongside + the keys. The name of the macro starts with |\collargs| and continues with + the name of the key, without spaces, each word capitalized; if the key is + boolean, this convention applies to the base of the \hologo{TeX} + conditional. For example, + \begin{tcblisting}{listing only, listing options app={escapechar=|}} +\CollectArguments[caller=\foo, tags, verbatim]|\marg{argument specification}\marg{next-code}| + \end{tcblisting} + is equivalent to + \begin{tcblisting}{listing only, listing options app={escapechar=|}} +\CollectArgumentsRaw{% + \collargsCaller{\foo}% + \collargsBeginTagtrue\collargsEndTagtrue + \collargsVerbatim +}|\marg{argument specification}\marg{next-code}| + \end{tcblisting} + + Withing the option-setting code, the programmer may also deploy macro + \docaux{cmd}{collargsSet}, which processes the \meta{options} in the keylist + format. One idea could be to execute this macro at the end of the low-level + options; this would set the ``defaults'' using the fast programmer's + interface, but still allow for user customization. +\end{doc} + + +\begin{doc}[ + pi={\docaux{cmd}{collargsCaller}}, + ]{ + key={name=caller, par=\meta{control sequence (name)}, + desc={no default, initially \cs{CollectArguments}}}, + } + Set the control sequence to refer to in error messages. + + If \meta{tokens} do not match the \meta{argument specification}, + \refcmd{CollectArguments} throws an error. By default, the error message + contains a reference to \refcmd{CollectArguments} itself, for example % + |! Argument of \CollectArguments has an extra }.| However, this might not be +very informative to the author. When \refcollargs{caller}|=\cs| is in effect, +the error messages will refer to the given |\cs| instead. + +If the value of this key is not a control sequence, it is assumed to be an +environment name, but as the caller must be a macro, this name will be +converted into a control sequence. Setting \refcollargs{caller}|=foo| will +result in error messages referencing |\foo| in \hologo{plainTeX}, |\startfoo| +in \hologo{ConTeXt} and |\begin{foo}| (a single control sequence!) in + \hologo{LaTeX}. +\end{doc} + +\begin{doc}[ + pi=\docaux{cmd}{collargsEnvironment}, + ]{key={name=environment, par=\meta{environment name}, + desc={applicable to type \docref{xparse:b}, no default, initially empty}} + } + + Set the name of the environment collected by argument type + \docref{xparse:b}. +\end{doc} + +\begin{doc}[ + pi={\docaux{cmd}{ifcollargsBeginTag}, \docaux{cmd}{ifcollargsEndTag}; + \refcollargs{tags} has no corresponding low-level command} + ]{ + key={name=begin tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false}, + key={name=end tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false}, + key={name=tags, par=\meta{boolean}, % not a conditional + desc={applicable to type \docref{xparse:b}, style, default |true|}}, + } + + When \refcollargs{begin tag}\slash \refcollargs{end tag} is in effect, the + begin\slash end tag will be will be prepended/appended to the collected + environment body. Style \refcollargs{tags} is a shortcut for setting + \refcollargs{begin tag} and \refcollargs{end tag} simultaneously. + + In \hologo{LaTeX}, using \refcollargs{tags} will thus dress up the + collected body in a pair or \refcmd{begin}\marg{environment name} and + \refcmd{end}\marg{environment name}. CollArgs will automatically use the tags + appropriate to the format. + + In the verbatim modes, the added tags are verbatim as well, with the detail + that in \hologo{LaTeX}, there is a slight difference between the full + \refcollargs{verbatim} and the partial \refcollargs{verb} mode. In the + full verbatim mode, the braces surrounding the environment name are verbatim + (the characters used as braces are actually determined by key + \refcollargs{braces}). In the partial verbatim, as well as the + non-verbatim mode, the environment name is surrounded by a pair of actual + braces of category 1 and 2, regardless of which characters are of these + categories in the calling code. +\end{doc} + +\begin{doc}[ + pi=\docaux{cmd}{ifcollargsIgnoreNesting}, + ]{ + key={name=ignore nesting, desc={applicable to type \docref{xparse:b},{ }}, + conditional=false}, + } + When this key is \emph{not} in effect, CollArgs respects the hierarchical + structure created by tag pairs such as \refcmd{begin}\bracestt{foo} and + \refcmd{end}\bracestt{foo}. Given the situation below on the + left, argument type |b{foo}| will collect everything up until the + \emph{second} |\end{foo}|. Now this is what we usually want, because +\hologo{LaTeX} keeps track of environment embedding as well. However, all +\emph{verbatim} environments I know of, starting with the standard +\hologo{LaTeX} \refenv{verbatim}, will ignore the nesting and simply scoop up +everything up to the first \cs{end}|{verbatim}|. In CollArgs, we can replicate +their behaviour by setting \refcollargs{ignore nesting}, as shown below on the +right. (Of course we also need to set \refcollargs{verbatim} if we want to +grab the environment body in the verbatim mode.) + + \begin{tcbraster}[raster columns=2] + \begin{tcblisting}{listing only, mark region={1}{5}, + example title=\texttt{ignore nesting=false}} + ... + \begin{foo} + ... + \end{foo} + ... +\end{foo} + \end{tcblisting} + \ExampleName{_collargs-verbatim} + \makeexample{\examplename.tex.c1} + \tcbinputexample{listing only, no attachment, mark region={1}{3}, + example title=\texttt{ignore nesting=true}} + \end{tcbraster} + + This key applies not only to argument type \docref{xparse:b} (in either + normal or verbatim mode), but also to the verbatim argument type + \docref{xparse:v} and to argument types \docref{xparse:m} and + \docref{xparse:g} in the \refcollargs{verbatim} (but not normal, or + \refcollargs{verb}) mode. With these keys, the relevant structure markers + are braces, |{| and |}|. +\end{doc} + +\begin{doc}[ + pi=\docaux{cmd}{ifcollargsIgnoreOtherTags}, + ]{ + key={name=ignore other tags, desc={applicable to type \docref{xparse:b},{ }}, + conditional=false}, + } + In \hologo{LaTeX}, the environment tags, \refcmd{begin}\marg{name} and + \refcmd{end}\marg{name}, contain braces, which retain their usual category codes + in the non-verbatim and in the partial verbatim mode. Consequently, CollArgs + cannot easily search for the full tags to delimit the + environment. + + When this key is \emph{not} in effect, CollArgs takes the easy path, and + determines the end of the environment only by inspection of \cs{begin}s and + \cs{end}s, without reference to what \meta{name} they begin or end. Only + when this key \emph{is} in effect does CollArgs inspect these \meta{name}s, + effectively ignoring any \cs{begin}s and \cs{end}s not followed by the name of + environment being collected. The effect of absence vs.\ presence of this key + is shown below, where the shaded area marks the code collected into + environment |foo|. + + \begin{tcbraster} + \ExampleName{_collargs-ignore-other-tags} + \makeexample{\examplename.tex.c1 N=2} + \tcbinputexample{listing only, no attachment, mark region={3}{3}, + example title=\texttt{ignore other tags=false}} + \tcbinputexample[.tex][.c2]{listing only, no attachment, mark region={3}{5}, + example title=\texttt{ignore other tags=true}} + \end{tcbraster} + + This key does not have any effect the full verbatim mode, which always + behaves as if this key was set to true, because braces are of category + ``other'' as well. Similarly, it is as if this key was always true in + \hologo{plainTeX} and \hologo{ConTeXt}, simply because environment tags in + these formats don't contain braces. +\end{doc} + +\begin{doc}[ + pi={\raggedright + \docaux{cmd}{collargsAppendPreprocessor}, + \docaux{cmd}{collargsPrependPreprocessor}, + \docaux{cmd}{collargsAppendPostprocessor}, + \docaux{cmd}{collargsPrependPostprocessor}}, + ]{ + key={name=append preprocessor, par=\meta{code}, desc={style, no default}}, + key={name=prepend preprocessor, par=\meta{code}, desc={style, no default}}, + key={name=append postprocessor, par=\meta{code}, desc={style, no default}}, + key={name=prepend postprocessor, par=\meta{code}, desc={style, no default}}, + } + These keys declare processors which will transform the collected argument + before appending it to the argument list. + + A collected argument undergoes the following transformations: + \begin{itemize} + \item First, the argument is processed by any \emph{pre}processors, in the + order indicated by |append| and |prepend|. + \item Next, the processed argument is dressed up in the delimiters according + to its type. For example, an optional argument of type \docref{xparse:o} + will be surrounded by square brackets. + \item Finally, the delimited argument is processed by any + \emph{post}processors, again in the order indicated by |append| and + |prepend|. + \end{itemize} + + \meta{code} will typically consist of a single control sequence pointing to a + one-argument macro, which will receive the collected argument (possibly + modified by the processors already applied). In general, however, the value + of this key may be any code; Collargs will execute \meta{code}\marg{collected + argument}. + + The processed argument should be returned by storing it into token register + \docaux{cmd}{collargsArg}. + + The following example illustrates how one could go about reimplementing Bruno + Le Floch ingenious package \pkg{cprotect}.\footnote{This example is merely a + proof of concept. For the bells and whistles which would make it useful in + real life, see the documentation of \pkg{cprotect}.} We define processor + |\writetofile| which dumps the argument into a file, replacing it with the + |\input| statement. (Of course, to allow for verbatim content in the + footnote, we also have to mark the argument as \refcollargs{verbatim}. + And we use \refcollargs{no delimiters} to get rid of the braces around the + footnote text.) + + \ExampleName{collargs-processor} + \makeexample{\examplename.tex.c1} + \tcbinputexample{listing and text, listing file=\examplepath.tex.c1} +\end{doc} + +\begin{doc}[ + pi={\docaux{cmd}{collargsClearPreprocessors}, + \docaux{cmd}{collargsClearPostprocessors}}, + ]{ + key={name=clear preprocessors}, desc=style, + key={name=clear postprocessors}, desc=style, + } + Clear the list of pre- or post-processors. +\end{doc} + +\begin{doc}[ + pi={\docaux{cmd}{collargsAppendExpandablePreprocessor}, + \docaux{cmd}{collargsPrependExpandablePreprocessor}, + \docaux{cmd}{collargsAppendExpandablePostprocessor}, + \docaux{cmd}{collargsPrependExpandablePostprocessor} + }, + ]{ + key={name=append expandable preprocessor, par=\meta{code}, desc={style, no default}}, + key={name=prepend expandable preprocessor, par=\meta{code}, desc={style, no default}}, + key={name=append expandable postprocessor, par=\meta{code}, desc={style, no default}}, + key={name=prepend expandable postprocessor, par=\meta{code}, desc={style, no default}}, + } + These keys may be used to add fully expandable processors. A processor added + with one of these keys will end up among the processors declared by + \refcollargs{append preprocessor} et al. + + A processor declared by one of these keys will define the processed argument + as the full expansion (|\edef|) of \meta{code}\marg{collected argument}. + \meta{code} will typically consist of a single control sequence pointing to a + fully expandable one-argument macro. + + For example, |\trim@spaces@noexp| from package \pkg{trimspaces} could be used + as an expandable processor of environment body to remove the spaces around + the grabbed environment body. +\end{doc} + +\begin{doc}[ + pi={\docaux{cmd}{collargsAppendPrewrap}, + \docaux{cmd}{collargsPrependPrewrap}, + \docaux{cmd}{collargsAppendPostwrap}, + \docaux{cmd}{collargsPrependPostwrap}}, + ]{ + key={name=append prewrap, par=\meta{macro definition}, desc={style, no default}}, + key={name=prepend prewrap, par=\meta{macro definition}, desc={style, no default}}, + key={name=append postwrap, par=\meta{macro definition}, desc={style, no default}}, + key={name=prepend postwrap, par=\meta{macro definition}, desc={style, no default}}, + } + These keys add processors which transform the collected argument in a single + expansion. + + The declared processor will use \meta{macro definition} to define a temporary + one-argument \meta{macro}, and then set the \meta{processed argument} to be + the single expansion of \meta{macro}\marg{collected argument}. + + For example, to add quotes around the collected argument, write + \refcollargs{append prewrap}|={``#1''}| (doubling the hash when executing + \refcmd{CollectArguments} from a macro, of course). Or, perhaps more + usefully, \refcollargs{append prewrap}|={\scantokens{#1}}| can be used to + retokenize a verbatim argument (during the execution of the + \meta{next-code}). +\end{doc} + + +\begin{doc}[ + pi=\docaux{cmd}{ifcollargsNoDelimiters} + ]{ + key={name=no delimiters, conditional=false}, + } + When this key is in effect, the collected argument will not be dressed up + into delimiters that it was dressed up in \meta{argument tokens}. For + example, an optional argument, encountered as [\meta{argument}] inside + \meta{argument tokens}, will be spit out simply as \meta{argument}. + + Any user-specified pre- or post-processing will still be applied. + \ExampleName{collargs-nodelimiters} + \makeexample{\examplename.tex.c1} + \tcbinputexample{comment=\input{\examplepath.tex.c1}} +\end{doc} + +\begin{doc}[ + pi=\docaux{cmd}{ifcollargsBraceCollected} + ]{ + key={name=brace collected, conditional=true}, + } + When this conditional is set to false, the collected arguments are not + enclosed in braces when passed on to \meta{next-code}. This probably only + makes sense when wrapping the individual arguments, e.g.\ by + \refcollargs{append postwrap}|={{#1}}|. +\end{doc} + +\begin{doc}[ + pi={\docaux{cmd}{collargsVerbatim}, + \docaux{cmd}{collargsVerb}, + \docaux{cmd}{collargsNoVerbatim}. + To ensure the same effect as with the keys, + place these macros at the end of the option code}, + ]{ + key={name=verbatim, desc=style}, + key={name=verb, desc=style}, + key={name=no verbatim, desc={style, the initial mode}}, + } + Select the full verbatim, the partial verbatim, or the non-verbatim mode of + argument collection. + + In the full \refcollargs{verbatim} mode, the arguments are collected under + a category code regime in which all characters are of category 12, ``other''. + The same goes for the partial \refcollargs{verb} mode, except that in this + case, the grouping characters --- usually the braces |{| and |}| --- retain + their usual category codes 1 and 2. Key \refcollargs{no verbatim} selects + the normal, non-verbatim mode. + + The partial \refcollargs{verb} mode can be useful for verbatim collection + of an optional argument. To pass |]| as an optional argument to command +|\foo|, we normally enclose it in braces: ˇ\foo[{~]~}]ˇ. However, if we try to +collect |[{]}]| with \refcmd{CollectArguments}|[|\refmmz{verbatim}|]{o}|, we +will get |{| (and most likely an error, as well), because in the + \refmmz{verbatim} mode, braces do not have their grouping function. Using + the \refmmz{verb} mode solves the problem: occuring within braces, the first + |]| is ``invisible'' to \refcmd{CollectArguments}|[|\refmmz{verb}|]|, so the +optional argument is correctly recognized as ending at the second |]|. + +The partial \refcollargs{verb} mode is also useful for collecting the bodies +of \emph{\hologo{LaTeX}} environments. The full \refcollargs{verbatim} mode +will only correctly collect these bodies when the relevant \refcmd{begin} +and\slash or \refcmd{end} control sequences are followed by the grouped +environment name without any intervening spaces. The partial +\refcollargs{verb} mode has no such restriction. + +In the verbatim modes, modifier \docref{xparse:+} has no effect. The arguments +are always collected as if they were long. + +To correctly collect arguments in the verbatim modes, CollArgs has to mimic the +many details of \hologo{TeX}'s tokenization and argument delineation. These +details depend on the category code regime, and CollArgs automatically adapts +to the ``outside'' category code regime, i.e.\ the regime in effect at the time +of invoking \refcmd{CollectArguments}. In particular, CollArgs remembers which +characters were of category codes 0, 1, 2, 5, 10 and 11, and adapts the +argument collection accordingly. For example, it will correctly pick up a +control sequence as a single-token \docref{xparse:m}-type (\hologo{TeX}'s undelimited) +argument even when it begins with a non-standard character of category code 0. +The single caveat is that only a single pair of characters can function as the +grouping characters in the full verbatim mode; to compensate for the +deficiency, this character pair is customizable via key +\refcollargs{braces}. +\end{doc} + +\begin{doc}[ + pi={\docaux{cmd}{collargsFixFromVerbatim}, + \docaux{cmd}{collargsFixFromVerb}, + \docaux{cmd}{collargsFixFromNoVerbatim}}, + ]{ + key={name=fix from verbatim, desc=style}, + key={name=fix from verb, desc=style}, + key={name=fix from no verbatim, desc=style}, + } + Key \refcollargs{fix from no verbatim} should be used when the first + argument should be collected in a verbatim mode, but the outside code has + already tokenized the first character of the subsequent input stream (most + probably by a |\futurelet|) in the non-verbatim category code regime. Using + this key will trigger a CollArgs' ``mode transition'' (described below) which + will fix the situation. (This key is used in the implementation of + \refcmd{mmz}.) + + The other two keys should be used in the unlikely reverse situation, where + the outside code has tokenized the following character in a verb(atim) mode, + while CollArgs is requested to collect the first argument in the non-verbatim + mode. +\end{doc} + +\begin{doc}[ + pi=\docaux{cmd}{collargsBraces} + ]{ + key={name=braces, par={\meta{begin-group char}\meta{end-group char}}, + desc={no default, for the initial value see below}}, + }% here + This key sets the verbatim begin-group and end-group characters. The setting + affects collection of argument types \docref{xparse:m}, \docref{xparse:g}, + \docref{xparse:v}, \docref{xparse:l}, \docref{xparse:e} and (in + \hologo{LaTeX}) \docref{xparse:b} in the \emph{full} verbatim + mode.\footnote{The choice of the verbatim grouping characters also affects + the effect of \refcollargs{begin tag} and\slash or \refcollargs{end tag}; + see the documentation of these keys for details.} + + For example, in the non-verbatim and the partial verbatim mode, an + \docref{xparse:m}-type argument may be delimited by any characters of + category code 1 (``begin-group'') and 2 (``end-group''). In the full + verbatim mode, there are of course no characters of these categories, so + CollArgs internally assigns the grouping function to some pair of characters. + When entering the full verbatim mode, CollArgs automatically sets the + verbatim grouping characters to characters which were of categories 1 and 2 + in the ``outside'' category code regime, i.e.\ the regime in effect at the + time of invoking \refcmd{CollectArguments}. However, in contrast to + \hologo{TeX}'s internal argument parser, only one pair of characters may + serve as the begin-group and the end-group character in CollArgs' full + verbatim mode. In case multiple characters were of category 1 or 2 on the + outside, CollArgs therefore has to make a choice, and it chooses the + candidate with the lowest character code. This choice may be overridden by + the user by invoking key \refcollargs{braces}; the user may even choose + characters which did not belong to categories 1 and 2 in the outside regime. + + When \meta{begin-group char} and \meta{end-group char} are of categories 1 + and 2 in the outside category regime, they must be enclosed in a triple + group. For example, if both |()| and |{}| have the grouping function on the + outside, and the user wants to select |{}| as the verbatim grouping + characters (CollArgs would go for |()|, as this pair has lower character + codes), the correct way to invoke this key is |braces={{{{}}}}| or + |braces=((({})))|. \footnote{This complication is due to the details of + \pkg{pgfkeys}' keylist processing, and does not apply to + \refcmd{collargsBraces}.} +\end{doc} + +\begin{doc}[ + pi=\docaux{cmd}{collargsVerbatimRanges} + ]{ + key={name=verbatim ranges, + par={\bracestt{\meta{from}\texttt{-}\meta{to}(\texttt{,} \meta{from}\texttt{-}\meta{to})*}}, + desc={no default, initially \texttt{0-255}}}, + } + If run under the \hologo{pdfTeX} or \hologo{XeTeX} engine, this key + determines which characters will be assigned category code 12 in the verbatim + mode. In \hologo{pdfTeX}, the range should remain at the initial |0-255|, + but in \hologo{XeTeX}, some rare situations might require extending this + range (don't attempt to set the full range of |0-1114111|, as this would be + very slow and you would most likely run out of save stack). + + In \hologo{LuaTeX}, we switch the category code regime using category code + tables, so this key has another meaning: it determines the range in which + CollArgs will scan for characters of category codes 1, 2 and 14, whose + identity it needs to know, for internal reasons. +\end{doc} + + + +\paragraph{Mode transition limitations} + +\refcmd{CollectArguments} has some minor limitations regarding the transition +from a verbatim into non-verbatim mode, or vice versa. The gist of the issue +is best illustrated with the optional argument type \docref{xparse:o} collected +in the verbatim mode. CollArgs determines whether an argument of this type is +present by peeking ahead (using \hologo{TeX}'s |\futurelet| primitive) into the +input stream. If the argument is present (i.e.\ if the input stream continues +with an open bracket, |[|%] +), all is well. But when the optional argument is absent, the peek-ahead will +tokenize the following character, which presents a problem when no more +arguments are present in the input stream, like in the example below, where the +verbatim \docref{xparse:o} is the (only and) final type in the argument +specification. In this case, the peek-ahead ``incorrectly'' assigns category +code 12 (``other'') to the first |$|. %$ +This character was intended to be tokenized as the math shift character of +category 3, to start the math mode after \refcmd{CollectArguments} is finished, +but having been assigned category code 12, it cannot perform this function, +resulting in error |! Missing $ inserted| %$ +once \hologo{TeX} encounters the superscript character |^|. + +\ExampleName{collargs-transition-ok} +\makeexample{\examplename.tex.c1} +\tcbinputexample{sidebyside, lefthand ratio=0.6, + comment=\input{\examplepath.tex.c1}} + +Well --- this is what \emph{would} happen if CollArgs didn't address the +transition issue described above. In fact, the above example compiles just +fine, because CollArgs \emph{does} address this issue, but unfortunately, +certain transition problems simply cannot be resolved --- read on to learn what +can go wrong. + +For example, you can typeset the name of the document author via +\hologo{LaTeX}'s internal command |\@author|, but to use this command in the +document, you have to precede it by |\makeatletter|. As shown by the first +line of the example below, this works rather nicely: |\makeatletter| sets the +category code of |@| to 11 (``letter''), so |@| may help form the control word +|\@author| --- importantly, |\makeatletter| sets the category code of |@| +\emph{before} control sequence |\@author| is constructed, even if it precedes +it immediately. + +\ExampleName{collargs-transition-cs} +\makeexample{\examplename.tex.c1} +\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''} +\tcbinputexample{comment=\input{\examplepath.tex.c1}} + +In the second line of the example, our clever invocation of |\@author| is +immediately preceded by a call to \refcmd{CollectArguments}, which tries to +collect a verbatim argument of type \docref{xparse:o}. It doesn't find it, which results in +the wrong, verbatim tokenization of the escape character of |\makeletter|. +CollArgs realizes the problem and tries to fix it. But while it is searching +for the end of control sequence |\makeletter| (which it successfully +constructs), it triggers the tokenization of what follows --- which, as |@| is +at that point of category 12 (``other''), yields the control symbol |\@| (later +followed by word ``author'', typeset in the example). + +In short, the solution has created another, delayed instance of the problem --- +an instance which cannot be addressed any further. But we're nevertheless +better off, as this particular issue will bite only in the case when the +``corrupted'' control sequence immediately following the invocation +\refcmd{CollectArguments} changes category codes in a way that affects the +tokenization of what immediately follows it. + +This was an example of what can go wrong in the transition from the +\meta{argument tokens} to the \meta{rest} of the tokens following an invocation +of \refcmd{CollectArguments}. As the ``outside world'' is non-verbatim, this +transition can only be problematic if the argument which corrupted the first of +the \meta{rest} tokens was verbatim, so if the transition was from the verbatim +to the non-verbatim mode. Such a transition can also occur within the +\meta{argument tokens}, but the good news here is that CollArgs successfully +solves any problems that occur there, so you should only worry about the +end-of-arguments situation. + +The other direction of the transition, from the non-verbatim to the verbatim +mode, however, can affect both the internal and the external transitions. Let +us illustrate the problem with the internal transition. Say you want to +collect an optional argument (\docref{xparse:o}) in the non-verbatim mode, and +then a mandatory argument (\docref{xparse:m}) in (either full or partial) +verbatim mode. In the first invocation of \refcmd{CollectArguments} below, the +optional argument is present, and we get what we expect: the percent sign is +collected, verbatim, into the mandatory argument. In the second invocation, +however, the percent character retains its usual commenting function --- +despite the fact that we have requested verbatim mode for the mandatory +argument --- which results in the group in the second line being picked up as +the mandatory argument. Again, this happens because CollArgs has to peek ahead +in the input stream when determining whether the optional argument is present. +Having requested non-verbatim mode for the optional argument, the peeking is +performed in the non-verbatim mode, and as the optional argument is not +present, it finds the comment character, which fulfills its regular function of +disappearing along with the rest of the line. Once CollArgs sets to find the +(verbatim) mandatory argument, the rest of the line is already gone, so it +searches for, and finds, this argument in the next line. + +\ExampleName{collargs-transition-comment} +\makeexample{\examplename.tex.c1} +\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''} +\tcbinputexample{listing and text} + +Nothing can be done here --- commenting deletes information, irrevocably --- +and in a similar fashion, nothing can be done to catch a verbatim end-of-line +character when preceded by an absent optional argument in the non-verbatim mode +(because it was already tokenized into a space token). + +Finally, the transition issues are not limited to transits from argument type +\docref{xparse:o}. The full list of argument types which give rise to +transition problems (when transiting \emph{from} arguments of these types) is +as follows: \docref{xparse:o}, \docref{xparse:d}, \docref{xparse:s}, +\docref{xparse:t}, \docref{xparse:g}, \docref{xparse:e}. + + +\endgroup % /collargs + + +\section{Varia} + +\subsectionclearpagefalse + +\subsection{Known issues} +\label{sec:known-issues} + +\paragraph*{Bitmap graphics export} is coming up in the next release of the +package. + + +\paragraph{An error occurs, but disappears in the next compilation} +If a non-fatal internal \hologo{TeX} error occurs during memoization, the memos +and externs may be nevertheless produced and utilized in subsequent +compilations. In such a case, the erroneous code won't be compiled again, and +therefore won't yield any errors, giving the mistaken impression that the code +is error-free. + +This problem does not apply to errors which trigger \refcmd{errmessage}, +because that control sequence is advised by \refmmzauto{abort}. +Note that internal \hologo{TeX} errors like |Undefined control sequence| are +\emph{not} reported through \refcmd{errmessage}, and will therefore cause the +issue. + +This problem does not affect \hologo{LuaTeX}, because this engine allows +Memoize to detect errors and abort memoization if it encounters any. + + +\paragraph{A minimal issue with \hologo{XeLaTeX}} +If the very first page of a document of class |minimal|, compiled by +\hologo{XeLaTeX}, happens to be an extern page, we have a problem: all the +regular pages of the document will be of the same size as that extern page. +\hologo{pdfLaTeX} and \hologo{LuaLaTeX} do not exhibit this behaviour, nor do +\hologo{LaTeX} classes other than |minimal|, even when compiled with +\hologo{XeLaTeX}. + + +\paragraph{CollArgs} + +Due to an unfortunate design decision, CollArgs does not accept a dot |.| as +the \meta{token} argument of types, \docref{xparse:r}, \docref{xparse:R}, +\docref{xparse:d}, \docref{xparse:D}, and \docref{xparse:t}. Furthermore, +\refcollargs{append prewrap} and friends (and their macro counterparts) do not +accept a parameter symbol consistently. These issues will be fixed in the next +release. + + +\subsection{Troubleshooting} +\label{sec:troubleshooting} + +\paragraph{Extern extraction does not work} +\label{sec:fadings-problem} +\label{sec:loading-order} + +Upon an unsuccessful extern extraction, you should get one of the errors or +warnings listed below. They should appear both in the terminal output and in +the log file. All but the final error can also appear when extern extraction +is performed externally. + + +Errors: + +\begin{enumerate} + +\item \code{Error: Python module 'pdfrw' was not found'},\\ + \code{Error: Perl module 'PDF::API' was not found"}, or\\ + \code{Error: Perl module 'PDF::Builder' was not found"} + + You have not installed the PDF processing library, at least not + successfully. The installation instructions can be found in + section~\ref{sec:install}. Perhaps you have multiple instances of Perl\slash + Python on your system and you have installed the library into the wrong + instance? + +\item\label{item:warning:extract:cannot-read} \code{Error: File + "\meta{jobname}.pdf" seems corrupted. Perhaps you have to load Memoize + earlier in the preamble?} + + While the PDF could be corrupted for various reasons, the most probable + reason is the package loading order, as indicated in the error message + itself. (Note that the absence of \meta{jobname}|.pdf| does not yield this + error. This is so that a fatal error unrelated to Memoize doesn't make + Memoize throw another error in the next compilation.) Embedded extern + extraction requires an intact document PDF from the previous compilation, so + Memoize must be loaded before the document PDF is opened for writing the + results of the ongoing compilation. In particular, the PDF is opened by \PGF + library |fadings|, included by \TikZ's libraries |fadings| and |shadows|, so + Memoize must be loaded before any of these libraries. With \pkg{beamer}, the + problem is particularly acute because the PDF is opened while loading the + class. In this case, simply moving |\usepackage{memoize}| up the preamble, + as suggested, won't help: you have to write |\RequirePackage{memoize}| + \emph{before} |\documentclass{beamer}|! + + It is also possible that the PDF processing library deployed by the + extraction script chokes on your particular document. Inspecting the error + message original error message produced by Perl\slash Python, which can be + found at the end of the long error message in the log, should help you see + whether this is the case. For example, a couple of times, I got + \code{%\myfontexpansionon + Invalid dictionary key at /.../perl5/site\_perl/PDF/API2/Basic/PDF/File.pm + line $N$\myfontexpansionoff} when extracting with the Perl-based script. + The issue might be related to PDF version --- by default, \hologo{TeX} + produces PDFs of version 1.5, while the PDF library + \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} officially only + supports versions up to 1.4 --- but I'm afraid I haven't identified the exact + circumstances yet (possibly, the externalizing a picture containing an + embedded PDF file might be the culprit). In general, the workaround is to + use another extraction method or PDF processing library, see \refmmz{extract} + and \refscript{memoize-extract.pl--library}. + +\item\label{item:error:openin_any} + \code{Error: I'm not allowed to write to '{\meta{filename}}' (openout\_any = + {\meta{mode}})"} + + Your \hologo{TeX} distribution does not allow this file to be written to. + The relevant setting is called \docref{openin_any} in \hologo{TeX}Live and + |[Core]AllowUnsafeOutputFiles| in MiK\hologo{TeX}. + +\item \code{Error: I'm not allowed to read from '{\meta{filename}}' (openin\_any + = {\meta{mode}})"} + + Your \hologo{TeX} distribution does not allow this file to be read from. The + relevant setting is called \docref{openout_any} in \hologo{TeX}Live and + |[Core]AllowUnsafeInputFiles| in MiK\hologo{TeX}. + +\item \code{Error: "Semi-absolute" paths are disallowed: '\meta{filename}'} + + This error can only occur on Windows. The extraction scripts are even more + paranoid than \hologo{TeX} and don't allow paths such as |C:foo\bar| or + |\foo\bar|, neither as path to a \dmmz, memo or extern file nor as an output + or temporary directory. + +\item \code{Error: I cannot extract page {\meta{n}} '\meta{filename}.pdf', as + it contains only \meta{N} pages} + + The document PDF is too short. This could happen if you are accidentally + triggering the extern extraction twice, and the first extraction was called + with \refscript{memoize-extract.pl--prune}. + +\item\label{item:error:runtime} \code{Error: Python error: \meta{error + message}} or \code{Error: Perl error: \meta{error message}} + + A runtime error occurred during the execution of the extraction script. The + error message should be followed by the traceback information which includes + the line number (in the extraction script) where the error occurred. + + One possible source of this error are insufficient filesystem permissions. + For example, it will occur if the current directory is marked as read-only + and the temporary output directory (\texttt{\$TEXMFOUTPUT}) is not set. + +\item \code{Error: Extraction of externs from document "\meta{jobname}.pdf" + using method "\meta{extraction method}" was unsuccessful.} + + This is a generic error produced when the extraction script was either not + executed at all, or the execution didn't finish properly.\footnote{How does + Memoize know whether this happened? When invoked given option + \refscript{memoize-extract.pl--format}, the extraction script is supposed + to write a log (actually, status) file called + % todo: \mbox is a workaround preventing "font expansion: using fonts with + % different limit of expansion in one paragraph is not allowed" error, + % which arises in combination with "pi=..." in the CollArgs section. + \mbox{\meta{jobname}}|.mmz.log|. Upon a clean completion, the final line + of this log reads \cs{endinput}. If this marker is missing, Memoize + produces the error under discussion. Feel free to see what's up with this + file if all else fails.} + + A couple of reasons for the failure to execute the script: + \begin{itemize} + \item Is the shell escape configured properly? It should be, as both + Memoize's extraction scripts are listed among restricted shell escape + commands in both \hologo{TeX}Live and MiK\hologo{TeX}, but it never hurts + to check. + + The easiest way to see if shell escape is the culprit is to compile the + document with command option \docref{-shell-escape} (on \hologo{TeX}Live) + or \docref{--enable-write18} (on MiK\hologo{TeX}). If this resolves the + problem, check your shell escape mode; see + footnote~\ref{fn:shell-escape-commands} on + page~\pageref{fn:shell-escape-commands} for how to do this. + \item Is environment variable PATH set correctly, so that the system can find + the extraction script? + \item Is Perl\slash Python installed on your system, and accessible to + \hologo{TeX}? + \end{itemize} + + Normally, you should get error~\ref{item:error:runtime} if a runtime error + occurs during the execution of the extraction script. You can only get the + generic error if the runtime error occurred before the script could set up + the log file, perhaps due to insufficient filesystem permissions in + combination with the paranoid \docref{openout_any} setting (which is the + default). + + If the source of the error remains a mistery, I suggest inspecting the + following sources of information, to help you with your investigation: + \begin{itemize} + \item Can you can run the extraction script by hand? Open the terminal, go + into the directory containing your document, write |memoize-extract.pl + |\meta{document name} (or |memoize-extract.py |\meta{document name}), and + see what happens. + \item Inspect \meta{jobname}|.log| --- search for + \code{runsystem(memoize-extract.pl ...)}, it will tell you whether the + script was executed. + \item Inspect the \hologo{TeX} terminal output --- if the script was + executed, it should've announced itself by \code{Extracting externs from + \meta{jobname}.pdf}; are there any further messages between this header + and the error message? + \item Inspect \meta{the path to the extern}|.log|, if you are using + \hologo{TeX}-based extraction + (\code{\refmmz{extract}=\refmmz{extract=tex}}). + \end{itemize} + +\end{enumerate} + +Not every failure to extract externs results in an error. When a warning is +produced, the compilation will succeed, it's just that as Memoize cannot +extract the externs, they will be produced, and dumped into your document, at +each and every compilation. For warnings other than the missing document PDF, +Memoize extracts as many externs as possible. + +\begin{enumerate} + +\item \code{Warning: Cannot open '\meta{filename}.pdf'} + + This is not an error because what if you deleted the document PDF on purpose? + +\item\label{item:warning:extract:unexpected-size} \code{Warning: I refuse to + extract page $n$ from "\meta{jobname}.pdf", because its size + ($\meta{width}\times\meta{height}$) is not what I expected\\ + ($\meta{expected width}\times\meta{expected height}$)} + + If the compilation which produced the offending extern pages yielded any + errors, you should probably disregard this warning, fix the errors, and + compile again. Otherwise, you have somehow winded up with mismatched + \meta{jobname}|.pdf| and \meta{jobname}\dmmz (the latter file contains + instructions on which pages to extract, complete with the expected + dimensions). Are you sure that they were produced by the same compilation, + and have remained untouched since? Are you perhaps trying to perform the + extraction the second time, after the first extraction + \refscript{memoize-extract.pl--prune}d the PDF? + + If the warning stubbornly persists, but you are sure that the page the script + is refusing to extract is correct, you can force the extraction by adding + option \refscript{memoize-extract.pl--force} to the script invocation, which + can be set by \refmmz{perl extraction options}. However, as such a situation + probably indicates a bug in Memoize, please let me know about it. + +\item \code{Warning: I refuse to extract page $n$ into extern \meta{extern + filename}, because the associated (c)c-memo does not exist.} + + Assuming that you haven't deleted the memos (|.memo| files), either manually + or via \refscript{memoize-clean.pl}, could it be that they were never created + in the first place? Check where they should be written to (the configuration + commands are listed in section~\ref{sec:ref:dirs}). Is that directory + writeable, both in the sense of the system and for \hologo{TeX} (the + |openout| setting in |texmf.cnf|)? + + +\end{enumerate} + + +% warning('I failed to execute "kpsewhich"; assuming openin_any = openout_any = +% "p" ' + +\paragraph{An extern won't be included} + +Did you receive a warning or error message? +\begin{enumerate} +\item \code{Package memoize Warning: Unexpected size of extern "\meta{extern + path}.pdf"; expected\\ $\meta{expected width}\times\meta{expected + height}$, got $\meta{width}\times\meta{height}$} + + This warning is related to warning~\ref{item:warning:extract:unexpected-size} + above, only that it occurs once the extern is extracted. The same + investigative methods apply. +\item \code{!pdfTeX error: pdflatex (file \meta{extern path}.pdf): reading + image file failed}, or something similar for engines other than + \hologo{pdfTeX} + + This is a fatal error. The extern file got corrupted, somehow --- inexistent + and even empty extern files merely trigger recompilation. +\item \code{pdfTeX warning: pdflatex (file \meta{extern path}.pdf): PDF + inclusion: found PDF version <$m$>, but at most version <$n$> allowed} + + When you produced the externs, a higher \docref{reg:pdfmajorversion} and/or + \docref{reg:pdfminorversion} was in effect than now. I guess you shouldn't worry + about this warning if the output looks fine. +\end{enumerate} + +If there was no warning or error --- are you certain that Memoize is +\refmmz{enable}d, and that it is not in the \refmmz{recompile} mode? Remember +that these settings can also apply only to a part of the document; search for +any stray \refcmd{mmzset} or \refcmd{mmznext} commands. + + +\paragraph*{Warnings about duplicate labels, indices, etc.} may be safely +disregarded. + +Externalization causes any (non-immediate) |\write| commands in the extern to +be executed twice, once upon the shipout of the regular page, and once upon the +shipout of the extern page. This results in warnings about doubly defined +labels, hyperreferences, indices, etc. For example, you might get +\texttt{LaTeX Warning: Label `' multiply defined} or \texttt{warning (pdf + backend): ignoring duplicate destination with the name ''}. You can +safely disregard these warnings; they will disappear once the extern is +utilized. + + +\paragraph{Memoization was aborted} + +This warning means that either: +\begin{itemize} +\item you are trying to (auto)memoize a \refenv{tikzpicture} with + \refkey{/tikz/remember picture} set, or more generally, some code which + contains \refcmd{savepos} --- this can't be done, see + section~\ref{sec:tut:good-to-know}; or +\item an error occurred during memoization --- in this case, Memoize cowardly + refuses to proceed with memoization, see section~\ref{sec:known-issues} for + details. +\end{itemize} + + +\subsection{License} + +Copyright \textcopyright\ 2020- Sašo Živanović. + +This work may be distributed and/or modified under the conditions of the +\hologo{LaTeX} Project Public License, either version 1.3c of this license or +(at your option) any later version. The latest version of this license is +in \url{https://www.latex-project.org/lppl.txt} and version 1.3 or later is +part of all distributions of \hologo{LaTeX} version 2008 or later. + +This work comprises of the sources, generated files, accompanying scripts, +auxiliary files and scripts, documentation, documentation sources, examples and +example sources of packages |memoize|, |nomemoize|, |memoizable|, |auto| and +|collargs|. The files belonging to this work and covered by LPPL are listed in +\meta{texmf}|/doc/generic/memoize/FILES|. + +This work has the LPPL maintenance status `maintained.' The Current Maintainer +of this work is Sašo Živanović. The work is available on CTAN +at \url{https://ctan.org/pkg/memoize} and on GitHub +at \url{https://github.com/sasozivanovic/memoize}. + + +\subsection{Acknowledgments} + +First and foremost, my gratitude goes to Stefan Müller of Language Science +Press for his encouragement and patience --- without him, this package might've +remained an idea forever! + +But there were others as well. Shunsaku Hirata of the \hologo{XeTeX} team +promptly resolved an issue with forward references in |xdvipdfmx|, and thereby +made Memoize work with \hologo{XeTeX}. Joseph Wright kindly didn't object to +me misappropriating \pkg{etoolbox} for other formats. Petra Rübe-Pugliese +(CTAN) and Karl Berry (\hologo{TeX}Live) offered good advice on packaging the +package; I would never get to call Advice Advice without their advice. Karl +also performed a security review of the extraction scripts, providing some very +useful comments along the way, and agreed to include the scripts among the +shell escape commands allowed in \hologo{TeX}Live. Christian Schenk worked +hard to support Memoize in MiK\hologo{TeX}. After the package was first +published, amonakov, mbert, Qrrbrbirlbel, and Ulrike Fischer provided valuable +feedback, suggestions and\slash or bug reports; cfr did all that, and also +advertised the package extensively. + +Thank you all! + + +\printindex + +\end{document} + + +% Local Variables: +% TeX-engine: luatex +% TeX-master: t +% End: diff --git a/macros/generic/memoize/doc/memoize-extract.1.md b/macros/generic/memoize/doc/memoize-extract.1.md index 434f2fea2f..f3f3d52ad0 100644 --- a/macros/generic/memoize/doc/memoize-extract.1.md +++ b/macros/generic/memoize/doc/memoize-extract.1.md @@ -2,8 +2,8 @@ title: memoize-extract section: 1 header: User Manual -footer: memoize-extract of Memoize v1.0.0 -date: October 10, 2023 +footer: memoize-extract of Memoize v1.1.0 +date: January 02, 2024 hyphenate: false --- @@ -34,17 +34,17 @@ Memoize communicates with **memoize-extract** through file *document.mmz*. When *document.mmz*, which records which pages in the produced document are extern pages and to which extern files they should be extracted. Therefore, after compiling *document.tex*, the externs should be extracted by -**memoize-extract** *document.mmz*. +**memoize-extract** *document.mmz*. + +*document.mmz* may also be given as document* or *document.tex*. When +environment variable *TEXMF_OUTPUT_DIRECTORY* is set, this filename is relative +to the output directory specified by this variable. *document.mmz* also records the expected width and height of each extern. In case of a mismatch, **memoize-extract** refuses to extract the page and removes the extern file if it already exist, and prints a warning message to the -standard error. The script furthermore refuses to extract the page if a -(c)c-memo associated to the extern does not exist, and to write to any file -whose absolute path does not occur under the current directory or the directory -set by TEXMFOUTPUT (in *texmf.cnf* or as an environment variable); TEXMFOUTPUT -is also not allowed to point to the root directory, except on Windows, where it -can only point to a drive root. +standard error. The script also refuses to extract the page if a (c)c-memo +associated to the extern does not exist. See also section SECURITY. The Perl (.pl) and the Python (.py) version of the script are functionally equivalent. The Perl script requires library @@ -64,30 +64,68 @@ library [pdfrw2](https://pypi.org/project/pdfrw2). : Do not modify the *document.mmz* to mark the externs as extracted. By default, they are commented out to prevent double extraction. +**-F, \--format** *latex*|*plain*|*context* +: When this option is given, the script assumes that it was called from within + a TeX compilation of a document in the given format: it prefixes all output + by the script name, and creates a log file *document.mmz.log*, which receives + any extraction-related warnings and errors. + **-f, \--force** : Extract the extern even if the size-check fails. -**-l, \--log** *filename* -: Log size mismatches to *filename* rather than to the standard error. - -**-w, \--warning-template** *string* -: Wrap the size mismatch warning text into the given template: \\warningtext in - the template will be replaced by the warning text. - **-q, \--quiet** : Don't describe what's happening. -**-e, \--embedded** -: Prefix all messages to the standard output with the script name. - **-m, \--mkdir** -: A paranoid *mkdir -p*. (No extraction occurs, *document.mmz* is interpreted as a directory name.) +: A paranoid *mkdir -p*. (No extraction occurs, *document.mmz* is interpreted as a directory name, which may end in any suffix; no suffix mangling is performed.) + +**-V, \--version** +: Show the Memoize version number and exit. **-h, \--help** : Show help and exit. -**-V, \--version** -: Show the Memoize version number and exit. +# SECURITY + +This script respects the restrictions on file input and output imposed by the +TeX configuration, more precisely, the variables *openin_any* and *openout_any* +of the **kpathsea** library (https://tug.org/kpathsea). You can inspect the +values of these variables by executing '**kpsewhich** -var-value=openin_any' and +'**kpsewhich** -var-value=openout_any'. The interpretation is as follows: + +**a** (or **y** or **1**) any +: Allows any file to be opened. + +**r** (or **n** or **0**) restricted +: Means disallowing special file names. + +**p** (or any other value) paranoid +: Means being really paranoid: disallowing special file names and restricting + input/output files to be in or below the working directory or the directory + specified by *TEXMFOUTPUT* or *TEXMF_OUTPUT_DIRECTORY*. *TEXMFOUTPUT* may be + set either in *texmf.cnf* (e.g. by **tlmgr**) or as an environment + variable. *TEXMF_OUTPUT_DIRECTORY* may only be set as an environment + variables; it is automatically set by TeX when called by *-output-directory* + option (starting in TeXLive 2024). + +# EXIT STATUS + +**0** +: The externs were successfully extracted. This exit code is returned even if + no externs need to be extracted, or if *document.mmz* does not exist. + +**10** +: A warning also reported back to the compilation when given option + **\--format**. Currently, either: (i) size-mismatch; (ii) a non-existing + associated (c)c-memo file; or (iii) unavailable *kpsewhich*. + +**11** +: An error also reported back to the compilation when given option + **\--format**. Currently, either: (i) a currupted document PDF, or (ii) + a kpathsea permission error. + +Other exit codes are as produced by the underlying scripting language (Perl of +Python). # SEE ALSO diff --git a/macros/generic/memoize/doc/memoize.mst b/macros/generic/memoize/doc/memoize.mst deleted file mode 100644 index 68a11283a5..0000000000 --- a/macros/generic/memoize/doc/memoize.mst +++ /dev/null @@ -1 +0,0 @@ -encap '`' diff --git a/macros/generic/memoize/doc/memoize.pdf b/macros/generic/memoize/doc/memoize.pdf deleted file mode 100644 index 5642f641b6..0000000000 Binary files a/macros/generic/memoize/doc/memoize.pdf and /dev/null differ diff --git a/macros/generic/memoize/doc/memoize.tex b/macros/generic/memoize/doc/memoize.tex deleted file mode 100644 index 8bc647f0ba..0000000000 --- a/macros/generic/memoize/doc/memoize.tex +++ /dev/null @@ -1,9094 +0,0 @@ -\documentclass[a4paper,11pt]{article} -\usepackage[margin=2cm,top=1cm,bottom=1cm,nohead,includefoot]{geometry} - -\usepackage{nomemoize} -\usepackage{memoize-doc} -\mmzset{ - % recompile, - % readonly, - % disable, - % inverse search=false, - % include context in ccmemo, - % direct ccmemo input, - % trace, -} -\def\exampledir{examples/} - -\title{\pkg[white]{Memoize}} -\author{Sašo Živanović\\[2mm] - \emailsymbol~\url(mailto:){saso.zivanovic@guest.arnes.si}\\ - \homepagesymbol~\url(http://){spj.ff.uni-lj.si/zivanovic}\\ - \faGithub~\url(https://){github.com/sasozivanovic}} -\datefrompackageversion{memoize} -\preto\packagever{v} - -\hypersetup{ - pdftitle={Memoize}, - pdfauthor={Sašo Živanović}, - pdfsubject={externalization}, - pdfkeywords={LaTeX, externalization, memoization} -} - -\usepackage{makeidx} -\pdfsystem{makeindex -s memoize.mst \jobname.idx} -\makeindex -\usepackage[hangindent=1em,justific=raggedright,font=small]{idxlayout} -\appto\indexfont{\def\pkgcolor{gray}} - -\AddToHook{begindocument}{% so that the region compilation takes these in - \docForeign{ - scope={ - after/.style={index annotation/.prefix={\hologo{LaTeX}\space}}, - cmd={name=begin}, cmd={name=end}, - cmd={name=label}, cmd={name=@currentlabel}, - cmd={name=ref}, cmd={name=pageref}, - cmd={name=index}, - cmd={name=NewDocumentCommand}, cmd={name=GetDocumentCommandArgSpec}, - env={name=verbatim}, - cmd={name=ReadonlyShipoutCounter}, - hook={name=begindocument}, - cmd={name=usepackage}, - }, - scope={ - before/.style={index annotation=\hologo{TeX} primitive}, - cmd={name=shipout}, - cmd={name=hbox}, cmd={name=vbox}, - cmd={name=ignorespaces}, cmd={name=unskip}, - cmd={name=errmessage}, - cmd={name=catcode}, - cmd={name=mag}, - }, - scope={ - before/.style={index annotation=\hologo{eTeX} primitive}, - cmd={name=scantokens}, - cmd={name=currentgrouplevel}, - }, - scope={ - before/.style={index annotation=\hologo{pdfTeX} primitive}, - cmd={name=quitvmode}, - }, - scope={ - before/.style={index annotation=\hologo{LuaTeX} primitive}, - pdfcmd={name=savepos}, - cmd={name=pdfvariable}, - }, - pdfvariable={name=compresslevel}, - pdfvariable={name=horigin}, - pdfvariable={name=vorigin}, - scope={ - after/.style={index annotation/.prefix={\hologo{ConTeXt}\space}}, - cmd={name=realpageno}, - }, - pdfcmd={name=primitive, index annotation=\hologo{LuaTeX}/\hologo{XeTeX} primitive}, - register={name=pdfmajorversion}, - register={name=pdfminorversion}, - scope={ - after/.style={index annotation/.prefix={\hologo{plainTeX}\space}}, - cmd={name=llap}, cmd={name=rlap}, - }, - scope={ - pkg=forest, - env={name=forest}, - cmd={name=Forest}, - }, - scope={ - pkg=tikz, keypath=/tikz, - cmd={name=tikz}, - env={name=tikzpicture}, - key={name=remember picture}, - key={name=overlay}, - cmd={name=tikzexternalize}, - cmd={name=pgfsys@getposition}, - }, - file={name=texmf.cnf}, - env={name=tcblisting, pkg=tcolorbox}, - cmd={name=vref, pkg=varioref}, - cmd={name=cref, pkg=cleveref}, - cmd={name=crefrange, pkg=cleveref}, - xparse type={name=l, index annotation/.append={\ (up to begin-group)}}, - xparse type={name=u, index annotation/.append={\ (until tokens)}}, - xparse type={name=r, index annotation/.append={\ (required delimited)}}, - xparse type={name=R, index annotation/.append={\ (required delimited with default)}}, - xparse type={name=d, index annotation/.append={\ (optional delimited)}}, - xparse type={name=D, index annotation/.append={\ (optional delimited with default)}}, - xparse type={name=o, index annotation/.append={\ (standard optional)}}, - xparse type={name=O, index annotation/.append={\ (standard optional with default)}}, - xparse type={name=t, index annotation/.append={\ (optional token)}}, - xparse type={name=s, index annotation/.append={\ (optional star)}}, - xparse type={name=m, index annotation/.append={\ (mandatory)}}, - xparse type={name=g, index annotation/.append={\ (optional group)}}, - xparse type={name=G, index annotation/.append={\ (optional group with default)}}, - xparse type={name=v, index annotation/.append={\ (verbatim)}}, - xparse type={name=e, index annotation/.append={\ (embellishments)}}, - xparse type={name=E, index annotation/.append={\ (embellishments with defaults)}}, - xparse modifier={name=+, index annotation/.append={\ (long)}}, - xparse modifier={name=!, index annotation/.append={\ (don't ignore spaces)}}, - xparse modifier={name=>, index annotation/.append={\ (argument processor)}}, - generic={name=shell-escape, name prefix=-, label prefix:=-, - index annotation=option of \hologo{TeX} binaries}, - generic={name=enable-write18, name prefix=--, label prefix:=--, - index annotation=option of \hologo{TeX} binaries}, - generic={name=TEXMFOUTPUT, index annotation={variable in - \reffile[into index=false]{texmf.cnf}}}, - }% - \docindex{% - cmd={name=pdfprimitive, see={\refcmd[into index=false,short]{primitive}}}, - cmd={name=pdfsavepos, see={\refcmd[into index=false,short]{savepos}}}, - cmd={name=pdfhorigin, see={\docref[into index=false]{pdfvar:horigin}}}, - cmd={name=pdfvorigin, see={\docref[into index=false]{pdfvar:vorigin}}}, - cmd={name=pdfcompresslevel, see={\docref[into index=false]{pdfvar:compresslevel}}}, - pdfvariable={name=majorversion, see={\docref[into index=false,long]{reg:pdfmajorversion}}}, - pdfvariable={name=minorversion, see={\docref[into index=false,long]{reg:pdfminorversion}}}, - }% -} - -\begin{document} - - -\maketitle -\thispagestyle{empty} - -\begin{abstract} - Memoize is a package for externalization of graphics and memoization of - compilation results in general, allowing the author to reuse the results of - compilation-intensive code. - Memoize - \begin{enumerate*}[(i)] - \item induces very little overhead, as all externalized graphics is produced - in a single compilation. It features - \item automatic recompilation upon the change of code or user-adjustable - context, and - \item automatic externalization of \TikZ pictures and Forest trees, easily - extensible to other commands and environments. Furthermore, Memoize - \item supports cross-referencing, \TikZ overlays and Beamer, - \item works with all major engines and formats, and - \item is adaptable to any workflow. - \end{enumerate*} -\end{abstract} - -\bigskip - -\ExampleName{titlepage} -\makeexample{\examplename.pdf N=2} % N=2, because we show the .mmz file later -\begingroup -\tcbset{page box/.style={ - box align=center,boxsep=2mm,top=0pt,bottom=0pt,left=0pt,right=0pt, - enhanced, remember as=#1, - }}% -\begin{tcboxedraster} - [raster columns=1] - {title=The two steps of externalization of graphics in Memoize} - \tcbinputexample{ - title=\tt doc.tex, - listing only, - remember as=tex, - } - \begin{tcolorbox}[example title=doc.pdf, no attachment, remember as=pdf1] - \includeexamplepdf[page box=extern1]{page=1,trim=1in 1in 1in 1in}\hfill - \includeexamplepdf[page box=extern2]{page=2,trim=1in 1in 1in 1in}\hfill - \includeexamplepdf[page box=extern3]{page=3,trim=1in 1in 1in 1in}\hfill - \includeexamplepdf[page box=docpage]{page=4}% - \end{tcolorbox} - \vspace{10mm} - \begin{tcboxedraster}[raster columns=3, raster column skip=1cm]{blankest} - \begin{tcolorbox}[example title=.pdf]\centering - \includeexamplepdf[page box=externfile1]{page=1,trim=1in 1in 1in 1in}\hfill - \end{tcolorbox} - \begin{tcolorbox}[example title=.pdf]\centering - \includeexamplepdf[page box=externfile2]{page=2,trim=1in 1in 1in 1in}\hfill - \end{tcolorbox} - \begin{tcolorbox}[example title=.pdf]\centering - \includeexamplepdf[page box=externfile3]{page=3,trim=1in 1in 1in 1in}\hfill - \end{tcolorbox} - \end{tcboxedraster} -\end{tcboxedraster} - -\medskip - -\begin{tcolorbox}[title=Using the externalized graphics] - \begin{tcolorbox}[example title=doc.pdf, - halign=center, remember as=pdf2] - \tcbox[remember, enhanced]{\footnotesize - We all love Ti\emph{k}Zlings! - \def\mytikzling#1#2{% - \tcbox[blankest, baseline=1mm, remember as=used extern #1] - {\tikz[x=0.3cm,y=0.3cm]{#2;}}}% - \mytikzling1{\penguin} is a penguin, - \mytikzling2{\koala} is a koala, and - \mytikzling3{\owl} is an owl. - } - \end{tcolorbox} -\end{tcolorbox} - -\begin{tikzpicture}[remember picture, overlay] - \draw[->, snake arrow, thick, red] (tex.west) to[out=-90-60, in=90+60] (pdf1.west); - \draw[->, snake arrow, thick, red] (tex.east) to[out=-90+19, in=90-19] (pdf2.east); - \draw[->, thick, red] (extern1.south) to[out=south, in=north] (externfile1); - \draw[->, thick, red] (extern2.south) to[out=south, out looseness=1.1, in=north, in looseness=0.9] (externfile2); - \draw[->, thick, red] (extern3.south) to[out=south, out looseness=0.7, in=north, in looseness=0.9] (externfile3); - \draw[->, thick,blue] (externfile1) to[out=south, in=north] ([yshift=0.8mm]used extern 1.north); - \draw[->, thick,blue] (externfile2) to[out=south, in=north] (used extern 2.north); - \draw[->, thick,blue] (externfile3) to[out=south, in=north] ([yshift=0.3mm]used extern 3.north); -\end{tikzpicture} -\endgroup - -\begin{tcolorbox}[colback=emphcolor, fontupper=\footnotesize] - This manual also documents packages Advice - ({\datefrompackageversion{advice}v\packagever}) and CollArgs - ({\datefrompackageversion{collargs}v\packagever}). These are auxiliary - packages which were developed alongside Memoize, but are distributed as - independent packages as they might be useful outside the context of Memoize, - as well. See sections~\ref{sec:advice} and \ref{sec:ref:advice} for Advice, and - sections~\ref{sec:collargs} and \ref{sec:ref:collargs} for CollArgs. -\end{tcolorbox} - - - -\section*{Introduction} -\label{sec:intro} - -\begin{tcolorbox}[ - title={\introboxtitle{What} is externalization and why you might want it?}, - ] - If you have ever worked on a long document full of \TikZ pictures and maybe - Forest trees, you have probably had some compilation-enforced coffee breaks - --- even on modern computers, compiling pictures, trees and such takes a lot - of time. And you might have wondered, why do I need to compile these - pictures over and over again? --- after all, I'm not changing them anymore! - Enter \emph{externalization}, a mechanism designed to deal precisely with - your issue, by saving the produced pictures into separate PDFs and including - those PDFs in subsequent compilations --- in no time at all! -\end{tcolorbox} - -\begin{tcolorbox}[ - title={\introboxtitle{Why} yet another externalization package?}, - ] - \TikZ, the popular and all-powerful graphics language for \TeX, ships - including an externalization library (described in \PGFmanual{53}). \TikZ's - library does an excellent job, but with one caveat. Assume you're using it - for the first time (or after a clean-up) on that long document full of \TikZ - pictures and Forest trees. It will take ages to produce all the externalized - graphics. Why? Because to get you up to speed, your document has to be - compiled many many times --- once for each and every externalized picture. - Even with \TikZ's advanced mechanism for skipping the parts of the document - irrelevant for the picture at hand, the first externalization can be a - daunting task. -\end{tcolorbox} - -\begin{tcolorbox}[ - title={\introboxtitle{How} does Memoize save your time?}, - ] - There is a reason why \TikZ uses an entire compilation cycle to produce a - single externalized picture: \TeX\ itself can only produce a single PDF per - compilation (at least at the moment). Memoize evades this limitation by - dumping the externalized pictures right in the middle of the document - (ouch!). More precisely, an externalized picture occurs in the PDF twice, - first on a special page of its own and then on a regular page, where you - intended it to be. The daring dump obviously necessitates a second step of - the procedure, when those special pages are \emph{extracted} into separate - PDFs, called \emph{externs}, which are then included into the document in - subsequent compilations. - - This two-step procedure, illustrated on the cover page of this manual, is - very fast. The first step, which externalizes \emph{all} the pictures into - the document itself (the squiggly red arrow), takes virtually no more time - than a regular compilation. The time needed for the second step, extraction - (the normal red arrows), depends on the system setup, but it ranges from - little to almost none. -\end{tcolorbox} - -\begin{tcolorbox}[ - title={\introboxtitle{When} should I use Memoize?}, - ] - In short, whenever you are writing a document containing lots of \TikZ - pictures, Forest trees, or other time-consuming constructs, and you are bored - waiting for the compilation to finish. - - Using Memoize on a paper containing a single picture does not make much of a - difference. But with more complex documents, the speed-up can be immense. - For example, the compilation time of my 400-page book containing about 160 - Forest trees was reduced by more than half, and the compilation time of a - 260-page Beamer presentation with a hundred complex dynamic trees went from - four minutes to a mere half minute! -\end{tcolorbox} - - -\begin{tcolorbox}[ - title={\introboxtitle{How} much extra work does Memoize require?}, - ] - In principle, none. - - For one, while allowing for manual memoization of selected document chunks - (\S\ref{sec:tut:memoize}), Memoize features a system which automatically - triggers memoization at each invocation of selected commands and - environments. Out of the box, Memoize \emph{automemoizes} \TikZ - pictures\noprint{\refenv{tikzpicture}} and Forest - trees\noprint{\refenv{forest}}, but the author can easily submit (almost) any - command or environment to automemoization (\S\ref{sec:tut:automemoization}). - Memoize also does its best to automatically prevent memoization of code that - cannot be externalized, like \TikZ pictures with \refkey{/tikz/remember - picture}, and to abort memoization in case the memoized code yields any - errors. -\end{tcolorbox} - - -\begin{tcolorbox}[ - title={\introboxtitle{Why} is Memoize not called Externalize?}, - ] - Fundamentally, Memoize is about producing and utilizing \emph{memos} --- - pieces of \hologo{TeX} code replicating the effect of the compilation of a - document chunk in a computationally less intensive manner. Typically, each - memo has an associated extern, which is where the effect of - \emph{typesetting} is stored, but conceptually, memos come first. For - example, the extern is included back into the document by the memo, and a - memo may be associated with any number of externs, including zero. - - Memos solve several externalization-related problems in a generic fashion, - allowing for a multitude of applications. For example, they store the - information about the associated externs, so that an extern can be integrated - back into the document as a box with the original orientation, width, height - and depth. They solve the problem of cross-referencing from and into the - memoized code by storing its \emph{context} (\S\ref{sec:cross-referencing}) - and replicating any \cs{label}s which occur in it. They are also crucial for - externalizing pictures in Beamer frames overlay by overlay. - - Incidentally, the term ``memoization'' is used with some programming - languages to refer to the process of remembering the result of the function, - along with the given arguments, so that on subsequent invocations of the - function with the same arguments, the result can be returned from memory - rather than recomputing it. I would say that, give or take the functions, - what Memoize does fits the bill. -\end{tcolorbox} - - -\begin{tcolorbox}[ - title={\introboxtitle{How} can I make my command (auto)memoizable?}, - ] - Any command which interacts with the rest of the document, like a command - which produces a float, must receive special treatment. Some issues can be - resolved from within Memoize. Other issues require a memoization-compatible - (re)implementation of the command. It is in the hope that package writers - will adapt their ``difficult'' commands to Memoize that this package offers a - documented interface to the memoization process, fully described in - sections~\ref{sec:tut:package}, \ref{sec:memoization-drivers} - and~\ref{sec:tut:automemoization-details}. - - An advanced user might also want to know that Memoize ships with two - auxiliary packages, which form the base of Memoize's automemoization feature. - Package Advice implements a generic framework for extending the functionality - of selected commands and environments, while package CollArgs provides a - command which can determine the argument scope of any command whose argument - structure conforms to \pkg{xparse}'s argument specification. -\end{tcolorbox} - -\begin{tcolorbox}[ - title={\introboxtitle{What else} is out there?}, - ] - Not long before submitting Memoize to CTAN, I became aware of another new - externalization package, \pkg{robust-externalize}, and it seems that the same - happened to the author of that package \Smiley, who found the - proof-of-concept version of Memoize, which was available - at \href{https://github.com/sasozivanovic/memoize}{GitHub} for a while. - - The key idea behind \pkg{robust-externalize} seems to be to extract the code - submitted to externalization into separate files, and add the necessary - preamble. While a compilation from scratch takes more time than with Memoize - (but less than with \TikZ library), the approach allows for parallel - compilation of externs --- nice! -\end{tcolorbox} - -\newpage -\tableofcontents - - -\section{Before you start} -\label{sec:setup} - -\subsection{Installing the extern extraction software} -\label{sec:install} - -Good news: using Memoize can be as easy as writing -\EmphVerbatim{\usepackage{memoize}} in the preamble. - -Bad news: Memoize won't work out of the box. The culprit is the extern -extraction --- the process which ships the externalized graphics from the main -document into separate extern files; for details, see the title page -illustration and the ``How'' box in the Introduction. For the extraction to -work, you will probably have to install some additional software, and you might -also have to allow your \hologo{TeX} to execute the extraction script. But -there's a silver lining: once Memoize is set up, it is set up for good. - -\begin{tcolorbox}[title=What do I have to do?] - - In principle, all you have to do for Memoize to work \emph{under the default - configuration} is install Perl libraries - \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} and - \hreftt{https://metacpan.org/pod/Path::Class}{Path::Class}; see the Perl - section below for the installation guideline. - -\end{tcolorbox} - -Consult section~\ref{sec:extraction} if you want to use an extraction method -other than the default \refmmz{extract=perl}-based method or adapt Memoize to -your particular workflow (for example, if you're compiling via a Makefile). - -If you installed Memoize through the package manager of your \hologo{TeX} -distribution, your system should be already set up to allow the execution of -Memoize's extraction scripts. If this is not the case, please contact either -me or the maintainer of your distribution; until the issue is resolved, you -have to either -\begin{enumerate}[(a)] -\item compile documents loading Memoize with command-line option - \docref{-shell-escape} (on \hologo{TeX}Live) or - \docref{--enable-write18} (on MiK\hologo{TeX}), or -\item set up the restricted shell escape mode to allow for the execution of - \refscript{memoize-extract.pl}.\footnote{\label{fn:shell-escape-commands}On - \hologo{TeX}Live, execute \code{tlmgr conf texmf shell\_escape\_commands} - to get the list of currently allowed commands \meta{current}, then add the - script by executing \code{tlmgr conf texmf shell\_escape\_commands - \meta{current},memoize-extract.pl}. On MiK\hologo{TeX}, you get the - \meta{current} list by \code{initexmf - --show-config-value=[Core]AllowedShellCommands[]}, and add to it by - \code{initexmf - --set-config-value=[Core]AllowedShellCommands[]=\meta{current};memoize-extract.pl}.} -\end{enumerate} - -Once you have set up your system, I advise you to follow the instructions in -section~\ref{sec:tut:test} to test if the setup was successful. - - -\begingroup -\setlength\intextsep{0pt} - -\paragraph{Perl} If you're running GNU/Linux or macOS, Perl is most likely -already installed on your system. On Windows, you might have to install it. I -tested Memoize with Strawberry Perl, available -at \url(https://){strawberryperl.com}; see \url(https://){www.perl.org} for -other options. - -\begin{wrapfigure}[3]{r}{0.25\linewidth} - \raisebox{0.5mm}[\dimexpr\height-0.5mm][\depth]{% - \mmznext{no verbatim} - \begin{tcolorbox}[ - % example title=Installing the required libraries in Perl, - % attach shifted boxed title to top right, - % top=2.5mm, - fontupper=\small, - left=2mm, - force nobeforeafter, - ] - |cpan PDF::API2| - - |cpan Path::Class| - \end{tcolorbox}% - }% -\end{wrapfigure} -Once Perl is installed on your system, you will need to install two Perl -libraries as well: the PDF processing library -\hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2}, and the cross-platform -path specification manipulation library -\hreftt{https://metacpan.org/pod/Path::Class}{Path::Class}. On some GNU/Linux -distributions, these libraries are included as packages --- just use your -package manager to install them. Otherwise, install them from -\hreftt{https://www.cpan.org}{CPAN} using -\hreftt{https://metacpan.org/pod/CPAN}{cpan} tool, as shown in the box. - - -\paragraph{Python} Installing (Python and) the required Python library is only -necessary if you decide to use the Python-based extraction script; see -section~\ref{sec:extraction}. - -If you're running GNU/Linux, install Python using your package manager. -Otherwise, download the installer from \url(https://){www.python.org}. You can -install Python even if you don't have administrator privileges: simply uncheck -the ``Install launcher for all users'' when running the installer. - -\begin{wrapfigure}[2]{r}{0.25\linewidth} - \raisebox{0.2mm}[\dimexpr\height-0.2mm][\depth]{% - \mmznext{no verbatim} - \begin{tcolorbox}[ - % example title=Installing the required library in Python, - % attach shifted boxed title to top right, - % top=2.5mm, - fontupper=\small, - left=2mm, - nobeforeafter, - ] - |pip install pdfrw2| - \end{tcolorbox}% - }% -\end{wrapfigure} - -Once Python is installed on your system, you will also need to install the -|pdfrw2| library (or its predecessor, |pdfrw|, which will work just as well). -Some GNU/Linux distributions offer this library as a package; if this is not -the case, and on other operating systems, install it -from \href{https://pypi.org}{The Python Package Index} -using \href{https://pip.pypa.io}{pip} tool (if you -run \href{https://pip.pypa.io}{pip} as a superuser, it will install the library -system-wide, otherwise locally), as shown in the box. - -\endgroup % \intextsep - - -\subsection{The configuration commands} -\label{sec:mmzset} - -Memoize can be configured using command \Emph{\refcmd{mmzset}\marg{keylist}} -(and friends). The \meta{keylist} argument is a comma-separated list of -configuration settings, each setting having the form \meta{key}|=|\meta{value}, -or sometimes just \meta{key}. The \meta{keylist} argument is processed by package -\pkg{pgfkeys} (see \PGFmanual{88}), so you can use all the bells and whistles -of that fantastic PGF utility (like easily defining your own styles). - -Here's some examples of \refcmd{mmzset}. For one, to whet your appetite to learn -about the various keys in the \refkeypath{/mmz} path, but more importantly now, to show -you that white-space is irrelevant in the \meta{keylist} argument, so you can -format the keylist as you wish --- as long as it does not contain an empty -line.\footnote{I like to add a comma after the final key as well, as shown in - the bottom left example, because if I don't, I often forget to insert it when - I add more keys.} - -\begin{tcbraster}[raster columns=4, raster valign=bottom] - \begin{tcboxedraster}[raster columns=1]{blankest, raster multicolumn=3, valign=bottom} - \begin{tcblisting}{listing only} -~\mmzset~{memo dir,recompile,memoize=circuitikz} - \end{tcblisting} - \begin{tcblisting}{listing only} -~\mmzset~{memo dir, recompile, memoize=circuitikz} - \end{tcblisting} - \end{tcboxedraster} - \begin{tcblisting}{listing only} -~\mmzset~{ - memo dir, - recompile, - padding=2in} - \end{tcblisting} -\end{tcbraster}\par -\begin{tcbraster}[raster columns=4, raster valign=top] - \begin{tcblisting}{listing only, raster multicolumn=2} -~\mmzset~{ - memo dir, readonly, - memoize=\qrcode{om}, - deactivate=\label, - disable, -} - \end{tcblisting} - \begin{tcblisting}{listing only} -~\mmzset~{ - memo dir, - recompile, - % comment - padding=2in -} - \end{tcblisting} - \begin{tcblisting}{listing only, bad} -~\mmzset~{ - memo dir, - recompile, - - padding=2in -} - \end{tcblisting} -\end{tcbraster} - -Command \refcmd{mmzset} can be used any time after loading the package. It is -very common in the preamble, but also useful in the document body, where its -effect is local to the \hologo{TeX} group.\footnote{Any keys with a non-local - effect are explicitly marked as such in the reference section.} For example, -you can use the idiom on the left below to force recompilation of a single -Forest tree somewhere in the middle of the document (in the code listings -below, highlighting marks the resulting scope of the \refmmz{recompile} -directive). However, as applying some setting to a single piece of -automatically memoized code is common, Memoize provides a special command for -the occasion: the keys given as the argument of \Emph{\refcmd{mmznext}} will be -applied only at the next instance of automemoization, overriding any keys set -by \refcmd{mmzset} in case of a conflict. If the command is given more than -once, only the final invocation takes effect. - -\begin{tcbraster}[raster columns=2, raster equal height=rows, raster force size=false] -\begin{tcblisting}{listing only, mark region={3}{6}} -~{~ - ~\mmzset~{recompile} - \begin{forest} - [VP[V][DP]] - \end{forest} - % ... -~}~ -\end{tcblisting} -\begin{tcblisting}{listing only, add to width=-1em, mark region={3}{5}} -% ... -~\mmznext~{recompile} -\begin{forest} - [VP[V][DP]] -\end{forest} -% ... - -\end{tcblisting} -\end{tcbraster} - -I like to follow |\usepackage{memoize}| by \refcmd{mmzset}, but if you prefer, you -can also provide the document-wide configuration as \Emph{package options.} -Note, however, that \hologo{LaTeX} removes spaces from package -options, so keys such as \refmmz{memo dir} won't work. That said, the following are -equivalent --- both will force re-externalization of all the externs in the -document. - -\begin{tcbraster}[raster columns=2, raster equal height=rows] - \begin{tcblisting}{listing only} -\usepackage{memoize} -\mmzset{recompile} - \end{tcblisting} - \begin{tcblisting}{listing only} -\usepackage[recompile]{memoize} - \end{tcblisting} -\end{tcbraster} - - -\subsection{The configuration file} -\label{sec:memoize.cfg} - -Memoize allows you to configure settings which apply to more than just one -document. It does that by attempting to load file \Emph{\reffile{memoize.cfg}} -just before executing package options. Given how \hologo{TeX} searches for -files, the location of this file determines whether it applies system-wide, -user-wide or directory-wide. The directory-wide location is clearly the -directory itself. The user-wide and system-wide location depend on the -\hologo{TeX} distribution and which format(s) you want to use -\reffile{memoize.cfg} with:\footnote{The |memoize| subfolder is not - obligatory.} - -\begin{center} - \meta{the relevant texmf tree root}|/tex/|\meta{format}|/memoize/memoize.cfg| -\end{center} - -You can say |generic| for \meta{format} if you want the configuration file to -be accessible by all formats, otherwise \meta{format} should be one of the -formats supported by Memoize: |plain|, |latex| or |context|. The texmf root -directory depends on the distribution, here's how to figure out what it is: - -\begin{tcboxedraster} - [raster columns=1, raster valign=top, raster force size=false] - {title=The roots of TEXMF trees} -\begin{tcolorbox}[title={\hologo{TeX} Live\hfill\url[white](https://){tug.org/texlive}}] - \begin{description}[itemsep=0pt] - \item[user-wide] - |tlmgr conf texmf TEXMFHOME| - or |kpsewhich -var-value TEXMFHOME|\\ - the default (on Linux): |/home/|\meta{username}|/texmf| - \item[system-wide] - |tlmgr conf texmf TEXMFLOCAL| - or |kpsewhich -var-value TEXMFLOCAL|\\ - the default (on Linux): |/usr/local/texlive/texmf-local| - \end{description} - - \smallskip - Don't forget to run |texhash| or |mktexlsr| after creating \reffile{memoize.cfg}. -\end{tcolorbox} -\begin{tcolorbox}[title={MiK\hologo{TeX}\hfill\url[white](https://){miktex.org}}, before upper={\parskip0.3\baselineskip plus 2pt\relax},] - Open the MiK\hologo{TeX} Console, select the ``Settings'' page and then the - ``Directories'' tab. - - If there is a folder marked with the ``Generic'' purpose with attribute - ``User'' (for a user-wide \reffile{memoize.cfg}) or ``Common'' (for a system-wide - \reffile{memoize.cfg}), that's the folder you are looking for. Otherwise, create a - folder following MiK\hologo{TeX}'s instructions and add it to the list. - - Don't forget to click ``Refresh file name database'' (in the ``Tools'' menu - of the MiK\hologo{TeX} Console) after creating \reffile{memoize.cfg}. -\end{tcolorbox} -\end{tcboxedraster} - -Note that a directory config file will override the user config, and the user -config will in turn override the system-wide one. This should not concern you -too much, because you will probably only want to use the user-wide config -anyway,\cprotect\footnote{If you want to have a directory-wide configuration - based on (rather than overriding) the user-wide configuration, your could - write down the real user-wide config in, say, |memoize.user.cfg| (located - user-wide), and then |\input| this file by both the user-wide and the - directory-wide \reffile{memoize.cfg}. Of course, the same logic can be used to base - a user-wide config on the system-wide one.} which might look something like -this: - -\ExampleName{memoize} -\makeexample{\examplename.cfg.c1} -\tcbinputexample[.cfg]{listing only} - -I recommend including \refmmz{memo dir} in your \reffile{memoize.cfg}, -as shown in the first line. It reduces the clutter in the document -directories; see section~\ref{sec:tut:memodir} for details. - -The \refmmz{extract} line concludes the story on ``permanently'' selecting the -extraction method, started in chapter~\ref{sec:setup}. As the final note, -observe that key \refmmz{extract} only makes sense as a package option or as a -\refcmd{mmzset} key in \reffile{memoize.cfg}. It will have no effect as a -\refcmd{mmzset} key in the document, because the extraction happens while the -package is loaded. - - - -\section{Your first memoized documents} -\label{sec:tut} - -\subsection{Let's see if it works!} -\label{sec:tut:test} - -\ExampleName{test} -\noprint{\refcmd{tikz}\refenv{tikzpicture}\refenv{forest}}% - -Take example file \texttt{\examplename.tex},\footnote{Where can you find - the example files? For one, they are integrated into this manual, so if your - PDF viewer supports attachments, you can simply click on the paperclip icon - on the top right of the example box (even if you're offline). Otherwise, - visit the |examples| subdirectory of wherever you found this document - \Smiley. Online, the Memoize documentation can be found at CTAN: - \url{https://ctan.org/pkg/memoize}; and if your \hologo{TeX} installation - includes the documentation, you should also find it in directory \meta{the - root of your \hologo{TeX} installation}|/doc/generic/memoize|. - - Incidentally, while we present a full example document in this section, many - code listings will only present the parts of the file relevant for the - discussion, for brevity. The example \emph{files}, however, will remain - full, compilable documents, including the document preamble etc.} or some -simple document containing a \TikZ picture or a Forest tree, add -\EmphVerbatim{\usepackage{memoize}} to the preamble,\footnote{Memoize likes to - be loaded early. If you get the error \texttt{Extern extraction from - document \meta{jobname}.pdf was unsuccessful}, move - \cs{usepackage}\bracestt{memoize} up the preamble; see - section~\ref{sec:loading-order} for details.} and compile it twice. - -\begin{itemize} -\item The first compilation of the example should produce a three-page PDF. - The first two pages are the \emph{extern pages} holding the externalized - graphics, while the final page is the (sole page of the) real document. Note - that you can see each extern twice: first on a page of its own, and then - wherever it belongs to in the real document. -\item At the second compilation, the extern pages should have disappeared from - the PDF, meaning they were successfully extracted into extern files, which - are now embedded into the main document. -\end{itemize} - -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, - comment={\centering - \includeexamplepdf[extern page,left=2mm,right=2mm]{page=1,trim=1in 1in 1in 1in}\quad - \includeexamplepdf[extern page,left=2mm,right=2mm]{page=2,trim=1in 1in 1in 1in}\\[2ex] - \includeexamplepdf[document page]{page=3} - }, -} - -You might want to play with this example a bit now. -For example, if you reverse the order of the \TikZ picture and the Forest -tree, you should notice that the externs don't get recompiled. You won't see -any extern pages again until you change the actual code of the picture or the -tree --- or until you add some other picture or tree, of course. - -If you \emph{don't} want to automatically memoize \TikZ pictures and/or Forest -trees, you can switch off their \emph{automemoization} using key -\Emph{\refmmz{deactivate}}. This key takes a list of command and environment -names. As you can see below, the command and the environment must be -deactivated separately. - -\begin{tcblisting}{listing only} -\mmzset{ - deactivate={\tikz, tikzpicture}, % deactivate automemoization of all TikZ pictures - deactivate=\tikz, % deactivate only the command - deactivate=tikzpicture, % deactivate only the environment -} -\end{tcblisting} - - -\subsection{Memoizing by hand} -\label{sec:tut:memoize} - -In the previous section, we have compiled our very first document which used -Memoize. In that document, we only had to load the package, as Memoize knows -how to externalize \TikZ pictures and Forest trees without any help from the -author. But what if you want to externalize some other code? The manual way of -doing this is by surrounding the code by a \Emph{\refenv{memoize}} environment, -or by making it the argument of the \Emph{\refcmd{mmz}} command. The only -difference between the two is that the environment, but not the command, -ignores any spaces surrounding the given code. - -\ExampleName{manual} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, lefthand ratio=0.528, - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\hfill - \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in} - }, -} - -Both the \refenv{memoize} environment and the \refcmd{mmz} command take a -configuration keylist as the optional argument, so their full syntax is -\cs{begin}\marg{\refenv{memoize}}\oarg{keylist}\meta{code to be - externalized}\cs{end}\marg{\refenv{memoize}} and and -\refcmd{mmz}\oarg{keylist}\marg{code to be externalized}. The keys given in -this optional argument take precedence over the keys set by \refcmd{mmzset}. -Note that \refcmd{mmznext} does not apply to manual memoization. - -Manual memoization is great for one-shot memoizations, but you can use it -within your own macros as well. For example, assume that you don't want to -externalize \TikZ pictures in general (so you have \refmmz{deactivate}d -automemoization of the \cs{tikz} command, as explained at the end of -section~\ref{sec:tut:test}), but that you want to easily memoize selected -pictures. You could define a memoized variant of the \cs{tikz} command, as shown -below (and similary for the environment). (Note the |%| comment characters in -the definition of |\mmztikz|. The definition was intentionally broken into -several lines to remind you that the spaces around the argument of \refcmd{mmz} -matter.) - -\ExampleName{mmztikz} -\makeexample{\examplename.pdf} -\tcbinputexample{ - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} - \quad - \includeexamplepdf[document page]{page=2} - }, -} - -\clearpage -\subsection{Memoizing automatically} -\label{sec:tut:automemoization} - -Out of the box, Memoize automatically externalizes \TikZ pictures and Forest -trees. Let us see how other commands and environments can be submitted to this -\emph{automemoization} process. - -We start with the simpler case of environments (fortunately, externalizing -environments also makes sense more often than externalizing commands). You can -submit an environment to automemoization by writing -\EmphTT{\refmmz{auto}=\marg{environment name}\braces{\refmmzauto{memoize}}} (as -a key in \refcmd{mmzset}). The natural (but not the only possible) location -for this instruction is the preamble. Below, we automemoize environment -\env{circuitikz} of package \pkg{circuitikz}, used for drawing electronic -circuits.\footnote{In section~\ref{sec:tut:test}, we learned that - automemoization can be switched off using key \refmmz{deactivate}. Memoize - also offers key \refmmz{activate}, but you probably won't have to use it, as - an \refmmz{auto} declaration automatically activates the submitted command.} - -\ExampleName{automemoize-environment} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, lefthand ratio=0.6, - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.8} - }, -} - -Commands are a bit harder to automemoize, because Memoize cannot possibly know -how far the arguments of a command extend (in contrast, the end of an -environment is clearly marked). With commands, we must inform Memoize about -their argument structure, which we achieve using key \refmmzauto{args} in the -second argument of key \refmmz{auto}: -\EmphTT{\refmmz{auto}=\meta{command}\braces{\refmmzauto{memoize}, - \refmmzauto{args}=\marg{argument specification}}}. We can only leave out -\refmmzauto{args} if the command was defined by \refcmd{NewDocumentCommand} or -similar; in this case, Memoize can retrieve the argument specification on its -own. - -And how does the \meta{argument specification} look like? It is a sequence of -letters, each letter determining an argument type. Memoize recognizes the same -argument types and their letters as package \pkg{xparse} (which defines -\refcmd{NewDocumentCommand} and friends), so you should look at (section 1 of) -the documentation of that package for details, if and when you need them. -Here, we focus on the two most commonly used types, \docref{xparse:m} and \docref{xparse:o}, and add the -optional star for good measure: -\begin{center} - \begin{tabular}{lll} - \toprule - letter & argument type & example\\ - \midrule - \docref{xparse:m} & mandatory argument \\ - & --- either surrounded with braces & |\foo{arg}|\\ - & --- or a single token & |\foo a|\\ - \docref{xparse:o} & optional argument, surrounded with brackets & |\foo[arg]|\\ - \docref{xparse:s} & optional star & |\foo*|\\ - \bottomrule - \end{tabular} -\end{center} - -Below, we write |args=om| because command \cs{qrcode} (of package \pkg{qrcode}) -takes two arguments: a bracketed optional argument, followed by a mandatory -argument (in braces). - -\ExampleName{automemoize-command} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, lefthand ratio=0.603, top=0mm, - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad - \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in} - }, -} - -This should get you started with automemoization. We'll provide some further -basic information in sections~\ref{sec:tut:working-on-a-picture} -and~\ref{sec:tut:verbatim} of the tutorial, but only package writers will -probably ever need the gory details from -section~\ref{sec:tut:automemoization-details}. It is my sincere hope that they -will support Memoize in their packages, where necessary, so that you don't even -have to write the \refmmz{auto} declarations, but there is one thing you should -know if you encounter a package supporting Memoize: you should load it -\emph{after} Memoize! - - - -\subsection{Working on a picture} -\label{sec:tut:working-on-a-picture} - - -Memoize automatically recompiles a picture when the code producing the picture -changes. However, sometimes we can modify a picture without changing its code, -like when we modify the definition of a command used in the code. In the -example below, a predefined style |emph| is applied to the node, producing a -node with a red background. Let's say we compile the document (with -memoization) and then change this style to set the yellow background. -Curiously, the node will remain red. - -\ExampleName{recompile} -\makeexample{\examplename.pdf N=3} -\begin{tcbraster}[raster columns=2, raster valign=top, raster column skip=4mm] - \tcbinputexample{ - after title pre={\ (version~1)}, - comment={\centering - \includeexamplepdf[document page]{page=2} - }, - } - \tcbinputexample[.tex][.c2]{ - no attachment, - after title pre={\ (version~2)}, - comment={\centering - \includeexamplepdf[document page]{page=1} - }, - } -\end{tcbraster} - -The curious thing happens (or rather, doesn't happen) because Memoize doesn't -keep track of how commands and styles are defined; it just uses the extern file -it created when the old style was in effect. To get a yellow node, we must ask -Memoize to reexternalize the picture. The simplest way to do that is by using -the \Emph{\refmmz{recompile}} key; below, we write -\refcmd{mmznext}\bracestt{\refmmz{recompile}} just before the picture and -compile the document again (remember from section~\ref{sec:mmzset} that -whatever keys we provide through \refcmd{mmznext} only apply to the instance of -automemoization). After the compilation, we may (and should) remove the -\refmmz{recompile} directive (otherwise, Memoize will produce the extern page -again and again). - -\tcbinputexample[.tex][.c3]{ - no attachment, - after title pre={\ (version~3)}, - sidebyside, lefthand ratio=0.6, - comment={\centering - \includeexamplepdf[document page]{page=2} - }, -} - -It is also common to put (again, for the space of a single compilation) -\refcmd{mmzset}\braces{\refmmz{recompile}} in the preamble, or to use -\refmmz{recompile} as the package option. Either will remake all the -externalized graphics in the document, so you can be sure all of them use the -latest version of your macros and styles. - -We'll revisit the issue of memoized code depending on macros and styles defined -elsewhere in section~\ref{sec:tut:redefinitions}. In this section, we will -learn how the issue can be \emph{avoided}, at least to some extent. One idea -is to turn off memoization for the picture(s) we are currently working on; -another idea is to let Memoize know which definitions the picture relies on. -The simplest way to achieve the former is by using key \Emph{\refmmz{disable}}; -by putting it into a \hologo{TeX} group, we can localize its effect to the -selected pictures. - -\ExampleName{disable} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, - lefthand ratio=0.6, - top=0mm, % manual top, bottoms and \\[...] to fit the page - comment={\centering - \includeexamplepdf[extern page,bottom=0.8mm]{page=1,trim=1in 1in 1in 1in}\\[0.5mm] - \includeexamplepdf[extern page,bottom=0.8mm]{page=2,trim=1in 1in 1in 1in}\\[0.5mm] - \includeexamplepdf[document page,bottom=0.8mm]{page=3} - }, -} - -\begin{tcolorbox}[warning] - As you can imagine, key \refmmz{disable} is complemented by \refmmz{enable}, - but it is perhaps worth mentioning a problem that can arise if you disable - memoization for a part of your document by enclosing it in a pair of - \refcmd{mmzset}\braces{\refmmz{disable}} and - \refcmd{mmzset}\braces{\refmmz{enable}}. Yes, it might work at the moment, - but say you later (e.g.\ when you are preparing the final version of the - document) decide to disable memoization for the entire document, and say you - try to do this by writing \refcmd{mmzset}\braces{\refmmz{disable}} in the - preamble. As shown below on the left, you're in for a surprise: memoization - will still be enabled in the part of the document following - \refcmd{mmzset}\braces{\refmmz{enable}}! The solution is to always disable - memoization for a part of the document by using - \refcmd{mmzset}\braces{\refmmz{disable}} in a \hologo{TeX} group (i.e.\ the - braces), as shown on the right. (In the examples below, the shaded areas - mark the parts of the document where memoization is \emph{disabled}.) -\end{tcolorbox} - -\makeexample{disable-bad.tex.c1} -\makeexample{disable-good.tex.c1} -\makeexample{disable-nomemoize.tex.c1} -\makeexample{disable-nommz.tex.c1} -\makeexample{disable-auto-cmd.tex.c1} -\makeexample{disable-auto-env.tex.c1} - -\begin{tcbraster}[raster columns=2, raster equal height=rows] - \tcbinputexample(disable-bad){ - listing and comment, bad, mark region={4}{7}, - comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}} - does not have the intended effect, - i.e.\ it doesn't apply to the whole document!}} - \tcbinputexample(disable-good){ - listing and comment, mark region={4}{10}, - comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}} applies - to the entire document, as expected.}} -\end{tcbraster} - -In fact, it might be better to disable memoization using environment -\Emph{\refenv{nomemoize}} or macro \Emph{\refcmd{nommz}}. I also like these -commands because it is easy to add and remove prefix |no| to switch manual -memoization (triggered using environment \refenv{memoize} or macro -\refcmd{mmz}) off and on. - -\begin{tcbraster}[raster columns=2, raster equal height=rows] - \tcbinputexample(disable-nomemoize){listing and comment, mark region={2}{2}, - comment=\raggedright Disable using the dedicated environment.} - \tcbinputexample(disable-nommz){listing and comment, mark region={2}{2}, - comment=\raggedright Disable using the dedicated command.} -\end{tcbraster} - -It is also possible to disable memoization for all occurrences of a selected -command or environment. In fact, we're already familiar with the procedure -from section \ref{sec:tut:automemoization}, where we used key -\refmmzauto{memoize} inside the second argument of \refmmz{auto} to -automatically memoize all instances of the command or environment given as the -first argument. All we have to do to auto-disable rather than auto-memoize, is -substitute \refmmzauto{nomemoize} for \refmmzauto{memoize}. Note that this -prevents memoization of not only the given command or environment, but also of -any (manual or automatic) memoization which would otherwise occur during its -execution; for example, if |\foo| executes \cs{tikz} under the hood, -autodisabling |\foo| prevents memoization of the inner \cs{tikz}, even though -that command is normally automemoized. - -\begin{tcbraster}[raster columns=2, raster equal height=rows] - \tcbinputexample(disable-auto-cmd){ - listing and comment, - mark region={3}{3}, mark region={5}{5}, - comment=\raggedright Autodisable within a command., - } - \tcbinputexample(disable-auto-env){ - listing and comment, - mark region={3}{5}, - comment=\raggedright Autodisable within an environment., - } -\end{tcbraster} - -All that said, Memoize actually offers a neater way to switch off the -externalization for the picture I'm currently working on. The -\Emph{\refmmz{readonly}} key instructs Memoize to use whatever externs it had -already produced (thereby reducing the document compilation time), but to -abstain from producing any new externs. In effect, the stuff we are currently -working on does not undergo memoization and therefore does not produce the -clutter which can potentially lead to trouble described in the \refmmz{recompile} -examples above. - -What I like to do is load the package using |\usepackage[readonly]{memoize}|, -work on stuff, and once I'm happy with the most recent pictures, remove -\refmmz{readonly} from the package options for one compilation. - -\ExampleName{readonly} -\makeexample{\examplename.pdf N=2} - -\begin{tcbraster}[raster columns=5, raster valign=top, raster column skip=4mm] - \tcbinputexample{ - raster multicolumn=2, - after title pre={\ (work in progress)}, - comment={\centering - \includeexamplepdf[document page]{page=1} - }, - } - \tcbinputexample[.tex][.c2]{ - raster multicolumn=3, - no attachment, - title={\tt readonly.tex} (the final version), - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad - \includeexamplepdf[document page]{page=2} - }, - } -\end{tcbraster} - -We're now ready to tackle a completely different way of avoiding the issue, by -informing Memoize which definitions the externalized picture depends on. We do -this by appending these definitions to \emph{context} --- when the context of a -picture changes, Memoize recompiles the picture, same as if the code of the -picture itself was changed. (We will talk about context in more detail in -section~\ref{sec:cross-referencing}.) - -A command can be added as a dependency using key \Emph{\refmmz{meaning to - context}}. Below, we make the following externalized picture depend on the -definition of macro \cs{answer}; changing this definition will result in the -recompilation of the extern. In general, \refmmz{meaning to context} accepts a -comma-separated list of command and environment names, e.g.\ -\code{\refmmz{meaning to context}=\bracestt{\cs{foo},bar}} (note the braces). - -Memoize offers several variants of \refmmz{meaning to context}, applicable to -various types of commands. For example, the easiest way of making the picture -depend on the definition of a \pkg{pgfkeys} style is to use handler -\Emph{\refkey{/handlers/.meaning to context}} --- note the dot in the name, and -observe that \code{emph/.meaning to context} below is executed within -\cs{tikzset}, not \refcmd{mmzset}; see \PGFmanual{87} to learn about key -handlers. - -\ExampleName{meaning-to-context} -\makeexample{\examplename.pdf N=2} -\tcbinputexample{ - after title pre={\ (version~1)}, - sidebyside, - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} - }, -} - -\tcbinputexample[.tex][.c2]{ - after title pre={\ (version~2)}, - sidebyside, - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} - }, - no attachment, -} - -All variants of \refmmz{meaning to context} (see reference -section~\ref{sec:ref:memoization:basic} for the full list) may be used within -the externalized picture itself, e.g.\ \code{\cs{node}[emph, - emph/\refkey{/handlers/.meaning to context}]\bracestt{...}} is perfectly -valid --- and also handy when you want to limit the effect of the handler to a -single picture, as \refkey{/handlers/.meaning to context} cannot be used within -\refcmd{mmznext}. - - - -\subsection{Keeping a clean house} -\label{sec:tut:memodir} - -Memoize produces lots of auxiliary files. For each piece of memoized code, it -produces two \emph{memo} files (we will learn more about these in -section~\ref{sec:memos}), which will be joined by the extern PDF upon the next -compilation.\footnote{If you're using the \hologo{TeX}-based extraction, each - extern (|.pdf|) is also accompanied by a log file (|.log|) produced during - the compilation that extracted the extern.} You can recognize these files -easily: their names start with the name of your document and include one or two -long hexadecimal numbers. - -\ExampleName{dirty-house} -\makeexample{\examplename.pdf} -{ - \relaxmmzcommands - \edef\marshal{% - \noexpand\def\noexpand\mynewextern.//\examplename - }\marshal.#1-#2.pdf\relax{% - \forestset{ - append={[\examplename.#1.memo\vphantom{p},pointcolor]}, - append={[\examplename.#1-#2.memo\vphantom{p},pointcolor]}, - append={[\examplename.#1-#2.pdf,pointcolor]}, - }% - } - \tcbinputexample{ - comment={% - \begin{tcolorbox}[reset, left=2mm, bottom=0mm, - my boxed title=the folder contents (after two compilations), - attach shifted boxed title to top right, - ] - \begin{forest} - before typesetting nodes={for tree={ - font=\scriptsize\tt, grow'=0, folder, s sep=0}}, - [\meta{the document folder}, inner xsep=0, - TeX={% - \def\mmzNewExtern##1##2##3##4##5{\mynewextern##1\relax}% - \input{\examplepath.mmz}% - }, - [\examplename.tex] - [\examplename.pdf] - [\normalsize\dots, no edge] - [\examplename.mmz, tikz={\draw[dotted](.east)--+(5em,0ex) node [anchor=south west, yshift=-1ex, align=justify, text width=14em, font=\footnotesize]{This is another auxiliary file produced by Memoize. We will mention it at the end of the section.};}] - ] - \end{forest} - \end{tcolorbox} - }, - } -} - -To top it off, changing the memoized code will produce new memo (and extern) -files, with the old files staying in place. This is all by design --- the -first hexadecimal number in these filenames is the MD5 sum of the memoized code -and that's how Memoize knows which memo belongs to which piece of -code\footnote{The second hexadecimal number in the memo and extern filenames is - the MD5 sum of the \emph{context}. The context is crucial for properly - externalizing code containing cross-references, see - section~\ref{sec:cross-referencing} for details.} --- but it has the downside -that the folder containing your document can get quite cluttered (imagine the -directory listing as above, but for a document with a hundred externalized -pictures which you have been working on for a month). - -\ExampleName{clean-house} - -I like to keep a clean house by instructing Memoize to put memos and friends -into their own directory. This can be achieved by writing -\refcmd{mmzset}\braces{\Emph{\refmmz{memo dir}}} into the preamble (anytime -after loading the package).\footnote{The \refmmz{memo dir} key is in fact - merely an abbreviation for a sequence of \refmmzpath{dir} and - \refmmzpath{prefix} within \refmmz{path}; use these keys if you need more - control over the name and location of the auxiliary files. Furthermore, there - is also the \refmmz{no memo dir} key, which reverts the configuration back to - the dirty default.} This will put the memo and the extern files into folder -\meta{document name}|.memo.dir| (and it will also omit the \meta{document name} -prefix in their filenames, because it makes no sense to repeat it there). - -\makeexample{\examplename.pdf} -{% - \relaxmmzcommands - \edef\marshal{% - \noexpand\def\noexpand\mynewextern./\examplename - }\marshal.memo.dir/#1-#2.pdf\relax{% - \forestset{ - append={[#1.memo\vphantom{p},pointcolor]}, - append={[#1-#2.memo\vphantom{p},pointcolor]}, - append={[#1-#2.pdf,pointcolor]}, - }% - } - \tcbinputexample{ - comment={% - \begin{tcolorbox}[reset, left=2mm, bottom=0mm, - my boxed title=the folder contents (after two compilations), - attach shifted boxed title to top right, - ] - \begin{forest} - before typesetting nodes={for tree={ - font=\scriptsize\tt, grow'=0, folder, s sep=0}}, - [\meta{the document folder}, inner xsep=0 - [\examplename.tex] - [\examplename.pdf] - [\normalsize\dots, no edge] - [\examplename.mmz] - [\examplename.memo.dir,pointcolor, - % tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=south west, yshift=-1ex, font=\footnotesize, emphshade=black!5!white, text width=15em, align=justify]{You need to create this directory by hand, or allow Memoize to create it by adding |mkdir| to the list of restricted shell escape commands.};}, - tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=west, font=\footnotesize, emphshade=black!5!white]{This directory must be created, somehow.};}, - TeX={% - \def\mmzNewExtern##1##2##3##4##5{\mynewextern##1\relax}% - \input{\examplepath.mmz}% - }, - ] - ] - \end{forest} - \end{tcolorbox} - }, - } -} - -\begin{tcolorbox}[title={Why is \refmmz[link color=white]{memo dir} not the default?}] - The \code{clean-house} example most likely compiled just fine, and you are - wondering why \refmmz{memo dir} is not in effect by default. Well, out of - the box, \hologo{TeX} cannot create directories, so it is the fact that - Memoize \emph{can} create them (at least under the default settings) which - requires explanation. By default, Memoize triggers extraction by executing - the Perl extraction script \refscript{memoize-extract.pl}, and it is this - script which actually creates the memo directory. However, not everybody - will necessarily use this script \dots\ so \refmmz{memo dir} should not be - the default. - - The Python extraction script \refscript{memoize-extract.pl}, used when - \refmmz{extract}|=|\refmmz{extract=python}, works the same as the Perl - variant. With \hologo{TeX}-based extraction - \refmmz{extract}|=|\refmmz{extract=tex}, things are different. If you are - compiling the document with a full shell escape mode - (\docref{-shell-escape}), Memoize successfully creates the directory with the - system command |mkdir|.\footnote{|mkdir| is the default value of key - \refmmz{mkdir command}, but executing the extraction method - \refmmz{extract=perl} or \refmmz{extract=python} overrides this default.} - However, if you're using the restricted shell escape mode, the attempt to - create the directory won't succeed unless you include |mkdir| among the - restricted shell escape commands (see footnote~\ref{fn:shell-escape-commands} - on page~\pageref{fn:shell-escape-commands} for how to do this, but note that - it is not recommended). - - If you are using external extraction, you have to create directory - \texttt{\examplename.memo.dir} by hand, prior to the first compilation of the - document (with Memoize). This is the case even if you are performing the - extraction using one of the shipped extraction methods, and it is due to the - fact that Memoize needs the memo directory to be present even before extern - extraction, because it writes the |.memo| files into the same directory. - (When Memoize uses the extraction script to create the memo directory, it - does so completely independently of extraction, and prior to creating any - |.memo| files.) -\end{tcolorbox} - -I actually suggest adding \refcmd{mmzset}\braces{\refmmz{memo dir}} into your -user-wide \reffile{memoize.cfg} (see section~\ref{sec:memoize.cfg} for details -on this file). This will keep all your houses clean --- without work! --- as -Memoize will automatically use the memo directory for any document your create. - -It is always safe to delete memos (|.memo|) and externs (|.pdf|s residing next -to memos), in the sense that you cannot lose data this way.\footnote{The same - goes for the extern |.log| files produced by the \hologo{TeX}-based - extraction.} - -\begin{itemize} - -\item Many memos and externs are typically stale anyway, i.e.\ they reflect - some previous state of your document and are not needed anymore. These files - can be deleted without any repercussions whatsoever (unless you later revert - to a previous version of the document, of course). In fact, you might - \emph{want} to delete them periodically, or at least once you finish writing - the document. As it is hard to figure out which memos/externs are stale, - Memoize ships with a clean-up script: writing \refscript{memoize-clean.pl} - \meta{document name}\dmmz (replace |.pl| with |.py| if you use Python rather - than Perl) into the command line will delete all the \emph{stale} auxiliary - files belonging to the document. - -\item If you delete a memo or an extern currently in use, you will trigger - recompilation of their code --- so deleting a memo or an extern is actually a - perfectly legal alternative to using the \refmmz{recompile} key!\footnote{For the - users of the \hologo{TeX}-based extraction: deleting the |.log| file does - \emph{not} trigger recompilation.} - -\end{itemize} - -It is also safe to delete the \dmmz file (or any other kind of record file, see -section~\ref{sec:record-files}) residing next to your document's |.pdf|. The -\dmmz file contains the information about which externs should be extracted -from the |.pdf|. Deleting it before this is done (by default, before compiling -the document again) will prevent the extraction (same as if providing the -package option \code{\refmmz{extract}=\refmmz{extract=no}}) and ultimately -result in the recompilation of the externs produced in the previous run. -Deleting it after the extraction will have almost no effect: it will only only -prevent the clean-up script from working (the \dmmz file also lists the -currently active memos and externs, and thereby indirectly informs the clean-up -script which files are stale). For further information on the \dmmz file, see -section~\ref{sec:.mmz}. - - - -\subsection{Writing a book?} -\label{sec:tut:multifile} - -Books and other long documents are usually produced from sources which reside -in more than a single file, and to speed up the editing process, authors -usually use some system which allows them to compile each chapter separately. -Can Memoize --- designed for virtually the same task of speeding up the editing -process --- work sensibly in this kind of situation? More precisely, can the -book and the individual chapters share the memos and the externs? Yes they -can! If we instruct Memoize to use the same memo directory for both the book -job and the chapter jobs, then we can externalize graphics when compiling a -chapter and have the externs included when compiling the book (and vice -versa).\footnote{Package \pkg{docmute} makes \hologo{LaTeX} ignore the preamble of - the chapter file when including this file into the main document.} All we -need to do is use our old friend \refmmz{memo dir} from section~\ref{sec:tut:memodir} ---- we see now that this setting is good for more than just keeping a clean -house! - -\ExampleName{book} -\makeexample{\examplename.tex.c1} -\makeexample{chapters/chapter1.tex.c1} - -\begin{tcbraster}[raster columns=2, raster valign=top] - \tcbinputexample{listing only} - \tcbinputexample(chapters/chapter1){listing only} -\end{tcbraster} - -In the above example, the individual chapters reside in files stored in the -|chapters| subdirectory, and that's why the |book.tex| preamble uses -\code{\refmmz{memo dir}=chapters/book} (rather than \code{\refmmz{memo - dir}=book} or just \refmmz{memo dir}). However, Memoize has no trouble -with a situation where the main file and the chapters reside in the same -folder; the setup is even simpler, as we then say \code{\refmmz{memo dir}=book} -in both the book and the chapter preamble. The more complicated situation was -chosen to point out the following potential problem with the setup where the -chapters reside in a subdirectory. - -If you're anything like me, you would first go for having a memo directory -immediately contained in the project directory (so |examples/book.memo.dir| -above) and set up \refmmz{memo dir} as shown below. Well, this won't work, or -at least it won't work with the vanilla \hologo{TeX} Live, because \hologo{TeX} -will refuse to \emph{write} into (memo) files outside the directory where it -was executed,\cprotect\footnote{In \hologo{TeX} Live, the \reffile{texmf.cnf} - option controlling this behaviour is called |openout_any|. By default, it is - set to |p| (paranoid), which ``disallow[s] opening dot files [and] - \emph{going to parent directories}, and restrict[s] absolute paths to be - under \texttt{\$TEXMFOUTPUT}'' (emphasis mine).} and this is precisely what -the chapter compilation is asked to do below. - -\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest, bad} - \begin{tcblisting}{example title, title=the main file, listing only} -\mmzset{~memo dir=book~} -% ... -\input{chapters/chapter1.tex} - \end{tcblisting} - \begin{tcblisting}{example title, title=a chapter file, listing only} -\mmzset{~memo dir=../book~} - \end{tcblisting} -\end{tcboxedraster} - -\pagebreak % manual - -Section~\ref{sec:tut:working-on-a-picture} presented some ideas on how to work -on a single picture. Those ideas can be all easily applied to the multi-file -situation. For example, you could use \refmmz{readonly} on the chapter that you're -working on (and that chapter only). This way, the preview of the chapter will -not be tarnished by the extern pages, and if you periodically compile it -without \refmmz{readonly}, or compile the book (which does not have the \refmmz{readonly} -set), you will have a reasonably up-to-date set of externs. - -\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest} - \begin{tcblisting}{example title, title=the main file, listing only} -\mmzset{memo dir=chapters/book} -% ... -\input{chapters/chapter1.tex} - \end{tcblisting} - \begin{tcblisting}{example title, title=the current chapter file, listing only} -\mmzset{memo dir=book, ~readonly~} - \end{tcblisting} -\end{tcboxedraster} - -\ExampleName{memoize.cfg._region_} -\makeexample{\examplename.tex.c1} -\begin{tcolorbox}[title=For \Emacs users] - I often use this \refmmz{readonly} trick myself, but with a twist. As an Emacs user, - I don't use a \hologo{TeX}-based mechanism (such as the \pkg{docmute} package) to - compile a chapter, but rely on the region compilation feature of Emacs' - AUC\hologo{TeX} package. AUC\hologo{TeX} offers a way to compile the current - buffer (if you don't know what an Emacs buffer is, read ``file'') or region - (roughly speaking, the selected text). It does that by putting the buffer or - the region into a file called |_region_.tex| while dressing it up in the - preamble of the original document (when I'm working on a multi-file document, - it correctly pulls the preamble from the main document). This results in a - compilable region file. My trick is to detect whether I'm compiling a region - (this is the job of |\ifregion|), and if so, put Memoize into the \refmmz{readonly} - mode (an alternative trick would be to \refmmz{disable} it). - - This is the trick in a nutshell, but to make it really work we have to - address one further issue: the original document and the region have to share - memos and externs. This happens automatically if the original document sets - \refmmz{memo dir} explicitly (e.g.\ if a document called |doc.tex| contains - \code{\refmmz{memo dir}=doc} in the preamble), but I'm lazy and don't want to - write this in every document --- if I have to do that, what's the point of - \refmmz{memo dir} I put into my \reffile{memoize.cfg} in - section~\ref{sec:tut:memodir}? Fortunately, the region file starts with - |\message{ !name(|\meta{original document name}|.tex)}| to indicate the - origin. The complicated part of the code below (everything following - \refcmd{mmzset}\braces{\refmmz{readonly}} parses this header to extract the - \meta{original document name}, which is then fed to \refmmz{memo dir}. Now, - the trick works automatically for any document.\footnote{The assumption here - is that \refmmz{memo dir} is in effect for the original document. If not, - the trick can be adapted to use (\refmmzpath{dir} and) \refmmzpath{prefix} - within \refmmz{path}.} - - \tcbinputexample{ - title=\tt memoize.cfg, - attachment name=memoize-region.cfg, - listing only, - } -\end{tcolorbox} - - -\subsection{Writing a presentation?} -\label{sec:tut:beamer} - -Memoize ships with built-in support for the most widespread \hologo{LaTeX} -presentation class, Beamer, in the sense that it can externalize a picture -which changes from overlay to overlay. Before we learn how to use that -functionality, however, there's a peculiarity about loading Memoize in Beamer -to address. - -\ExampleName{beamer} -\makeexample{\examplename.pdf} - -\tcbinputexample{ - comment and listing, - listing options app={lastline=2}, - warning, no attachment, title=, - % - comment={Beamer opens the document PDF while loading the class, while - Memoize requires the PDF from the previous compilation intact in order to - extract the externs (when extraction is triggered internally, which is the - default setting). The solution is to load Memoize (a package) before - Beamer (a class), which can be done by using \cs{RequirePackage} instead of - the usual \cs{usepackage}. Easy, if hacky.}, - % -} - -To memoize a piece of code which produces different results on different -overlays --- by virtue of containing |\pause|, |\only|, and\slash or related -commands --- apply key \Emph{\refmmz{per overlay}}. Without this key, -externalization of the picture will end badly, with a single extern (the final -one) appearing on all overlays. The key may be invoked either from a prior -\refcmd{mmznext} command,\footnote{Of course, \refmmz{per overlay} may also be - invoked from \refcmd{mmzset}, but I guess this won't make sense often. For - example, if you set it for the entire presentation, and the presentation - contains static memoized pictures as well, you will compile those pictures - more times than necessary: once for each overlay, whereas once per frame - would suffice. It might occasionally make sense, however, to use \refmmz{per - overlay} as an \refmmz{auto} option --- consult - section~\ref{sec:tut:verbatim} to learn what that is.} or executed in the -memoized code itself. The example below illustrates the latter option, and -also shows that we may invoke it via its full path, -\refkeypath{/mmz}|/|\refmmz{per overlay}, when listed among options processed by -\pkg{pgfkeys}.\footnote{Read section~\ref{sec:per-overlay} to learn how the - Beamer support is implemented. The implementation only uses Memoize's public - interface, so understanding it should help if you need to support some other - presentation package.} - -\tcbinputexample{ - listing options app={firstline=3}, - middle=1mm, - comment={\def\dpwidth{0.27\linewidth}\def\epwidth{0.15\linewidth}% - \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=1,trim=1in 1in 0.4in 0.6in}}\hfill - \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=2,width=\dpwidth}}\hfill - \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=3,trim=1in 1in 0.4in 0.6in}}\hfill - \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=4,width=\dpwidth}}% - }, -} - -\begin{tcolorbox}[warning] - If the memoized code changes the value of Beamer's pause counter - |beamerpauses|, e.g.\ by issuing a |\pause|, take care that (i) \refmmz{per - overlay} is executed prior to any changes of |beamerpauses|, and that (ii) - the final value of this counter in the memoized code is the same for all - overlays. -\end{tcolorbox} - - -\subsection{When stuff sticks out} -\label{sec:tut:padding} - -Some constructs --- like plain \hologo{TeX}'s \refcmd{llap} and \refcmd{rlap}, -and, notably, \TikZ overlays --- fool \hologo{TeX} into thinking that the -``size'' of the typeset material is different than what it actually is. This -can cause trouble for externalization: a piece of your picture might disappear! -In a sentence, the solution is to manually set the \emph{padding} of the -externs, but let's slow down a bit. - -The \TikZ picture in the following example consists of node with a pin on the -right, but let's say we want to horizontally center this picture so that only -the node rather than the entire picture (including the pin) will be centered. -This can be achieved by adding key \refkey{/tikz/overlay} to the pin (actually, -we need to add it to both the pin and its edge). \TikZ normally updates the -extents (called the bounding box) of the picture every time it puts something -in it; when \refkey{/tikz/overlay} is in effect, however, these updates are -temporarily disabled. In effect, the \refkey{/tikz/overlay} key on the pin -below will fool \hologo{TeX} into thinking that the node is all there is to the -picture, so centering will work as desired. - -\ExampleName{overlay} -\makeexample{\examplename.pdf N=4} -\tcbinputexample{ - after title pre={\ (no memoization)}, - comment={\centering - % \includeexamplepdf[document page]{page=1,trim=1.8in 7.4in 1.8in 1.7in} - \includeexamplepdf[document page]{page=1} - }, -} - -What happens when we try to externalize this picture? The example below shows -what would happen if Memoize had no concept of \emph{padding} --- which we -simulate by setting \code{\refmmz{padding}=0pt}.\footnote{Unlike in the rest of - the manual, the extern pages in this section are shown without trimming the - whitespace.} Along with the rest of \hologo{TeX}, Memoize would be fooled -into thinking that the picture comprises of the node only, so the pin would -never make it into the extern. You would end up with a document missing the -pin!\footnote{On the first compilation, the document page containing the figure - without padding looks fine, as it uses the result of the compilation rather - than the extern file. But on the second compilation, when Memoize actually - uses the extern, the pin disappears.} - -\tcbinputexample[.tex][.c2]{ - after title pre={\ (memoization without padding)}, - attachment name=\examplename-no-padding.tex, - sidebyside, lefthand ratio=0.3, - comment={\centering - \includeexamplepdf[size=minimal,boxrule=0.5mm,extern page, - attach boxed title to top right={xshift=1.5mm,yshift=-1mm}, - ]{page=1} - }, -} - - -By default, Memoize puts an inch of space around (what it thinks is) the -externalized picture, and if the overlayed parts of the picture fit into this -inch of space, you will find them in the extern and therefore also in the -document. In our example, however, the default padding is not enough --- the -pin is only partially visible.\footnote{You might wonder why I didn't make the - default padding much bigger, like 10 inches. \hologo{TeX} wouldn't be - bothered (unless the resulting extern size exceeded its maximum dimension), - but you might be, because with such a large default padding, all the externs - would be huge, most often bigger than the document pages, and remember that - the externs are first dumped into the document, where they can bother you.} - -\tcbinputexample[.tex][.c3]{ - after title pre={\ (memoization with default padding)}, - attachment name=\examplename-default-padding.tex, - listing and comment, - middle=0.9mm, - comment={\centering - \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1} - }, -} - -The solution is to set the padding manually. Below, I used \refmmz{padding - right} to only increase the padding on the right side (clearly, we also have -\refmmz{padding left}, \refmmz{padding top} and \refmmz{padding bottom}), but -if you're not bothered by a large extern, you can just use -\Emph{\refmmz{padding}}, which sets all four sides at once. By the way, having -too much padding (almost) never hurts, and as you see, you can use (simple) -arithmetics in the value of these keys. - -\tcbinputexample[.tex][.c4]{ - after title pre={\ (memoization with extra padding)}, - attachment name=\examplename-extra-padding.tex, - listing and comment, - middle=0.9mm, - comment={\centering - \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1} - }, -} - -\enlargethispage{1ex} % manual -Incidentally, the \refmmz{padding} keys only change how -the externalized picture is \emph{stored}. Memoize remembers the size of the -extern as seen by \hologo{TeX} (e.g.\ the bounding box of the picture as -reported by \TikZ, with overlayed parts of the picture protruding out of it), -and it uses that size when integrating the extern into the document --- so -everything works as it should! - - - -\subsection{The verbatim mode} -\label{sec:tut:verbatim} - -Not all code will peacefully submit to memoization. In particular, this is the -case for environments which process the environment body verbatim (or perform -some other kind of \refcmd{catcode} magic). A simple environment of this kind is the -standard \hologo{LaTeX} \refenv{verbatim}, but let us illustrate the issue with -\refenv{tcblisting}, which typesets a code listing alongside its compiled effect. -(This environment is defined by the |listings| library of package \pkg{tcolorbox} -and was used extensively during the production of this manual.) To manually -memoize a \refenv{tcblisting} environment, we enclose it in a \refenv{memoize} -environment with a \Emph{\refmmz{verbatim}} key in the optional argument --- -without this key, the example below would produce nothing but -errors.\footnote{Memoize also offers a \emph{partial} verbatim mode, triggered - by key \refmmz{verb}; in this mode, the braces retain their usual category - codes. Also note that the effect of \refmmz{verbatim} can be ``undone'' by - key \refmmz{no verbatim}.} - -\ExampleName{verbatim-manual} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\\[1ex] - \footnotesize - (The document page is the same as for the \texttt{verbatim-auto} example below.) - }, -} - -Using \refmmz{verbatim} from \refcmd{mmzset} or \refcmd{mmznext} works just as -well, and the latter can be very useful with automemoization, when some -environment (say, \env{tcolorbox}) generally does not require the verbatim mode, -but a specific occurrence does (say, because it contains some verbatim -construction such as \verb!|!\meta{verbatim text}\verb!|! of the \pkg{ltxdoc} -class). - -However, for an environment such as |tcblisting|, it makes the most sense to -declare it verbatim in general, so that all instances of the environment will -be processed in the verbatim mode. This is simple to do: add \refmmz{verbatim} -to the \refmmz{auto} keylist. - -\ExampleName{verbatim-auto} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, % lefthand ratio=0.62, - comment={\centering - \footnotesize - (The extern page is the same as for the \texttt{verbatim-manual} example above.)\\[1ex] - \includeexamplepdf[document page]{page=2}\\ - }, -} - -In fact, you can add any \refkeypath{/mmz} key to the \refmmz{auto} keylist, -and the key will be applied to all occurrences of the command or the -environment. For example, adding \refmmz{recompile} to the declaration of -|tcblisting| above would recompile all and only the |tcblisting| environments; -and as an \refmmz{auto} declaration only updates (rather than completely -replaces) a previous declaration, you can also say things like -\code{\refmmz{auto}=\refcmd{tikz}\braces{\refmmz{recompile}}} to recompile all -\TikZ pictures produced by the \cs{tikz} command (handy, as you don't know how -automemoization for \cs{tikz} was declared unless you've read -section~\ref{sec:tut:automemoization-details} or looked at the Memoize's source -code). - - -\subsection{The final version of your document} -\label{sec:tut:final} - -Bluntly put, you might want to disable Memoize when compiling the final version -of your document, at least if you intend to distribute it in electronic form, -for two reasons: -\begin{itemize} -\item An externalized picture cannot contain hyperlinks. Any hyperlinks (or - hyperlink anchors) contained in the original picture will silently disappear - during the production of the extern. -\item When the document contains many externs, the size of the resulting PDF - can be several times the size of the PDF compiled without externalization. -\end{itemize} - -Below, we list several ways of fully disabling Memoize. You're of course -already familar with the first two ways, but what's this \refpkg{nomemoize} -package? The rationale behind this package is that if you want to be -absolutely sure that there is no trace of memoization in your document (for -example, see the \refmmz{disable} -- \refmmz{enable} pitfall in -section~\ref{sec:tut:working-on-a-picture}), the best thing to do is to not -load the package at all. However, you have all those \refcmd{mmzset}s etc.\ in -your source, so the document won't compile without -\cs{usepackage}\braces{\refpkg{memoize}}, right? Right, but wrong. Enter -\Emph{\refpkg{nomemoize}}, a dummy package which accepts all the commands that -Memoize does, but does nothing. In effect, your document will compile, but you -can be sure that not a single memo or extern was loaded or produced. - -\begin{tcbraster}[raster columns=3, raster force size=false, raster valign=top] - \begin{tcblisting}{listing only, add to width=2.4em, enhanced} -\usepackage[~disable~]{memoize} - \end{tcblisting} - \begin{tcblisting}{listing only, add to width=-1.6em, enhanced} -\usepackage{memoize} -\mmzset{~disable~} - \end{tcblisting} - \begin{tcblisting}{listing only, add to width=-0.8em, enhanced} -\usepackage{~no~memoize} - \end{tcblisting} -\end{tcbraster} - -There is one issue you might need to resolve manually before package -\refpkg{nomemoize} works as intended, though. If you have used any -\refkeypath{/mmz} keys outside \refcmd{mmzset}, you need to list them in -\refcmd{nommzkeys}. For example, if you used \refmmz{per overlay} in the manner -illustrated in section~\ref{sec:tut:beamer}, i.e.\ as -\refkeypath{/mmz}|/|\refmmz{per overlay} among the \TikZ keys, you need to -write \refcmd{nommzkeys}\braces{\refmmz{per overlay}} into the document -preamble. - -Another thing you might want to do once you have produced the final version of -the document (in fact, just before you disable Memoize for good) is clean up. -As we saw in sections~\ref{sec:tut:memodir} and~\ref{sec:tut:redefinitions}, -Memoize produces a lot of auxiliary files (memos and externs) and it keeps the -old versions around! Once your document is prepared, you can reduce the -clutter (and save some disk space) by deleting memos and externs belonging to -the work-in-progress versions of your document, and keep only those used in the -final version. - -You could achieve this by deleting all the memos and externs (if you're using -the \refmmz{memo dir} directive, this amounts to the entire contents of the -memo directory) and compiling your document for the final couple of times. -However, there is an easier (but \hologo{TeX}-external) way: on the command -line, change into the directory containing your (main) document and write -\EmphTT{\refscript{memoize-clean.pl}} \meta{document name}\dmmz (substitute -|.py| for |.pl| to use Python rather than Perl). The script will inspect the -contents of the \dmmz record file to see which memos and externs were used in -the final compilation, and delete all other memos and externs belonging to the -given document. - -\begin{tcolorbox}[warning] - Deleting memos and externs is never an irreversible operation, as you can - always recreate them, but it is still wise to be cautious when cleaning up. - For one, avoid cleaning up after a compilation which produced errors; a - failed compilation can lead to an incomplete \dmmz file, which can in turn - lead to over-deletion. Another bad idea is cleaning up after disabling - Memoize for a part of a document, for the same reason. - - All that said, Memoize takes some precautions itself. It will cowardly - refuse to perform the clean-up when the \dmmz file is missing the - end-of-file marker (|\endinput|), assuming that this indicates a fatal error - in the previous compilation. It will do the same in case the \dmmz file is - absent or empty. The latter is assumed to be a result of a globally - \refmmz{disable}d memoization, but note that clean-up will be performed if - memoization was disabled using package \refpkg{nomemoize}: that package does not - touch the \dmmz file, so cleaning up should work as intended. -\end{tcolorbox} - -As the final note, memos and externs (cleaned-up or not) may be copied (along -the document source) to another directory or machine, where they should be -picked up by Memoize. There is no need to copy the \dmmz file (assuming that -the document PDF contains no extern pages waiting for extraction). - - -\section{Digging deeper} -\label{sec:potential-pitfalls} - - - -\subsection{Good to know} -\label{sec:tut:good-to-know} - - -\paragraph{Line- and page-breaking} An extern can't be broken across lines or -pages. - -Externalization of a chunk of code produces a PDF, which is included into the -document at subsequent compilations as a picture --- an unbreakable object (a -horizontal box) with fixed width and height. Therefore, the original code -should produce an unbreakable object as well. For example, this means that you -cannot externalize some text and expect \hologo{TeX} to break it across lines -or pages on subsequent compilations. If you try, the compilation will succeed ---- without an error! --- but your externalized text will end up in a single -line, as shown below. - -\ExampleName{no-linebreaking} -\makeexample{\examplename.pdf N=2} -\tcbinputexample{ - bad, - comment={\centering - \includeexamplepdf[extern page,to be continued on right] - {page=1,trim=1in 1in 14in 1in}\\[1ex] - \includeexamplepdf[document page]{page=2}\\[1ex] - \includeexamplepdf[document page,title=the expected document page] - [\examplepath.c2.pdf]{page=1} - }, -} - -That said, you \emph{can} externalize a paragraph or some other vertical mode -material using \refmmz{capture}|=|\refmmz{capture=vbox}, but beware that the -vertical spacing between the memoized material and its surroundings might -change. - - -\paragraph{\env{remember picture}} -\TikZ pictures using this key cannot be externalized. - -Memoize will silently refuse to externalize any \TikZ picture using -\refkey{/tikz/remember picture} (see \PGFmanual{17.13}). Such pictures -interact with the outside world --- they either reference or are referenced by -other pictures --- and are as such unsuitable for externalization. For -example, while the colored boxes in this manual are generally externalized --- -out of principle \Smiley\ --- the title page illustration is not, and it cannot -be, because of the arrows connecting the various \TikZ pictures composing that -illustration. Some packages use the \refkey{/tikz/remember picture} mechanism -under the hood, and are thus subject to this limitation; one example is package -\pkg{todonotes}, but in general, any package dealing with absolute positions on the -page will be limited in this way. - -How does Memoize deal with this situation? Well, by cowardly refusing to -externalize any code which uses \refkey{/tikz/remember picture} or a similar -mechanism for dealing with absolute positions. Luckily, any such mechanism -eventually boils down to the \hologo{TeX} primitive \refcmd{savepos}, so -Memoize hacks --- or as we will say in this manual, \emph{advises} --- this -primitive to abort any ongoing memoization. Initializing and then aborting the -memoization takes some time, to be sure, but the overhead is negligible, -especially in the light of the fact that \emph{not} aborting wreaks real havoc. - -Memoize actually provides a user interface for aborting memoization. -Memoization can be aborted either manually, by executing \refcmd{mmzAbort}, or -automatically. The latter is a generalization of the automemoization idea: a -command such as \refcmd{savepos} can be advised to abort memoization by -\refmmz{auto}|=|\refcmd{savepos}\bracestt{\refmmzauto{abort}}. In Memoize, -commands take your advice seriously, so memoization will be aborted whenever -the advised command or environment is encountered. - - -\paragraph{Indirectly embedded environments} Such environments cannot be -memoized. - -Read this if you got an error message such as |Environment "tikzpicture" ended -as "foo"|. - -Some environments are defined so that they embed another environment using the -idiom shown on the left: the begin-code of the outer environment opens the -inner environment, and the end-code of the outer environment closes the inner -environment. While this is a fine, and common, idiom, it messes up the -memoization of the inner environment. In the example on the right, trying to -\emph{auto}memoize a \env{minipage} environment (not recommended at all!) causes -trouble with package \pkg{sectionbox}.\footnote{This is a \texttt{Package - collargs Error} because Memoize outsources the actual work of collecting - the environment body to the auxiliary package CollArgs, described in - section~\ref{sec:collargs}.}\fncomma\cprotect\footnote{Why does this happen? - As mentioned in section~\ref{sec:tut:memodir}, Memoize keeps track of memos - and externs by the MD5 sum of the memoized code. But to compute that sum for - an environment, Memoize has to \emph{grab} the environment body, meaning it - has to collect the body in advance. This presents no problem when - \refcmd{end}\meta{environment name} is already present in the input stream at the -time \refcmd{begin}\meta{environment name} is executed, like when you use the - environment normally in your document, or when some macro expands so that it - produces both \cs{begin}\meta{environment name} and \cs{end}\meta{environment - name} simultaneously --- so there would be no problem above if - |\end{minipage}| occured in the beginning code of \env{sectionbox}. The idiom -presented above is problematic for memoization because at the time \hologo{TeX} -executes |\begin{sectionbox}|, putting |\begin{minipage}| into the input - stream, |\end{sectionbox}| is not yet executed and remains as it is. The - input stream therefore contains a pair of |\begin{minipage}| and - |\end{sectionbox}|. In the normal, non-memoizing course of events this - would not be a problem, because |\end{sectionbox}| would eventually expand to -|\end{minipage}|. During memoization, however, this \emph{is} a problem, -because, as we said, Memoize needs to grab the environment body: upon -encountering |\begin{minipage}|, it looks in the input stream for - |\end{minipage}| --- but there is no |\end{minipage}| in the input stream, -there is only |\end{sectionbox}|, and this results in the |Environment -"minipage" ended as "sectionbox"| error.} - -\begingroup -\ExampleName{sectionbox} -\makeexample{\examplename.tex.c1} -\mmzset{disable} -\begin{tcbraster}[raster equal height] -\begin{tcblisting}{listing only, bad, valign=center} -\newenvironment{foo}% ... args -{% the begin-code of foo - % ... - \begin{tikzpicture} - % ... -} -{% the end-code of foo - % ... - \end{tikzpicture} - % ... -} -\end{tcblisting} -\tcbinputexample{ - bad, - comment={\small\tt - ! Package collargs Error: Environment\\"minipage" ended as "sectionbox". - }, - } -\end{tcbraster} -\endgroup - -What are your options in this kind of a situation? -\begin{enumerate} -\item The only way to perform any memoization here is to memoize the - \emph{outer} environment --- if that makes sense.\footnote{This avoids the - error because Memoize grabs and memoizes the outer environment, and while - it is memoizing it, further memoization is switched off.} You can do this - either on a case-by-case basis, by enclosing it in the \refenv{memoize} - environment, or automemoize it: \code{\refmmz{auto}=\marg{outer - environment}\braces{\refmmzauto{memoize}}}. -\item But what if memoizing the outer environment is out of the question? - Then, the only way to avoid the error is to prevent the automemoization of - the inner environment. - \begin{enumerate} - \item If you are facing a single occurrence of the problem, it is perhaps - easiest to use key \refmmz{disable} just before the start of the outer - environment. - \item Otherwise, you can automatically disable memoization for the span of - each occurrence of the outer environment: \code{\refmmz{auto}=\marg{outer - environment}% - \braces{\refmmzauto{nomemoize}}}. - \item To deactivate the automemoization of the inner environment for the span - of the outer environment, but otherwise allow for memoization inside the - outer environment: \code{\refmmz{auto}=\marg{outer environment}% - \braces{\refmmzauto{noop}, \refmmz{deactivate}=\meta{inner - environment}}}. Key \refmmzauto{noop} does nothing but apply the - given options (in this case, \refmmz{deactivate}=\meta{inner environment}) - to the advised command or environment. - \end{enumerate} -\end{enumerate} - -\subsection{Extraction methods and modes} -\label{sec:extraction} - -Remember that in Memoize, externalization is a two-step process. First, -externs are typeset on separate pages of the main document, called extern -pages. Then, these extern pages are extracted out of the main document PDF -into extern files. The process is illustrated on the title page. - -Memoize is flexible in terms of which piece of software is used to perform -extern extraction. It ships with three \emph{extraction methods}: - -\begin{description} -\item[\refmmz{extract=perl}] A Perl script, \refscript{memoize-extract.pl}. - This method is the default because it is fast and because Perl is usually - already installed on a system running \hologo{TeX}. However, you will most - likely still need to install two pieces of software that the script depends - on: Perl libraries \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} and - the \hreftt{https://metacpan.org/pod/Path::Class}{Path::Class}, the - installation guidelines can be found in section~\ref{sec:install}. -\item[\refmmz{extract=python}] A Python script, \refscript{memoize-extract.py}. - This method is even faster than the Perl script, though not by much. Try it - out if you have problems installing Perl or the required libraries, or if the - Perl script chokes on your document (see section~\ref{sec:troubleshooting} - for the list of known issues). Besides Python ($\geq 3.8$), you will also - need the Python library \hreftt{https://pypi.org/project/pdfrw/}{pdfrw} or - \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2}. For the installation - guidelines, see section~\ref{sec:install}. -\item[\refmmz{extract=tex}] \hologo{TeX}-based extraction requires no - additional software, but it is much slower than the scripts. As \hologo{TeX} - can only produce a single PDF per compilation, an instance of \hologo{TeX} - (loading the entire document PDF) has to be invoked for each extern, and this - takes time (although the entire process is still much faster than the - venerable \TikZ externalization library). -\end{description} - -Memoize is also flexible in terms of how extern extraction is triggered, -providing two \emph{extraction modes}: - -\begin{description} -\item[internal]\label{item:setup:who} By default, extern extraction is - triggered internally, i.e.\ by Memoize during the compilation of the - document; more precisely, any externs produced in a compilation are extracted - during the next compilation. To choose an extraction method other than the - default Perl script, load Memoize with the package option - \EmphTT{\refmmz{extract}=\meta{extraction method}}. - -\item[external] Loading Memoize with with package option - \EmphTT{\refmmz{extract}=\refmmz{extract=no}} instructs Memoize to \emph{not} - trigger the (internal) extraction. When instructed to use extraction - ``method'' \refmmz{extract=no}, Memoize expects you to trigger the extraction - yourself, in any way that is convenient to you: manually from the command - line, or automatically through your editor, a Makefile, etc. --- all Memoize - cares about is that the extraction takes place before the next compilation of - the document. -\end{description} - -Summing up, the extraction mode and method are selected by providing the -appropriate value to package option key \refmmz{extract}; the possible values -are listed in the table below. Note that this key can only be used as a -package option, or in \refcmd{mmzset} within \reffile{memoize.cfg}. In -particular, it is disabled in the document preamble, because Memoize performs -extraction while it is loaded. - -\medskip -\begin{center} - \begin{tabular}{lll} - \toprule - extraction method&external program&Memoize invocation\\ - \midrule - \refmmz{extract=perl}&\refscript{memoize-extract.pl}&\refcmd{usepackage}\braces{\refpkg{memoize}}\footnotemark\\ - \refmmz{extract=python}&\refscript{memoize-extract.py}&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=python}}\braces{\refpkg{memoize}}\\ - \refmmz{extract=tex}&|pdftex|&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=tex}}\braces{\refpkg{memoize}}\\ - \refmmz{extract=no}&none (external extraction)&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=no}}\braces{\refpkg{memoize}}\\ - \bottomrule - \end{tabular}% - \footnotetext{Or - \refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=perl}}% - \braces{\refpkg{memoize}}. This is useful if you have changed the default - using \reffile{memoize.cfg}.}% -\end{center} -\smallskip - -For internal extraction, \hologo{TeX} must be allowed to execute the external -program implementing the chosen extraction method. Both |memoize-extract| -scripts should be listed among restricted shell escape mode commands in your -\hologo{TeX} distribution; their execution should therefore be allowed under -the default, restricted shell escape mode. However, the |pdftex| program, -executed by extraction method \refmmz{extract=tex}, is not listed there, nor -should it be. If you are forced to use this fallback method, I suggest you -compile documents loading Memoize under the full shell escape mode, by adding -command-line option \docref{-shell-escape} (on \hologo{TeX}Live) or -\docref{--enable-write18} (on MiK\hologo{TeX}) to the invocation of the -\hologo{TeX} program. The answers linked from question -``\href{https://tex.stackexchange.com/q/598818/16819}{How can I enable - shell-escape?}'' on \href{https://tex.stackexchange.com}{\hologo{TeX} - StackExchange} will tell you how you can ask your editor to do this for you. - -You may use any extraction method to perform external extraction. The simplest -option is to use the Perl or the Python script. Supposing you are doing this -manually from the command line, change into the directory containing your -document, which should contain the auxiliary \dmmz file produced by Memoize, -and execute: - -\begin{enumerate}[(a)] -\item \refscript{memoize-extract.pl} \meta{document name}\dmmz \hfill (for - the Perl script) -\item \refscript{memoize-extract.py} \meta{document name}\dmmz \hfill (for - the Python script) -\end{enumerate} - -See sections~\ref{sec:.mmz} and \ref{sec:ref:extraction-perl-python} for -further details on the \dmmz file and the extraction scripts. - -Things are a bit more complicated if you want to use the \hologo{TeX}-based -extraction method externally, because an instance of |pdftex| needs to be -invoked for each extern (and these have unwieldy names and can be many in -number), but Memoize can help you here by producing a shell script or a -makefile, executing which will extract all the externs at once. To have Memoize -produce a shell script, use package option -\code{\refmmz{record}=\refmmz{record=sh}} (or -\code{\refmmz{record}=\refmmz{record=bat}} on Windows); package option -\code{\refmmz{record}=\refmmz{record=makefile}} will make a makefile. By -default, these files are named |memoize-extract.|\meta{document name} plus the -|.sh|, |.bat| or |.makefile| suffix. If neither a shell script nor a makefile -works for you, you can also define your own kind of \emph{record file}, to be -processed by the external tool of your choice (and implementation) in order to -extract the externs; see section~\ref{sec:new-record-file} to learn how to do -this. - - -\subsection{From cross-references to the context} -\label{sec:cross-referencing} - -Cross-referencing presents a challenge to externalization, because without -special provisions, the ``communication channel'' between the \cs{label} and the -\cs{ref} is broken once we start utilizing the extern. - -One direction of the issue occurs when a \refcmd{label} within the memoized -code is referenced by a \cs{ref} on the outside. Without the (built-in) -workaround, the \cs{label} command would only be executed when the extern is -being produced, but not on subsequent compilations of the document, when it is -merely included. Memoize addresses this problem by generalizing -externalization (which can only produce a picture, the extern) to memoization -(which can additionally produce arbitrary code). When Memoize is externalizing -code which contains a \cs{label}, it automatically replicates it into the -\emph{memo}, which is input into the document on subsequent compilations. In -effect, the memo--extern team will continue to produce the label even when it -is utilized rather than compiled. As far as the author is concerned, \cs{label}s -in memoized code ``just work,'' without any observable differences to the -situation without memoization. This is why we will not discuss this direction -of the issue here; a reader interested in how precisely the system works is -invited to read section~\ref{sec:memos}. - -The other direction of the issue occurs when a \refcmd{ref} within the memoized -code references \cs{label} on the outside. In this situation, the extern -should be recompiled when the value of the label it refers to changes. Again, -Memoize addresses this problem in full generality, by associating with each -extern a \emph{context}, and recompiling the extern whenever the value of the -context changes.\footnote{The dependency of an extern upon prior definitions - and such can also be addressed in a more \emph{ad hoc} manner, by recompiling - manually; we have already touched upon this subject in - section~\ref{sec:tut:working-on-a-picture}, and will revisit it in - section~\ref{sec:tut:redefinitions}.} All that needs to be done for \cs{ref} -and friends, specifically, is to advise them to add their reference keys to the -context. - -As we shall see presently, for the author, the only difference between a -non-memoized and a memoized \cs{ref} is that the latter will take one more -compilation cycle to ``stabilize'' the resulting document. (More precisely, -the memoized situation will take one more cycle if the reference is undefined -on the first compilation.) Then, we will show how we can teach Memoize about -cross-referencing commands other than \cs{ref} and \cs{pageref}. Finally, we will -learn about key \refmmz{context}, the backbone of the cross-referencing support -in Memoize. (The inner workings of the context are further explained in -section~\ref{sec:c-memos}.) - -\ExampleName{ref} -\makeexample{\examplename.pdf N=7} - -When the memoized code contains a \cs{ref} referring to a label given in another -part of the document, the code is recompiled when (and only when) the reference -changes. Let us look at the following example, jumping in at the point where -it was already compiled enough times that the resulting PDF had stabilized into -a single (document) page with correct references. (Environment -\refenv{nomemoize} disables memoization -of \href{https://ctan.org/pkg/tikzlings}{Ti\emph{k}Zlings}, so that their -externs don't disturb us, and we can focus on the \cs{tikz} command, which does -get externalized and contains a \cs{ref}.) - -\tcbinputexample[.tex][.c3]{ - after title pre={\ (with stable output after three compilations)}, - middle=1mm, - comment={\centering - \includeexamplepdf[document page]{page=1} - } -} - -Let us add an owl in front of the penguin. In the next compilation, neither -the ``normal'' nor the memoized reference is yet updated, as expected --- in -this compilation, the new value of the penguin label only makes it into the -|.aux| file. - -\tcbinputexample[.tex][.c4]{ - after title pre={\ (after the first compilation with the added owl)}, - no attachment, - comment={\centering - \includeexamplepdf[document page]{page=1} - } -} - -During the following compilation, the \refcmd{ref}s pick up the new value of the -penguin label, and the \cs{ref} inside the automemoized \cs{tikz} command -forces recompilation of the extern (\emph{how} this is done will be explained -later). - -\tcbinputexample[.tex][.c5]{ - after title pre={\ (after the second compilation with the added owl)}, - comment only, no attachment, - comment={\centering - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad - \includeexamplepdf[document page]{page=2} - } -} - -In the next compilation, the resulting PDF is finally stabilized, as the -updated extern is (extracted and) included into the document. - -\tcbinputexample[.tex][.c6]{ - after title pre={\ (after the third compilation with the added owl)}, - comment only, no attachment, - comment={\centering - \includeexamplepdf[document page]{page=1} - } -} - -The message to take home? When some memoized code contains a reference and -that reference changes, it will take three compilation cycles (so, one more -cycle than without memoization) for the resulting document to ``stabilize.'' - -Out of the box, Memoize supports the standard \hologo{LaTeX} cross-referencing -commands \cs{ref} and \refcmd{pageref}. To automatically recompile code -containing some other cross-referencing command, like \refcmd{vref} of package -\pkg{varioref}, we use the advising framework implemented by package Advice. -This framework is a generalization of automemoization: we use the familiar -\refmmz{auto}, but with advice offered by \Emph{\refmmzauto{ref}} rather than -\refmmzauto{memoize}. - -{\ExampleName{vref} -\makeexample{\examplename.tex.c1} -\tcbinputexample{listing only}} - -Key \refmmzauto{ref} only works for commands which operate on a single -reference key. However, that single key (which must be enclosed in braces) may -be preceded by optional argumen(s) of any kind. Extensions to \refcmd{ref}, -e.g.\ the \pkg{hyperref}'s variant, which accepts an optional |*|, work out of -the box. Furthermore, Memoize offers support for cross-referencing commands -which work on multireferences and reference ranges, such as \pkg{cleveref}'s -\refcmd{cref} and \refcmd{crefrange}. Those commands should be advised by -\refmmz{auto} keys \refmmzauto{multiref} and \refmmzauto{refrange}, -respectively. - -We have jumped into first example of this section with the assumption that it -had already been compiled several times, allowing the resulting PDF to -stabilize. Let us now take a look at what happens at the very first, fresh -compilation of our original example (the one without the owl). (Removing the -|.aux| file before compiling the example again will start afresh.) The curious -thing is that we don't get the extern page containing -\tikz[baseline]\node[draw=red,thick,fill=yellow,anchor=base,font=\bf]{??};. -This is so because by default, Memoize aborts a memoization containing an -undefined reference. - -\tcbinputexample[.tex][.c1]{ - after title pre={\ (after the fresh compilation of the original example)}, - comment only, no attachment, - comment={\centering - \includeexamplepdf[document page]{page=2} - } -} - -Now sometimes you might want to produce an extern even if it contains an -undefined reference --- for example, you might intend to write the code -containing the \cs{label} much later but enjoy the speed-up offered by Memoize -until then. In that case, apply the \refmmz{auto} key \Emph{\refmmzauto{force - ref}} to \cs{ref}. - -\tcbinputexample[.tex][.c7]{ - after title pre={\ (after the fresh compilation with \refmmzauto{force ref})}, - attachment name=\examplename-force.tex, - comment={\centering - \includeexamplepdf[extern page][\examplepath.c1.pdf]{page=1,trim=1in 1in 1in 1in}\quad - \includeexamplepdf[document page][\examplepath.c1.pdf]{page=2} - } -} - -However, note that when you use \refmmzauto{force ref}, \hologo{LaTeX} will -\emph{not} complain about the undefined reference once the extern containing it -is included (unless that reference also occurs in some non-memoized piece of -code). Using \refmmzauto{force ref} is therefore a tiny bit dangerous, and -this is why \refmmzauto{ref}, with the abortion mechanism, is the default -handler for \cs{ref} and \refcmd{pageref}. - -As already noted in the previous section, \refcmd{ref} works by appending the -cross-reference to the \emph{context}, the change of which triggers -recompilation. Memoize initializes the context to contain the four -\refmmz{padding}s --- as a result, an extern recompiles if we change the -padding --- but we can append stuff to the context by ourselves, as well. -Below, we use key \Emph{\refmmz{context}} to append the font size (we'll talk -about the value given to this key a bit later); as a result, the picture is -recompiled whenever the font size changes. Below, we change the font size -using command |\small|; changing the default size with a class option such as -|12pt| works as well. - -\ExampleName{fontsize} -\makeexample{\examplename.pdf N=2} -\tcbinputexample{ - sidebyside, lefthand ratio=0.7, after title pre={\ (the first version)}, - comment={ - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} - } -} - -\tcbinputexample[.tex][.c2]{ - sidebyside, lefthand ratio=0.7, after title pre={\ (the second version)}, - no attachment, - comment={ - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in} - } -} - -How does this work? Key \refmmz{context} appends the given tokens to the -\emph{context expression}. When creating an extern or trying to use it, -Memoize (fully) expands this expression and computes the MD5 sum of the -expansion. This \emph{context MD5 sum} then serves as a part of the extern's -filename (see sections~\ref{sec:tut:memodir} and~\ref{sec:memos}). In effect, -Memoize will only find (and utilize) the extern if \emph{the context MD5 sum - computed during (attempted) utilization matches the one computed during - memoization}. - -As revealed by looking at the \hologo{LaTeX} source code, \hologo{LaTeX} holds -the current font size in macro |\f@size|, and above, we have effectively added -the contents of this macro to the context. Now, why didn't we simply write -\refmmz{context}|=\f@size|? First, we used |\csname ... \endcsname| because we -were under the normal \hologo{LaTeX} catcode regime, where |@| cannot be a part -of the command name. Of course, we could have temporarily changed the catcode -of |@| using |\makeatletter| and |\makeatother|, but I would advise against -that, because the approach does not work in general: it fails when key -\refmmz{context} is used \emph{within} memoized code (we will explain why in -section~\ref{sec:memos}). Another reason why I recommend the |\csname -... \endcsname| approach is that it does not result in an error when the -control sequence is not defined (|\csname ... \endcsname| will expand to -|\relax| then); this behaviour is handy for undefined cross-references, for -example. Second, why did I write |fsize={...}| around the control sequence? -Well, because I'm being paranoid, really. Writing \refmmz{context}|={\csname - f@size\endcsname}| would work just as well, but I like to explicitly -``announce'' the value to prevent any possibility of a conflict with an -alternative context. Imagine that we don't use the ``announcements'' and we -decide to add some other dimension instead of the font size to the context. -Now if that dimension happened to have the same value as the font size, Memoize -would incorrectly pick up the ``font size extern'' instead of producing a new -one. - -It bears emphasizing that whatever you add to the context expression must be -fully expandable, and also not merely declared as robust. So writing -\code{\refmmz{context}=\cs{ref}\marg{key}}, for example, would be unwise, since -it would not work as intended when package \pkg{hyperref} is loaded. (This -package declares \refcmd{ref} as robust, so it won't expand to the -cross-reference value.) You have to look up where the cross-references are -stored internally; the cross-reference for \meta{key} turns out to be stored in -the internal control sequence |\r@|\meta{key}, so it is |\csname -r@|\meta{key}|\endcsname| that the \refmmzauto{ref} handler actually appends to -the context. - -The padding and font-size contexts are useful quite generally. However, the -context can be pretty command-specific, as well. Consider the \pkg{skak} -package used for typesetting chess games. The board is drawn using command -|\showboard|, but this command has no arguments, because it draws the state of -the board that is reached by the moves given by command |\mainline|. Memoizing -|\showboard| as such will therefore yield the wrong result --- all the boards -will be one and the same board! The solution is to provide the correct -context: we dig into the \pkg{skak} sources and realize that the current -board is stored in macro |\csname chessgame.skak.mainline\endcsname|. - -\ExampleName{skak} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, lefthand ratio=0.48, - comment={ - \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.45}\quad - \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in,scale=0.45} - } -} - -If you remove \code{\refmmz{context}=\bracestt{...}} from the code above, you -will end up with a document where the final board drawn takes place of all the -boards. This is so because in that case, all externs are written into the same -file, so the final extern overwrites the previous ones, but note that you will -only observe this after the second compilation, when the externs are actually -used. - - - -\subsection{More on redefinitions and stale externs} -\label{sec:tut:redefinitions} - -In this subsection, we elaborate on an issue touched upon at the beginning of -section~\ref{sec:tut:working-on-a-picture}: what happens if the memoized code -depends on some macro or style which gets redefined? The answer was -``nothing,'' and one solution was to \refmmz{recompile} the code. Let us take -the example from that section a bit further. We will propose no new solution -or workaround, but deepen our understanding of the issue. - -\ExampleName{redefinitions} -\makeexample{\examplename.pdf N=6} -\tcbset{ - recompile step/.style={ - sidebyside, lefthand ratio=0.74, title=, - comment={\includeexamplepdf[blankest][\examplepath.c\theenumi.pdf]{#1}}, - /utils/exec={\addtocounter{enumi}{1}}, - enhanced, overlay app={\node[at=(frame.north west), font=\scriptsize, circle, fill=black!80!white, text=white, inner sep=1pt]{\theenumi};}, - }, -} -\begin{tcboxedraster}[raster columns=1]{% - title=Working on \texttt{\examplename.tex}, enhanced, breakable} - \setcounter{enumi}{0}% - \tcbox[blankest]{I like red. My emphasized nodes will have red background.} - \tcbinputexample[.tex][.c1]{recompile step={page=2}} - \tcbox[blankest]{Hmm, this particular node is really important, - let me put the text in italics as well!} - \tcbinputexample[.tex][.c2]{recompile step={page=2}, no attachment} - \tcbox[blankest]{You know what? Perhaps yellow background would work better - --- in general.} - \tcbinputexample[.tex][.c3]{recompile step={page=1}, bad, no attachment} - \tcbox[blankest]{How come my node is still red?! Oh yes, I changed the - style, so I have to recompile the extern!} - \tcbinputexample[.tex][.c4]{recompile step={page=2}, no attachment} - \tcbox[blankest]{Ahh, yellow background, that's much better. But you know - what, this double emphasis won't do after all, let me go back to the - upright shape.} - \tcbinputexample[.tex][.c5]{recompile step={page=1}, bad, no attachment} - \tcbox[blankest]{Red???!???!? Ok, I know that recompiling will help, but what - happened here?} - \tcbinputexample[.tex][.c6]{recompile step={page=2}, no attachment} -\end{tcboxedraster} - -What happened is that the externs from steps~1 and~5 share the very same code. -In step~1, this code was compiled when the red |emph| style was in effect, and -that extern lingered and was eventually picked up again in step~5, Memoize -having no idea that it is including an extern produced with the obsolete -definition of the style. - -There are two points to this story. First (and forgetting for a moment about -the context, which we started discussing in -section~\ref{sec:cross-referencing}), Memoize identifies externs (and memos) by -the code that produced them --- or more precisely, by the MD5 sum of the code, -as each piece of code has a unique (well, unique enough) MD5 sum. Each extern -is saved in a file whose name contains this MD5 sum; see -section~\ref{sec:tut:memodir} for illustration. Generally, this is a very -useful feature. You can move your picture to another location in the document, -insert some other (externalized) picture in front of it, and so on, all without -triggering recompilation of the extern(s). (None of this is possible with -\TikZ's externalization library, which identifies the externs by the order in -which they appear in the document.) - -The downside of the MD5 sum approach is the potential pitfall illustrated -above, and the downside comes about because of the second point of the story: -Memoize does not attempt to delete the ``old'' externs. (However, as described -in section~\ref{sec:tut:memodir}, stale memos and externs can be easily removed -using the \refscript{memoize-clean.pl} script.) That would be not only -dangerous (as any deletion inherently is) but also potentially wasteful: what -if you have only temporarily removed some code, or compiled only a portion of -the document --- you surely wouldn't want your hard-won externs to disappear in -such a situation! - -The pitfall described above applies to any command which depends on parameters -which can be set prior to the invocation of the command, like \TikZ pictures, -which depend on the settings given in \cs{tikzset}. After customizing the -settings, you will have to recompile the externs: -\refcmd{mmznext}\braces{\refmmz{recompile}} is useful when you only have to -recompile a single extern; use \refcmd{mmzset}\braces{\refmmz{recompile}}, or -the package option \refmmz{recompile}, to recompile all externs in the -document; and there is also the middle road: if you have changed only Forest's -settings, you can write -\code{\refmmz{auto}=\braces{forest}\braces{\refmmz{recompile}}} to recompile -all and only the Forest trees. - -Above, we have seen the ``same code, same extern'' issue manifested ``through -time,'' i.e.\ Memoize was (incorrectly) reusing externs produced in previous -compilations, but the issue can also manifest ``through space.'' This can -happen if the same code appears twice in the same document --- but, crucially, -with some parameters which it depends on changed from one occurrence to the -next. Observe what happens in the following example, where the settings for -|\progressbar| are changed by |\progressbarchange|. After the first -compilation, everything looks fine. But as both extern pages were produced by -the same code, they will be stored into the same file, the second one -overwriting the first one. The second compilation pulls in the externs --- or -rather, the single extern --- resulting in the document containing the second -progressbar in the place of the first one as well. - -\ExampleName{progressbar} -\makeexample{\examplename.pdf N=2} -\tcbinputexample{ - bad, float, - comment={ - \begin{tcbitemize}[raster columns=3,raster equal height,halign=center,valign=center] - \tcbitem[title=After 1\textsuperscript{st} compilation] - \includeexamplepdf[extern page,remember as=extern1]{page=1,trim=1in 1in 1in 1in}\\ - \includeexamplepdf[extern page,remember as=extern2]{page=2,trim=1in 1in 1in 1in}\\ - \includeexamplepdf[document page]{page=3} - \tcbitem[title=After extern extraction\vphantom{p\textsuperscript{nd}}] - \includeexamplepdf[my boxed title=the extern file,attach shifted boxed title to top right,remember as=externfile]{page=2,trim=1in 1in 1in 1in} - \tcbitem[title=After 2\textsuperscript{nd} compilation] - \includeexamplepdf[document page,remember as=doc][\examplepath.c2.pdf]{page=1} - \end{tcbitemize} - \begin{tikzpicture}[remember picture,overlay] - \draw[->, thick, red] (extern1.east) to[out=east, in=west] ([yshift=0.5ex]externfile.west); - \draw[->, thick, red] (extern2.east) to[out=east, in=west] ([yshift=-0.5ex]externfile.west); - \draw[->, thick, blue] (externfile.east) to[out=east, in=west] (doc.west); - \end{tikzpicture} - }, -} - -The same ``extern duplication'' can arise due to how a particular command is -implemented. Say we deactivate automemoization of Forest trees -(\code{\refmmz{deactivate}=forest}), but keep on automemoizing \TikZ pictures. -Forest uses \refenv{tikzpicture}\noprint{\refenv{forest}} under the hood (a -lot); in particular, the tree itself is typeset as a \env{tikzpicture} -environment. But the code that typesets it is the same for all trees, -regardless of their content (the actual content of the tree is hidden in -various macros and boxes, rather than ``pasted'' into the \env{tikzpicture}). -Consequently, the final tree of the document will overwrite all other trees in -the document, just as the second (and thus final) progress bar overwrote the -first one above.\footnote{That is assuming that \hologo{TeX} doesn't simply - spew a bunch of errors. This can happen as well. In the interest of full - disclosure, compiling a Forest tree in the situation described above would - actually also produce --- but only in the first compilation --- a number of - small empty extern pages, one for each node of the tree. A promise: Forest - will soon fully support Memoize and (among other things) avoid this pitfall. - But the principle will remain.} Ouch! - -Generally speaking, this final sort of extern duplication issue can arise -whenever we have an ``outer'' command that we don't want to (auto)memoize which -uses an ``inner'' command that we \emph{do} want to automemoize. The solution -is to use the \refmmz{auto} key \refmmzauto{nomemoize} on the outer command; -remember that this key disables memoization for the space of the command or -environment. For example, the correct way to ``deactivate'' automemoization of -\env{forest} environments (but keep automemoizing \TikZ pictures) is -\code{\refmmz{auto}=\bracestt{forest}\braces{\refmmzauto{nomemoize}}}. - - - - -\subsection{Supporting Memoize in your package} -\label{sec:tut:package} - -\subsubsection{Loading Memoize?} -\label{sec:loading-memoize} - -So you want to support Memoize in your package? That's great! - -What form precisely this support will take of course depends on your package. -For some commands, a simple \refmmz{auto} declaration will suffice; for other -commands, you might need to write a dedicated \emph{memoization driver}, as -explained in section~\ref{sec:memoization-drivers}. However, one thing is -clear: you \emph{don't} want to require Memoize's presence by -\cs{RequirePackage}\braces{\refpkg{memoize}} in your package. That would -trigger memoization, but triggering memoization should be left at the sole -discretion of the author. The question is, if you're not allowed to load -Memoize, how can you even issue the \refmmz{auto} declaration? - -Well, it's not that you really want to \emph{memoize} anything; you want to -make the commands of your package \emph{memoizable}. So: -\cs{RequirePackage}\braces{\Emph{\refpkg{memoizable}}} --- and note that in -`memoizable', the final `e' of `memoize' is dropped, apparently this is the -correct way to spell it. - -Loading \refpkg{memoizable} does nothing if Memoize is already loaded, and -behaves like package \refpkg{nomemoize} otherwise --- remember from -section~\ref{sec:tut:final} that \refpkg{nomemoize} is a dummy package which -accepts all the commands that Memoize does, but does nothing. - -I have decided to require that Memoize must be loaded before any package that -supports it. Allowing for an arbitrary loading order would complicate the -implementation (and possibly even turn out to be problematic), and furthermore, -Memoize likes to be loaded early anyaway, because it needs to be loaded before -the document PDF is opened if it is to perform the embedded extern extraction. -I don't think the ordering requirement will cause any problems --- let me know -if it does! --- but perhaps it is wise to inform the author about it in the -documentation of your package (I did so at the end of -section~\ref{sec:tut:automemoization}). Anyway, I have enforced the -requirement by raising an error and refusing to load the package in case -Memoize detects \refpkg{memoizable} to be loaded. - -Note that the loading order requirement implies that you can use -|\@ifpackageloaded{memoize}| to specifically react to the presence Memoize, if -necessary. - - -\subsubsection{Memoizable design} -\label{sec:memoizable-design} - -Many commands and environments can be submitted to externalization with a -single-line \refmmz{auto} declaration, as illustrated in -section~\ref{sec:tut:automemoization}, perhaps requiring an addition to the -context (section~\ref{sec:cross-referencing}), or some pre- or post-memoization -code (section~\ref{sec:tut:beamer}). In some situations, however, these simple -approaches won't work. Most often, this will happen when the extern must be -integrated into the document in some special way. For example, a command might -internally create floating material, or surround the core typeset material with -some stretchable space.\footnote{Commands and environments of package - \pkg{tcolorbox} exhibit both these issues (see \pkg{tcolorbox} options |float|, - |before| and |after|), and were in fact the inspiration for several technical - details of Memoize.} None of these behaviours can be replicated by merely -including the extern; with respect to the stretchable space, remember that an -extern, being a picture, has fixed size, so if our extra space ended up in the -extern, it would lose the stretchability. - -The key to successful memoization of problematic commands is their design. In -a nutshell, the idea is to \emph{break up the command's definition into two - parts, the outer command and the inner command, and only submit the inner - command to automemoization}. We will illustrate this with a simple -environment --- |poormansbox| --- which produces a potentially framed box of -the given width, and surrounds this box with some configurable material --- by -default, this material will be stretchable vertical space, and this will be the -source of our memoization problem. (In terms of user experience, the solution -in this section will leave something to be desired, but we will revisit the -example in section~\ref{sec:memoization-complex-single-driver} and make things -right.) - -Let us first take a look at a document using our to-be-developed box -environment. The |poormansbox| environment takes one optional argument, a -keylist of options, which can also be set with the |\pmbset| command. This -being a \textbf{p}oor \textbf{m}an's \textbf{b}ox, it doesn't recognize many -options. One can set the |width| of the box, or request that it occurs in a -|frame|, and the surrounding material can be configured using keys |before| and -|after|. As we will see later in the listing of the package, the box is -|\linewidth| wide by default, has no frame, and is surrounded by vertical glue -|\vskip 2ex plus 1ex minus 1ex| (|2ex| of natural vertical space which may both -stretch and shrink for |1ex|); furthermore, the default value of |before| -contains |\centering| to center the box horizontally (centering is of course -only observable when we change the width of the box). - -\ExampleName{poormansbox} -\makeexample{\examplename.sty} -\makeexample{\examplename.pdf} -\tcbinputexample{ - sidebyside, lefthand ratio=0.482, - comment={\centering - \includeexamplepdf[document page,left=0pt,right=0pt] - {page=1,scale=0.5,trim=0mm 3mm 0mm 3mm} - }, -} - -You might want to play with the example to see that the surrounding vertical -space is indeed stretchable. The example is set up so that the surrounding -space is shrunk a bit to fit all the material onto one page. But if you remove -the final |\lipsum[144]|, the natural amount of all vertical space can be -accommodated on the page, so you should observe an increase of vertical -spacing. - -You might have noticed that the example contains nested |poormansbox|es: the -second box (the one which contains |\lipsum[66]|) is nested within the first -one (between |\lipsum[101]| and |\lipsum[75]|). This is intentional: when we -will revisit the |poormansbox| example in -section~\ref{sec:memoization-complex-single-driver}, the implementation will -have to pay special attention to nesting (which presents no problem to the -implementation in this section). - -As you can see in the package listing below (\texttt{\examplename.sty}), the -implementation of our environment is straightforward. We first define the -configuration command |\pmbset| and the option keys (we're using -\pkg{pgfkeys}), and set the option defaults. Then, we move on to the -environment itself: we apply the given options, execute the pre-code, typeset -the box (which is a |minipage| of the given width, potentially wrapped in a -|\fbox|), and execute the post-code. - -\tcbinputexample[.sty]{listing only, float} - -\ExampleName{poormansbox-memoizable} -Now let's make our |poormansbox| externalizable (\texttt{\examplename.sty}). As -announced above, the idea is to split the definition of the environment into -the outer part (below, the user-level environment |poormansbox|), which -(applies the options and) executes the pre- and the post-code, and the inner -part (below, the macro |\@poormansbox|), which typesets the actual box. If we -then then submit |\@poormansbox|, rather than |poormansbox|, to -automemoization, the outer part will be executed at every compilation (giving -us stretchable space if we request it), while the inner command will be -executed (and memoized) at the first compilation, and substituted for the -extern (the fixed-size box) at subsequent compilations.\cprotect\footnote{You - might have wondered why our definition of the |poormansbox| environment grabs - the body into an argument (|+b|, yielding |#2|), necessitating the use of - |\NewDocumentEnvironment| over the venerable |\newenvironment|. One reason - was that having the environment body as an argument simplifies wrapping the - |\fbox| around the |minipage|, but there is a more important reason. If we - did not grab the environment body, we would have to implement the internal - part of the definition as an environment (|@poormansbox|) as well, and embed - it into the user-level environment using the following idiom: - |\newenvironment{poormansbox}[2][]{...\begin{@poormansbox}}{\end{@poormansbox}...}|. - However, as illustrated in section~\ref{sec:tut:good-to-know}, automemoizing - an environment indirectly embedded in such a way produces an error, because - Memoize is prevented from collecting the environment body.} - -\makeexample{\examplename.sty} -\makeexample{\examplename.pdf}% just to know if it compiles fine -\tcbinputexample[.sty]{listing only, float} - -Looking at the definition of the internal |\@poormansbox| command, it might -strike you as weird that we have equipped this command with an optional -argument (|#1|) it never uses. However, this optional argument is crucial for -memoization. It will become a part of the memoized code (note -\code{\refmmzauto{args}=om} in the \refmmz{auto} declaration) and thereby -ensure that Memoize will produce separate externs for invocations of -|\@poormansbox| with the same environment body but different options; or in -other words, it will ensure that changing the options recompiles the -extern.\cprotect\footnote{Of course, this only holds for options given in the - optional arguments; if the user changes an option value using a prior - \cs{pmbset} (and that option does not occur in the optional argument), - Memoize won't detect the change. But the end-user knows about this issue, as - it was addressed in sections~\ref{sec:tut:working-on-a-picture} - and~\ref{sec:tut:redefinitions}, and she is also aware of two workarounds: - manual recompilation, or setting the context - (section~\ref{sec:cross-referencing}). - - While we're on the subject of the context, note that it is also possible to - deploy context to trigger recompilation of the inner command upon change of - parameters it depends on. We could simply omit the optional argument of - \cs{@poormansbox} and add - \refmmz{context}|={width=\pmb@width,frame=\ifpmb@frame true\else false\fi}|, - to the \refmmz{auto} declaration. The advantage of such an approach is that - Memoize reacts to the change of parameters regardless of whether they are set - using the optional argument or \cs{pmbset}. However, the approach is - unfeasible for commands depending on many parameters: can you imagine listing - all the \TikZ options in the context? Not to mention that a particular - picture usually only depends on a small subset of these options --- by and - large, \TikZ externs would get recompiled too often if the context contained - all \TikZ options.}\footnote{I have toyed with the idea of - splitting (using \pkg{pgfkeys} key filtering) the given options into outer - options, relevant for the outer command, and inner options, relevant for the - inner command, and only passing the inner options to the inner command. The - thought was that would (i) avoid recompiling the extern when only outer - options change, as these options don't affect the inner command, and (ii) - avoid applying the inner options when utilizing the extern, as these options - don't affect the outer command. However, it then hit me that the end-user - might define a style which incorporated both inner and outer options --- I - know I do this with my |tcolorbox|es.} - -The downside to automemoizing an internal command is that this might be -counter-intuitive for the author. For example, to deactivate automemoization -of |poormansbox|, the author will have to write -\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{@poormansbox}} (note the -|\@|), but they will have no clue they have to do this unless they have -carefully read |poormansbox|'s documentation. Even worse, the above -\refcmd{mmzset} command will not work unless surrounded by |\makeatletter| and -|\makeatother|, as it refers to an internal control sequence containing |@|. -Well, Memoize offers \refmmz{auto csname}, \refmmz{activate csname} and -\refmmz{deactivate csname}, so that |@| catagory code manipulations can be -omitted by writing \refcmd{mmzset}\bracestt{\refmmz{deactivate - csname}|=@poormansbox|}, but still. - -Another downside could occur when you use the same (automemoized) internal -command in service of several user interface commands. For the sake of -illustration, assume we have also defined an UI-macro |\pmb| which again relies -on |\@poormansbox|. How is the author to deactivate automemoization of |\pmb| -but leave the |poormansbox| environment intact? This is how: -\refcmd{mmzset}\bracestt{\refmmz{auto}=\cs{pmb}\bracestt{\refmmzauto{args}=m, - \refmmzauto{nomemoize}}}. Again, counter-intuitive; the author expects -\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{pmb}} to work. - -One other consequence of this approach is that the code included in the c-memo -(if \refmmz{include source in cmemo} is in effect) will not faithfully reflect -the source: as shown in the c-memo listing below, it will contain -|\@poormansbox{...}| instead of |\begin{poormansbox}...\end{poormansbox}| --- -even if this might actually count as an advantage, as the discrepancy will at -least inform the author who refuses to read the fine material accompanying our -|poormansbox| that something funky is going on. - -\begingroup -\relaxmmzcommands -\def\mmzNewCMemo#1{% fetch the last c-memo filename - \def\mycmemo{#1}% -} -\input{\examplepath.mmz.c1} -\sed{% - s/[]]\lbrace/]\n\space\space\space\space\space\space\space - \space\space\space\space\space\space\space\lbrace/; -}{\exampledir\mycmemo} -\tcbinputlisting{ - listing only, - listing file=\exampledir\mycmemo, - example title, - title=the c-memo of the last \texttt{poormansbox} environment, - left=0.5em, right=0em, -} -\endgroup - -In a nutshell, automemoizing an internal command might be counter-intuitive for -the author. But the core idea --- to support memoization of a resistant -command by splitting its definition into the outer and the inner command --- is -sound, and we will elaborate on this idea in -section~\ref{sec:memoization-complex-single-driver}, where we will revisit our -|poormansbox| example and develop a variant of this environment which is both -memoizable and user-friendly. - - -\section{Under the hood} -\label{sec:under-the-hood} - -This chapter is written for three audiences: a curious user who wants to -know how Memoize does what it does; a package writer who wants to support -Memoize in a tricky situation; and myself, lest I forget why I made the design -decisions that I made. - -\subsection{The entry point} -\label{sec:Memoize} - - -From the author's perspective, the functionality of this package is entered -either through the manual memoization commands (macro \refcmd{mmz} and -environment \refenv{memoize}), or via automemoization. And while that is -correct, those user interface entry points merely determine what code is -submitted to memoization, and set any options specific to the upcoming -memoization. The real fun starts with command \Emph{\refcmd{Memoize}}, which -is eventually executed by both manual and automatic memoization. - -Not every call to \refcmd{Memoize} results in memoization. Calling this macro -has three possible outcomes. It can result in \emph{memoization}, which -produces the memos and externs; in \emph{utilization} of the result of an -earlier memoization (which boils down to inputting the memos); or in -\emph{regular compilation}, whereby the code is compiled -(almost)\footnote{\label{fn:regular-compilation-almost}This is absolutely true - for memoized code which is ``contained'' in the sense of not peeking into the - input stream following the memoized code. In general, code which fails to - satisfy this containment requirement is most likely simply not memoizable; - but there are borderline cases. For example, \refcmd{ignorespaces} at the - end of some code will have the expected effect in the absence of Memoize, but - no effect when executed either during memoization or regular compilation - under Memoize, simply because it will hit some code belonging to Memoize - rather than the continuation of the document. Memoize offers the - \refmmz{ignore spaces} provision to work around this specific problem.} as if -Memoize was not there. Which outcome obtains depends on several factors. The -decision logic is depicted below, and note that you can \refmmz{trace} the -action on the terminal. - -\begin{center} - \begin{forest} - yes/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,left,#1]{yes}}}, - no/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,right,#1]{no}}}, - use/.style={content=utilization\\(of the cc-memo)\vphantom{dj},fill=orange,align=center}, - memoize/.style={content=memoization\vphantom{dj},fill=green}, - compile/.style={content=regular compilation\vphantom{dj},fill=emphcolor}, - for tree={ - l sep*=1.5, - if n children={0}{}{align=center}, - child anchor=north, - }, - [Is Memoize enabled?\\(\refcmd{ifmemoize}), - [Are we currently undergoing memoization?\\(\refcmd{ifmemoizing}), yes - [,compile, yes] - [Is the\\\refmmz{recompile}\\mode in effect?, no - [,memoize, yes] - [Do the relevant\\memos and externs exist?, no - [,use, yes] - [Is the\\\refmmz{readonly}\\mode in effect?,no - [,compile, yes] - [,memoize, no] - ] - ] - ] - ] - [,compile, no] - ] - \end{forest} -\end{center} - -As the memoization options were already set by the user interface entry points, -you might expect, quite reasonably, that \refcmd{Memoize} takes a single -argument, the code submitted to memoization. After all, what more does it -need? Clearly, executing this code is what produces the typeset material, and -to detect whether the code has ``changed'' (in order to recompile the memos and -externs), we compute the MD5 sum of this very code, don't we? Well, the -reality is a bit more complicated. When it comes to automemoized commands, the -code which the MD5 sum is computed off of (and which is displayed in the c-memo -if \refmmz{include source in cmemo} is in effect) is not exactly the same as -the code we compile (during either memoization or regular compilation). We'll -see what the difference is in section~\ref{sec:tut:automemoization-details}; -what matters here is that we must provide \refcmd{Memoize} with both and that -this macro therefore takes two arguments: the \emph{identification code}, which -the MD5 sum is computed off of, and the \emph{executable code}, which, well, is -the code that gets executed during memoization (or regular compilation). - -Let's illustrate this with an example which is probably entirely useless (but -don't worry, we'll get to a realistic example in -section~\ref{sec:tut:automemoization-details}). We first memoize some text -manually, using command \refcmd{mmz}, and then do something very stupid: we use -this very text as the identification code for the following \refcmd{Memoize}, -even if the executable code of that command is completely different. The -second line of the typeset output should convince you that the first argument -to that command was really used to produce the extern; and one further -compilation should convince you that the first argument was indeed used to -identify the extern: the extern produced by \refcmd{mmz} was overwritten by the -extern produced by \refcmd{Memoize}, in the fashion of the |progressbar| -example from section~\ref{sec:tut:redefinitions}. - -\ExampleName{memoize-internal} -\makeexample{\examplename.pdf N=2} -\tcbinputexample{ - comment={% - \includeexamplepdf[document page, - after title pre={\ (after the first compilation)} - ]{page=3}\hfill - \includeexamplepdf[document page, - after title pre={\ (after the second compilation)} - ][\examplepath.c2.pdf]{page=1} - }, -} - -The example above also illustrates a(nother) peculiar feature of -\refcmd{Memoize}. \refcmd{Memoize} does not open a new \hologo{TeX} group, but -it \emph{expects a group to be opened prior to calling it}, as it will issue an -|\endgroup| at some point. Specifically, the memoization group will be closed -before regular compilation or utilization, but after memoization. If you want -to know why, read the boxed text below. - -\begin{tcolorbox}[title=\string\Memoize\ and grouping, enhanced, breakable] - One important desideratum behind the design of Memoize was that using the - package should disrupt the original, Memoize-less compilation as little as - possible. In particular, if the memoized code contains local assignments - whose effect (in the original compilation) persists into the rest of the - document (until the end of the surrounding \hologo{TeX} group, of course), - wouldn't one want these local effects to persist when Memoize is around, as - well? Fortunately, most memoized code does not have persistent local effects - (at least for me, it is usually environments, like \env{tikzpicture}s, that I - want to memoize, and environments introduce a group anyway) --- fortunately, - because there are design reasons for enclosing memoization in a \hologo{TeX} - group (or two), and this enclosure will of course cancel the effect of local - assignments in the memoized code. - - For one, the user interface memoization commands, such as \refcmd{mmz} and - automemoized commands, allow for options specific to a particular piece of - memoized code (the options given as the optional argument to manual - memoization, the next-options and the auto-options), and to delimit their - effect, it makes most sense to apply them in a group. I have toyed with the - idea of working around the introduction of a group by manually saving and - restoring all the options, but I quickly gave up on this line of thought. - For one, manually saving and restoring the options would be cumbersome and - error-prone, and probably also slower than using the group. But even worse, - all that work would not really solve the problem of the persistence of local - effects, because memoization itself introduces a group, as well: during - memoization, the typeset material is collected into a box, and opening a box - introduces a group. In some particular situations, this could be avoided by - typesetting the memoized code as-is and collecting the resulting material - using |\lastbox|, but this approach cannot work in general. In general, - memoization will take place in a group, so the issue of local effects must be - addressed in some other way. Memoize offers the following workaround: during - memoization, the memoized code can (globally) add code to the \refmmz{after - memoization} hook, which gets executed immediately after closing the - memoization group. - - Does this mean it would be best if the user interface memoization commands - straightforwardly surrounded \refcmd{Memoize} by |\begingroup| and - |\endgroup|? For example, \refcmd{mmz} would open the memoization group, let - \refcmd{Memoize} do its work, and then close the group. Not really. - Remember that memoization is not the only possible outcome of calling - \refcmd{Memoize}. Perhaps we can at least retain the local effects of a - regular compilation, and of utilization? - - We can, by finely tuning the timing of the memoization group closure within - \refcmd{Memoize}. This command is designed to close the memoization group - after memoization, but before regular compilation and utilization. Closing - the group after memoization makes sure that the given options are in effect - during this process. By closing the group prior to regular compilation, - regular compilation of the memoized code (which takes place when Memoize is - disabled, for example) is guaranteed to have (almost, see - footnote~\ref{fn:regular-compilation-almost}) exactly the same effect as the - compilation of that code in absence of Memoize; in particular, the effect of - any local assignments will persist into the rest of the document. Finally, - closing the group before utilization simplifies the construction of the memo - in the cases where we need to replicate local effects of the memoized code - --- the group closed, there is no need to smuggle local assignments out of a - memo. -\end{tcolorbox} - -\subsection{Memos} -\label{sec:memos} - -Up until now, we have pretended that there is a single kind of a memo file. In -truth, there's two kinds: \emph{code memos}, or \emph{c-memos} for short; and -\emph{code--context memos}, or \emph{cc-memos} for short. In this section, we -will learn what they are for, and how they look like --- and also a bit on how -they are produced, even if the details on that will have to wait until -section~\ref{sec:memoization-drivers}. - -We will see that when Memoize utilizes memos, c-memos are processed first. But -conceptually, cc-memos are more important, so we will start the discussion with -these. - -\subsubsection{Cc-memos (and extern inclusion)} -\label{sec:cc-memos} - -When it is input, a cc-memo replicates the effect of the memoized code. This -includes the reproduction of its visual output, which takes the form of -inclusion of any externs produced by memoization. And yes, you got the -implication right: a cc-memo can have any number of associated externs, -including zero, even if the most common case is that of exactly one extern per -cc-memo. The number of externs mostly depends on the memoization driver (see -section~\ref{sec:memoization-drivers}); the default driver always produces -exactly one extern. - -A cc-memo is located in the directory given via subkey \refmmzpath{dir} (and -\refmmzpath{relative}) of key \refmmz{path}. You can recognize it by its filename, -which has the following form (\meta{prefix} is set via subkey \refmmzpath{prefix}): -\begin{center} - \code{\meta{prefix}\meta{code md5sum}-\meta{context md5sum}.memo} -\end{center} - -In fact, this is how Memoize recognizes --- or rather, searches for --- a -cc-memo as well: Memoize will utilize a cc-memo when the code and the context -MD5 sum computed during an attempted utilization match the code and the context -MD5 sum computed during some previous memoization (for details on the context -MD5 sum, see section~\ref{sec:c-memos}). In detail, a cc-memo is created at -the end of memoization, at which point Memoize computes the MD5 sum of the -memoized code and the MD5 sum of the context, and writes the results of -memoization into the cc-memo identified by (the prefix and) these two MD5 sums. -And when Memoize, on a subsequent compilation, encounters a piece of memoized -code, it again computes the MD5 sum of that code and the MD5 sum of the -context, and tries to input the cc-memo identified by (the prefix and) these -two MD5 sums. If the inputting is successful, we have utilized the cc-memo -(which in the typical case amounts to including the one associated extern); if -the cc-memo cannot be found, Memoize starts the memoization process, which -creates the memos and the externs. - -Let us take a look at the contents of a cc-memo in detail. Here's a typical -cc-memo (it belongs to the titlepage penguin): - -\begingroup -\relaxmmzcommands -\def\mmzNewCCMemo#1{% fetch the first cc-memo filename - \def\myccmemo{#1}% - \endinput -} -\input{\exampledir titlepage.mmz.c1} -\sed{% - s/\cmd{quitvmode} \cmd{mmzIncludeExtern}/\cmd{quitvmode}\n\cmd{mmzIncludeExtern}/; - s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; -}{\exampledir\myccmemo} -\tcbinputlisting{ - listing only, - listing file=\exampledir\myccmemo, - example title=\myccmemo, -} -\endgroup - -A cc-memo begins by listing the externs which the memo will (actually, might) -attempt to include into the document. When the cc-memo is input, each -\refcmd{mmzResource} command checks if the given extern exists. If some -existence check fails, Memoize enters the memoization mode, same as if the -cc-memo itself did not exist. If all the resources pass the existence check, -Memoize inputs the core of the cc-memo, i.e.\ everything following the -\refcmd{mmzMemo} marker. - -The core might contain arbitrary code, but most often, it will consist of only -two commands. The first one is \refcmd{quitvmode} and it is included if the -extern was \refmmz{capture}d into a horizontal box (which is the usual -situation). The second one is \refcmd{mmzIncludeExtern}, and it is this -command which actually includes the extern into the document upon inputting the -cc-memo. The core code is executed without introducing any groups, i.e.\ the -effect of any local assignments in the cc-memo will persist into the code -following the memoized code. - -Command \refcmd{mmzIncludeExtern} takes nine parameters. The first is the -sequential number of the extern associated with the cc-memo, starting with 0; -usually, this is simply 0 as most memos are associated with a single extern. -The second one is a \refcmd{hbox} or \refcmd{vbox}, noting the type of the box -the memoized code was externalized into. The next three numbers are the -expected width, height and the depth of the extern. Finally, we have the four -\refmmz{padding} amounts (left, bottom, right and top). We should arrive at -the expected size after trimming the extern PDF by the -padding amounts; Memoize will complain if we don't. - -Let's look at a more interesting cc-memo. Using the advising framework, -described in section~\ref{sec:tut:automemoization-details}, Memoize hacks -\refcmd{label} to support \cs{label}s inside memoized code --- the following -code ``just works.'' - -\ExampleName{label} -\makeexample{\examplename.pdf N=3} -\tcbinputexample{ - comment={\centering - \includeexamplepdf[document page, after title pre={\ (compilation 1)}, - left=1.5mm, right=1mm]{page=2}\quad - \includeexamplepdf[document page, after title pre={\ (compilation 2)}, - left=1.5mm, right=1mm][\examplepath.c2.pdf]{page=1}\quad - \includeexamplepdf[document page, after title pre={\ (compilation 3)}, - left=1.5mm, right=1mm][\examplepath.c3.pdf]{page=1} - }, -} - -Everything seems normal --- after the first compilation, we get ``\textbf{??}'' -because the label has not made it into the |.aux| file yet, but in subsequent -compilations, we learn where the penguin lives --- but it is far from normal -under the hood. If we de-hacked \cs{label} by writing -\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{label}}, the third compilation -(and subsequent compilations) would revert to ``\textbf{??}''. Why would that -happen? The memoized code containing the \cs{label}s is only executed in the -first compilation; in the subsequent compilations, we're simply inputting the -cc-memo, so the memoized code, including any \cs{label}s in contains, is not -compiled, and the labels don't get into the |.aux| file anymore. - -The \cs{label} hack deploys Memoize's ability to put arbitrary code into the -cc-memo. During memoization, the memoized code may add arbitrary code to -register \refcmd{mmzCCMemo}, and the contents of this register at the end of -the memoization form the free-form part of the cc-memo.\footnote{This is also - how the above-described code containing \refcmd{mmzIncludeExtern} gets into - the cc-memo. The code is produced by \refcmd{mmzExternalizeBox} and appended - to \refcmd{mmzCCMemo} by the default memoization driver - \refcmd{mmzSingleExternDriver}; see section~\ref{sec:memoization-drivers} for - details.} When the hacked \cs{label} is encountered during memoization, it -appends \refcmd{mmzLabel}\marg{label name}\marg{current label value} to -\refcmd{mmzCCMemo}, so this command winds up in the cc-memo. It is then a -simple job for \refcmd{mmzLabel}, executed when the cc-memo is input at -subsequent compilations, to temporarily store \meta{current label value} (i.e.\ -the contents of \refcmd{@currentlabel} at the time the \cs{label} was invoked) back -into \cs{@currentlabel} and to execute \cs{label}\marg{label name}. In effect, any -\cs{label} command contained within the memoized code is executed at every -compilation, even if the memoized code itself is not compiled. - -\begingroup -\relaxmmzcommands -\def\mmzNewCCMemo#1{% fetch the first cc-memo filename - \def\myccmemo{#1}% - \endinput -} -\input{\examplepath.mmz.c1} -\sed{% - s/~//g; - s/\cmd{quitvmode} \cmd{mmzLabel}/\cmd{quitvmode}\n\cmd{mmzLabel}/; - s/\(\marg\)\cmd{mmzIncludeExtern}/\1\n\cmd{mmzIncludeExtern}/; - s/\(\cmd{mmzLabel}\) *\(\marg\marg\) */~\1\2~\space/g; - s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; -}{\exampledir\myccmemo} -\tcbinputlisting{float, - listing only, right=1.5mm, - listing file=\exampledir\myccmemo, - example title=\myccmemo, -} -\endgroup - -We will continue the discussion of \refcmd{label} in section~\ref{sec:label+} using -a funkier example. - - -\subsubsection{C-memos (and context)} -\label{sec:c-memos} - -As explained in the previous section, a cc-memo belonging to a piece of -memoized code is identified by two MD5 sums: the MD5 sum of the memoized code, -and the MD5 sum of the associated context. However, when Memoize encounters -some code submitted to memoization, the context expression is not yet fully -known, as it may be adjusted by the memoized code itself during memoization --- -and this potential adjustment is crucial for \cs{ref} and friends to work as -advertised (see section~\ref{sec:cross-referencing}). Upon being invoked, -Memoize therefore cannot immediately attempt to input the cc-memo; it needs to -first learn about the context adjustments. Here's where c-memos enter the -picture: \emph{the primary job of a c-memo is to store the context adjustments - made by the memoized code}. Let's see how this works in detail. - -Same as cc-memos, c-memos are located in the directory given via subkey -\refmmzpath{dir} (and \refmmzpath{relative}) of key \refmmz{path}. However, a c-memo -belonging to some memoized code is identified by the MD5 sum of that code -alone; its filename has the following form, where \meta{prefix} is again set -via subkey \refmmzpath{prefix}: -\begin{center} - \code{\meta{prefix}\meta{code md5sum}.memo} -\end{center} - -The c-memo is created at the end of the memoization process. At that time, the -context expression is fully known, as the memoized code was already processed. -Even more, Memoize keeps track of both the state of the context expression -prior to memoization, stored in token register \refcmd{mmzContext}, and of the -\emph{additions} to the context expression made by the memoized code, which are -stored in token register \refcmd{mmzContextExtra}. (Incidentally, key -\refmmz{context} automatically adapts to the situation by appending to -\refcmd{mmzContext} outside memoization and to \refcmd{mmzContextExtra} during -memoization.) The complete context expression is the concatenation of the -contents of these two registers, but it is only the context expression -additions, i.e.\ the contents of \refcmd{mmzContextExtra}, which Memoize stores -into the c-memo, with the idea that during subsequent compilations, the initial -context (\refcmd{mmzContext}) will be set up again via ``normal'' compilation, -while inputting the c-memo will restore the additions, jointly reconstructing -the complete context expression associated with a piece of memoized code to -what it was at the end of memoization. - -We can now complete the picture of a utilization attempt started in -section~\ref{sec:cc-memos}. Memoize begins by trying to input the c-memo; this -can be done as the c-memo can be identified based solely on (the MD5 sum of) -the memoized code. If the c-memo does not exist, Memoize starts the -memoization process, which will produce the memos and the externs. But if it -does exist, inputting it reconstructs the context expression to the state at -the end of memoization. Therefore, as the MD5 sum of the \emph{expansion} of -the context expression \emph{at the end of memoization} is baked into the -cc-memo filename, trying to load the cc-memo identified by (the prefix, the -code MD5 sum and) the MD5 sum of the \emph{expansion} of the context expression -\emph{at attempted utilization} will succeed precisely when the context -remained unchanged from memoization to attempted utilization. - -All this might have sounded very complicated, but in the end, most c-memos are -quite boring, the titlepage penguin's c-memo shown below being no exception. A -c-memo starts with the \refcmd{mmzMemo} marker, which is always followed by a -(global) assignment to token register \refcmd{mmzContextExtra}, holding the -context expression additions. As promised, the c-memo below is boring: it -assigns an empty token list to this register, leaving the context expression -as-is. Next comes the free-form part of the memo. Below, it is boringly empty -as well (just the percent sign), but in principle, it will contain any code -gathered in register \refcmd{mmzCMemo} during memoization; see -\ref{sec:per-overlay} for an example. A c-memo is concluded by an optional -part consisting of the \refcmd{mmzSource} marker, followed by the memoized -code. The source code section is not used by Memoize in any way and can be -switched off by \code{\refmmz{include source in cmemo}=false}; it is included -by default so that an interested user can know which code produced which memo, -which can be useful if one wants to trigger recompilation of an extern by -deleting the corresponding memo. Incidentally, any newlines in the source code -are lost in the c-memo replica (unless \refmmz{verbatim} is in effect), but we -will only see this once we arrive at the \pkg{beamer} example below. - -\begingroup -\relaxmmzcommands -\def\mmzNewCMemo#1{% fetch the first c-memo filename - \def\mycmemo{#1}% - \endinput -} -\input{\exampledir titlepage.mmz.c1} -\tcbinputlisting{ - listing only, - listing file=\exampledir\mycmemo, - example title=\mycmemo, -} -\endgroup - -A c-memo of code containing a cross-reference will prove more interesting. The -following c-memo was produced by the |ref| example from section -\ref{sec:cross-referencing}. As we know from that section, when a \refcmd{ref} (as -hacked by Memoize's \refmmzauto{ref} key) occurs in some memoized code, it -appends the cross-reference to the context. In the c-memo below, this is -reflected by the (global) assignment of an expression containing the -cross-reference macro to token register \refcmd{mmzContextExtra}, holding -the context expression additions. - -\begingroup -\relaxmmzcommands -\def\mmzNewCMemo#1{% fetch the first c-memo filename - \def\mycmemo{#1}% - \endinput -} -\input{\exampledir ref.mmz.c1} -\sed{% - s/~//g; - s/\cmd{global}.*\marg/~\0~/; -}{\exampledir\mycmemo} -\tcbinputlisting{ - listing only, - listing file=\exampledir\mycmemo, - example title=\mycmemo, -} -\endgroup - - -\subsubsection{More on \texorpdfstring{\cs{label}}{\textbackslash label}} -\label{sec:label+} - -A \refcmd{label} inside memoized code works out of the box in the usual situation -when label value is fully determined by the memoized code, as in the example in -section~\ref{sec:cc-memos}, where the memoized code contained the outermost -(and only) \env{enumerate} environment. However, the out of the box approach does -not work if the label value is (fully or partially) determined outside the -memoized code. To illustrate the problem, and some potential solutions, we -define two very simple enumeration environments, |listi| and |listii|, which -use counters |counti| and |countii|, and which are intended as the outer and -the inner environment, respectively. Our interest here is in the inner -environment, |listii|. While it prefixes each item by an indented -|\thecountii)|, the label is a composite of both counters: -|\thecounti\thecountii|. The label is stored into \refcmd{@currentlabel}, so -referencing works as usual. However, problems arise when we automemoize the -inner environment. - -\ExampleName{label+} -\makeexample{\examplename.tex.c1 N=6} -\makeexample{\examplename.pdf N=2} -\tcbinputexample{ - sidebyside, lefthand ratio=0.53, - comment={\centering\includeexamplepdf[document page][\examplepath.c2.pdf]{page=1}}, -} - -While the result looks fine at first, changing the order of |listii| -environments, for example by moving ``pets'' below ``domestic'', will result in a -problem: the reference at the bottom will remain unchanged. This is so because -the reference text is baked into the cc-memo, as shown below. - -\begingroup -\relaxmmzcommands -\def\mmzNewCCMemo#1{% fetch the first cc-memo filename - \def\myccmemo{#1}% - \endinput -} -\input{\examplepath.mmz.c1} -\sed{% - s/~//g; - s/\cmd{mmzLabel} \marg\marg/~\0~/; - s/\(\marg\)\(\cmd{mmzIncludeExtern}\)/\1 \2/; - s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; -}{\exampledir\myccmemo} -\tcbinputlisting{ - listing only, - listing file=\exampledir\myccmemo, - example title=\myccmemo, -} -\endgroup - -How can we remedy this? The manual option is to force the recompilation of the -extern by putting an (invisible) reference to the outer item into the inner -item: add |\label{item:pets}| to item ``pets'' and refer to it at ``dog'' by -\refcmd{mmzNoRef}|{item:pets}|.% -\attachexample[\examplename mmzNoRef.tex][\examplepath.tex.c3.attachment] - -An automatic variant of the recompilation solution is to add \cs{@currentlabel} -to the context upon memoizing |listii|. This can be achieved by adding -\refmmz{context}|={@currentlabel={\csuse{@currentlabel}}}| to the \refmmz{auto} -declaration for |listii|. The downside of this approach is that every |listii| -will get reexternalized upon movement, whether it actually contains a label or -not.\attachexample[\examplename context.tex][\examplepath.tex.c4.attachment] - -In fact, given that the externs produced by the inner environment do not -contain the value of the outer counter, it seems wasteful to recompile any -extern just to change the reference. And indeed, it is possible to avoid this, -but the approach unfortunately requires adapting the inner environment code -(and this is why I have not illustrated the problem using an environment of an -elaborate package like \pkg{enumitem}). The idea is to ``unbake'' the -reference to the outer item in the cc-memo. We can achieve this by changing -|listii| to define \cs{@currentlabel} to be |\unexpanded{\thecounti}\thecountii|. -Under this definition, the cc-memo will contain |\mmzLabel -{item:dog}{\thecounti a}|, and rearranging |listii| environments will produce -(upon two compilations, of course) the correct reference without recompiling -the extern. Note again, however, that this solution can only work when the -value of the outer counter does not appear in the extern, i.e.\ it would not -work the ``dog'' item was prefixed by |1a)| rather than simply |a)|. In those -cases, one should deploy one of the other solutions.% -\attachexample[\examplename listii.tex][\examplepath.tex.c5.attachment] - -The final solution, presented below, is an elaboration on the second one. -Rather than append \cs{@currentlabel} to the context immediately upon beginning -to memoize environment |listii|, we will at that point redefine \cs{label} to do -that. In effect, changing the location of |listii| will only recompile it if -it contains a \cs{label}. - -At announced, we redefine \cs{label} once the memoization of |listii| begins, -so within \refmmz{at begin memoization}.\footnote{Another generally good - location for such redefinitions is among the auto-options of |listii|. We - could include an \refmmz{auto}\cs{label}\marg{...} there, or a |/utils/exec| - with \refcmd{AdviceSetup}. However, in this particular case this would be - wasteful, as it would be applied regardless of whether memoization will take - place or not, whereas we only need the redefined \cs{label} when memoizing.} -However, we do not redefine \cs{label} directly, as Memoize advises this -control sequence out of the box (see -section~\ref{sec:tut:automemoization-details} for details). What we redefine -is the command which the advising framework executes instead of \cs{label} --- -its so-called \refmmzauto{outer handler} --- and we do this by calling -\refcmd{AdviceSetup}, the low-level variant of the familiar key -\refmmz{auto}.\footnote{We could have also used \refmmz{auto}, but we don't, - because (a) \refcmd{AdviceSetup} is faster, (b) it is easier to prepend - material to a handler using the low-level interface, and (c) I wanted to - showcase \refcmd{AdviceSetup}.} The first argument of \refcmd{AdviceSetup} -is the installation path (|/mmz|), the second one the command or environment we -are submitting to the framework (\cs{label}), and the third one the setup -\emph{code} --- here lies the biggest difference between \refmmz{auto} and -\refcmd{AdviceSetup}: the former expects a keylist, and the latter \hologo{TeX} -code which directly manipulates settings macros like -\refcmd{AdviceOuterHandler} (for the full list, see -section~\ref{sec:tut:automemoization-details} or~\ref{sec:ref:advice}). - -\tcbinputexample[.tex][.c6]{listing only, float, - attachment name=\examplename auto.tex, -} - -Within \refcmd{AdviceSetup}, we prefix (using macro \cs{preto} of package -\pkg{etoolbox}) the original outer handler \refcmd{AdviceOuterHandler} by code -which causes \cs{outerlabeltocontext} to be executed at the end of memoization -(by globally appending this macro to \refcmd{mmzAtEndMemoizationExtra}; -``Extra'' because we're appending during memoization). It is -\cs{outerlabeltocontext} which then appends \refcmd{@currentlabel} to the -context (\refcmd{mmzContextExtra}; again, ``Extra'' because we're appending -during memoization),\footnote{As a courtesy, we clear out macro - \cs{outerlabeltocontext} once it did it's job, so that multiple \cs{label}s - do not include multiple \cs{@currentlabel}s into the context. But the code - would work even without this addendum, can you see why?} and it is crucial -that this happens at the end of memoization rather than when \cs{label} is -executed. When \cs{label} is executed, we're inside an inner item, and -\cs{@currentlabel} refers to that item, while at the end of memoization, the -value of this macro equals the value at the beginning of memoization, namely -the label of the outer item. While the outer list remains unshuffled, the -value of \refcmd{@currentlabel} that contributes to the context MD5 sum during -the utilization attempt will therefore match the value which contributed to the -context MD5 sum during memoization, resulting in matching MD5 sums and -therefore in actual utilization of the extern; once the outer list is shuffled, -this will cease to be the case and the extern will be recompiled. - - -\subsubsection{The Beamer support explained} -\label{sec:per-overlay} - -The implementation of \refmmz{per overlay}, which makes memoization sensitive -to Beamer overlays, provides an example of a complex interaction between -various components of memoization. At the core, the Beamer support works by -adjusting the context, but we will also have the occasion to observe the -free-form part of the c-memo, add a bit of extra code to the cc-memo, and -deploy several memoization hooks. We will show the complete Beamer support -code later on; let us build our understanding of that code step by step. -(Before you read on, you might want to refresh your memory about the \pkg{beamer} -example from section~\ref{sec:tut:beamer}, as we will refer to it in the -present section.) - -The core idea behind \refmmz{per overlay} is to append the current beamer -overlay number to the context:\cprotect\footnote{As we saw in - section~\ref{sec:tut:beamer}, it is convenient to execute \refmmz{per - overlay} inside memoized code. But remember, from - section~\ref{sec:cross-referencing}, that when \refmmz{context} is executed - from within the memoized code, its argument winds up in the c-memo. As the - c-memo is processed under the normal category code regime, where |@| is not a - letter, we have to access |\beamer@overlaynumber| using the |\csname - ... \endcsname| construct.} \refmmz{context}|={overlay=\csname - beamer@overlaynumber\endcsname}|. This makes Memoize produce a separate -extern for each overlay. However, only the first of these externs will get -utilized on subsequent compilations, in general at least. Even worse, we will -lose each frame whose creation is driven solely by the memoized code. We will -lose the second overlay in our example (i.e.\ in the \pkg{beamer} example from -section~\ref{sec:tut:beamer}) as the second overlay was only created because -Beamer encountered |only={2}{...}| (resolving to |\only<2>{...}| under the -hood) inside the picture code; once we utilize the extern instead of compiling -the picture on the first overlay, the |\only| command is not executed anymore, -so Beamer thinks it is done with the frame. - -As the compilation of our picture is substituted by utilization of its -cc-memos, we have to somehow drive the creation of the necessary overlays from -these files. An easy way to achieve this is to furnish them with a dummy -|\only<|\meta{final overlay number}|>{}|,\footnote{`The final overlay' here - should be understood as relative to our memoized picture, i.e.\ as the final - overlay containing the memoized picture.}\footnote{Actually, putting this - \cs{only} command only into the first cc-memo would suffice, but would be - harder to implement.} but there is a problem: the final overlay number is -unknown when we're memoizing our picture --- it is unknown even when we're -memoizing the picture on final overlay itself (we simply don't know yet that -this overlay will end up being the final one), let alone during the memoization -on the first overlay. - -The solution exploits the fact that \emph{the c-memo is rewritten at each - memoization}: at each memoization of our picture, we store the the -\emph{current} overlay number to the c-memo; after all memoizations, the c-memo -will thus contain the number of the \emph{final} overlay containing our -memoized picture. To access this number from the cc-memo, we store it as a -macro definition, and then use the defined macro, |\mmzBeamerOverlays|, in the -overlay specification of the dummy |\only|. Below, you can see all this in -code, as the argument to \refmmz{at begin memoization}. - -\ExampleName{per-overlay-v1} -\makeexample{\examplename.sty} -\tcbinputexample[.sty]{ - listing only, - title={The implementation of \refmmz{per overlay} (first attempt)}, -} - -A couple of remarks are in order here. First, the definition of -|\mmzBeamerOverlays| in the c-memo is global, because it will be accessed from -the cc-memo, but the cc-memo is input after closing the memoize group (which -the c-memo \emph{is} processed in). Second, using \refmmz{at begin - memoization} makes it possible to use \refmmz{per overlay} both outside and -during memoization: if \refmmz{at begin memoization} is executed outside -memoization, its argument is (locally) stored into hook -\refcmd{mmzAtBeginMemoization}, to be called at the beginning of each -memoization; if the key executed outside memoization, the argument is executed -immediately. Third, the Memoize keys in the definition of \refmmz{per overlay} -are prefixed with \code{\refkeypath{/mmz}/}, so that this key can be called from -\pkg{pgfkeys} option lists of other packages, for example the option list of -the \env{tikzpicture} environment, as shown in the example in -section~\ref{sec:tut:beamer}. - -I used the above version of \refmmz{per overlay} for quite a while. In -general, it worked as I expected, but there were glitches. Occasionally, the -picture would appear on the wrong overlay, or I would get an extra overlay, or -perhaps lose an overlay. Eventually, I figured out this happens when I play -with the overlay structure of the frame: when I add or remove a \cs{pause} or -similar. In hindsight, it is easy to see what was happening. Once the picture -is memoized, it is fixed, forever, which extern will appear on which overlay. -I cannot expect the extern--overlay correlation to change just because I added -a \cs{pause} in front of the picture. Furthermore, the number of overlays the -memos will drive to be created is fixed as well. If I memoize the picture -while it follows a \cs{pause}, and the picture creates 10 overlays, the c-memo -will define |\mmzBeamerOverlays| to 11. So what, if I then remove that -\cs{pause}! The c-memo will still define |\mmzBeamerOverlays| to 11, and drive -the creation of 11 overlays --- one too many. - -By now, the road ahead is probably clear --- we put the |beamerpauses| counter -into the context --- but we will see there are still obstacles on the way. The -issue is that the context is evaluated at the \emph{end} of memoization (so -that those cross-references from section~\ref{sec:cross-referencing} actually -get into it). However, the memoized code might contain a \cs{pause} or similar -itself, and change the value of |beamerpauses|. For one, this means that we -have to write down the changed value of |beamerpauses| into the cc-memo; below, -we do this using key \refmmz{at end memoization} (the code given code to this -key is executed after the driver but before Memoize writes down the memos and -ships out the extern pages; the key itself may be executed either before or -during memoization). Furthermore, if the memoized code changes the value of -|beamerpauses|, the value of |beamerpauses| at the attempted utilization, which -would nicely match the value from the start of memoization, will never match -the changed, final value from the end of memoization, in effect preventing our -hard-won externs from ever getting utilized. - -We therefore have to invent a way to get the memoization-\emph{initial} value -of |beamerpauses| into the context.\footnote{This imposes a requirement on the - in-code usage of \refmmz{per overlay}, namely, that it should be executed - prior to any changes of |beamerpauses|.} Fine, we store it in a -macro,\footnote{We must define \cs{mmzBeamerPauses} globally, because - \refmmz{per overlay} can be arbitrarily deeply embedded in the memoized - code.} |\mmzBeamerPauses|, when we start the memoization (\refmmz{at begin - memoization}), and put |\mmzBeamerPauses| rather than |beamerpauses| into the -context. Will this work? Not yet, because |\mmzBeamerPauses| is undefined at -utilization. We need to set up the context expression so that it will expand -to the value of |\mmzBeamerPauses| at (the end of) memoization, and to the -value of |beamerpauses| at utilization. This leads to the -|pauses=\ifmemoizing...| part of the context expression in (the final version - of) the Beamer support code below. - -\medskip % manual, to avoid an orphan - -\ExampleName{per-overlay} -\makeexcerpt{per-overlay} -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, - left=0pt, % manual indentation gobble - title={The implementation of \refmmz{per overlay}}, -} - -Are we done? Almost. The final issue is that once we have introduced support -for pauses, we have to relativize |\mmzBeamerOverlays| (the final overlay -number) to |beamerpause|. So instead of a simple |\gdef\mmzBeamerOverlays| in -the first version, we define |\mmzSetBeamerOverlays|\nohyphen\marg{beamer - pauses}\marg{final overlay number}, which sets |\mmzBeamerOverlays| only if -\meta{beamer pauses} argument matches the value of |beamerpauses| (at its -invocation in the c-memo). Well, the macro has some other housekeeping to do -as well: it is self-replicating, so that during potential memoization, the -|\mmzBeamerOverlays| values belonging to non-current |beamerpauses| values get -rewritten into the c-memo.\footnote{As replication should only occur during - memoization (actually, it \emph{can} only occur then, anyway), the - instruction to append to the c-memo is appended to the - \cs{mmzAtBeginMemoization} hook (the low-level interface to \refmmz{at begin - memoization}). Note that the assignment to this hook must be local (once - local, always local), and it \emph{can} be local because of a little - implementation detail: while the c-memo is processed in the memoize - \hologo{TeX} group, we don't open an additional group to process it; so the - local effects from c-memo will persist into memoization (but not into - utilization, because remember that the memoize group is closed before - inputting the cc-memo).} (Fine, there is another fine detail, regarding -anti-pollution: the macro also ensures that, relative to \marg{beamer pauses}, -only the instance with the greatest \meta{final overlay number} is replicated.) - -We are now truly done, and we can look at the final result, the c-memo and the -cc-memo belonging to the extern on the first overlay of the example from -section~\ref{sec:tut:beamer}. Specifically, look at the |\mmzSetBeamerOverlays -{1}{2}|, which says that the extern chain started when |beamerpauses| equals 1 -should continue up to overlay 2, and at the (expanded) context included at the -end of the cc-memo, courtesy of \refmmz{include context in ccmemo}, where you -can see that the cc-memo will be used when on the first overlay (|overlay=1|) -when preceded by no \cs{pause} command (|pauses=1|). - -\begingroup -\relaxmmzcommands -\def\mmzNewCMemo#1{% fetch the first c-memo filename - \def\mycmemo{#1}% -} -\def\mmzNewCCMemo#1{% fetch the first cc-memo filename - \def\myccmemo{#1}% - \endinput -} -\input{\exampledir beamer.mmz.c1} -\sed{% - s/~//g; - s/\cmd{mmzSetBeamerOverlays} \marg\marg/~\0~/; -}{\exampledir\mycmemo} -\sed{% - s/~//g; - s/overlay=[0-9]*/~\0~/; - s/pauses=[0-9]*/~\0~/; - s/\(\cmd{quitvmode}\) \(\cmd{only}\)/\1\n\2/; - s/\rbrace\(\cmd{mmzIncludeExtern}\)/\rbrace\percentchar\n\1/; - s/\rbrace\(\cmd{setcounter}\)/\rbrace\percentchar\n\1/; - s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; -}{\exampledir\myccmemo} - -\tcbinputlisting{ - listing only, - listing file=\exampledir\mycmemo, - example title=\mycmemo, -} - -\tcbinputlisting{ - listing only, - listing file=\exampledir\myccmemo, - example title=\myccmemo, -} -\endgroup - - -\subsection{Record files} -\label{sec:record-files} - -We have seen that externalization is a two-step process in Memoize: as it is -impossible for \hologo{TeX} to create multiple PDFs during a single -compilation, the externs are first dumped into the document PDF as special -extern pages, and only later extracted from the main document into separate PDF -files. But extraction requires a complete PDF, which is unavailable even at -the very end of the compilation which produces the externs. The externs can -therefore only be extracted \emph{after} that compilation (either before or at -the beginning of the next one), and this necessitates some form of -communication whereby the memoization step informs the extraction step which -pages should be extracted from the document PDF and into which (PDF) files they -should be stored. This communication is implemented through auxiliary files -called \emph{record files}. - -\subsubsection{The \texttt{.mmz} file} -\label{sec:.mmz} - -By default, Memoize records the information needed for the extraction in a file -named \meta{document name}\dmmz,\footnote{For \hologo{TeX}perts: the - \meta{document name} is of course the expansion of \cs{jobname}.} henceforth -a \dmmz file. In fact, this file contains more than information on externs -created during the last compilation: it records which memos and externs were -either used or created during the compilation. The full information contained -in the \dmmz file is used by the clean-up script \refscript{memoize-clean.pl} -to safely remove stale memos and externs. Let us take a look at the \dmmz file -produced by the titlepage illustration. In fact, we have two versions of this -file, as it changes upon the second compilation. - -\ExampleName{titlepage} -\makeexample{\examplename.pdf N=2} -\sed{% - s/~//g; - s/\cmd{mmzNewExtern}/~\0~/; -}{\examplepath.mmz.c1} -\tcbinputexample[.mmz][.c1]{ - listing only, one file, no attachment, float, - after title pre={\ (after the first compilation)}, - listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar}, -} -\tcbinputexample[.mmz][.c2]{ - listing only, one file, no attachment, float, - after title pre={\ (after subsequent compilations)}, - listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar}, -} - -As you can see, the \dmmz file takes the form of a \hologo{TeX} script (the -format was chosen because it facilitated the implementation of the internally -triggered \hologo{TeX}-based extraction). The crucial lines in this file, and -the only lines used by the extraction script, occur in the first version of the -file: they contain command \refcmd{mmzNewExtern}, which informs the extraction -script that it should extract the document page given by the second argument -into the extern file given by the first argument.\footnote{If you look at the - \dmmz file after extracting the externs using \refscript{memoize-extract.pl} - without the \refscript{memoize-extract.pl--keep} option, you will find that - the \refcmd{mmzNewExtern} commands are commented out; this is to prevent - multiple extractions (even if they are harmless).} (The following two -arguments provide the expected width and height of the extern; the -extraction script may check whether the extern size conforms to these -expectations, but this is not crucial, as the extern size is checked every time -it is included anyway.) - -A \dmmz file also contains a record of the memos (both c-memos and cc-memos) -created in the last compilation; this information is provided by the sole -argument of commands \refcmd{mmzNewCMemo} and \refcmd{mmzNewCCMemo}. And once -memos and externs get used in subsequent compilations, the \dmmz file will -reflect this with \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and -\refcmd{mmzUsedExtern}, as shown in the second version of the file above. - -Finally, both versions illustrate that a \dmmz file always begins with command -\refcmd{mmzPrefix} and ends with the |\endinput| marker. The argument of -\refcmd{mmzPrefix} is the path prefix to the memo and extern files, as -determined by the invocation of key \refmmz{path} (and its subkeys -\refmmzpath{relative}, \refmmzpath{dir} and \refmmzpath{prefix}). The initial -\refcmd{mmzPrefix} line is written to the \dmmz file at the beginning of the -document, but an additional \refcmd{mmzPrefix} line will occur for every -invocation of \refmmz{path} in the document body. Finally, the |\endinput| -marker signals that the \dmmz file is complete. - -As mentioned above, the full contingent of \dmmz file commands is only used by -the clean-up script \refscript{memoize-clean.pl}. By default, this script -removes all memos and externs with the prefix given by \refcmd{mmzPrefix} -(relative to the directory hosting the \dmmz file) but those listed by any of -\refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo}, \refcmd{mmzNewExtern}, -\refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and \refcmd{mmzUsedExtern}. -Furthermore, the clean-up script will cowardly refuse to delete anything if the -\dmmz file does not end with |\endinput|, as this means that the compilation -ended prematurely and that the \dmmz file might not mention all memos and -externs actually used in the document. If given option -\refscript{memoize-clean.pl--all}, the clean-up script removes even the memos -and externs mentioned in the \dmmz file, and as this option is intended to -bring Memoize to a clean slate after any ``disasters,'' the -\refscript{memoize-clean.pl--all} mode also ignores the potential absence of -the |\endinput| marker. Incidentally, the \refscript{memoize-clean.pl--all} -mode is also the raison d'être for \refcmd{mmzPrefix}: while the prefix is -usually recognizable from \refcmd{mmzNewCMemo} and friends, these commands -might not make it into the \dmmz file in a fatally failed compilation, but it -is precisely such compilations that could occasionally require the full -clean-up. - -\subsubsection{Defining a new record type} -\label{sec:new-record-file} - -The \dmmz file is not the only kind of a record file that can be produced by -Memoize. Out of the box, it can also write down the extraction instructions -into a makefile or a shell script. These are useful on systems which have to -employ the \hologo{TeX}-based extraction but cannot trigger it internally. -Running the \hologo{TeX}-based extraction manually would be painful, as it must -be done on extern-by-extern basis, so Memoize offers to automate the extraction -by a makefile or a shell script; here, the record file is named -\code{memoize-extract.\meta{document name}.\meta{record type}} by default, -where \meta{record type} is either \refmmz{record=makefile}, -\refmmz{record=sh} (for shell scripts on Linux), or \refmmz{record=bat} (for -shell scripts on Windows). - -To turn on recording of an alternate record type, use key -\refmmz{record}|=|\meta{record type}. Memoize can record any number of files -simultaneously, so saying \code{\refmmz{record}=\refmmz{record=sh}} will -produce the shell script alongside \dmmz (Memoize internally executes -\code{\refmmz{record}=\refmmz{record=mmz}} to start recording the \dmmz file); -this should not be a problem, but if you really want to disable the \dmmz file -production, you can say \refmmz{no record}. - -The predefined record types are defined through a generic system open to the -user. To define an additional record type, one needs to define, using -\pkg{pgfkeys}, the relevant hooks of the form \refkeypath{/mmz/record/record - type}|/|\meta{hook}. The following \meta{hook}s can be defined (the hooks -not needed for the record file type may be left undefined): -\begin{itemize} -\item Key \refmmz{record/record type/begin} will be executed at the beginning - of the document; it will receive no argument. Use it to open the record - file. -\item Key \refmmz{record/record type/end} will be executed at the end of the - document; it will receive no argument. Use it to close the record file. -\item Key \refmmz{record/record type/prefix} will be executed at the end of the - document and at every invocation of key \refmmz{path} in the document body; - it will receive a single argument, the path prefix determined by key - \refmmz{path}. -\item Keys \refmmz{record/record type/new cmemo}, \refmmz{record/record - type/used cmemo}, \refmmz{record/record type/new ccmemo} and - \refmmz{record/record type/used ccmemo} will be executed after creating or - inputting a memo; they will receive a single argument, the full path to the - memo. -\item Key \refmmz{record/record type/used extern} will be executed after an - extern was included into the document; it will receive a single argument, the - full path to the extern. -\item Key \refmmz{record/record type/new extern} will be executed after - creating creating an extern, more precisely at the end of memoization, right - after shipping out the extern page. It will receive a single argument, the - full path to the extern, but additionally, Memoize prepares the following - macros: - \begin{itemize} - \item \refcmd{ne:externbasepath} holds the full path to the extern, but (unlike |#1|) - without the |.pdf| suffix; - \item \refcmd{ne:pagenumber} holds the ``physical'' page number of the extern page in - the document (the numbering starts by $1$); - \item \refcmd{ne:expectedwidth} and \refcmd{ne:expectedheight} hold the width and - the height (total height, i.e.\ the sum of \hologo{TeX}'s height and depth) - of the extern page. - \end{itemize} -\end{itemize} - -Below, we present two simple examples of a record file. The first type simply -records the names of all memos and externs used or created by Memoize; the -resulting file could be included by |.gitignore| to have -\hreftt{https://git-scm.com}{git} automatically ignore all files produced by -Memoize. The second type lists the new externs, each preceded by its page -number in the |.pdf|; this file could be fed to a custom extern extraction -tool. - -\ExampleName{record-files} -\makeexample{\examplename.tex.c1} -\tcbinputexample{listing only} - -\ExampleName{record-extern-pages} -\makeexample{\examplename.tex.c1} -\tcbinputexample{listing only} - -Finally, note that (unlike memos and externs) record files are auxiliary files -and may be deleted at any time after the extraction of the externs produced in -the final compilation --- actually, even if these externs were not yet -extracted, deleting the record file(s) will merely force their recompilation. - - -\subsection{The memoization process} -\label{sec:memoization-drivers} - -We now turn to the memoization process itself. The job of memoization is to, -while compiling the given code in a regular fashion, prepare the cc-memo -(which, when it is input, will replicate the effect of the given code), -alongside any externs that the cc-memo will include (these hold the typeset -material to be replicated). Clearly, merely compiling the code cannot have -this effect (unless that code was written specifically to support memoization; -more on this later), and this is why the memoized code is typically wrapped by -a \emph{memoization driver}, which can be set using key \Emph{\refmmz{driver}}. -We'll inspect the default memoization driver, \refcmd{mmzSingleExternDriver}, -in the first subsection, and we will learn how to write specialized drivers in -the remaining subsections. But first, let us say some words about a -grouping-related \hologo{TeX}nical detail we need to take care about during -memoization. - -During memoization, we have to collect certain information, like build the -contents of the cc-memo. Some of that information might be contributed by the -memoized code itself. For example, a \cs{label} ``adds itself'' to the cc-memo -(by appending to token register \refcmd{mmzCCMemo}); a -\refkey{/tikz/remember picture} aborts memoization (by issuing -\refcmd{mmzAbort}); etc. The issue is that the memoized code might open any -number of \hologo{TeX} groups; we have no idea how deeply embedded the -\cs{label} or \refkey{/tikz/remember picture} might be. Therefore, we have -to collect all the information about the ongoing memoization \emph{globally}: -all assignments to \refcmd{mmzCCMemo} must be global; \refcmd{mmzAbort} sets -the underlying conditional globally; etc. (Clearly, all these global variables -are initialized at the start of memoization.) - -This was the easy part. An additional complication arises with some options -which may be set either outside memoization, or during this process. For -example, you can append the font size to the context expression in the preamble -(see section~\ref{sec:cross-referencing}), so that the externs will be -automatically recompiled when the font size changes, and clearly, this context -adjustment should respect \hologo{TeX} grouping; but a \cs{ref} or some other -cross-referencing command in the memoized code needs to append to the context -as well, and as this \cs{ref} occurs \emph{within} the memoized code, the -assignment must be global, as explained above. - -Mixing the local and global assignments to the token register -\refcmd{mmzContext}, which holds the (in the actual implementation, local) -context expression, will not do. For one, we \emph{do} want to restore the -pre-memoization context expression after we have memoized the code, and -furthermore, mixing local and global assignments to the same variable is not -recommended for save stack reasons anyway. - -Memoize addresses this issue by having \emph{two} context registers, -\refcmd{mmzContext} and \refcmd{mmzContextExtra} --- when computing the context -MD5 sum (which happens at the end of memoization), the two registers are -concatenated (the local one comes first). A package writer should know when to -use which register, and how. Outside memoization, one should assign to -\refcmd{mmzContext} --- \emph{locally}. During memoization, one should assign -to \refcmd{mmzContextExtra} --- \emph{globally}. The user interface key -\refmmz{context} respects this requirement automatically: it locally appends to -\refcmd{mmzContext} outside memoization, and it globally appends to -\refcmd{mmzContextExtra} during memoization. (The same idea is applied to the -post-memoization hooks \refmmz{at end memoization} and \refmmz{after - memoization}.) - - -\subsubsection{The default memoization driver} -\label{sec:memoization-driver-default} - -The default memoization driver, \refcmd{mmzSingleExternDriver}, produces -exactly one extern, which contains whatever is typeset by the code submitted to -memoization. The driver compiles the code into a horizontal or vertical box -depending on the value of key \refmmz{capture}. Let us look at the definition -of the driver line by line: - -\makeexcerpt{single-extern-driver} -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, - listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em}, - title=The default memoization driver, - float, -} - -\begin{enumerate} -\item Macro \refcmd{mmzSingleExternDriver} (and in fact any memoization - driver) takes a single argument, the code to compile. Memoize will call the - driver with the code given as the second argument to \refcmd{Memoize}, but - wrapped in a macro which re-reads it using \refcmd{scantokens} when - \refmmz{verbatim} is in effect. -\item If we're capturing into a horizontal box - (\refmmz{capture}|=|\refmmz{capture=hbox}), we put \refcmd{quitvmode} into - the cc-memo --- putting it to the very beginning should make sure that any - replicated \cs{label} and \cs{index} commands refer to the correct page. -\item We compile the given code, storing the typeset material into a box - (above, a temporary box called |\mmz@box|). |\mmz@capture| resolves into a - box construction command, depending on the value \refmmz{capture}. -\item Macro \refcmd{mmzExternalizeBox} instructs Memoize to externalize the box - given as its first argument. However, this macro does not directly produce - an extern page or write any instructions into the cc-memo; the road to this - final destination is indirect. \refcmd{mmzExternalizeBox} has two effects. - First, it adds the contents of the given box (above, |\mmz@box|) to an - internal box dedicated to holding all the externs produced in this - memoization (the contents of |\mmz@box| remain as they are) --- it is only at - the end of memoization that the contents of this internal box are shipped off - to extern pages. Second, \refcmd{mmzExternalizeBox} produces the code which - will include the extern into the document on subsequent compilations (this - will be a call to \refcmd{mmzIncludeExtern}, potentially prefixed by - \refcmd{quitvmode}; see section~\ref{sec:cc-memos} for details). This code - is stored into the token register gives as the second argument (above, - |\mmz@temptoks|), and it is the responsibility of the driver to include it - into the cc-memo. (In the interest of full disclosure, - \refcmd{mmzExternalizeBox} also updates the list of externs produced in this - memoization. At the end of memoization, this list is written to the - beginning of the cc-memo, resulting in the \refcmd{mmzResource} lines - preceding the \refcmd{mmzMemo} marker.) -\item The construction of the cc-memo is indirect as well. In the third line - of the definition, we globally append the extern-inclusion code residing in - |\mmz@temptoks| to token register \refcmd{mmzCCMemo}. At the end of - memoization, the contents of \refcmd{mmzCCMemo} are written into the cc-memo, - preceded by the \refcmd{mmzMemo} marker. -\item We put the typeset material into the document, again preceded by - \refcmd{quitvmode} when capturing in a horizontal box. -\end{enumerate} - -You might wonder why the construction of the extern pages and the cc-memo (and -actually, of the c-memo as well) is indirect, as described above. -\begin{itemize} -\item For one, the indirect construction facilitates potential abortion of - memoization (see section~\ref{sec:tut:good-to-know}). With the indirect route, - aborting is easy --- as nothing was permanently written anywhere yet, Memoize - simply skips the final part of the process, where extern boxes are shipped - into extern pages and the memo registers written into memo files --- and also - clean: if \refcmd{mmzExternalizeBox} immediately shipped out the extern - pages, these pages would remain in the document even in the case of abortion. -\item Even more importantly, the cc-memo filename contains the \meta{context - md5sum} (see section~\ref{sec:cc-memos}), but the context expression is not - yet fully known when memoization starts --- remember (from - section~\ref{sec:cross-referencing}) that a \cs{ref} in the memoized code will - update the context! The cc-memo can therefore only be opened at the end of - memoization, which necessitates a buffer (i.e.\ the \refcmd{mmzCCMemo} - register) for storing its contents during memoization. -\end{itemize} - - -\subsubsection{Pure memoization} -\label{sec:pure-memoization} - -The default memoization driver discussed above is really an externalization -driver: it produces a single extern. We now move to examples of drivers with -other functions, starting with a pure memoization driver, which does not -externalize any typeset output --- simply because it does not call -\refcmd{mmzExternalizeBox} at any point --- but rather remembers the result -of a (pgfmath) computation (let's pretend that the computation is -time-consuming). - -\ExampleName{pgfmathparse} -\makeexample{\examplename.pdf} -\begingroup -\relaxmmzcommands -\def\mmzNewCCMemo#1{% fetch the first cc-memo filename - \def\myccmemo{#1}% - \endinput -} -\input{\examplepath.mmz.c1} -\tcbinputexample{% - comment={% - \tcbinputlisting{ - listing only, width=.4\linewidth, nobeforeafter, - listing file=\exampledir\myccmemo, - example title, title=the cc-memo, - }\quad - \includeexamplepdf[document page]{page=1} - }, -} -\endgroup - -Command \refcmd{mmz} above memoizes its mandatory argument with the memoization -\refmmz{driver} set to the previously defined macro \cs{mmzPgfmathDriver}. -Just as the default driver above, \cs{mmzPgfmathDriver} first executes the -given code. However, there is no need to do this in the context of a -|\setbox|, as the memoized code, which is obviously expected to consist of a -single \cs{pgfmathparse} call, does not typeset anything: \cs{pgfmathparse} -evaluates the given expression and stores the result into macro -\cs{pgfmathresult}. The driver has two jobs: first, it must store this result -into the cc-memo, to be utilized in subsequent compilations; second, because -the assignment to \cs{pgfmathresult} (within \cs{pgfmathparse}) is local, the -driver also needs to somehow smuggle the result out of the \cs{endgroup} issued -by \refcmd{Memoize} and thereby make it into the following code (the final line -of the example, which typesets the equation). Both jobs are easy enough: the -expansion of |\def\noexpand\pgfmathresult{\pgfmathresult}| (in this case, -|\def\pgfmathresult{42.0}|) is (globally) appended both to the token register -\refcmd{mmzCCMemo}, which Memoize later writes into the cc-memo, and to the -macro underlying the \refmmz{after memoization} hook, whose contents are -executed \emph{after} closing the memoization group. - -Let us consider an alternative implementation of the same goal of memoizing the -result of a pgfmath computation, showcasing a couple of useful tricks. - -\ExampleName{pgfmathparse-embellished} -\makeexample{\examplename.pdf} -\tcbinputexample{listing only} - -For one, this ``embellished'' example reminds us that we can list the -\refmmz{driver} key among the auto-options (even if I don't really recommend -automemoizing \cs{pgfmathparse}). But even more importantly, the example shows -that the driver consist of more than a single control sequence; the only -requirement is that the given driver code will consume the memoized code. In -this example, we have developed a generic smuggling driver and applied it to -\cs{pgfmathresult} in particular --- \cs{pgfmathresult} will become the first -argument of \cs{mmzSmuggleOneDriver}, and the memoized code will become its -second argument. - -In the first version of the example, we have appended the same code to macro -\refcmd{mmzCCMemo} and to macro \refcmd{mmzAfterMemoizationExtra} --- no -surprise here, as we want the effect of memoization and utilization to be the -same. In the embellished version, we advertise another way to achieve the same -effect, a way which might be useful for complicated drivers: we simply smuggle -out the entire cc-memo. The idea works even when memoization procudes externs; -in that case, however, the driver also has to say \cs{mmzkeepexternstrue} --- -conditional \refcmd{ifmmzkeepexterns} decides whether Memoize keeps the externs -around, in memory, even after shipping them out (but they are always gone at -the start of the next memoization). - -Finally, remember that the default context expression contains the -\refmmz{padding} values. However, these really have no place in the context -expression of some purely memoized code. We have therefore emptied out the -context expression using \refmmz{clear context}. - - -\subsubsection{Multiple externs per memo} -\label{sec:memoization-multiple-externs} - -In the next example, we show how to produce multiple externs for a single piece -of memoized code. The usage case I find most appealing is breaking the typeset -material, like a table, across pages --- but of course, table-breaking is too -complicated an example, so we illustrate the idea by defining command -|\countdown|, which counts down from the given number, typesetting each number -into its own line. Clearly, if we were to externalize a call to this command -using the default memoization driver, page breaking would stop working, as the -entire countdown would be seen as a single, unbreakable box. To externalize it -properly, the chunks of the countdown that should appear on separate pages must -be externalized into separate externs, as shown below. - -\ExampleName{countdown} -\makeexample{\examplename.sty N=2} -\makeexample{\examplename.pdf} -\tcbinputexample{% - listing options app={ - basicstyle=\ttfamily\footnotesize, - }, - middle=1mm, bottom=0mm, - comment={\centering - \relaxmmzcommands - \def\mmzUsedExtern##1{% - \saveexpandmode\fullexpandarg - \StrGobbleRight{##1}{4}[\mytemp]% - \StrRight{\mytemp}{5}[\mytemp]% - \restoreexpandmode - \relaxmmzcommands - }% - \def\mmzNewExtern##1##2##3##4##5{\mmzUsedExtern{##1}} - \def\myexternname##1{extern \dots\texttt{\mytemp##1.pdf}}% - \input{\examplepath.mmz}% - \begin{tabular}{ccc} - \includeexamplepdf[extern page,title/.expand once=\myexternname{}] - {page=2,trim=1in 1in 1in 1in,scale=0.333}& - \includeexamplepdf[extern page,title/.expand once=\myexternname{-1}] - {page=3,trim=1in 1in 1in 1in,scale=0.333}& - \includeexamplepdf[extern page,title/.expand once=\myexternname{-2}] - {page=4,trim=1in 1in 1in 1in,scale=0.333}\\ - \includeexamplepdf[document page]{page=1,scale=0.333}& - \includeexamplepdf[document page]{page=5,scale=0.333}& - \includeexamplepdf[document page]{page=6,scale=0.333} - \end{tabular} - }, -} - -To achieve this, we will have to integrate the memoization driver into the very -code of |\countdown|. This approach contrasts sharply with the standard -memoization driver, which is simply wrapped around the memoized code. Let's -say we have implemented a non-memoization-aware variant of |\countdown| as a -loop which gathers the countdown lines into a vertical box, and periodically, -when this box holds all the material that will fit onto the page, places it -into the main vertical list (i.e.\ on the page).\footnote{Of course, - implementing \cs{countdown} this way would be idiotic; a sane implementation - would simply spit out the countdown lines one by one, and let \hologo{TeX} - deal with page-breaking. However, remember that we are pretending that we - are typesetting (and page-breaking) some complex material, like a table; in - such a case, the loop outlined in the main text would make perfect sense.} -To have this command support memoization, we have to externalize our box every -time we're placing it into the main vertical list. This is precisely what we -do in the definition of |\countdowntypeset| below:\footnote{We omit the - definition of the core algorithm of \cs{countdown} in the listing, because it - is mostly irrelevant for our discussion, and only show the - memoization-related code.} the final line of this macro adds the material to -the main vertical list, and the preceding lines externalize it (the two lines -inside \cs{ifmemoizingcountdown} should be familiar from the definition of the -standard memoization driver; we'll explain about the conditional below); note -that Memoize automatically deals with the fact that our box is vertical. As a -result of having our memoization driver integrated into the loop of the core -command, we can create as many externs as necessary, complete with the code in -the cc-memo for including each and every one of them on subsequent -compilations. Each extern eventually makes it into its own extern file, and -note that the filenames of the non-first externs have their sequential number -(we start counting at $0$) appended to the basename, as shown in the -example.\cprotect\footnote{The \refmmz{auto} declaration of |\countdown| adds - some relevant parameters to the context (see - section~\ref{sec:cross-referencing}). The countdown will be recompiled upon - change of either the font size (|f@size|), the text height (|\textheight|), - or the height of the material in the main vertical list collected so far - (|\pagetotal|). The |\pagetotal| parameter is especially important; - including it makes sure that the countdown will be recompiled when it is - pushed up or down the page. Also note that we want the context to record the - value of |\pagetotal| when the (automemoized) |\countdown| is encountered - (rather than at the end of memoization), so we expand it when applying the - auto-options.} - -\tcbinputexample[.sty]{listing only, after title pre={\ (version 1)}} - -Of course, the chunks of the countdown should only be externalized when the -code is actually being memoized, and not, say, when Memoize is disabled or -performing regular compilation. (Note that this is a problem that only affects -integrated drivers and not wrapped drivers such as the default driver.) The -first thought is to detect whether we're undergoing memoization using -conditional \refcmd{ifmemoizing}, which Memoize sets to true at the start of -every memoization. This conditional is used in the run conditions of advice -for \cs{ref} and \cs{label}, the idea being that they should add stuff to the -context (\cs{ref}) and the cc-memo (\cs{label}) only when undergoing -memoization. However, deploying \refcmd{ifmemoizing} in the current example -would not be exactly right. It would work well with the main document as it -is, but it would fail if |\countdown| was called from a piece of code that was -independently submitted to memoization, e.g.\ -\refcmd{mmz}|{\countdown{30}}|.\footnote{Such embedding occurs more often than - you might think. For example, \env{forest} calls \env{tikzpicture} under the - hood, and both environments are automemoized.} In that case, both the -\refcmd{mmz} driver and the |\countdown| integrated driver would get executed, -resulting in the creation (and in subsequent compilations, utilization) of four -externs: first, the |\countdown| driver would externalize each countdown chunk -separately, and then, the \refcmd{mmz} driver would externalize them, all -together, yet again. You can try this out by replacing -\cs{ifmemoizingcountdown} in \cs{countdowntypeset} by \refcmd{ifmemoizing} (and -wrapping the \cs{countdown} call in \cs{mmz}). - -The solution to the \refcmd{ifmemoizing} problem deployed in the example is to -declare a new, \cs{countdown}-specific memoization conditional, and set it to -true in \cs{countdown}'s formal driver, i.e.\ the macro set as the -\refmmz{driver} in the \refmmz{auto} declaration for \cs{countdown}. In fact, -Memoize can do most of this for you: when we write \refmmzauto{integrated - driver}|=countdown|, Memoize creates the countdown-specific memoization -conditional and declares the formal driver which sets this conditional to true; -you only have to access this conditional in your code, and you should do this -using the \hologo{LaTeX}-style conditional \refcmd{IfMemoizing}, as shown -below.\footnote{You shouldn't directly use the plain \hologo{TeX} - countdown-specific conditional created by \refmmzauto{integrated driver} --- - to prevent accidental access, Memoize doesn't actually name it - \cs{ifmemoizingcountdown} --- because this conditional is undefined when - Memoize is not loaded, i.e.\ when only package \refpkg{memoizable} is in - effect. Furthermore, \refcmd{IfMemoizing} addresses a problem faced by - integrated drivers of potentially recursive commands; we will talk about this - in section~\ref{sec:memoization-complex-single-driver}.} - -\tcbinputexample[.sty][.c2]{listing only, after title pre={\ (version 2)}, - attachment name=\examplename-integrated-driver.sty, -} - - -\subsubsection{Driver-based memoizable design} -\label{sec:memoization-complex-single-driver} - -In the previous section, we used the integrated driver approach to produce -memos including multiple externs, but the approach can be useful for one-extern -memos as well, when the extern must be integrated into the document in some -special way. We already discussed such situations in -section~\ref{sec:memoizable-design}, where we suggested to split a -``difficult'' command into the outer command and the inner command, and only -submit the inner command to automemoization. However, the vanilla flavour of -this approach had a negative impact on the user interface to automemoization. -In this section, we will deploy the memoization driver to overcome the issue. - -\ExampleName{poormansbox-driver} -\makeexample{\examplename.sty} -\makeexample{\examplename.pdf N=2} - -Let us revisit the |poormansbox| example from -section~\ref{sec:memoizable-design}. Remember that that environment produced -a potentially framed box of a certain width, surrounded by some pre- and -post-code, and that the issue was that the pre- and the post-code should not be -memoized, but rather executed at every invocation of the command, as it was -primarily intended to put some stretchable vertical space around the box. - -The document source\attachexample\ and the resulting PDF of the example are the -same as in section~\ref{sec:memoizable-design}, so we will not repeat them -here, but jump directly into a revised definition of the environment. We will -retain the core idea from the original implementation: the outer command will -execute the pre- and the post-code, and the inner command will typeset the box. -But unlike in the original implementation, we will not automemoize the inner, -internal command (this was the source of the author's discomfort) but the outer, -user-level command --- and we will equip it with a custom memoization driver. -The major idea here is to have the driver compose a cc-memo which not only -includes the extern, but also executes the outer command. - -\tcbinputexample[.sty]{listing only, float} - -In detail, the implementation (partially shown in the |.sty| listing) is as -follows. The outer command (|\poormansbox@outer|) first applies the options -(|#1|) and then wraps the pre-code (|\pmb@before|) and the post-code -(|\pmb@after|) around some arbitrary code (|#2|). During memoization or -regular compilation, the outer command is invoked through the |poormansbox| -environment, and you can see that in the definition of that environment, the -second argument to |\poormansbox@outer| is a call to the inner command -(|\poormansbox@inner|; this command takes two arguments, the options and the -environment body). During utilization, the outer command is invoked from the -cc-memo,\cprotect\footnote{As you can see, in the cc-memo the outer command is invoked - by |\csuse{poormansbox@outer}|. A straight |\poormansbox@outer| would not - work because we're in the middle of the document where |@| is not a letter, - and including a |\makeatletter| in front of it (and in a group) only works if - \refmmz{direct ccmemo input} was in effect. Under the default, indirect - cc-memo input regime, the core cc-memo is tokenized before |\makeatletter| can - take effect.} and as you can see in the cc-memo listing below, the second -argument to |\poormansbox@outer| there is a call to \refcmd{mmzIncludeExtern}. - -\begingroup -\relaxmmzcommands -\def\mmzNewCCMemo#1{% - \def\mmzNewCCMemo##1{% fetch the second cc-memo filename - \def\myccmemo{##1}% - \endinput - }% -} -\input{\examplepath.mmz.c1} -\sed{% - s/~//g; - s/\(\cmd{csuse} *{poormansbox@outer}\) *\({\nobrace*\marg\nobrace*}\)\({.*}\rbrace\)/~\1~\2~\3~/; - s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/; -}{\exampledir\myccmemo} -\tcbinputlisting{ - listing only, left=1.5mm, right=1mm, - listing options app={basicstyle=\tt\small}, - listing file=\exampledir\myccmemo, - example title, title=the second poor man's box's cc-memo, -}% -\endgroup - -And how does |\poormansbox@outer| get into the cc-memo, which normally only -includes a call to \refcmd{mmzIncludeExtern}, you ask? This is the job of the -memoization driver, which is in this case integrated into the inner command. -The overall shape of the driver is the same as the shape of the standard -driver, discussed in section~\ref{sec:memoization-driver-default}: typeset the -extern material into a box, externalize this box, append the extern-inclusion -code to the cc-memo, and put the extern box into the document. It is the -cc-memo part which interests us right now: unlike the standard driver, we don't -simply append the contents of |\mmz@temptoks|, i.e.\ a -\refcmd{mmzIncludeExtern} call; we rather append a call to -|\poormansbox@outer|, which gets the \refcmd{mmzIncludeExtern} call as its -second argument (and the options as its first argument). - -The core part of the driver, which externalizes the box and appends to the -cc-memo, is embedded inside the true branch of conditional -\refcmd{IfMemoizing}|[1]{pmb}|. We already used this conditional in -section~\ref{sec:memoization-multiple-externs}, but without the optional -argument. Such usage will not work here, because it is not recursion-safe. -Unlike in the \cs{countdown} situation, one |poormansbox| environment can be -embedded in another one (and, in our example, it is). If we deployed -\refcmd{IfMemoizing}|{pmb}| in the inner command, the driver would be executed for -\emph{both} the outer and the inner instance of the environment, whereas it -should really be executed only for the outer instance. - -When used in a recursion-safe way, i.e.\ with the optional argument, -\refcmd{IfMemoizing} first tests whether the auxiliary command-specific -conditional from the previous section is true, and then proceeds to compare the -current group level (\hologo{eTeX}'s \refcmd{currentgrouplevel}) to the group -level at the start of memoization (which Memoize stored in -\refcmd{memoizinggrouplevel}). Only if these group levels match do we know -that we're working on the outer instance of the environment, and that we should -therefore execute the memoization driver. Importantly, though, the two group -levels are compared modulo the offset, given as the optional parameter to -\refcmd{IfMemoizing}: in our example, the offset is 1, because the driver is -located inside the |poormansbox| environment, which opens a group --- note that -0 zero (no offset) is not the default optional parameter; the absence of the -optional parameter indicates that the non-recursion safe method should be -used.\footnote{Even this approach is not completely bullet-proof. It will only - work when the inner instance of the command is guaranteed to occur in an - additional group, i.e.\ when our command opens up a group for any free-form - code. I will assume that situations which require externalization of a - potentially recursive command which, for some reason, cannot open the group - before processing a free-form argument, are rare enough to not warrant a - generic solution here.} - - -\FloatBarrier - - -\subsubsection{Shipout} - -Memoize is a hypocrite: when it is creating extern pages, it uses -\refcmd{primitive}\refcmd{shipout} to bypass the regular shipout routine of the -format, but it is offended if anyone else does that. - -Memoize bypasses the regular shipout because the extern pages should really not -be modified or discarded by a foreign package. But using the primitive -\refcmd{shipout} means that extern shipouts can't be detected by another -package, at all. To facilitate peaceful coexistence with a potential package -which needs to know about our extern pages, we offer public counter -\refcmd{mmzExternPages} holding the number of externs shipped out so far. And -if anyone really needs to \emph{do} something at every extern shipout, they can -always (ab)use \refmmz[show keypath]{record/record type/new extern} as a -post-extern-shipout hook. - -The other side of the story is about Memoize needing to know the ``physical'' -page numbers of its externs in the document PDF --- how else are we to extract -them? Memoize computes these page numbers by adding the values of several -counters: \refcmd{mmzRegularPages}, which holds the number of regular shipouts; -the above-mentioned \refcmd{mmzExternPages}, which holds the number of extern -shipouts; and \refcmd{mmzExtraPages}, which holds the number of other shipouts. -The latter counter should be advanced by a package which, like Memoize, -bypasses the regular shipout routine. - -\hologo{LaTeX} and \hologo{ConTeXt} kindly provide the number of regular -shipouts as publicly a accessible counter, so we define -\refcmd{mmzRegularPages} as synonymous with their -\refcmd{ReadonlyShipoutCounter} and \refcmd{realpageno}. In \hologo{plainTeX}, we -have to hijack the \refcmd{shipout} control sequence and count regular shipouts -ourselves; as we have to hijack it while it still refers to the -\refcmd{shipout} primitive, this format provides another reason for preferring -Memoize to be loaded early. - - -\subsection{Automemoization} -\label{sec:tut:automemoization-details} - -Automemoization is a mechanism that automatically memoizes the result of the -compilation of certain commands and environments. Writing Memoize, I went to -great lengths to make it flexible, yet easy to use. This resulted in -automemoization deploying two specifically developed auxiliary packages: -package Advice, which provides a generic framework for extending the -functionality of selected commands and environments, and package CollArgs, -which provides a command for collection of the arguments conforming to the -given (slightly extended) \pkg{xparse} argument specification. - -This section lists the considerations which went into designing the system, -followed by short tutorials on both auxiliary packages, which include several -examples of how Memoize uses the underlying advising framework. - -\paragraph{(De)activation} -Ideally, all commands and environments where memoization makes sense would -support Memoize (or Memoize would support them) and nothing would ever go -wrong. In this dream world, memoization would be completely transparent to the -author. However, things will go wrong, so at the very least, we need to -offer the author a \emph{simple} way to \emph{selectively} switch automemoization -on and off. This is achieved by keys \refmmz{activate} and -\refmmz{deactivate}. - -\paragraph{Submission} -Of course, there will be commands without official support by either Memoize or -the package which defines them; clearly, at the moment when I write this, all -commands but \cs{tikz}, \env{tikzpicture} and \env{forest} are such. Or, the -author might want to automemoize his or her own command. Ideally, submitting a -new command to automemoization would be as simple as -|memoize=|{\meta{command}}, and for environments, this is in fact achievable, -although the actual interface is -\refmmz{auto}|=|\braces{\meta{environment}}\braces{\refmmzauto{memoize}}. But -simply submitting the name cannot work for commands, because commands are where -we encounter the major \hologo{TeX}nical problem with automemoization: we need -to somehow collect the arguments of the command --- without executing the -command itself. - -\paragraph{Argument collection using CollArgs} -\hologo{TeX} being \hologo{TeX}, automatically determining the scope of a -command in general is just plain impossible. Note that inspecting the -|\meaning| is not enough in general, because the ``real'' and the formal -arguments of a command can, and quite often do, differ wildly. The author (or the -package writer) will need to tell Memoize about the argument structure of the -command. And as there is already a nice and general argument specification on -the market --- I'm obviously referring to the argument specification of package -\pkg{xparse}, which was recently even integrated into the core \hologo{LaTeX} ---- why not use that? Memoize comes with an auxiliary package CollArgs, which -(given the slightly extended \pkg{xparse}-style argument specification) -collects the arguments of a command into a single entity. All the user needs -to write to enable automemoization for a command is thus -\refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize}, - \refmmzauto{args}|=|\braces{\meta{argument specification}}}. Even simpler, -when it comes to commands defined by \pkg{xparse}'s \refcmd{NewDocumentCommand} -or friends, writing \refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize}} -will suffice, as the argument specification of these commands can be retrieved -by \refcmd{GetDocumentCommandArgSpec}. - -\paragraph{Weird commands} -Not every argument structure can be described using \pkg{xparse}'s argument -specification, a case in point being \refcmd{tikz} with its totally -idiosyncratic syntax --- and if Memoize won't support \cs{tikz}, why have it at -all? The interface to automemoization must be flexible enough to cover even -the craziest commands, and this is why Memoize allows for arbitrary argument -collectors. These are defined by the advanced user or package writer and then -declared to be used for parsing the argument structure of a command by writing -\code{\refmmz{auto}=\marg{command}\bracestt{...,\refmmzauto{collector}=\meta{argument - collector}}}. - -\paragraph{Over and above automemoization: handlers} -The framework facilitating automemoization must cover more than just that. For -one, it sometimes makes sense to automatically \emph{prevent} memoization -during the execution of certain commands (as in the \refmmzauto{nomemoize} -example in section~\ref{sec:tut:redefinitions}). It follows that the action -performed to an invocation of a command should not be fixed. In the advising -framework, implemented by the auxiliary package Advice, we assign each -advised command a handler --- a command which does the real work of -memoizing or whatever. Crucially, the handler and the collector are -independent of each other, allowing a single memoization handler to handle -commands with both standard and non-standard argument structure, and allowing a -single collector to serve either the memoization or the no-memoization handler. - -\paragraph{Over and above automemoization: the outer and the inner handler} -Second, unlike the memoization handler set by \refmmzauto{memoize}, not all -handlers work with the entire argument list of the advised command. Some -handlers don't care about the arguments of the advised command at all: -\refmmzauto{abort} simply aborts memoization whenever the advised command is -executed. Other handlers are only intended to advise a single command or a -small family of commands, and need to inspect specific arguments of the advised -command: for example, \refmmzauto{ref} needs to append the internal -cross-reference macro to the context, and it of course constructs the name of -this macro from the reference key. For all such handlers, it would be plain -wasteful to first collect the arguments and then tear them apart to inspect -them (or not). The advising framework therefore recognizes two kinds of -handlers. The abortion and the cross-reference handler are examples of an -\emph{outer} handler, which is simply placed in front of the arguments of the -handled command as they are, without invoking the collector. The memoization -handler, on the other hand, is an example of an \emph{inner} handler, which -receives the entire argument list from the collector (as a single argument) --- -or more precisely, even \refmmzauto{memoize} sets up an outer handler, but this -outer handler doesn't do much more than invoke the collector, which in turn -invokes the inner handler. - -\paragraph*{Run conditions} are another, if minor, lego piece of the -advising framework. Using key \refmmzauto{run conditions}, the user can set the -conditions under which the (outer) handler is executed; for example, -cross-reference commands are only advised when memoization is underway. And -the same goes for \refcmd{label}, and for the \refmmzauto{replicate}d -\refcmd{index}, and for \refcmd{savepos}, upon which memoization must be -aborted. The bottom-line is that run conditions repeat across handlers, so -it makes sense to separate them out as an independent component of the -framework, with the added bonus that the system can make sure that an -invocation of a advised command which does not satisfy the run conditions -will incur as litte overhead as possible. - -\paragraph{Bailout handler} -An automemoized command applies the next-options (set by \refcmd{mmznext}), but -what happens when the run conditions are not satisfied? If nothing happened, -the existing next-options might apply to the next instance of -(auto)memoization, which would not be what the author intended. This is why -Advice introduces the bailout handler, a piece of code executed before the -original command when the run conditions are not met. Obviously, the bailout -handler for memoization clears out the next-options (and does not process -them). - -\paragraph{The structure of advice} -Together, the components mentioned above form a piece of \emph{advice}: - -\begin{center} - \begin{forest} - /tikz/edge node/.style={midway, font=\scriptsize, yshift=0.5ex}, - for tree={l sep=5ex, % manual, to avoid a page break - child anchor=north, s sep+=2em}, - [activated? - [\refmmzauto{run conditions}?, - edge=dashed, edge label={node[edge node, left]{yes}} - [\refmmzauto{outer handler}\meta{args}, - edge=->, edge label={node[edge node, left]{yes}}, - [default:, - edge=dashed, - [\refmmzauto{collector}\meta{args}, edge=->, - [\refmmzauto{inner handler} - \textcolor{red}{\bracestt{\textcolor{black}{\meta{args}}}}, - edge=-> - ] - ] - ] - [other:\\whatever\rlap\footnotemark, - align=center, edge=dashed] - ] - [\refmmzauto{bailout handler}, - edge=->, edge label={node[edge node, right]{no}}, - [\meta{original command}\meta{args}, edge=->] - ] - ] - [\meta{original command}\meta{args}, - edge=dashed, edge label={node[edge node, right]{no}}, - ] - ] - \end{forest}% - \footnotetext{The handler may do whatever as long as it consumes all and - only the arguments of the original command.} -\end{center} - - - -\paragraph{Deferred activation} -Memoize needs to be loaded early, but activation should take place late, so -that it can surely override the submitted commands; a case in point, -\pkg{hyperref} redefines \refcmd{ref} very late. To address the issue, the -advising framework implements the deferred \refmmz{activation} regime, under -which (de)activation commands are not executed but collected in a special -style, \refmmz{activate deferred}. Memoize deploys the deferred activation -regime throughout the preamble, and executes \refmmz{activate deferred} at the -latest possible \docref{hook:begindocument} hook; as a bonus, it also offers -the author a way to avoid automatic activation completely by invoking key -\refmmz{manual}. - - -\paragraph{Install anywhere} -Once all this machinery is developed, why not offer it to others as well? - -Once I decided to offer Advice (at the time, still called Auto) as a standalone -package (and I freely admit that the framework got much cleaner once I -separated its code out of Memoize) it became immediately clear that if it is -to serve as a generic framework, it should be possible for multiple packages to -use it without interfering with each other. The package thus allows any number -of installations into different namespaces, each namespace a \pkg{pgfkeys} -keypath. The installation is a breeze: -\cs{pgfkeys}\bracestt{\meta{namespace}/\refkey{/handlers/.install advice}}. - - -\subsubsection{Using package Advice} -\label{sec:advice} - -In this section, we will provide some examples of handler declarations, mainly -based on how Memoize deploys the advising framework. - -\paragraph{\refmmzauto[show keypath]{memoize}} - -In section~\ref{sec:tut:automemoization}, the author was instructed to submit a -command to automemoization by writing -\code{\refmmz{auto}=\meta{command}\bracestt{\refmmzauto{memoize},...}}. The -auto-key \refmmzauto{memoize} is a style (defined by Memoize rather than -Advice) which sets the appropriate components of the automemoization advice. -Residing in keypath \refkeypath{/mmz/auto}, it is effectively defined as -follows: - -\begin{tcblisting}{listing only} -\mmzset{ - auto/memoize/.style={ - run if memoization is possible, - bailout handler=\mmz@auto@bailout, - outer handler=\mmz@auto@outer, - inner handler=\mmz@auto@memoize - } -} -\end{tcblisting} - -The heart of this advice is its inner handler, which actually triggers -memoization by executing \refcmd{Memoize}. Remember that the first argument of -\refcmd{Memoize} is the code which the md5sum is computed off of. This -argument must therefore be identical to the author's invocation of the -automemoized command or environment. Given what Advice offers, this is easy to -construct: \refcmd{AdviceReplaced} holds the code replaced by the advice, and -the \meta{arguments} of the automemoized command are waiting for us in |#1|. -The second argument of \refcmd{Memoize} will be similar, but as this is the -code which will get compiled, we have to execute the original definition of the -command, followed by the (unbraced!) \meta{arguments} as |#1|; this is a job -for \refcmd{AdviceOriginal}. (Note that \emph{executing} -\refcmd{AdviceReplaced} would run the auto-handler again, resulting in an -infinite loop! Or at least a pile of errors.) - -However, the overly simplistic approach shown below won't necessarily work. -The issue is that the arguments of \refcmd{Memoize} contain -\refcmd{AdviceReplaced} and \refcmd{AdviceOriginal} themselves, instead of their -contents, i.e.\ (first) expansions. - -\begin{tcblisting}{listing only, bad} -\long\def\mmz@auto@memoize#1{% - \Memoize{\AdviceReplaced#1}{\AdviceOriginal#1}% -} -\end{tcblisting} - -Regarding the first argument, the problem is that the code md5sum will be -computed off of the token list \refcmd{AdviceReplaced}\meta{arguments} --- -exactly as you see it.\footnote{If you inspected a c-memo, you would find - \refcmd{AdviceReplaced}\meta{arguments} in the \refcmd{mmzSource} section.} -This implies that two commands sharing exactly the same \meta{arguments} will -receive the same (c-)memo. For example, if you automemoized first -|\textit{foo}| and then |\textbf{foo}|, both would come out as a bold ``foo'' -upon utilization. - -The second argument illustrates a general issue about the lifespan of -\refcmd{AdviceOriginal} and other |\Advice...| commands.\footnote{The list of - all commands available only within the handler can be found in the - documentation of key \refmmzauto{outer handler} in - section~\ref{sec:ref:advice}.} By executing \refcmd{Memoize}, we leave the -advice and thereby cannot be sure that when expanded, \refcmd{AdviceOriginal} -will mean what it means at the moment. In general, another piece of advice might be -triggered until its expansion, or the group might be closed, etc. For example, -the author may have issued \refcmd{mmznext}\bracestt{\refmmz{at begin - memoization}=\cs{label}\marg{key}}, and as the pre-memoization code is -executed before the memoization driver, the \cs{label}, which is submitted to -Advice so that any \cs{label}s inside the memoized code ``just work'', would -execute another piece of advice, redefining \refcmd{AdviceOriginal} and friends. -Effectively, you'd end up memoizing an invocation of the \cs{label} command. - -The bottom line is that while the code following the above template -\emph{might} sometimes work, Advice offers no guarantees that it will, so I -advise against using it. The actual definition of the memoization inner -handler is shown below. In this definition, we expand \refcmd{AdviceReplaced} -and \refcmd{AdviceOriginal} --- exactly once! --- into the respective arguments -of \refcmd{Memoize}; of course, as the entire invocation of \refcmd{Memoize} is -expanded, we have to guard against expanding the collected arguments (|#1|) and -\refcmd{Memoize} itself.\footnote{As the final touch, the handler also contains - \refcmd{ignorespaces} after the invocation of \refcmd{Memoize}, if this was - requested using \refmmz{ignore spaces}. Note that this could not be done - without pre-expanding the \cs{ifmmz@ignorespaces} conditional, as - \refcmd{Memoize} closes the group in which the auto- and the next-options are - applied.} The result of the expansion is shown under the code: the first and -the second line assume we are automemoizing command |\foo| and environment -|bar|, respectively. Note that \refcmd{AdviceOriginal} expands into an -invocation of \refcmd{AdviceGetOriginal}, a command which may be safely used -outside the advice; the first argument of this command is the auto-namespace -(in our case, \refkeypath{/mmz}), and the second argument is the advised -command. For a \hologo{LaTeX} environment, the advised command is actually -\refcmd{begin}, and this is why the call of \refcmd{AdviceGetOriginal} is of -course followed by the environment name. - -\makeexcerpt{_auto-memoize-inner} -\tcbinputexample[.tex][.excerpt]{% - listing and comment, one file, no attachment, - title={The implementation of the \refmmzauto{inner handler} - for automemoization}, - comment={% - $\rightarrow$ \refcmd{Memoize}\braces{\cs{foo}\Arg1}% - \braces{\refcmd{AdviceGetOriginal}\bracestt{/mmz}\bracestt{\cs{foo}}\Arg1}\\ - $\rightarrow$ \refcmd{Memoize}\braces{\cs{begin}\bracestt{bar}\Arg1}% - \braces{\refcmd{AdviceGetOriginal}% - \bracestt{/mmz}\bracestt{\cs{begin}}\bracestt{bar}\Arg1} - } -} - -Let us now move backwards in time and look at the outer handler installed by -\refmmzauto{memoize}. It is very simple, but performs an important function of -applying the auto- and the next-options (in this order), which also -necessitates opening a group (closed by \refcmd{Memoize}). The final line -invokes the argument collector, which then calls the inner handler; remember -that the invocation of \refcmd{AdviceCollector} is the sole function of the -default outer handler. - -\makeexcerpt{_auto-memoize-outer} -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, no attachment, - title={The implementation of the \refmmzauto{outer handler} - for automemoization}, -} - -Moving even further back in time, we arrive at the run conditions. The -\refmmzauto{memoize} style invokes \refmmzauto{run if memoization is possible}, -defined as \code{run conditions=\cs{mmz@auto@rc@if@memoization@possible}}, with -the installed macro as shown below. Indeed, memoization only makes when -Memoize is \refmmz{enable}d (which we test using \refcmd{ifmemoize}), but we're -not already ``inside \refcmd{Memoize}'' (which we test using -\refcmd{ifinmemoize}). The latter condition is true when we're either -memoizing or regularly compiling some code submitted to memoization (see the -diagram in section~\ref{sec:Memoize}). Note that it is not necessary to invoke -\refcmd{AdviceRunfalse} in branches where the run conditions are not satisfied. - -\makeexcerpt{_auto-run-if-memoization-is-possible} -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, no attachment, - title={The implementation of - \refmmzauto{run if memoization is possible}}, -} - -While it is clear that double memoization is a no-no, why should we avoid -memoizing inside a regular compilation? Imagine that Memoize decides not to -memoize a Forest tree, perhaps because \refmmz{readonly} is in effect. Under -the hood, Forest creates many \env{tikzpicture}s. Should all of them be -(auto)memoized now? Certainly not. - -Finally, what happens when the run conditions are not met? Not much, but -something important nevertheless: by consuming the next-options, the -\refmmzauto{bailout handler} makes sure they will not erroneously apply to the -next instance of (auto)memoization. - -\makeexcerpt{_auto-memoize-bailout} -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, no attachment, - title={The implementation of the \refmmzauto{bailout handler} - for automemoization}, -} - -The only component of the automemoization advice not determined by style -\refmmzauto{memoize} is the argument collector, which allows the user to submit -a command with a weird argument structure to automemoization simply by setting -key \refmmzauto{collector} in addition to executing \refmmzauto{memoize}. For -example, Memoize submits \refcmd{tikz} to automemoization by loading -\reffile{advice-tikz.code.tex}, which contains Advice's definition of the -\cs{tikz} collector \refcmd{AdviceCollectTikZArguments}, and issuing the -following declaration. - -\makeexcerpt{_auto-tikz-collector} -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, no attachment, - title={Declaring automemoization of command \cs{tikz}}, -} - -\paragraph{\refmmzauto[show keypath]{ref}} - -The cross-reference advice presents an example of an outer handler radically -different from the default outer handler. This outer handler does not invoke -the collector at all. As shown below, it grabs the argument of \refcmd{ref} (or -whichever cross-referencing command) on its own --- remember that the outer -handler receives the arguments of the handled command ``as they are,'' i.e.\ -uncollected. It then asks \refcmd{mmzNoRef} to do the real job of getting the -reference key into the context, and finally executes the original \cs{ref}. - -\ExampleName{_auto-ref} -\makeexample{\examplename.tex.c1} -\tcbinputexample{% - listing only, no attachment, - title={\hypercolor{link}{white}A simplified\footnotemark{} definition - of \refmmzauto{ref}}, - % This only works if there is a single footnote in the memoized code. - /mmz/context/.expanded={footnote=\numexpr\thefootnote-1}, -}\footnotetext{The real outer handler allows for arbitrary optional arguments of - the cross-referencing command, and shares code with \refmmzauto{force - ref}.} - -The run conditions of this style are agonizingly simple: \refmmzauto{run if - memoizing} sets \refmmzauto{run conditions} to a macro defined as -\refcmd{ifmemoizing}\hyp\refcmd{AdviceRuntrue}\hyp\cs{fi}. - -\paragraph{\refmmzauto[show keypath]{abort}} - -The advice for aborting memoization is very simple --- it merely executes -\refcmd{mmzAbort} --- but also very sneaky. Here, the \refmmzauto{run - conditions} do the real work of aborting memoization, while the ``real,'' -i.e.\ outer handler, never even gets executed; note the absence of -\refcmd{AdviceRuntrue}, which implies \refcmd{AdviceRunfalse}, which triggers -the execution of the original command after the run conditions are ``checked.'' - -\makeexcerpt{_auto-abort} -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, no attachment, - title={The definition of \refmmzauto{abort}}, -} - -The point here is that executing \refcmd{mmzAbort} (itself a single-liner -setting an internal conditional) is cheaper than testing for the real run -conditions (\refmmzauto{run if memoizing}) and aborting only if they are -satisfied. Of course, the trick only works because (i) the advice doesn't -need to inspect any arguments of advised command, and because (ii) setting the -internal abortion conditional outside memoization does no damage. - -\paragraph{Advice in chains} - -A command may be submitted to several instances of the advising framework, -i.e.\ instances installed under different keypaths. In the example below, we -submit |\foo| both to the instance of Advice installed in keypath |/one| and to -the one installed in keypath |/two|. Under |/one|, the result of |\foo{...}| -will be boxed (|\fboxWrap|); under |/two|, in will be parenthesized -(|\parenWrap|). The order in which this happens depends on the order in which -|\foo| was activated under different keypaths. If we first activate it under -|/one| with the boxing effect and then under two with the parenthesizing -effect, the box will appear within parenthesis; if we reverse the activation -order, the parenthesis will appear inside the box. - -\ExampleName{chained-advice} -\makeexample{\examplename.tex.c1 N=2} -\tcbinputexample{comment=\input{\examplepath.tex.c2}} - -First of all, looking at the code above, you have probably noticed the absence -of key \refmmz{auto}. This is because by default, \refkey{/handlers/.install - advice} defines the \emph{setup key} to be |advice| --- Memoize overrides -this default by installing the framework with \refkey{/handlers/.install - advice}|=|\bracestt{\refkey{/advice/install/setup key}=auto, ...}. - -Next, |advice'| is a variant of |advice| which prevents automatic activation -upon setup (and the same holds for |auto'| vs.\ |auto| in Memoize). We have -used the bar variant above to make it clear that it is the order of activation, -rather than declaration by |advice|\slash \refmmz{auto}, which matters in -determining which handler is applied first. - -Finally, note that the deactivation order must be the reverse of the activation -order. So if we activate |\foo| first in |/one| and then in |/two|, we should -deactivate it in |/two| first and in |/one| next, otherwise Advice will -complain. - - -\paragraph{A simple collector} - -Let us implement a collector for a command which accepts one (standard -\hologo{LaTeX}) optional argument and one mandatory argument; in \pkg{xparse} -terms, a command with argument specification |om|. - -Using \cs{NewDocumentCommand}, such a collector is very easy to implement. We -simply define a command with signature |om| and distinguish two possibilities -regarding the presence of the optional argument, which we test using -\cs{IfValueTF}. If the true branch, we pass a \emph{braced} |[#1]{#2}| to the -inner handler, which we invoke by \refcmd{AdviceInnerHandler}; in the false -branch, we omit the optional argument, passing it an (additionally) braced -|{#2}|. - -\ExampleName{om-collector-NewDocumentCommand} -\makeexample{\examplename.tex.c1} -\tcbinputexample{listing only} - -Defining a functionally equivalent collector using \cs{newcommand} would be a -bit more involved, as \hologo{LaTeX2e} does not offer a standardized way to -test for the \emph{presence} of the optional argument. Consider the following -collector, whose optional argument has the same default value as the advised -command. Is it functionally equivalent to the one above? - -\ExampleName{om-collector-newcommand} -\makeexample{\examplename.tex.c1} -\tcbinputexample{listing only} - -While there will be no visual difference, there is a difference under the hood. -If you compile both documents, you will see that the first one creates three -memos\slash externs, while the second one only creates two: |\foo{...}| does -not have its own memo anymore, but creates and uses the same memo as -|\foo[green]{...}|. - -While the second version might sometimes be preferred, perhaps even in the -context of memoization, the initial collector, which deploys command -\refcmd{CollectArguments}, behaves like the \cs{NewDocumentCommand}-defined -collector above,\footnote{In fact, there is a slight difference after all. - While the above-defined collector won't distinguish between the single-token - mandatory argument given with or without braces, \refcmd{CollectArguments} - will again faithfully replicate the original argument tokens.} as it attempts -to perfectly replicate the command invocation. Furthermore, this behaviour -makes it unnecessary for the author to provide the default values of optional -arguments (and even allows them to replace |O{default}| in the argument -specification by \docref{xparse:o}). - -We now turn to the package CollArgs, which implements the actual argument -collection; we'll revisit the initial collector of Advice at the end of the -following subsection. - - - -\subsubsection{Using package CollArgs} -\label{sec:collargs} - -Automemoization is implemented on top of the framework offered by package -Advice, and that package in turn couldn't really work as intended without -package CollArgs. A regular user of Memoize shouldn't need to know anything -about CollArgs, but a package writer wanting to support Memoize might have to. - -The package provides two public commands, \refcmd{CollectArguments} and -\refcmd{CollectArgumentsRaw}; we'll focus on the former first. -\refcmd{CollectArguments} takes three arguments: optional \meta{options} in the -form of a \pkg{pgfkeys} keylist; a mandatory \meta{argument specification} in a -(slightly extended) \pkg{xparse} format; and the \meta{next-code}: -\begin{center} - \refcmd{CollectArguments}\oarg{options}\marg{argument specification}\marg{next-code}% - \textcolor{gray}{\meta{tokens}} -\end{center} -Following the three formal arguments of \refcmd{CollectArguments} are some -\meta{tokens} --- the rest of the document, really --- and the job of -\refcmd{CollectArguments} is to figure out the extent to which these -\meta{tokens} conform to the given \meta{argument specification}. In other -words, \refcmd{CollectArguments} will consume as many of the \meta{tokens} as a -\meta{command} defined by \refcmd{NewDocumentCommand}\meta{command}\marg{argument - specification}\bracestt{...} would. Once these \meta{argument tokens} are -collected, \refcmd{CollectArguments} executes the \meta{next-code} with the -\meta{argument tokens} given as a single, braced argument (clearly, the -\meta{rest} of the \meta{tokens}, i.e.\ the non-consumed tokens, will follow): -\begin{center} - \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}} -\end{center} - -In the example below, we define macro |\PrintAndDo|, which takes two arguments, -a command and the collected arguments of that command, prints out which command -we're about to execute and with what arguments, and then executes that command -with those arguments --- |#1#2| at the end of the definition. Note that |#2| -immediately following |#1| is not braced, so -|\PrintAndDo\makebox{[5em][r]{text}}| executes |\makebox[5em][r]{text}|. - -{\tolerance 1000 Executing |\PrintAndDo\makebox{[5em][r]{text}}| directly would - thus yield the first line of the result below --- and in fact, this is - precisely what gets executed to yield that line, but in a roundabout fashion. - Given the argument specification |oom| (two optional arguments followed by a - mandatory argument), \refcmd{CollectArguments} figures out how many tokens - following its formal arguments conform to this argument specification --- - below, these would be |[5em][r]{text}| following - \refcmd{CollectArguments}\hyp|{oom}{\PrintAndDo\makebox}|--- and puts them, - braced, behind its \meta{next-code} argument, |\PrintAndDo\makebox|, yielding - |\PrintAndDo|\hyp|\makebox{[5em][r]{text}}|.\par} - -\ExampleName{collargs-makebox} -\makeexample{\examplename.tex.c1} -\tcbinputexample{listing and compile} - -Seeing the arguments of |\makebox| without the immediately preceding |\makebox| -might seem strange, but remember that \refcmd{CollectArguments} is about the -arguments of a command, not about the command's control sequence. It doesn't -know or care which command the argument tokens ``belong'' to, as long as they -conform to the given specification. In the example above, it is only in |#1#2| -of |\PrintAndDo| that |\makebox| is ``reunited'' with its arguments, but note -that the reunion is far from obligatory. - -CollArgs supports all the argument types (and modifiers) that \pkg{xparse} -does, including the environment type \docref{xparse:b}, as exemplified below. -Again, the code below might seem strange, as it features an |\end{minipage}| -without the matching |\begin{minipage}|, but the logic is similar as for - commands: just as \refcmd{CollectArguments} occurs in front of the command - arguments, without the command itself, so it occurs in front of the - environment body, without the opening of that body. However, while - \refcmd{CollectArguments} never needs to know the command name, we need to - inform it of the environment name, so that it can find the end of the - environment. This can be achieved as shown below, using key - \refcollargs{environment} in the optional argument of the command, or by our - extension to the \pkg{xparse} argument specification, where the environment - argument type \docref{xparse:b} may be followed by a braced environment name. - In the example below, we could therefore also invoke argument collection by - \refcmd{CollectArguments}\braces - {\docref{xparse:+}\docref{xparse:b}\braces{\env{minipage}}} (we have preceded - \docref{xparse:b} with a \docref{xparse:+} to allow for an environment body - containing paragraph tokens).\footnote{CollArgs automatically adapts to the - format, i.e.\ it knows that environments are tagged by \cs{}\meta{name} and - \cs{end}\meta{name} in \hologo{plainTeX} and by \cs{start}\meta{name} and - \cs{stop}\meta{name} in \hologo{ConTeXt}.} - -\ExampleName{collargs-minipage} -\makeexample{\examplename.tex.c1} -\tcbinputexample{comment=\input{\examplepath.tex.c1}} - -You might wonder why didn't we provide \refcmd{CollectArguments} in the -previous example with argument specification |omb| --- after all, the -\env{minipage} environment takes an optional and a mandatory argument. While that -would work, and produce the same result,\footnote{Unless argument processing - was in effect; see section~\ref{sec:ref:collargs} for details.} note that -\refcmd{CollectArguments} is only interested in finding the scope of the -arguments, and grabbing everything until |\end{minipage}| is the same as first -grabbing the optional argument, maybe, then the mandatory argument, and finally -the argument body. - -\refcmd{CollectArguments} not only supports \pkg{xparse}'s verbatim argument -type \docref{xparse:v}, it can grab an argument of \emph{any} type in the -verbatim mode, triggered by option \refcollargs{verbatim}.\footnote{We refer to - the verbatim mode triggered by \refcollargs{verbatim} as the full verbatim - mode, where all characters are of category ``other''. There is also the - partial verbatim mode, triggered by \refcollargs{verb}, where braces retain - their normal category codes.} We illustrate this key below, where we also use -option \refcollargs{tags}, which makes CollArgs automatically surround the -grabbed environment body with the begin tag \refcmd{begin}\marg{environment - name} and the end tag \cs{end}\marg{environment name}, and use -\refcmd{scantokens} to execute the grabbed environment. Consult -section~\ref{sec:ref:collargs} for the full reference on the verbatim mode and -its limitations. - -% A harmless warning: -% Missing character: There is no ^M (U+000D) in font [lmmono10-regular]:! -% because \texttt in the example is fed a (verbatim) carriage return -\ExampleName{collargs-verbatim} -\makeexample{\examplename.tex.c1} -\tcbinputexample{listing and compile} - -Finally, CollArgs extends the \pkg{xparse} specification by modifier -\docref{xparse:amp}, which allows the user to specify options which apply only -to the following argument, as opposed to the options given as the optional -argument of \refcmd{CollectArguments}, which apply to all the arguments. A -third way to invoke the environment body collection in the above example is -thus \refcmd{CollectArguments}% -\bracestt{\docref{xparse:amp}\bracestt{\refcollargs{environment}=minipage}+b}. - -Both the single-argument and the common options can be given not only as -\pkg{pgfkeys} keys, but also in the raw, ``programmer's interface'' format. -Every option key has a corresponding macro; for example, key -\refcollargs{environment} is matched by macro \refcmd{collargsEnvironment}. -The macros are listed alongside their corresponding keys in the reference -section~\ref{sec:ref:collargs}; here, we merely learn how to use them. - -To use raw options for a single argument, double the ampersand in the argument -specification. Therefore, the fourth way to specify the environment name is -\code{\docref{xparse:amp}\docref{xparse:amp}\bracestt{% - \refcmd{collargsEnvironment}\bracestt{minipage}}\docref{xparse:+}\docref{xparse:b}}. - -To set the raw options for all arguments, use \refcmd{CollectArgumentsRaw}, the -second public command of the package. This command is exactly like -\refcmd{CollectArguments}, excepts that it expects the options in the raw -format and as a \emph{mandatory} argument: - -\begin{center} - \refcmd{CollectArgumentsRaw}\marg{raw options}\marg{argument specification}% - \marg{next-code}\textcolor{gray}{\meta{tokens}} -\end{center} - -This leads us to the fifth way to set the environment name (an overkill, I -know): \refcmd{CollectArgumentsRaw}\hyp -\bracestt{\refcmd{collargsEnvironment}\bracestt{minipage}}\bracestt{+b}\marg{next-code}. -Furthermore, you can use a mixture of raw and key--value options: the raw -option commands include \refcmd{collargsSet}, which applies the given option -keylist. The idea here (incarnated by both Auto and Memoize) is that the -package will provide CollArgs with the raw options, for speed, while the -author can supplement them in the friendly keylist format --- and this leads -us to the sixth, and thankfully final way to set the environment name: -\refcmd{CollectArgumentsRaw}\bracestt{\refcmd{collargsSet}% - \bracestt{\refcollargs{environment}=minipage}}\bracestt{+b}\marg{next-code}. - - -\paragraph{The initial collector} - -As the final example, let us study Advice's initial collector; this is a macro -which is used as the collector when key \refmmzauto{collector} is not given. -This macro is not really \refcmd{CollectArguments}, as we sometimes state to -simplify matters, but a macro which acts as the ``bridge'' between Advice and -CollArgs, by compiling an invocation of \refcmd{CollectArgumentsRaw} from the -given advice setup, and executing it. - -The bridge macro is shown below in its full glory, but it is really less -complicated than it might appear at first sight. In line~2, we use -\refcmd{AdviceIfArgs} to see whether the argument structure of the handled -command was given by the user. If it wasn't, we assume that the handled -command was defined using \cs{NewDocumentCommand} or similar, and use -\refcmd{GetDocumentCommandArgSpec} to retrieve it (line~3; note that -\refcmd{AdviceName} holds the handled control sequence) and store it into -\refcmd{AdviceArgs} (line~4), which also receives the argument specification -when given by the user via key \refmmzauto{args}. - -\makeexcerpt{_advice-CollectArgumentsRaw}(../../advice.edtx) -\tcbinputexample[.tex][.excerpt]{% - listing only, one file, no attachment, - listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em}, - title={The definition of the initial collector}, -} - -Lines~6--17 are somewhat of an expansion mess, because we have to construct the -invocation of the CollArgs' collector from the advice setup stored in various -macros. But once we think away all the (non-)expansion commands, we're left -with \refcmd{CollectArgumentsRaw} plus the following three arguments: -\begin{enumerate} -\item The raw options (lines 8--11): - \begin{enumerate} - \item In line 8, the advised command's control sequence is designated as the - \refcollargs{caller}. The effect is that if the given arguments don't - conform to the specification, the error thrown seems to come from the - advised command rather than some internal CollArgs macro. The author will - be grateful for this little detail. - \item In line 9, we add any \refmmzauto{raw collector options} set by Advice - (plus the package deploying Advice, like Memoize); user-given options are of - course possible, but not really expected here, because: - \item In lines 10--11, we add the user-given \refmmzauto{collector options}, - if there are any, embedded under \refcmd{collargsSet}. - \end{enumerate} -\item The argument specification (line 14). -\item The \refmmzauto{inner handler} (line 15). -\end{enumerate} - - - -\section{Reference} -\label{sec:reference} - -In this section, the extra information about keys, offered at the right of a -key in parenthesis, may contain the initial value of a key, and also a default -value of a key. In this context, the terms ``initial'' and ``default'' have -the meaning employed by the \pkg{pgfkeys} utility. Initial value refers to the -value that is set by the package. More often, we would call this the default -or the package-default value, but in the \pkg{pgfkeys} parlance, the default value -refers to the value that is passed to the key in the absence of the argument. -Honestly, I only kept to this convention in the reference section; elsewhere, -I often say ``default'' or ``package-default'' and mean the initial value. - -Another convention I kept to in this section is the color-coding of the keys -and commands. Green background indicates a basic key or command, which any -user might want to know about. Red background indicates other, more or less -advanced keys and commands. - - -\subsection{Loading} -\label{sec:ref:loading} -\paragraph{\hologo{LaTeX}} - -Load Memoize by \cs{usepackage}\braces{\docaux{package}{memoize}} or -\cs{usepackage}\oarg{options}\bracestt{memoize}. The latter form functions -almost identically to the former followed by -\refcmd{mmzset}\bracestt{\meta{options}}, with two exceptions. - -First, the \meta{options} as package options may not contain spaces. This is a -constraint imposed by \hologo{LaTeX} itself. To avoid it for a particular key, -define a spaceless variant of the key in \reffile{memoize.cfg}. For example, -to use \refmmz{memo dir} as a package option, put -\refcmd{mmzset}|{memodir/.style={|\refmmz{memo dir}|={#1}}}| into that file. - -Second, key \refmmz{extract} only make senses as a package option, or within -\reffile{memoize.cfg}. Internal extern extraction can only be performed before -the output PDF is opened, and to maximize chances of that, Memoize is designed -to extract as soon as it can: while it is being loaded. Consequently, changing -the extraction method in the document preamble has no effect. - -The necessity of performing extraction before the output PDF is opened is also -why Memoize prefers to be loaded early with respect to other packages. In -particular, it must be loaded before \PGF library |fadings| (see -section~\ref{sec:fadings-problem}) and before the \pkg{beamer} class (see -section~\ref{sec:tut:beamer}). - -If you're familiar with the \TikZ externalization library, you might wonder -whether Memoize has an equivalent of the \refcmd{tikzexternalize} command. It -doesn't. Memoize assumes that if you loaded it, you want to use it --- but you -can always disable it using key \refmmz{disable}, or even by loading package -\refpkg{nomemoize} in its stead. - -In \hologo{LaTeX}, initialization and finalization are completely automatic. -Memoize defines several initialization and finalization styles --- -\docaux{key}{begindocument/before}, \docaux{key}{begindocument}, -\docaux{key}{begindocument/end} and \docaux{key}{enddocument/afterlastpage} ----and executes them at the cognominal \hologo{LaTeX} hooks. - -\paragraph{\hologo{plainTeX}}{} - -Load Memoize by |\input memoize|. As package options cannot be provided in -\hologo{plainTeX}, the author must trigger extraction from \refcmd{mmzset} -using key \refmmz{extract}; I recommend doing this immediately after loading -the package. This key may be invoked with or without a value. In the latter -case, Memoize will extract using the package default method -\refmmz{extract=perl}, unless its has been overriden from -\reffile{memoize.cfg}. - -Furthermore, as \hologo{plainTeX} has no concept of a document body, the text -must be manually enclosed in \refcmd{mmzset}\bracestt{\docaux{key}{begin - document}} and \refcmd{mmzset}\bracestt{\docaux{key}{end document}}; this -is where the initialization and finalization hooks described above will be -executed. Note that \refmmz{extract}, when used, must precede this enclosure. - -Finally, \hologo{plainTeX} has another reason for preferring the early loading -of the package. In this format, Memoize must redefine \refcmd{shipout}, at a -time the meaning of this control sequence is still primitive. In particular, -this means that Memoize must be loaded before \pkg{atbegshi}. - -\begin{tcboxedraster}[raster column skip=1cm]{blankest} - \ExampleName{plainTeX}% - \makeexample{\examplename.tex.c1}% - \tcbinputexample{listing only, title=A minimal \hologo{plainTeX} example} - % - \ExampleName{ConTeXt}% - \makeexample{\examplename.tex.c1}% - \tcbinputexample{listing only, title=A minimal \hologo{ConTeXt} example} -\end{tcboxedraster} - -\paragraph{\hologo{ConTeXt}}{} - -Load Memoize by \cs{usemodule}\bracketstt{memoize} or -\cs{usemodule}\bracketstt{memoize}\oarg{options}. Unlike in \hologo{LaTeX}, -spaces are allowed in \meta{options}; the remarks on key \refmmz{extract} and -loading order are the same as for \hologo{LaTeX}. - -In \hologo{LaTeX}, Memoize automatically executes its initialization and -finalization code when at the beginning and the end of the document body. Due -to my very limited experience with \hologo{ConTeXt}, and its project structure -in particular, I don't know what the appropriate place for initialization and -finalization would be in \hologo{ConTeXt}. I therefore provisionally leave it -to the author to execute \refcmd{mmzset}\bracestt{\refmmz{begin document}} and -\refcmd{mmzset}\bracestt{\refmmz{end document}} manually, and hope for advice -on how to handle this properly. - - -\paragraph{Auxiliary packages} - -\begin{doc}{package={name=nomemoize}} - Loading this package instead of Memoize completely disables memoization, but - does not require removing any Memoize commands from the document (they all - become no-ops). -\end{doc} - -\begin{doc}{package={name=memoizable}} - This package is a programmer's stub: if Memoize is loaded, it does nothing; - otherwise, it provides the no-op variants Memoize commands. See - section~\ref{sec:loading-memoize} for details. -\end{doc} - - -\subsection{Configuration} -\label{sec:ref:configuration} - - -\begin{doc}{easy,cmd={name=mmzset, par=\marg{options}}} - - Update the Memoize configuration. - - The \meta{options} are a comma-separated list of \meta{key}|=|\meta{value} - pairs. They are processed using the \pkg{pgfkeys} utility of PGF/\TikZ (see - \PGFmanual{87}), with the default path set to \docaux{key path}{mmz}. - - The changes are local to the current \hologo{TeX} group, except for keys - where explicitly noted otherwise. -\end{doc} - -\begin{doc}{easy, - cmd={name=mmznext, par=\marg{options}}, - } - - This command accepts the same \meta{options} as \refcmd{mmzset}, but - interprets them as \emph{next-options} --- options which will be applied to - the next, and only the next, \emph{auto}memoized command or environment. - (Remember that a command or environment is submitted to automemoization by - \refmmz{auto}|=|\marg{command or - environment}\bracestt{\refmmzauto{memoize},...}; see - sections~\ref{sec:tut:automemoization}, \ref{sec:tut:working-on-a-picture} - and \ref{sec:ref:advicememoization} for details.) - - Remarks for the author: - - \begin{itemize} - \item Key \refmmz{enable} has no effect inside \refcmd{mmznext}. - \item If \refcmd{mmznext} is used more than once preceding an automemoized - command, only the final invocation takes effect. - \item The next-options also apply to commands and environments for which - memoization is autodisabled via \refmmz{auto}|=|\marg{command or - environment}\bracestt{\refmmzauto{nomemoize},...}. - \item It is safe to set the next-options in front of a command submitted to - automemoization which does not actually undergo memoization in this - particular instance. In other words, the absence of memoization will not - cause the next-options to ``leak'' to the next automemoized command. - \item Only the linear (execution) order of \refcmd{mmznext} and the - automemoized command matters. Key \refcmd{mmznext} will correctly apply to - a single following automemoized command even if it occurs outside the group - which that command is executed from; and it will apply to the following - automemoized command even if it is called within a group closed before that - command is executed. - \end{itemize} - - Remarks for the programmer: - \begin{itemize} - \item The next-options are set globally. - \item The effect of \refcmd{mmznext} is \emph{not} cumulative. Consequently, - \refcmd{mmznext}\braces{} clears the next-options. - \item The next-options are applied by executing \refcmd{mmzAutoInit} within - the advice. Any piece of advice applying the next options should also clear them - when the \refmmzauto{run conditions} are not met. This is streamlined by - style \refmmzauto{apply options}, intended for use within \refmmz{auto} - declarations. Out of the box, this style is deployed by - \refmmzauto{memoize}, \refmmzauto{nomemoize} and \refmmzauto{noop}, but it - may be used by any piece of advice. Note that the \refmmzauto{outer handler} - declared by this style opens a group (to apply the options in) but leaves - it to the (undeclared) \refmmzauto{inner handler} to close that group. - \item Key \refmmz{enable} has no effect inside \refcmd{mmznext} because when - Memoize is disabled upon encountering an automemoized command, the advice - bails out without ever applying the next-options. More generally, this - applies to any advised command whose run conditions require - \refcmd{ifmemoize} to be true. Key \refmmz{disable}, on the other hand, - takes effect, because \refcmd{ifmemoize} is checked within \refcmd{Memoize} - as well. - \end{itemize} -\end{doc} - -\begin{doc}{easy,file={name=memoize.cfg, desc=file}} - The configuration file, loaded just before processing the package options. - It will typically contain a \refcmd{mmzset} command, but it may contain any - \hologo{TeX} code. - - As for any other file loaded by \hologo{TeX}, the location of the file - determines whether it applies system-wide, user-wide or directory-wide. - - This file is also loaded by package \refpkg{nomemoize}, on the off chance it - defines some commands other than \refkeypath{/mmz} keys. (It is not loaded - by package \refpkg{memoizable}, though). -\end{doc} - -\begin{doc}{cmd={name=nommzkeys, par=\marg{key}}} - In package \refpkg{memoize}, this command is a no-op; in packages - \refpkg{nomemoize} and \refpkg{memoizable}, it defines key - \refkeypath{/mmz}|/|\meta{key} as a no-op. - - As explained in section~\ref{sec:tut:final}, use this command to declare any - \refkeypath{/mmz} keys you have used outside \refcmd{mmzset} when switching - to package \refpkg{nomemoize}. -\end{doc} - - -\subsection{Memoization} -\label{sec:ref:memoization} - -\subsubsection{Manual memoization commands} -\label{sec:ref:memoization:manual} - -\begin{doc}{easy, - cmd={name=mmz, par=\oarg{options}\marg{code}}, - env={name=memoize, par=\oarg{options}}, - } - Submit \meta{code} or \meta{environment body} to memoization. - - Prior to memoization, the configuration is locally updated by executing - \meta{options} given as the optional argument to this command, i.e.\ the - given options take precedence to options previously set via \refcmd{mmzset}. - Note that next-options, set by \refcmd{mmznext}, are not applied. - - The effect of the macro and of the environment version of this command is the - same, except that the command version memoizes \meta{code} exactly as-is, - while the environment version trims away any spaces at the beginning and the - end of the code.\footnote{What actually happens is that at the beginning of - the environment body, all space tokens will be discarded. At the end of - the body, no spaces are actually discarded; Memoize simply issues an - \refcmd{unskip}. This should not matter to a regular user who simply - writes down the environment.} The space-trimming feature of the environment - ensures that you can write |\begin{memoize}| and |\end{memoize}| in separate - lines (as shown above), but no extra space will creep into the extern. - - The space-trimming feature of the environment, which trims spaces at the - beginning and at the end of the \meta{environment body}, should not be - confused with the effect of \refmmz{ignore spaces}, which ignores spaces - following the environment end-tag (in \hologo{LaTeX}, |\end{...}|) --- and - which does not apply to manual memoization at all! - - The argument of \refcmd{mmz} \emph{must} be enclosed in braces. -\end{doc} - -\begin{doc}{easy, - cmd={name=nommz, par=\oarg{options}\marg{code}}, - env={name=nomemoize, par=\oarg{options}}, - } - Disable Memoize for the span of the compilation of \meta{code} or - \meta{environment body}. - - This command consumes the \meta{options} in the same way as - \refcmd{mmz}\slash \refenv{memoize} described above. The macro and the - environment version of the command exhibit the same space-trimming behaviour - as their \refcmd{mmz}\slash \refenv{memoize} counterparts, and the argument - of \refcmd{nommz} \emph{must} be enclosed in braces. -\end{doc} - - -\subsubsection{Basic configuration} -\label{sec:ref:memoization:basic} - -\begin{doc}{easy, - key={name=enable}, - key={name=disable}, - } - Enable or disable the functionality of the package. - - What happens when Memoize is enabled depends on the memoization mode - (\refmmz{normal}, \refmmz{readonly}, \refmmz{recompile}) and many other - factors. When the package is disabled, it neither creates new memos and - externs, nor uses the existing ones; this applies to both manual and - automatic memoization. The effect is close to not having Memoize loaded, or - to loading NoMemoize, but it is not completely the same; for example, the - record file (\dmmz) \emph{is} updated while Memoize is disabled, reflecting - the fact that nothing was memoized (or utilized) in the disabled - state.\footnote{Cleaning the folder (\S\ref{sec:cleanup-scripts}) after - disabling the package for the entire document is thus a bad idea.} - - If these keys are used in the preamble, their effect is delayed until the - beginning of the document, to ensure that Memoize is never enabled in the - preamble. Other than that, all these keys do is set the \hologo{TeX} - conditional \refcmd{ifmemoize}, which you may use in your code to test - whether Memoize is enabled. You may also use \refcmd{memoizetrue} and - \refcmd{memoizefalse}, as long as you never enable the package in the - preamble. - - Key \refmmz{enable} cannot be applied to automemoized commands via - \refcmd{mmznext}. It will take effect for manual memoization, though, and - key \refmmz{disable} will work for both, as expected. -\end{doc} - -\begin{doc}{easy, - key={name=normal, desc=the default mode}, - key={name=readonly}, - key={name=recompile}, - } - - Select the memoization mode. - - Each piece of code submitted to (either manual or automatic) memoization is - associated to several files: one c-memo, one cc-memo, and some externs (zero - or more, typically one). When Memoize encounters a piece of code submitted - to memoization, it takes one of the following actions: - - \begin{description} - \item[memoization] The code is compiled in a special way which produces the - associated memos and externs. - \item[utilization] The code is not compiled. Its effect is replicated by - processing the cc-memo; typically, this includes the single extern into the - document. - \item[regular compilation] The code is compiled as if Memoize was absent or - disabled (the memos and externs are neither utilized nor produced). - \end{description} - - \begin{minipage}[t]{0.48\linewidth} - The action taken depends on the memoization mode and on whether all the - memos and externs associated with the code exist, as shown in the table on - the right. Note that a single missing memo or extern implies the ``no'' - column of the table, and that memoization will create \emph{all} the - associated memos and externs, even those which already exist. - \end{minipage} - \hfill - \begin{tabular}[t]{@{}lll@{}} - \toprule - &\multicolumn{2}{c}{Do the memos and externs exist?}\\ - \cmidrule{2-3} - mode&yes&no\\ - \midrule - normal&utilization&memoization\\ - readonly&utilization®ular compilation\\ - recompile&memoization&memoization\\ - \bottomrule - \end{tabular} - - The memoization mode only has effect when Memoize is \refmmz{enable}d. Mode - selection is orthogonal to enabling\slash disabling the package; for example, - if you switch to a new mode while the package is disabled, the new mode will - be in effect once the package is enabled. -\end{doc} - - -\begin{doc}{easy, - desc={style}, - key={name=verbatim}, - key={name=verb}, - key={name=no verbatim}, - } - When \refmmz{verbatim} or \refmmz{verb} is in effect, the code submitted to - memoization is read verbatim; \refmmz{no verbatim} reverts to the normal, - non-verbatim collection of the code. This applies to both manual and - automatic memoization. - - The long version, \refmmz{verbatim}, switches to the full verbatim mode, - where all characters are assigned category code 12 (other). With the short - version, \refmmz{verb}, the braces, |{| and |}|, retain category codes 1 and - 2, which can be useful for verbatim collection of optional arguments. For - details, see the documentation of CollArgs' \refcollargs{verbatim} in - section~\ref{sec:ref:collargs}. - - Under the hood, these keys have two effects. First, they are passed on to the - argument collector (typically, \refcmd{CollectArguments} of the auxiliary - package CollArgs; for details, see section~\ref{sec:ref:advicememoization}), - instructing it to collect the code in the specified fashion, as described - above. Second, if the collected verbatim code is eventually compiled (either - regularly, or memoized), Memoize first rescans it using \refcmd{scantokens}. -\end{doc} - -\begin{doc}{easy, - key={name=padding left, par=\meta{dimension}, - desc={no default, initially |1| |in|}}, - key={name=padding right, par=\meta{dimension}, - desc={no default, initially |1| |in|}}, - key={name=padding top, par=\meta{dimension}, - desc={no default, initially |1| |in|}}, - key={name=padding bottom, par=\meta{dimension}, - desc={no default, initially |1| |in|}}, - } - Set the left/right/top/bottom padding of the extern in the extern PDF. - - Without padding, the (PDF) page holding the extern would tightly fit the - bounding box of the extern. These keys enlarge the extern page by the given - amounts, so that any parts of the extern lying outside the bounding box will - be correctly included when using the extern. See - section~\ref{sec:tut:padding} for details. - - \meta{dimension} is evaluated with \hologo{eTeX}'s |\dimexpr|, and may - contain control sequences \docaux{cmd}{width}, \docaux{cmd}{height} and - \docaux{cmd}{depth}, which will refer to the dimensions of the extern. - \refcmd{width} and friends behave like dimension registers, so it is ok to write - e.g.\ \refmmz{padding right}|=0.5\width|. - - The default padding is what \hologo{pdfTeX} puts around the page anyway, 1 - inch, but we use |1 in| rather than |1 true in|, which is the true default - value of PDF registers \docref{pdfvar:horigin} and \docref{pdfvar:vorigin}, - as we want the padding to adjust with \refcmd{mag}nification. -\end{doc} - -\begin{doc}{easy,key={name=padding, par=\meta{dimension}, desc={style, no default}}} - Set all the above |padding| keys to the given value. -\end{doc} - -\begin{doc}{ - easy, - key={name=context, par=\meta{tokens}, desc={cumulative, no default, - initially set by \refmmz{padding to context}}}, - key={name=clear context}, - } - These keys append the given \meta{tokens} to the \emph{context expression}, - or clear this expression. Memoized code gets recompiled whenever the - expansion of the context expression changes. - - The \meta{tokens} must be fully expandable (modulo protection); in - \hologo{LaTeX}, they will be expanded by |\protected@edef| when calculating - the md5sum of the context. - - The context expression is evaluated at the end of the memoization, and at - utilization attempts after inputting the c-memo (note that the c-memo - contains any additions to the context expression accumulated during - memoization). At evaluation, the given context expression is fully expanded, - yielding the \emph{context value}, whose md5 sum forms the \meta{context md5 - sum} part of the filename of the cc-memo and the extern. - - These keys may be used both prior to the memoization process, or during - memoization. In the former case, their effect is local to the current group; - in the latter case, the effect is global, so that the changes surely survive - until the end of memoization, when the c-memo, where the context expression - is stored, is written into a file. - - Under the hood, these keys manipulate token registers - \docaux{cmd}{mmzContext} and \docaux{cmd}{mmzContextExtra}, changing the - contents of the former while not memoizing, and the contents of the latter - during memoization. These token registers may also be manipulated directly - by the user, as long as one keeps to the convention of adjusting - \refcmd{mmzContext} locally and only while not memoizing, and adjusting - \refcmd{mmzContextExtra} globally and only during memoization. -\end{doc} - -\begin{doc}{ - key={easy, name=meaning to context, - par=\marg{comma-separated list of commands and environment names}}, - key={name=csname meaning to context, par=\marg{control sequence name}}, - key={name=key meaning to context, - par=\marg{full path to a pgfkeys command key}}, - key={name=key value to context, - par=\marg{full path to a pgfkeys value key}}, - } - These keys append the definition of the given construct to the context. - - Essentially, the ``meaning'' keys append \cs{meaning}\meta{control sequence} - to the context, for the appropriate \meta{control sequence}. For example, - \refmmz{meaning to context} appends \cs{meaning}\cs{foo} when given \cs{foo} - as in item in the list, and it appends the internal environment macros - appropriate to the format when given an environment name. Similarly, - \refmmz{key meaning to context} resolves \meta{control sequence} to the - internal macro holding the key command. - - Key \refmmz{key value to context} should be used on keys which store values, - e.g.\ keys initialized by \pkg{pgfkeys} handler \code{.initial}; see - \PGFmanual{87.3.4 and \S87.4.5}. - - All the keys prefix the meaning\slash value by the name of the command\slash - environment\slash key, in order to prevent ambiguous contexts, see - section~\ref{sec:cross-referencing} for details. Furthermore, they all - operate through \code{\cs{csname}...\cs{endcsname}} construct, allowing one - to safely add internal commands to the context. -\end{doc} - -\begin{doc}{ - keypath={/handlers}, - key={easy, name=.meaning to context}, - key={name=.value to context}, - } - These are the handler variants of \refmmz{key meaning to context} and - \refmmz{key value to context}. -\end{doc} - - -\begin{doc}{key={name=padding to context,desc=style}} - This key appends the values of the \refmmz{padding} keys to the context, - causing the memoized code to be recompiled whenever the padding values are - changed. This key is used to initialize \refmmz{context}, so the author - normally shouldn't have to use this key. -\end{doc} - -\begin{doc}{ - cmd={name=mmzNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only}, - cmd={name=mmzForceNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only}, - } - These macros append the current value of the \meta{reference key} to the - context,\footnote{More precisely, it is \meta{reference key}|=|\marg{current - value} which is appended.} causing the memoized code to be recompiled - when the reference changes. - - If the reference key is undefined, \refcmd{mmzNoRef} aborts memoization, - while \refcmd{mmzForceNoRef} uses \cs{relax} as the reference string. - - These commands are deployed in the implementation of \refmmzauto[show - keypath]{ref}, \refmmzauto{force ref} and friends. If the - cross-referencing commands you are using are advised by these keys, you most - likely have no need of these macros. -\end{doc} - -\begin{doc}{easy, key={name=per overlay,desc=style}} - This key is only defined in the Beamer class. When in effect, the memoized - code will produce a cc-memo (and extern) for each overlay of the frame. For - implementation, see section~\ref{sec:per-overlay}. -\end{doc} - -\begin{doc}{ - key={name=capture, par=\docAux{of=key:/mmz/capture, value={name=hbox}, text=\alt, - value={name=vbox,of=key:/mmz/capture}}, - desc={no default, initially \refmmz{capture=hbox}}} - } - Select the capture mode. This setting only applies to the default - memoization \refmmz{driver}. - - By default, it is assumed that the memoized code should be executed in the - horizontal mode, so the default memoization driver captures the output of the - memoized code in a \refcmd{hbox} --- and also issues a \refcmd{quitvmode} - (both in the document and in the cc-memo), just in case the memoized code - occurs at the start of the paragraph. - - Use \refmmz{capture}|=|\refmmz{capture=vbox} to execute the memoized code in - the vertical mode: Memoize will capture the output of the memoized code in a - \refcmd{vbox}, and avoid issuing \refcmd{quitvmode}. For example, this - capture mode is necessary to memoize a \refenv{verbatim} environment. -\end{doc} - - -\subsubsection{Inside the memoization process} -\label{sec:ref:memoization:process} - -\begin{doc}{easy, cmd={name=mmzAbort}} - This command aborts the ongoing memoization. - - The memoization will proceed as usual (i.e.\ the extern boxes and the cc-memo - code will be produced), but at the end of this process, no memos will be - produced, no externs shipped out to extern pages, and no record files - updated. -\end{doc} - -\begin{doc}{easy, cmd={name=mmzUnmemoizable}} - This command aborts the ongoing memoization, and marks the submitted code as - unmemoizable. - - The ongoing memoization produces a c-memo setting conditional - \docaux{cmd}{ifmmzUnmemoizable} to true. Upon utilizing this c-memo, the - system switches to regular compilation. - - For example, if you are automemoizing |tcolorbox|es of package - \pkg{tcolorbox}, you will want to refrain from memoizing boxes marked as - |breakable| or |float|ing. Simply aborting the memoization cannot do the - trick here, as memoization compiles the submitted code in a \hologo{TeX} box. - Marking a |breakable| or |float|ing |tcolorbox| as unmemoizable (either - manually using this macro, or automatically using \refmmz{auto key}) makes - sure that after the first compilation when memoization is attempted, the box - will be compiled regularly, and will have the intended ability to break - across pages, or float. -\end{doc} - - -\begin{doc}{ - cmd={name=Memoize, par=\marg{key}\marg{code}}, - } - Depending on various factors, this command either memoizes \meta{code} under - key \meta{key}, utilizes the results of a previous memoization, or performs a - regular compilation of \meta{code}. - - The outcome of executing this command --- memoization, utilization or regular - compilation --- depends upon the Memoize's state (\refcmd{ifmemoize}, - \refcmd{ifmemoizing}) and mode (\refmmz{normal}, \refmmz{readonly}, - \refmmz{recompile}), and the existence of the relevant memos and externs. - The decision process is depicted in section~\ref{sec:Memoize}. - - This command expects to be executed in a dedicated group, which it will close - itself. - - Invoking memoization through \refcmd{Memoize} might be useful for packages - which want to save the results of intensive computations, regardless of - whether the author loads (and enables) memoization or not. However, this - usage is not yet officially allowed, because there is currently no way to - load the core memoization routines without loading the entire package, - thereby forcing the author to use Memoize. -\end{doc} - -\begin{doc}{ - key={ - name=driver, par=\marg{code}, - desc={no default, initially \refcmd{mmzSingleExternDriver}} - }, - } - This key sets \meta{code} as the memoization driver. - - Given some code submitted to memoization, the memoization driver should - produce the memos and externs which will replicate the effect of that code - (while retaining its regular effect). For details, see - section~\ref{sec:memoization-drivers}. - - Typically, the \meta{code} argument of this key will consist of a single - control sequence (the driver control sequence), but any amount of tokens is - allowed. Memoize executes the driver followed by the code which it is - supposed to memoize, in braces, and only cares that the driver consumes that - code. -\end{doc} - - -\begin{doc}{ - key={name=at begin memoization, par=\meta{code}, desc={cumulative, initially empty}}, - key={name=at end memoization, par=\meta{code}, desc={cumulative, initially empty}}, - key={name=after memoization, par=\meta{code}, desc={cumulative, initially empty}}, - } - Use these keys to set up memoization hooks. - - These keys may be used both prior to the memoization process, or during - memoization. In the former case, their effect is local to the current group; - in the latter case, the code given to \refmmz{at begin memoization} is - executed immediately, while the assignment performed by the other two keys is - global, so that the changes surely survive until the end of memoization. - - The code given to hook \refmmz{at begin memoization} is kept in macro - \docaux{cmd}{mmzAtBeginMemoization}, while the content of the other two hooks - resides in two macros per hook: \docaux{cmd}{mmzAtEndMemoization} and - \docaux{cmd}{mmzAtEndMemoizationExtra}, and \docaux{cmd}{mmzAfterMemoization} - and \docaux{cmd}{mmzAfterMemoizationExtra}. All these macros may be - manipulated directly by the user,\footnote{Use \cs{appto}, \cs{eappto}, - \cs{gappto} and \cs{xappto} of package \pkg{etoolbox} (loaded by Memoize) - to easily append code to these macros.} as long as one keeps to the - convention of adjusting the macros without ``\texttt{Extra}'' locally and - only while not memoizing, and adjusting the macros with ``\texttt{Extra}'' - globally and only during memoization. The ``\texttt{Extra}'' macros require - global assignments as they might be manipulated by code residing within a - \hologo{TeX} group of any depth. - - These complications explained, let us take a look at how memoization proceeds - to learn when the hooks are used: - \begin{enumerate} - \item Initialize various conditionals, macros and token registers. (Here is - where the ``\texttt{Extra}'' hooks are cleared.) Remember that at this - point, we're inside a group opened by \refcmd{Memoize}. - \item Execute \refmmz{at begin memoization} hook, i.e.\ the contents of - macro \refcmd{mmzAtBeginMemoization}. - \item Execute the memoization \refmmz{driver}. - \item Execute \refmmz{at end memoization} hook, i.e.\ the contents of macros - \refcmd{mmzAtEndMemoization} and \refcmd{mmzAtEndMemoizationExtra} (in this - order). - \item Write out the memos and ship out the externs to extern pages (unless - memoization was aborted). - \item Close the memoization group. - \item Execute \refmmz{after memoization} hook, i.e.\ the contents of - macros \refcmd{mmzAfterMemoization} and \refcmd{mmzAfterMemoizationExtra} - (in this order). - \end{enumerate} -\end{doc} - -\begin{doc}{ - cmd={name=mmzCMemo, desc={token register, global, empty at the start of memoization}}, - } - This token register mediates the construction of the c-memo. During - memoization (and only during memoization), arbitrary code may be added to - this register; at the end of memoization, Memoize writes out its contents to - the free-form part of the c-memo. - - All assignments to this register should be global. Use \refcmd{gtoksapp} and - \refcmd{xtoksapp} to easily append tokens to the register. -\end{doc} - - -\begin{doc}{ - cmd={name=mmzCCMemo, desc={token register, global, empty at the start of memoization}}, - } - This token register mediates the construction of the cc-memo. During - memoization, the memoization driver should append cc-memo code to - \refcmd{mmzCCMemo}; at the end of memoization, Memoize writes out its - contents to the cc-memo (preceded by the list of produced externs). - - All assignments to this register should be global. Local assignments would - not work, because the memoized code may contain commands, like \cs{label} and - \cs{ref}, which contribute content to cc-memo as well, but these commands may - appear within a \hologo{TeX} group of any depth. - - Use \refcmd{gtoksapp} and \refcmd{xtoksapp} to easily append tokens to the - register. -\end{doc} - -\begin{doc}{ - par=\meta{token register}\marg{tokens}, - cmd={name=toksapp}, - cmd={name=gtoksapp}, - cmd={name=etoksapp}, - cmd={name=xtoksapp}, - } - These commands append the given \meta{tokens} to the \meta{token register}. - \refcmd{etoksapp} and \refcmd{xtoksapp} expand the \meta{tokens} before - appending them; \refcmd{gtoksapp} and \refcmd{xtoksapp} perform a global - assignment. - - These commands are actually provided by CollArgs, and they are defined only - if they don't already exist; in particular, note that \hologo{LuaTeX} - provides them as primitives. - - Unlike the \hologo{LuaTeX} primitive variant, these commands require the - \meta{token register} to be given by a (|\toksdef|fed) control sequence; it - cannot be given as |\toks|\meta{number}. -\end{doc} - -\begin{doc}{ - cmd={name=ifmemoize}, - cmd={name=memoizetrue}, - cmd={name=memoizefalse}, - } - Use the \hologo{TeX}-style conditional \refcmd{ifmemoize} to test whether - Memoize is currently enabled. Within the document body, the conditional may - be set using \refcmd{memoizetrue} and \refcmd{memoizefalse}, which are then - functionally equivalent to \refmmz{enable} and \refmmz{disable}. Do not set - the conditional in the preamble of the document (unless you really know what - you are doing). -\end{doc} - -\begin{doc}{ - cmd={name=ifmemoizing,desc={readonly}}, - } - Use this \hologo{TeX}-style conditional to test whether Memoize is currently - memoizing. It may be only inspected; you should \emph{never} set this - conditional yourself. -\end{doc} - -\begin{doc}{ - cmd={name=ifinmemoize,desc={readonly}}, - } - Use this \hologo{TeX}-style conditional to test whether Memoize is currently - active, in the sense of either memoizing or regularly compiling some code --- - so inside a call to \refcmd{Memoize}. The conditional may be only inspected; - you should \emph{never} set it yourself. -\end{doc} - -\begin{doc}{ - cmd={name=mmzSingleExternDriver, par=\marg{code}}, - } - This is the default memoization \refmmz{driver}, producing exactly one extern - containing whatever is typeset by the submitted \meta{code}. - - The \meta{code} is compiled either within a horizontal or vertical box, - depending on the value of key \refmmz{capture}. In the case of a horizontal - capture, the driver makes sure that the horizontal mode is entered prior to - both typesetting the resulting box in the document, or utilizing the extern. - - For the implementational details, see - section~\ref{sec:memoization-driver-default}. -\end{doc} - -\begin{doc}{ - cmd={name=mmzExternalizeBox, par=\marg{box}\marg{token register}}, - } - This macro is indended to be called by memoization drivers to produce an - extern page. The given \meta{box} is dumped into the document as a separate - extern page, while the \meta{token register} receives the cc-memo extern - inclusion code. - - The \meta{box} may be given either as a control sequence (declared via - |\newbox|), or as box number. The resulting extern page will contain a - \emph{copy} of the given box, padded by the \refmmz{padding} values in effect - at the time of invocation of \refcmd{mmzExternalizeBox}. - - An implementation detail is that \refcmd{mmzExternalizeBox} does not ship out - the extern page immediately. This action is delayed until the end of the - memoization process; more precisely, it is carried out (in tandem with - writing out the c-memo and the cc-memo) between execution of hooks \refmmz{at - end memoization} and \refmmz{after memoization}. This delay guarantees - that no extern pages are produced in the event of aborting memoization, even - if the abortion is triggered after executing \refcmd{mmzExternalizeBox}. - - The \meta{token register} may be given either as a control sequence (declared - via |\newtoks|) or as control sequence |\toks| followed by the register - number. The register will receive the code, which, when executed from the - cc-memo, includes the extern file into the main document. This code consists - of a single invocation of \refcmd{mmzIncludeExtern}. It is the - responsibility of the driver to include the code received by \meta{token - register} in the register \refcmd{mmzMemo}, whose contents are, unless - memoization is aborted, written into the cc-memo. (See - \refcmd{ifmmzkeepexterns} and \refmmz{after memoization} to learn about - another way to use the code received by \meta{token register}.) - - The invocation of \refcmd{mmzIncludeExtern} in the produced extern-inclusion - code is adapted to the type of the box (horizontal or vertical), which is - detected automatically --- the memoization driver does not need to inform - \refcmd{mmzExternalizeBox} about this type explicitly.\footnote{This does not - negate the need for key \refmmz{capture}, which applies to the default --- - and therefore generic --- memoization driver. This driver cannot know - whether the memoized code would prefer to be compiled in a horizontal or - vertical box. It is precisely key \refmmz{capture} which gives the user an - opportunity to inform Memoize about this preference. Only once the - memoized code is compiled into a box of the appropriate type, it is trivial - to detect the type of that box.} -\end{doc} - -\begin{doc}{ - cmd={name=ifmmzkeepexterns, desc={initially \cs{iffalse}}}, - cmd={name=mmzkeepexternstrue}, - cmd={name=mmzkeepexternsfalse}, - } - Setting this conditional to true makes Memoize keep the extern boxes in the - global temporary storage even after shipping them out as extern pages. (The - temporary storage is emptied at the start of the next memoization.) - - The extern inclusion code received by the \refcmd{mmzCCMemo} when executing - \refcmd{mmzExternalizeBox} is primarily meant to be executed by inputting the - cc-memo file; i.e.\ when the cc-memo is input, \refcmd{mmzIncludeExtern} is - defined to include the extern file into the document. However, it sometimes - makes sense to execute the cc-memo contents immediately after memoization; - for example, if memoization produces several externs, intricately integrated - into the surrouding environment, it might be cumbersome to replicate their - typesetting both in the memoizing compilation and in the cc-memo code --- - easier to build up the cc-memo code and execute it right after memoization. - This is why Memoize, just before executing the contents of \refmmz{after - memoization} hook, redefines \refcmd{mmzIncludeExtern} to include externs - from the temporary storage rather than from (at that point still - non-existing) extern files. However, as this mechanism requires Memoize to - keep the externs around even after memoization, it is not enabled by default: - it must be enabled by setting conditional \refcmd{ifmmzkeepexterns} to true. -\end{doc} - -\begin{doc}{ - key={keypath=/mmz/auto, name=integrated driver, par=\marg{name}} - } - Use this key to easily setup a memoization driver which is integrated into - the command itself. - - \begin{tcolorbox}[warning] - This is an auto-key residing in keypath \refkeypath{/mmz/auto}. - \end{tcolorbox} - - An integrated driver must have a way of telling whether it is memoizing or - regularly compiling the code. This key declares a driver-specific - conditional which may be inspected, using \refcmd{IfMemoizing}, to determine - this. The conditional is set to true by the formal driver of the command - (set up by the invocation of this key), executed at the start of memoization; - it should never be set elsewhere. See - section~\ref{sec:memoization-multiple-externs} for details and an example. -\end{doc} - -\begin{doc}{ - cmd={name=IfMemoizing, par=\oarg{offset}\marg{name}\marg{true code}\marg{false code}} - } - This \hologo{LaTeX}-style conditional is meant to be used by the - \refmmzauto{integrated driver} with the given \meta{name}. It tests whether - this particular driver is currently memoizing some code. - - Potentially recursive commands are supported via the optional argument - \meta{offset}. If given, the conditional will only execute the \meta{true - code} when the current \hologo{TeX} group level matches the \hologo{TeX} - group level at the time of the invocation of the formal driver (held in - \refcmd{memoizinggrouplevel}), plus the \meta{offset}. In effect, the inner - invocation of the integrated driver will perform a regular compilation. For - details, see section~\ref{sec:memoization-complex-single-driver}. -\end{doc} - -\begin{doc}{ - cmd={name=memoizinggrouplevel,desc={readonly}}, - } - During memoization, this macro holds the \hologo{TeX} group level in effect - at the start of the memoization. -\end{doc} - -\begin{doc}{ - cmd={name=mmzRegularPages, desc=readonly counter}, - } - This counter holds the number of pages shipped out (so far) by the format's - regular shipout routine. \emph{Do not change its value!} - - In \hologo{LaTeX}, this counter is synonymous with - \refcmd{ReadonlyShipoutCounter}, and in \hologo{ConTeXt}, it is synonymous - with \refcmd{realpageno}. Memoize does not touch its value. - - In \hologo{plainTeX}, Memoize hijacks the \refcmd{shipout} control sequence - to count (and only to count) regular shipouts. In order for its value to be - realistic, Memoize should be loaded before other packages which hack - \cs{shipout} --- in particular, before \pkg{atbegshi}. -\end{doc} - -\begin{doc}{ - cmd={name=mmzExternPages, desc=readonly counter}, - } - This counter holds the number of extern pages Memoize has shipped out (so - far). \emph{Do not change its value!} - - A third-party tool may inspect this counter to have a realistic count of - shipped-out pages. -\end{doc} - -\begin{doc}{ - cmd={name=mmzExtraPages, desc=counter}, - } - This counter holds the number of pages shipped out (so far) in a way not - tracked by either \refcmd{mmzRegularPages} or \refcmd{mmzExternPages}. It - \emph{should} be advanced by any code which performs such shipouts, or - Memoize won't work correctly. -\end{doc} - - -\subsubsection{Tracing} -\label{sec:ref:memoization:tracing} - -\begin{doc}{ - key={name=trace, conditional=false}, - } - When tracing is on, Memoize shows information about its decision processes on - the terminal. You can learn whether the memoized code is being memoized, - utilized or regularly compiled; find out the md5sum of the code and which - input line it comes from; etc. - - This key has the syntax of a conditional, but there is no underlying - \hologo{TeX} conditional. The low-level interface for switching the tracing - on and off consists of macros \docaux{cmd}{mmzTracingOn} and - \docaux{cmd}{mmzTracingOff}. - - To learn about tracing the ``auto'' part of the automemoization process, also - \refcmd{AdviceTracingOn}. -\end{doc} - -\begin{doc}{ - key={name=include source in cmemo, conditional=true}, - } - - As a courtesy towards a curious user and a debugging aid, Memoize can include - a copy of the memo source in the c-memo. This feature is switched on by - default, but as the package itself never uses that information, it can be - safely switched off at any time. -\end{doc} - -\begin{doc}{ - key={name=include context in ccmemo, conditional=false}, - } - When this key is in effect, the expanded context expression is appended to - the cc-memo, behind the \docaux{cmd}{mmzThisContext} marker. - - Memoize never uses the context information from the cc-memo; this information - is only for tracing purposes. -\end{doc} - -\begin{doc}{ - key={name=direct ccmemo input, conditional=false}, - } - When this key is set to false, a cc-memo is processed indirectly: it is first - read into a token register, and it is the contents of this register which are - executed. When the key is set to true, the cc-memo is simply |\input|ted. - - The indirect execution is implemented to facilitate inverse search. Under - the direct cc-memo input, inverse search pointed at an included extern will - visit the cc-memo, which is not practical; under the indirect regime, the - inverse search will work as expected, and this is why the indirect cc-memo - input is the default. - - The overhead produced by the default indirect input method seems negligible, - but there are other factors which might make the user switch to the direct - input. For one, a cc-memo changing some category codes will require direct - input (no such cc-memos are ever produced out of the box). Less crucially, - sometimes one would like to use the inverse search to figure whether a part - of the document was produced by regular compilation or utilization, and which - memos\slash externs were utilized if the latter. Figuring this out under the - indirect input regime is harder: (i) reading the tracing information shown by - \refmmz{trace} is the surest way to learn what's going on, although (ii) - visual inspection of the externs and (iii) grepping through the |.memo.dir| - folder for particular code often help, as well. - - Both input methods use the same cc-memos; there is no need to - \refmmz{recompile} the memos when switching the cc-memo input method. Note - that the default indirect input method crucially relies on cc-memos ending - with \refcmd{mmzEndMemo}; this macro should not appear in the cc-memo itself. -\end{doc} - - - -\subsubsection{Internal memo commands} -\label{sec:ref:memoization:internal} - -The end-user should never have to use these commands. They are not formally -marked as internal by a |@| in their name only because doing so would -complicate |\input|ting the memos due to the category code changes it would -require. - -\begin{doc}{ - cmd={name=mmzMemo}, - } - This macro marks the beginning of a c-memo and a cc-memo core. Without it, - utilization of a memo will not work. -\end{doc} - -\begin{doc}{cmd={name=mmzSource}} - This macro marks the beginning of the memoized source in the c-memo. That - source is not used by Memoize in any way. It's inclusion into the c-memo may - be switched off by \refmmz{include source in cmemo}|=false|. -\end{doc} - -\begin{doc}{ - cmd={name=mmzResource, par=\marg{filename}}, - } - This is an internal command, which only occurs in a cc-memo. It checks - whether file \meta{filename} exists and is non-empty, and triggers - recompilation of the memoized code if the check fails. -\end{doc} - -\begin{doc}{ - cmd={ - name=mmzIncludeExtern, - par=\marg{seq}\braces{\refcmd{hbox}\alt\refcmd{vbox}}% - \marg{expected width}\marg{expected height}\marg{expected depth}% - \marg{padding left}\marg{padding bottom}\marg{padding right}\marg{padding top}, - } - } - - This is an internal command, which only occurs in a cc-memo. It includes the - extern identified by the sequential number \meta{seq} into the document as a - box of the specified type (horizontal or vertical). The extern is trimmed by - the given padding values. After trimming, the command checks whether the - size of the resulting box matches the given expectations; if it doesn't, a - warning is yielded. - - Before this command is executed, the externs should be listed by a sequence - of \refcmd{mmzResource} commands; \meta{seq} refers to the sequential number - of an extern in this sequence. - - This command may also be executed by executing the entire contents of - \refcmd{mmzCCMemo} after memoization. -\end{doc} - -\begin{doc}{ - cmd={name=mmzLabel, par=\marg{label key}\marg{label value}} - } - This is an internal command written into the cc-memo by the auto-handler of - \refcmd{label}. It temporarily stores \meta{label value} into - \refcmd{@currentlabel} and then executes \cs{label}\marg{label name}. -\end{doc} - -\begin{doc}{ - cmd={name=mmzEndMemo}, - } - This macro marks the end of a cc-memo. It is used to grab the cc-memo core - (everything between \refcmd{mmzMemo} and \refcmd{mmzEndMemo}) under the - indirect cc-memo input regime, i.e.\ when \refmmz{direct ccmemo input} is not - in effect. -\end{doc} - - -\subsection{Location of memos and externs} -\label{sec:ref:dirs} - -\begin{doc}{easy, - key={name=memo dir,par=\meta{name}, desc={style, default \texttt{\string\jobname}}}, - } - A convenient way to store memos and externs in a dedicated directory (and - create this directory in case it does not exist). Without an argument, this - key places these files in subdirectory \meta{document name}|.memo.dir| of the - current directory. See section~\ref{sec:tut:memodir} for the tutorial. - - This key sets \refmmzpath{relative} to |false|, \refmmzpath{dir} to - \meta{name}|.memo.dir|, \refmmzpath{prefix} to empty and \refmmz{mkdir} to - |true|. In effect, memos and externs are placed in the subdirectory - \meta{name}|.memo.dir| of the directory containing the document; their - filenames contain no \meta{name} prefix, as \meta{name} already occurs in the - directory name. The latter feature also makes it easy for projects (or parts - of a project) to share memos and externs; see section~\ref{sec:tut:multifile} - for a typical usage case. -\end{doc} - -\begin{doc}{easy,key={name=no memo dir,desc=style}} - A convenient way to undo the effect of \refmmz{memo dir} and revert to the - initial settings where memos and externs are located in the current - directory. See section~\ref{sec:tut:memodir} for the tutorial. - - This key resets the subkeys of \refmmz{path} to their initial values, and - sets \refmmz{mkdir} to |false|. -\end{doc} - -\begin{doc}{ - key={name=path, par=\meta{keylist}, desc=style}, - } - - Set the \emph{path prefix} to memo and extern files by executing the given - \meta{keylist} in keypath \docaux{key path}{mmz/path}. - - The path prefix contains both the location of the memo\slash extern files - (set by \refmmzpath{relative} and \refmmzpath{dir}) and the initial, - fixed part of their filenames (set by \refmmzpath{prefix}): - \begin{center} - \meta{path prefix}=[|./|]\meta{directory}|/|\meta{prefix} - \end{center} - where ``|./|'' is only present when \refmmzpath{relative} is in effect. - Given the \meta{path prefix}, this is how the paths to memos and externs are - constructed: - \begin{center} - \begin{tabular}{r@{ }l} - c-memo:&\meta{path prefix}\meta{code md5 sum}|.memo|\\ - cc-memo:&\meta{path prefix}\meta{code md5 sum}|-|\meta{context md5 sum}|.memo|\\ - extern:&\meta{path prefix}\meta{code md5 sum}|-|\meta{context md5 sum}-$N$|.pdf|\\ - &(where ``-$N$'' is only present when $N\neq 0$, i.e.\ for non-first externs)\\ - \end{tabular} - \end{center} - - For example, the default \refmmz{no memo dir} \meta{path prefix} equals - \meta{jobname}|.| (note the dot), and after issuing \refmmz{memo dir}, the - \meta{path prefix} is \meta{jobname}|.memo.dir|. Executing - \refmmz{path}|=|\bracestt{\refmmzpath{dir}=project-memo-dir, - \refmmzpath{prefix}=\string\jobname.} would set the memo prefix to - |./project-memo-dir/|\meta{jobname}|.|, allowing the author to keep the memo - files of all documents compiled from the project directory in the same place. - - The following keys may occur in the \meta{keylist}: - \yadocset{keypath=/mmz/path} - - \begin{doc}{ - key={name=relative, conditional=true}, - } - - This key determines whether the location is relative to the current - directory, i.e.\ the directory where \hologo{TeX} is executed from; - usually, this will be the directory where the compiled |.tex| file resides. - When set to |true| (the default), the directory set by \refmmzpath{dir} - is prefixed by ``|./|''. - - You will probably need to set |openout_any=a| in \reffile{texmf.cnf} - to allow writing to an arbitrary directory --- but remember, this might be - dangerous! - \end{doc} - - \begin{doc}{ - key={name=dir, par=\meta{name}, desc={no default, initially empty}}, - } - - Set the \meta{directory} where Memoize will search for memos and externs - and\slash or create them in. - - Unless \refmmzpath{relative} is set to |false|, this location is - relative to the current directory. Given the empty default, memos and - externs are therefore created in the directory holding the source file by - default. - - In principle, the given memo directory must already exist, but see - \refmmz{mkdir}. - \end{doc} - - \begin{doc}{ - key={name=prefix, par=\meta{name}, - desc={no default, initially \texttt{\string\jobname.}}}, - } - - Set the \meta{prefix}, i.e.\ the initial part of the memo\slash extern - filename. Initially, it is set to the document name. - \end{doc} - - After processing the \meta{keylist}, \refmmz{path} records the new path - prefix by invoking \refmmz{record/record type/prefix}, which typically results - in a \refcmd{mmzPrefix} entry in the \dmmz file, and attempts to create the - \meta{directory} is \refmmz{mkdir} is in effect --- except when \refmmz{path} - is executed in the document preamble: then, these actions are only carried - out at the beginning of the document, for the final value of the keys. -\end{doc} - -\begin{doc}{key={name=mkdir, conditional=false}} - When this key is set to true, Memoize will attempt to create the directory - set by \refmmz{path} if it does not yet exist. The directory creation takes - place at the beginning of the document and at every subsequent invocation of - key \refmmz{path}, using the system command specified by \refmmz{mkdir - command}. -\end{doc} - -\begin{doc}{ - key={name=mkdir command, par=\meta{system command invocation}, - desc={no default, initially |mkdir "\#1"|}}, - } - Sets the command used to create \refmmzpath{dir} when \refmmz{mkdir} is in effect. - - This is a system command, so an appropriate shell escape mode must be in - effect to execute it successfully. - - The default should work on Linux, MacOS and Windows. On Linux systems, it - makes sense to change the default (using \reffile{memoize.cfg}) to |mkdir - -p "#1"|, which can create parent folders as necessary. - - Note that extraction methods \refmmz{extract=perl} and - \refmmz{extract=python} set \refmmz{mkdir command} to - \refscript{memoize-extract.pl} \refscript{memoize-extract.pl--mkdir} and - \refscript{memoize-extract.py} \refscript{memoize-extract.pl--mkdir}, - respectively. Unlike system |mkdir|, these commands are safe in the sense - that they conform to the paranoid openout regime (|openout_any=p|). -\end{doc} - -\begin{doc}{ - key={name=output-directory, par=\meta{directory}, - desc={no default, initially undefined, as package option only}}, - cmd={name=mmzOutputDirectory, par=\meta{directory}, desc={pre-defineable}} - } - If the \hologo{TeX} binary was invoked with option |-output-directory|, one - should use this key or define this macro to inform Memoize this, as there is - no way to learn about it automatically. - - The given \meta{directory} must end with a slash (|/|). - - The key is only allowed as a package option. But I imagine that defining the - macro from the command line, prior to inputting the document source, will be - more common: - \begin{center} - |pdftex -output-directory somedir '\def\mmzOutputDirectory{somedir/}\input doc'| - \end{center} -\end{doc} - - -\subsection{Extern extraction} -\label{sec:ref:externs} - -\begin{doc}{easy, - key={name=extract,par=\marg{extraction method}, - desc={preamble-only, initially \refmmz{extract=perl}, default: see below}}, - } - This key selects or executes the extern \meta{extraction method}, i.e. the - method which Memoize will use to extract the extern pages out of the document - PDF. - - Out of the box, Memoize recognizes the following \meta{extraction method} - keywords: \docAux{easy, of=key:/mmz/extract, value={name=perl}, comma, - value={name=python}, comma, value={name=tex}, and, value={name=no}}. The final - keyword (not available in \hologo{plainTeX}) instructs Memoize to \emph{not} - perform to extraction; it should be used when extraction is performed - externally (for details, see section~\ref{sec:setup}). Additional methods - may be installed by defining key \docaux{key - path}{mmz/extract}|/|\meta{extraction method}. - - When invoked from \reffile{memoize.cfg} or used as a package option, this key - \emph{selects} the extraction method. In this case, the key has no default - value, i.e.\ it is illegal to use it without an argument. The method - selected by the package option overrides the method selected in - \reffile{memoize.cfg}, which in turn overrides the package-initial value - \refmmz{extract=perl}. - - In \hologo{LaTeX} and \hologo{ConTeXt}, the selected method is executed at - the end of loading the package. Afterwards, the key is disabled. If you - really need to invoke extraction in the preamble, or again, write - \refmmz{extract}|/|\meta{method}. - - When invoked from a \refcmd{mmzset} in the document preamble, this key - immediately \emph{executes} the given extraction method. In the preamble - incarnation, the key may be invoked without a value to execute the previously - selected method. - - As we want to allow a \hologo{plainTeX} author to override the extraction - method specified by the package (\refmmz{extract=perl}) or in - \reffile{memoize.cfg}, Memoize does \emph{not} perform extern extraction - while loading the package in this format. In \hologo{plainTeX}, internal - extraction can only be triggered by an explicit invocation of - \refmmz{extract} in the ``document preamble'' --- i.e.\ between |\input - memoize| and \refcmd{mmzset}\braces{\refmmz{begin document}}. In this case, - the key does not require an argument; invoking it without the argument will - execute either the package-default, \refmmz{extract=perl}, or whatever the - user had selected in \reffile{memoize.cfg}. - - Executing extraction method \refmmz{extract=perl} or \refmmz{extract=python} - has an additional effect of setting the \refmmz{mkdir command} to the - extraction script with option \refscript{memoize-extract.pl--mkdir}. This - obviates the need to include |mkdir| among the restricted shell commands if - one is using the restricted shell mode. -\end{doc} - -\subsubsection{Perl- and Python-based extraction} -\label{sec:ref:extraction-perl-python} - -\begin{doc}{ - key={name=perl extraction command, par=\meta{system command}, - desc={no default, initially \refscript{memoize-extract.pl}}}, - key={name=perl extraction options, par=\meta{options}, - desc={no default, initially: see below}}, - key={name=python extraction command, par=\meta{system command}, - desc={no default, initially \refscript{memoize-extract.py}}}, - key={name=python extraction options, par=\meta{options}, - desc={no default, initially: see below}}, - } - These keys determine the system calls used for invoking the extraction - scripts \refscript{memoize-extract.pl} and \refscript{memoize-extract.py}. - All the details below apply both to the Perl and the Python version. - - Use |perl|\slash|python extraction command| to set the name of the extraction - script. If necessary, include the full path to the scrip, or |perl|\slash - |python| plus the path to the script. Whatever you set here must be allowed - by the shell escape mode. - - Use |perl|\slash|python extraction options| to set the options that the - script will receive; consult the documentation of - \refscript{memoize-extract.pl} for their meaning. - - The initial value of \refmmz{perl extraction options}, shown below, (i) sets - embedded mode, which prefixes each output line with the script name, (ii) - requests a log file named \meta{jobname}|.mmz.log| (|-l|; |\input|ting the - log after extraction informs Memoize, and the author, whether extraction was - successful), and (iii) sets the warning template suitable for the format - (|-w|; this way, any warning messages issued by the script can be reissued by - the compilation of the document). - - These keys were initialized using \pkg{pgfkeys} handler |.initial|, so their - values may be modified by handlers |.prefix|, |.append|, etc. During the - execution of the system call, their values are fully expanded --- thus the - |\string|s in the initial options value below. The guards - (|%| etc.) make sure that - each format asks for a warning message it understands (the code shown below - is an excerpt from the |.dtx| file; ultimately, each format receives a single - of the three guarded lines). - - \makeexcerpt{perl-extraction-options} - \tcbinputexample[.tex][.excerpt]{listing only, one file, no attachment, - title={The initial value of \refmmz{perl extraction - options}}, full width, left=-2mm, right=0mm, listing options app={ - morecomment={[s]{\%<}>} }, } -\end{doc} - -\begin{doc}{ - script={name=memoize-extract.pl, par=[\meta{options}] \meta{name}\texttt{.mmz}}, - script={name=memoize-extract.py, par=[\meta{options}] \meta{name}\texttt{.mmz}}, - } - These scripts extract the new externs recorded in \meta{name}\dmmz from - \meta{name}|.pdf|. Memoize invokes them when loaded with package option - \refmmz{extract}|=|\refmmz{extract=perl} (the default) or - \refmmz{extract}|=|\refmmz{extract=python}. - - The script inspects the given record file, \meta{name}\dmmz, for lines of - form \refcmd{mmzNewExtern}\marg{extern filename}\marg{extern page - number}\braces{\meta{expected width}\texttt{pt}}\braces{\meta{expected - height}\texttt{pt}}. For each such line, page number - \meta{extern page number} is extracted from \meta{name}|.pdf| into - \meta{extern filename}. Other lines are ignored (and so are commented - invocations of \refcmd{mmzNewExtern}). - - The \meta{extern filename} may contain a (relative or absolute) path to the - new extern file. The relative paths are relative to the location of the - \dmmz file, even when the script is invoked from some other directory. - - To guard against extracting a wrong page, the script checks whether the size - of each extracted page matches the \meta{expected width} and \meta{expected - height}.\footnote{\label{fn:tolerance}To avoid false positives, the match - need not be exact, a difference up to |0.01pt| is tolerated. Some PDF - tools, notably the |PDF::API2| library deployed by the Perl version of the - script, round the dimensions of a PDF page, recorded in |/MediaBox|, to two - digits.} If it does not, the script refuses to extract the page, yields a - warning and even removes the extern file if it exist. - - The extraction script's paranoia extends further. It will refuse to extract - the page, yielding a warning, if a (c)c-memo associated to the extern does - not exist. And it will refuse (yielding a runtime error) to write to any - file whose absolute path does not occur under the current directory or the - directory set by \docref{TEXMFOUTPUT} (in \reffile{texmf.cnf} or as an - environment variable); \docref{TEXMFOUTPUT} is also not allowed to point to - the root directory, except on Windows, where it can only point to a drive - root. - - \yadocset{ - of=script:memoize-extract.pl, - override/.style={ - index annotation={option of {% - \hypercolor{link}{gray}% - \hyperref[script:memoize-extract.pl]{\texttt{memoize-extract}}% - }}, - }, - } - \begin{doc}{ - option={name=pdf, short name=P, par=\meta{pdf}} - } - Extract the externs from the given \meta{pdf} instead of the default - \meta{name}\texttt{.mmz}. Note that file \meta{pdf}, despite the different - name, should be produced by the same compilation that produced - \meta{name}\dmmz, otherwise wrong pages might be extracted. - \end{doc} - \begin{doc}{ - option={name=prune, short name=p} - } - After extraction, remove the extracted extern pages from the document PDF. - \end{doc} - \begin{doc}{ - option={name=keep, short name=k} - } - By default, the script comments out the \refcmd{mmzNewExtern} lines in the - \dmmz file, to prevent multiple extractions. Specifying this option - prevents this behaviour. - \end{doc} - \begin{doc}{ - option={name=log, short name=l, parameters=\meta{filename}} - } - When given this option, the script creates a log file with the given name. - The log will receive any warnings yielded by the script, see also - \refscript{memoize-extract.pl--warning-template}. A well-formed log ends - with |\endinput|, indicating that the \dmmz file was completely processed. - - In absence of this option, the warnings are printed to the standard error. - \end{doc} - \begin{doc}{ - option={name=warning-template, short name=w, parameters=\meta{template}}, - } - When this option is given, a warning message is not logged (or printed to - standard error) as-is; it is wrapped by the template given as the argument - of this option. More precisely, the script will log the given - \meta{template}, with all occurrences of \docaux{cmd}{warningtext} - substituted by the actual warning message. - - The option can be used to propagate warnings to the upstream compilation, - simply by setting this option appropriately (see the initial value of - \refmmz{python extraction options}) and |\input|ting the log after - extraction. - \end{doc} - \begin{doc}{ - option={name=force, short name=f} - } - Force extern extraction even if the size-check fails. The failure will - still be logged. - \end{doc} - \begin{doc}{ - option={name=quiet, short name=q} - } - Normally, the script prints what it is doing to the standard output; in - particular, it prints out the page number and the filename of each extern - it is extracting. This option disables this behaviour. - \end{doc} - \begin{doc}{ - option={name=embedded, short name=e} - } - When given this option, everything printed to the standard output is - prefixed by the script name. The idea is to use this option when calling - the script from the \hologo{TeX} compilation, as it makes it easy to - identify the output of the script. - \end{doc} - \begin{doc}{ - option={name=mkdir, short name=m} - } - When given this option, the extraction script transforms into a paranoid - |mkdir| (|-p|). Argument \meta{name}|.mmz| is interpreted as a path to the - directory to create; all other options are ignored. - - The ancestors of the directory are created as needed. The script will - refuse to create any directory whose absolute path does not occur under the - current directory or a directory listed in \docref{TEXMFOUTPUT} (set in - \reffile{texmf.cnf} or as an environment variable). - - This option exists so that the author using the restricted shell mode does - not have to list |mkdir| among the restricted shell mode commands (and it - is also safer than a plain |mkdir|). - - Memoize automatically uses this script to create the memo directory when - option \refmmz{extract} is given value \refmmz{extract=perl} or - \refmmz{extract=python}. - \end{doc} - \begin{doc}{ - option={name=help, short name=h} - } - Show help. - \end{doc} - \begin{doc}{ - option={name=version, short name=V} - } - Show Memoize version. - \end{doc} - - Functionally, the Perl (|.pl|) and the Python (|.py|) version of the script - are almost equivalent. The minor differences, listed below, are mostly due - to the underlying PDF-processing library: - \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} in Perl, and - \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2} in Python. - \begin{itemize} - \item The Python script is about twice as fast as the Perl script. However, - both scripts are very fast compared to the \hologo{TeX}-based extraction. - On my computer, extracting 160 externs out of a 400-page book takes 1.4s - with Python, 3.7s with Perl, and 65s with \hologo{TeX}. But even when - using \hologo{TeX}-based extraction, externalization using Memoize is - extremely fast compared to the \TikZ's externalization library. Adding - the regular compilation time of one minute to the above numbers, we arrive - at the maximum externalization time of about two minues, whereas my - estimate for the production of all 160 externs using \TikZ's - externalization would be an \emph{hour} or more. - \item The Python script cannot extract externs out of PDFs created without - stream compression, i.e.\ with \refcmd{pdfvariable} - \docref{pdfvar:compresslevel} set to |0|. - \item Occasionally, the Perl script crashes during extraction externs; see - section~\ref{sec:troubleshooting} for details. - \end{itemize} -\end{doc} - -\subsubsection{\texorpdfstring{\hologo{TeX}}{TeX}-based extraction} -\label{sec:ref:extraction-tex} - -\begin{doc}{ - key={name=tex extraction command, par=\meta{system command}, - desc={no default, initially |pdftex|}}, - key={name=tex extraction options, par=\meta{options}, - desc={no default, initially: see below}}, - key={name=tex extraction script, par=\meta{\hologo{TeX} code}, - desc={no default, initially: see below}}, - } - \yadocset{ - before/.style={label prefix=cmd:te}, - index annotation={defined at \hologo{TeX} extraction}, - } - - Together, these keys determine the system call used for invoking the - \hologo{TeX}-based extraction script, \refscript{memoize-extract-one.tex}. - (They were initialized using \pkg{pgfkeys} handler |.initial|, so their - values may be modified by handlers |.prefix|, |.append|, etc.) - - Memoize uses the resulting system call string at the following occasions. - First, it executes it, once for each new extern in the \dmmz file, during the - internal extraction, i.e.\ when it is loaded with package option - \refmmz{extract}|=|\refmmz{extract=tex}. Second, it uses it to construct - record files of types \refmmz{record=sh}, \refmmz{record=bat} and - \refmmz{record=makefile} when recording of these types is requested. - - Key \refmmz{tex extraction command} sets the \hologo{TeX} binary used for - \hologo{TeX}-based internal extraction. By default, the \hologo{pdfTeX} - engine |pdftex| is used; other sensible values for this key are |luatex| and - |xetex|, or the program name including the path. Note that shell escape mode - must be configured appropriately, and that - \refscript{memoize-extract-one.tex} must be compiled with the - \hologo{plainTeX} format. - - The value of key \refmmz{tex extraction options} is passed as options to the - \hologo{plainTeX} binary. As shown in the initial value in the frame below, - it may use the temporary macro \docaux{cmd}{externbasepath}, which expands to - the path to the extern without the |.pdf| suffix. This macro is available - during internal \hologo{TeX}-based extraction and during the execution of - \refmmz[show keypath]{record/record type/new extern} key for and \meta{record - type}. - - \makeexcerpt{tex-extraction-options} - \tcbinputexample[.tex][.excerpt]{% - listing only, one file, no attachment, title={The initial value of - \refmmz{tex extraction options}}, - } - - Key \refmmz{tex extraction script} defines the command-line script executed - to extract the extern. The default value, shown in the frame below, invokes - \refscript{memoize-extract-one.tex} after setting its parameter macros. - Temporary macros \docAux{cmd={name=pagenumber}, comma, cmd={name=expectedwidth}, - and, cmd={name={expectedheight}}} are defined at the same occasions as - \refcmd{te:externbasepath} above, i.e.\ during internal \hologo{TeX}-based - extraction and during the execution of a \refmmz{record/record type/new - extern} key. The initial value requests the extern to be of the same version as - the main document, if possible;\footnote{As far as I know, it is impossible - to access the version of the PDF being produced in \hologo{XeTeX}, i.e.\ - there are no registers \docref{reg:pdfmajorversion} and - \docref{reg:pdfminorversion}. To request production of a specific version of - PDF, \hologo{XeTeX} must be invoked by with command-line option - \code{-output-driver 'xdvipdfmx -V $N$'}.} note that Memoize defines - \docref{reg:pdfmajorversion} and \docref{reg:pdfminorversion} in \hologo{LuaTeX}. - - \makeexcerpt{tex-extraction-script} - \tcbinputexample[.tex][.excerpt]{% - listing only, one file, title=The initial value of \refmmz{tex extraction - script}, no attachment} - - As the value of \refmmz{tex extraction script} is fully expanded when used, - the initial value shown above must prevent the expansion of much code. - Furthermore, the initial value varies with the \hologo{TeX} format, as - indicated by the |.dtx| guards in the definition of - \refcmd{meo:warningtemplate}. -\end{doc} - -\begin{doc}{ - script={ - head prefix={% - (\texttt{pdf}\alt\texttt{lua}\alt\texttt{xe})\texttt{tex -jobname} - \meta{extern filename} - \texttt{"}\meta{parameters} - \cs{input}\texttt{\textbraceleft} - }, - name=memoize-extract-one.tex, - head infix=\texttt{\textbraceright"}, - }, - } - Compiling \refscript{memoize-extract-one.tex} with \hologo{plainTeX} - produces an extern file containing a single (extern) page extracted from the - document PDF. - - Memoize invokes this script, once for each new extern appearing in the \dmmz - file, when loaded with package option - \refmmz{extract}|=|\refmmz{extract=tex}. - - The desired \meta{extern filename} is given as the value of option |-jobname| - of the \hologo{TeX} binary. To set the extraction \meta{parameters}, define - the following macros before |\input|ting the file: - \yadocset{ - index annotation=option of \texttt{memoize-extract-one.tex}, - before/.style={head prefix=\cs{def}, label prefix=cmd:meo}, - } - \begin{doc}{cmd={name=fromdocument, parameters=\marg{document pdf filename}}} - Defining this macro sets the filename of the PDF which the externs will be - extracted from. The filename is relative to the working directory. - \end{doc} - \begin{doc}{cmd={name=pagenumber, parameters=\marg{number}}} - Defining this macro sets the number of the page to extract. The first page - has number 1. - \end{doc} - \begin{doc}{ - cmd={name=expectedwidth, parameters=\marg{dimension}, desc=optional}, - cmd={name=expectedheight, parameters=\marg{dimension}, desc=optional}, - } - Defining these macros sets the expected width and height of the extracted - page. - - To guard against extracting a wrong page, the dimensions of the extracted - page are compared against the expected width and height. If the size check - fails,\footnote{The match need not be exact, see - footnote~\ref{fn:tolerance}.} the resulting extern PDF is empty (which - counts as non-existent when Memoize checks for its presence when it - attempts to utilize it), and a warning message (formatted via - \refcmd{meo:warningtemplate}) is printed to the log file, if logging was - requested via \refcmd{meo:logfile}. - - If any of these macros is undefined, the size check will be skipped. - \end{doc} - \begin{doc}{cmd={name=logfile, parameters=\marg{filename}, desc=optional}} - Defining this macro sets the name of the log file. If not defined, no log - file will be produced. - - The log file is intended to be used when the script is invoked from an - outer \hologo{TeX} compilation. In particular, it is intended to be - |\input| by that compilation to see whether the extraction was successful. - Upon a failed size check, it will contain a warning (formatted by - \refcmd{meo:warningtemplate}, if that macro is defined). The log file ends - with |\endinput| to signal that extraction actually took place. - \end{doc} - \begin{doc}{cmd={name=warningtemplate, parameters=\marg{code}, desc=optional}} - Defining this macro determines how to log the warning message in the case - of a failed size check. The macro should expand to a \hologo{TeX} - format-specific warning message code containing the warning text given in - \docAux{cmd={name=warningtext, label prefix=cmd:meo, into - index=false}}.\footnote{Macro \refcmd{meo:warningtemplate} is passed - the warning text by a macro rather than a formal parameter to avoid - category code problems with the parameter character when setting key - \refmmz{tex extraction script}.} - - While the script formats the warning message \emph{text} on its own (``I - refuse to extract page \dots''), the warning message is not written into - the log unadorned. The log file is intended to be \cs{input} by the outer - \hologo{TeX} compilation, and the idea is that inputting it should yield a - warning in that compilation (in the case of a failed size - check). Therefore, the content of the log file must contain an invocation - of the command used to produce warning messages in the \hologo{TeX} format - used by the outer compilation. - - For example, when this script is invoked from within a \hologo{LaTeX} - compilation, it makes sense to define something like - |\def\warningtemplate{\PackageWarning{memoize}{\warningtext}}|. - \end{doc} - \begin{doc}{cmd={name=force, - parameters=\bracestt{\texttt{true}\Alt\texttt{false}}, desc=optional}} - If this macro is defined to |true|, extern extraction will be carried out - even if the size-check fails. The failure will still be logged. - \end{doc} - \begin{doc}{ - cmd={name=mmzpdfmajorversion, parameters=\marg{number}, desc=optional}, - cmd={name=mmzpdfminorversion, parameters=\marg{number}, desc=optional}, - } - Defining (one or both of) these macros requests that the extern PDF be - produced with the given major\slash minor PDF version, i.e.\ the extraction - script will set registers \docref{reg:pdfmajorversion} and - \docref{reg:pdfminorversion}. - \end{doc} - - After extracting the extern, the script will end the compilation, i.e.\ - intentionally, only one page documents can be produced. -\end{doc} - -\subsubsection{The clean-up scripts} -\label{sec:cleanup-scripts} - -\begin{doc}{ - script={name=memoize-clean.pl, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]}, - script={name=memoize-clean.py, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]}, - } - This script removes memo and extern files whose filenames start with - \meta{path prefix}es mentioned in the given \dmmz files or by the - \refscript{memoize-clean.pl--prefix} option. Unless option - \refscript{memoize-clean.pl--all} is given, the script only deletes the - \emph{stale} files, i.e.\ the files not mentioned in any of the given \dmmz - files. - - A \meta{path prefix} of a memo or an extern is what was set by key - \refmmz{path}, or more commonly, one of the shortcut keys \refmmz{memo dir} - and \refmmz{no memo dir}; see section~\ref{sec:ref:dirs} for details on the - form of a memo/extern filename. - - In detail, the script scans the given \dmmz files for occurrences of - \refcmd{mmzPrefix}, and adds their \meta{path prefix} arguments to the list - of prefixes given on the command line by option - \refscript{memoize-clean.pl--prefix}; a \meta{path prefix} occurring in some - \dmmz file is interpreted relatively to the location of the \dmmz file. The - script removes all files whose full pathname (relative to the current - directory) matches pattern \meta{path - prefix}\meta{md5sum}(|-|\meta{md5sum})(|.memo|\alt(|-|$N$)|.pdf|\alt|.log|),% - \footnote{The |.log| files are produced by the \hologo{TeX}-based extraction - script.} except those which occur as the \meta{filename} argument to one - of \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo}, \refcmd{mmzUsedExtern}, - \refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo} and \refcmd{mmzNewExtern} in one - of the \dmmz files. - - The script is fairly paranoid. It refuses to delete anything if a \dmmz file - is malformed in any way (but not if it doesn't exist or is completely empty, - which facilitates its usage in clean-up scripts), or if it would remove a - file not residing under the current directory. Before removing the files, it - lists the files to be removed and asks for confirmation. - - Functionally, the Perl (|.pl|) and the Python (|.py|) version are completely - equivalent. - - \yadocset{ - of=script:memoize-clean.pl, - override/.style={ - index annotation={option of {% - \hypercolor{link}{gray}% - \hyperref[script:memoize-clean.pl]{\texttt{memoize-clean}}% - }}, - }, - } - \begin{doc}{ - option={name=prefix, short name=p, par=\meta{path prefix}} - } - Add \meta{path prefix} to the list of prefixes; the given prefix is - relative to the current directory. This option may be given multiple - times. - \end{doc} - \begin{doc}{ - option={name=all, short name=a} - } - When given this option, the script removes \emph{all} memos and externs - belonging to the document, not just the stale ones, i.e.\ it effectively - ignores the occurences of \refcmd{mmzUsedCMemo} and friends in the \dmmz - file. - \end{doc} - \begin{doc}{ - option={name=yes, short name=y} - } - When given this option, the script does not ask for confirmation before - removing the files. - \end{doc} - \begin{doc}{ - option={name=quiet, short name=q} - } - Normally, the script prints what it is doing to the standard output; in - particular, it prints out the filename of each file as it is deleting it. - This option disables this behaviour. - \end{doc} - \begin{doc}{ - option={name=help, short name=h} - } - Show help. - \end{doc} - \begin{doc}{ - option={name=version, short name=V} - } - Show Memoize version. - \end{doc} -\end{doc} - -\subsubsection{Record files} -\label{sec:ref:record-files} - -\begin{doc}{ - key={name=record, par=\marg{record type}, - desc={cumulative, initially \refmmz{record=mmz}, no default}}, - key={name=no record}, - } - Memoize records which externs were produced and used in the compilation, - producing a record file of every type found in the record-type list. These - keys add \meta{record type} to the record-type list, or clear this list. See - section~\ref{sec:record-files} for details. - - Note that passing an undefined \meta{record type} to this key will not yield - an error. - - Out of the box, the following \meta{record type}s are recognized: - \yadocset{of=key:/mmz/record} - \begin{doc}{value={name=mmz}} - This record type produces a \dmmz file recording new/used - externs/c-memos/cc-memos and changes in the \refmmz{path} prefix to these - files; see section~\ref{sec:.mmz} for details. - - The produced file is named \docAux{file={name=mmz, - name prefix=\meta{jobname}., - ref prefix=., - index prefix=., - }}. This name cannot be changed. - - The \dmmz file is a \hologo{TeX} file, but uses only a simple subset of - the \hologo{TeX} syntax, to be easily parsable by the external scripts such - as \refscript{memoize-extract.pl}. Each line of the file consists of a - (possibly commented) invocation of one of the commands listed below; the - final line is |\endinput|. The \meta{path prefix} below consists of the - path to memos/externs and the immutable \refmmzpath{prefix} of their filename. - \begin{doc}{ - cmd={name=mmzUsedCMemo, par=\marg{filename}}, - cmd={name=mmzUsedCCMemo, par=\marg{filename}}, - cmd={name=mmzUsedExtern, par=\marg{filename}}, - } - Record that the (c)c-memo or extern residing in file \meta{filename} was - utilized. - \end{doc} - \begin{doc}{ - cmd={name=mmzNewCMemo, par=\marg{filename}}, - cmd={name=mmzNewCCMemo, par=\marg{filename}}, - } - Record that a new (c)c-memo residing in file \meta{filename} was - produced. - \end{doc} - \begin{doc}{cmd={name=mmzNewExtern, par=\marg{filename}\marg{page number}% - \marg{expected width}\marg{expected height}}} - Record that a new extern was produced and dumped as page \meta{page - number} into the document, that it should be extracted into file - \meta{filename}, and that it should be \meta{expected width} wide and - \meta{expected height} high (modulo tolerance of |0.01pt|, see - footnote~\ref{fn:tolerance}), where the height is the total height - comprising both \hologo{TeX} height and depth. - \end{doc} - \begin{doc}{cmd={name=mmzPrefix, par=\marg{path prefix}}} - Record that the \refmmz{path} prefix of memo and extern files was - changed. - \end{doc} - \end{doc} - \begin{doc}{value={name=makefile}} - This record type produces a makefile which, when processed by the |make| - utility, triggers \hologo{TeX}-based extraction of the new externs. - \begin{doc}{ - key={name=makefile, par=\marg{filename}, desc={no default, - initially \texttt{memoize-extract.\jobname.makefile}}} - } - Use this key to change the filename of the produced makefile. - \end{doc} - \end{doc} - \begin{doc}{value={name=sh}, value={name=bat}} - These record types produce a shell script which, when executed, triggers - \hologo{TeX}-based extraction of the new externs. - - Use \refmmz{record=sh} on Unix-like systems, and \refmmz{record=bat} on - Windows. - - \begin{doc}{ - key={name=sh, par=\marg{filename}, desc={no default, - initially \texttt{memoize-extract.\jobname.sh}}}, - key={name=bat, par=\marg{filename}, desc={no default, - initially \texttt{memoize-extract.\jobname.bat}}} - } - Use these keys to change the filename of the produced shell script. - \end{doc} - \end{doc} -\end{doc} - -\begin{doc}{ - desc=definable, - keypath=/mmz/record/\meta{record type}, - keypath label=/mmz/record/record type, - name prefix={\meta{record type}/}, ref prefix=, - index annotation={key in \texttt{/mmz/\meta{record type}}}, - key={name=begin}, - key={name=prefix, par=\marg{path prefix}}, - key={name=new extern, par=\marg{filename}}, - key={name=new cmemo, par=\marg{filename}}, - key={name=new ccmemo, par=\marg{filename}}, - key={name=used extern, par=\marg{filename}}, - key={name=used cmemo, par=\marg{filename}}, - key={name=used ccmemo, par=\marg{filename}}, - key={name=end}, - } - A new record type can be implemented by defining these keys in keypath - \docAux{key path={name=mmz/record/\meta{record type}, label=mmz/record/record - type}} (using the standard \pkg{pgfkeys} handlers such as |.code| and - |.style|). The keys are invoked by Memoize where appropriate if recording - for the defined type is activated by \refmmz{record}|=|\meta{record type}, - just as for the predefined types. Only those keys which are required for - implementing the desired functionality need to be defined. - - The following macros are available during the execution of key - \refmmz{record/record type/new extern}: - \yadocset{before/.style={label prefix=cmd:ne}, - index annotation=defined at \refmmz{record/record type/new extern}} - \begin{doc}{cmd={name=pagenumber}} - This macro holds the number of the extern page. - \end{doc} - \begin{doc}{ - cmd={name=expectedwidth}, - cmd={name=expectedheight}, - } - These macros hold the width and the height of the extern page. - \end{doc} - \begin{doc}{cmd={name=externbasepath}} - This macro holds the filename of the extern, minus the |.pdf| suffix (but - including the path leading to the extern). - \end{doc} -\end{doc} - - - -\subsection{Automemoization} -\label{sec:ref:advicememoization} - - -\subsubsection{Package Advice} -\label{sec:ref:advice} - -Package Advice is a namesake of \Emacs's Advice. As such, it implements a -generic framework for extending the functionality of selected commands and -environments. Each \emph{advised} command and environment is assigned a piece -of \emph{advice} --- a command which is executed instead of the advised command -and environment, and which may, or may not, invoke the original command or -environment during its execution. The package offers an elegant way of -declaring advice, setting up the conditions upon which the advised command will -actually be replaced by the advice, collecting the arguments of the advised -command and invoking it, and (de)activating the advice. - -Before the advising framework can be used, it must be installed into a selected -\pkg{pgfkeys} keypath (multiple installations into different keypaths are -allowed, even if they handle the same commands). Memoize installs the -framework into keypath \refkeypath{/mmz} and (primarily) uses it to -automatically memoize the results of compilation of selected commands and -environments. - -\begin{doc}{ - keypath={/handlers}, - key={name=.install advice, - sort index=install advice, - par=\marg{configuration keylist}, - }, - } - This key is a \pkg{pgfkeys} key handler (see \PGFmanual{87.3.5}) which - installs the advising framework into the keypath which it was invoked from - --- henceforth, the \meta{namespace}. - - For example, \code{\cs{pgfkeys}\bracestt{/my/\refkey{/handlers/.install - advice}}} installs the framework into keypath |/my|. - - Argument \meta{configuration keylist} may contain the following keys: - \yadocset{keypath=/advice/install} - \begin{doc}{key={ - name=setup key, - parameters=\marg{name}, - description={no default, initially \texttt{advice}}}, - } - This key determines the names of the user-interface keys used to setup - advice for commands and environments in \meta{namespace}. - - The keys whose names are determined by this key are the following: - \meta{name}, \code{\meta{name} csname}, \code{\meta{name} key}, - \code{\meta{name}'}, \code{\meta{name} csname'} and \code{\meta{name} - key'}. Memoize sets \refkey{/advice/install/setup key}|=auto|, and - thereby defines \refmmz{auto}, \refmmz{auto csname}, \refmmz{auto key}, - \refmmz{auto'}, \refmmz{auto csname'} and \refmmz{auto key'}. - \end{doc} - \begin{doc}{key={ - name=activation, - par=\meta{initial activation type}, - desc={no default, initially \texttt{immediate}} - }, - } - This key sets the \meta{initial activation type} for \meta{namespace}. - - At the end of the installation, the system will execute - \meta{namespace}|/|\refmmz{activation}|=|\meta{initial activation type}; - consequently, \meta{initial activation type} must be one of - \refmmz{activation=immediate} and \refmmz{activation=deferred}. In - Memoize, the \meta{initial activation type} is - \refmmz{activation=deferred}. - - Setting the \refmmz{activation} type during the installation only matters - in \hologo{LaTeX}, where the installation ends by advising \refcmd{begin} - to implement advising of environments. - \end{doc} - - \begin{tcolorbox}[warning] - Writing the documentation for Advice, I was faced with a dilemma. Should - the documentation reflect the fact that the full names of keys defined by - the package depend on the installed instance of the framework, in - particular on \meta{namespace} and \meta{setup key}? For example, should - the reference headers contain things like \meta{namespace}|/activate| and - \meta{namespace}|/|\meta{setup key} |csname|? In my opinion, this would - make the reference hard to read, so I decided to have the reference headers - refer to the Advice keys of the Memoize installation, where - \meta{namespace}=|/mmz| and \meta{setup key}=|auto|, resulting in - friendlier headers such as \refkeypath{/mmz}|/|\refmmz{activate} and - \refkeypath{/mmz}|/|\refmmz{auto csname}. (Consequently, it also made - sense to document Advice within the Memoize documentation.) - - The bottomline: if you're reading this section with a non-Memoize - installation in mind, you have to mentally replace any \refkeypath{/mmz} - and \refmmz{auto} in the reference headers with the \meta{namespace} and - the \meta{setup key} selected by that installation. - (Section~\ref{sec:ref:advice:memoization} is another matter. Keys - described there are only available in Memoize.) - \end{tcolorbox} - - In more detail, key handler \refkey{/handlers/.install advice} performs the - following actions (as explained in the box above, we assume that the - advising framework was installed into keypath \refkeypath{/mmz} with the - setup key named |auto|): - \begin{itemize} - \item \edef\origtolerance{\the\tolerance}\tolerance=1000 - It defines the following keys in keypath \refkeypath{/mmz}: - \refmmz{auto}, - \refmmz{auto csname}, - \refmmz{auto key}, - \refmmz{auto'}, - \refmmz{auto csname'}, - \refmmz{auto key'}, - \refmmz{activation}, - \refmmz{activate deferred}, - \refmmz{activate}, - \refmmz{deactivate}, - \refmmz{activate csname}, - \refmmz{deactivate csname}. - \refmmz{activate key}, - \refmmz{deactivate key}. - \refmmz{force activate}, - \refmmz{try activate}. - \item - It defines the following keys in keypath \refkeypath{/mmz/auto}: - \refmmzauto{run conditions}, - \refmmzauto{outer handler}, - \refmmzauto{bailout handler}, - \refmmzauto{collector}, - \refmmzauto{args}, - \refmmzauto{collector options}, - \refmmzauto{clear collector options}, - \refmmzauto{raw collector options}, - \refmmzauto{clear raw collector options}, - \refmmzauto{inner handler}, - \refmmzauto{options}, - \refmmzauto{clear options}, - \refmmzauto{reset}. - \item \tolerance\origtolerance\relax - It defines the |.unknown| key handler for \refkeypath{/mmz/auto}. - This handler appends any unknown keys (and their values) to - \refmmzauto{options}. - \item It executes \refmmz[show keypath]{activation}|=|\meta{initial - activation type}. - \item In \hologo{LaTeX}, it submits \refcmd{begin} to advising, thereby - enabling environment support in this format. Consequently, advising of - environments can switched off by writing \refmmz{deactivate}|=\begin|. - \end{itemize} -\end{doc} - -\paragraph*{The keys installed into keypath \metabf{namespace}} are used to -declare and (de)activate advice. In the documentation in this subsection, -we assume that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup - key}=\refmmz{auto}. In particular, this also applies to the reference -headers. - -\begin{doc}{ - key={name=activation, par=\docAux{of=key:/mmz/activation, value={name=immediate}, - text=\Alt, value={name=deferred}}, desc={style, no default}}, - key={name=activate deferred, desc={style}}, - } - Key \refmmz{activation} selects the activation regime. Under the - \refmmz{activation=immediate} regime, keys \refmmz{activate}, - \refmmz{deactivate}, \refmmz{force activate} and \refmmz{try activate} behave - as described in their documentation below. Under the - \refmmz{activation=deferred} regime, however, those keys are not executed; - rather, their invocations are appended to style \refmmz{activate deferred}. - For example, writing \refmmz{activate}|=\foo| in the deferred activation - regime appends \refmmz{activate}|=\foo| to \refmmz{activate deferred}. It is - up to the user if and when to execute the keys collected in \refmmz{activate - deferred}; see the documentation of \refmmz{manual} to learn what Memoize - does with the contents of this style. -\end{doc} - - -\begin{doc}{easy, - par=\marg{list of commands and/or environments}, - desc={style}, - key={name=activate}, - key={name=deactivate}, - } - These keys activate or deactivate the advice for the given commands and - environments. When the advice is activated, it replaces the advised command; - when it is deactivated, the command is reverted to its original definition. - - In Memoize, these keys are most commonly used to activate or deactivate - automemoization for the given commands or environments. For example, write - \refmmz{deactivate}|={\tikz,tikzpicture}| to deactivate automemoization of - \TikZ pictures (which is declared and active by default). The curly braces - may be omitted if the list contains a single command or environment, e.g.\ - \refmmz{deactivate}|=\tikz| or \refmmz{deactivate}|=tikzpicture|. - - (De)activation of a piece of advice is completely orthogonal to its declaration with - \refmmz{auto}. For example, there is no need to deactivate a command before - redeclaring its advice, and reactivate it afterwards. A command may be - activated even before declaring its advice --- however, the command itself - must be defined at the time of activation. - - As the advice is normally automatically activated upon declaration with - \refmmz{auto}, explicit activation is rarely needed, but see \refmmz{auto'}. - The effect of these keys under the deffered activation regime is described in - \refmmz{activation}. - - Note that I sometimes speak of (de)activating a command, and sometimes of - (de)activating its advice. I mean the same thing. -\end{doc} - -\begin{doc}{ - par=\marg{control sequence name}, - desc={style}, - key={name=activate csname}, - key={name=deactivate csname}, - } - These keys activate and deactivate a command given by its \meta{control - sequence name}; for example, \refmmz{activate csname}|=foo| is equivalent - to \refmmz{activate}|=\foo|. Note that unlike the regular \refmmz{activate} - and \refmmz{deactivate}, their |csname| variants only accept a single command - at a time (otherwise, including a comma in the command name would be - impossible). -\end{doc} - -\begin{doc}{ - par=\marg{list of full key names}, - desc={style}, - key={name=activate key}, - key={name=deactivate key}, - } - These keys activate and deactivate |pgfkeys| keys. Note that \emph{full} key - names must be given, i.e.\ the names must include the keypath. - - Under the hood, these keys merely execute \refmmz{activate} and - \refmmz{deactivate} on the internal macros corresponding to the given keys. -\end{doc} - -\begin{doc}{ - key={name=try activate, conditional=false}, - } - When this conditional is set to true, \refmmz{activate} will not yield an - error if the advice is already activated, and \refmmz{deactivate} will not - yield an error if the advice is not yet activated. - - This key applies to the next, and only to the next, invocation of key - \refmmz{activate} or \refmmz{deactivate}, i.e.\ it is reset back to |false| - after invoking \refmmz{activate} or \refmmz{deactivate}. -\end{doc} - -\begin{doc}{ - key={name=force activate, conditional=false}, - } - When this conditional is set to true, \refmmz{activate} will activate even a - previously activated command, provided that, additionally, the command has - been redefined since the prior activation. - - In more detail, the original definition of the advised command is saved upon - activation (to provide the possibility of both deactivation and the usage of - the original command by the handler). Consequently, activation of an already - activated command would result in the saved original definition being - overwritten by the redefinition made during the first activation. However, - if the handled command was meanwhile redefined by a third party, reactivation - makes sense, under the assumption that the former original definition is - obsolete and should be replaced by the (third party) redefinition. As a - safeguard, however, \refmmz{activate} requires such reactivation to be - explicitly requested using conditional \refmmz{force activate}.\footnote{A - potential problem, not (yet) addressed, is that the third party might be - another incarnation of the advising framework. In this case, forced - reactivation will result in the loss of the original command and a circular - dependency between the two pieces of advice.} - - This key applies to the next, and only to the next, invocation of key - \refmmz{activate}, i.e.\ it is reset back to |false| after invoking - \refmmz{activate}. This key does not apply to \refmmz{deactivate}. -\end{doc} - -\begin{doc}{easy, - key={name=auto, par=\marg{command or environment}{\marg{keylist}}, desc=style}} - - This key sets up the advice for the given command or environment, or updates - the configuration of an existing piece of advice. - - In Memoize, this key is most commonly used to submit a command or environment - to automemoization. For an environment (say, |bar|), it suffices to write - \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{memoize}}; for a command, - we usually need to include its argument specification: - \refmmz{auto}|=\foo|\bracestt{\refmmzauto{memoize}, - \refmmzauto{args}=\bracestt{...}}. Another common usage is to prevent - memoization during the execution of a command or environment: - \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{nomemoize}}. For - details, see sections~\ref{sec:tut:automemoization}, - \ref{sec:tut:working-on-a-picture} and~\ref{sec:tut:verbatim}. - - The advice is configured by the given \meta{keylist}, which is executed with - the default keypath set to \docaux{key path}{mmz/auto}. Any unknown keys in - \meta{keylist} are passed on to key \refmmzauto{options}; for example, a - plain \refmmz{verbatim} or \refmmz{padding}|=2in| have the same effect as - \refmmzauto{options}|=|\refmmz{verbatim} or - \code{\refmmzauto{options}=\braces{\refmmz{padding}=2in}}. - - This key automatically activates the declared advice, unless it is already - activated; under the deferred activation regime, the automatic activation is - deferred as well. Use variant \refmmz{auto'} when you don't want to - automatically activate the advice. - - When this key is used on a command or environment with an existing piece of advice, - the advice is merely updated. This makes it easy to, for example, - temporarily switch to verbatim collection of an environment in Memoize: - \refmmz{auto}|=|\bracestt{tcolorbox}\braces{\refmmz{verbatim}}. Use key - \refmmzauto{reset} to setup the advice from scratch: - \refmmz{auto}|=|\bracestt{...}\bracestt{\refmmzauto{reset}, ...}. - - A piece of advice consists of several interlocked components, declared by keys - residing in path \refkeypath{/mmz/auto}: \refmmzauto{run conditions}, - \refmmzauto{bailout handler}, \refmmzauto{outer handler}, - \refmmzauto{collector} and \refmmzauto{inner handler}. During the execution - of the advice, these components are available through the following macros: - \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler}, - \refcmd{AdviceOuterHandler}, \refcmd{AdviceCollector} and - \refcmd{AdviceInnerHandler}. These macros are also defined during setup, and - it is possible to change the configuration by modifying them directly; in - that case, it likely also makes sense to use the low-level variant of this - key, macro \refcmd{AdviceSetup}. - - Control sequences used in the advice components do not need to be defined at - the time of invoking key \refmmz{auto}, or activating the advice; they must - only be defined at the time the advice is actually executed. It is thus - perfectly fine to declare \refmmzauto{inner handler}|=\myinnerhandler| before - defining |\myinnerhandler|, or to redefine |\myinnerhandler| between the - invocations of the advised command. - - This key configures not only the components, but also the options of the - advice. These options are set by keys \refmmzauto{args}, - \refmmzauto{collector options}, \refmmzauto{raw collector options} and - \refmmzauto{options}. Whether these options are used or not depends on the - advice components. (Same as the components, the options have their - corresponding low-level macros: \refcmd{AdviceArgs}, - \refcmd{AdviceCollectorOptions}, \refcmd{AdviceRawCollectorOptions} and - \refcmd{AdviceOptions}.) - - Parameter symbols (i.e.\ |#|) are not allowed in advice settings. - - A command or environment may be submitted to several instances of the - advising framework, i.e.\ instances installed under different keypaths. The - effect of such chained advice depends on the order of activation. If advice - $A$ is activated before advice $B$, it will also be applied before $B$. - - The advice setup takes place in a group. Use key \refmmzauto{after setup} to - execute code outside this group. - - In general, the name of this key equals whatever was submitted to - \refkey{/advice/install/setup key} during the installation of the advising - framework via \refkey{/handlers/.install advice}. -\end{doc} - -\begin{doc}{ - key={name=auto csname, par=\marg{control sequence name}{\marg{keylist}}, desc=style} - } - This key is a variant of \refmmz{auto}, but with the command of the first - argument given as a control sequence name, i.e.\ \refmmz{auto - csname}|={foo}{...}| is equivalent to \refmmz{auto}|=\foo{...}|. -\end{doc} - -\begin{doc}{ - key={name=auto key, par=\marg{full key}{\marg{keylist}}, desc=style}, - } - This key is a variant of \refmmz{auto}, but it works with \pkg{pgfkeys} keys. - The first argument should be a \meta{full key} like |/tcb/float|, i.e.\ it - must consist of both the keypath and the keyname. - - This key sets up advice for the internal command corresponding to the given - \meta{full key}, and also properly initializes the collector, so that - \refmmzauto{inner handler} will ``just work.'' -\end{doc} - -\begin{doc}{ - key={name=auto', par=\marg{command or environment}{\marg{keylist}}, desc=style}, - key={name=auto csname', par=\marg{control sequence name}{\marg{keylist}}, desc=style}, - key={name=auto key', par=\marg{full key}{\marg{keylist}}, desc=style}, - } - These keys are variants of \refmmz{auto}, \refmmz{auto csname} and - \refmmz{auto key} which do not attempt to activate the command after setting - it up. -\end{doc} - -\begin{doc}{ - cmd={ - name=AdviceSetup, - par=\marg{namespace}\marg{command or environment}{\marg{setup code}} - }} - This macro is the low-level variant of key \refmmz{auto}. The differences - between the two are the following: - \begin{itemize} - \item An invocation of the macro must provide the namespace (i.e.\ the - installation keypath) as the first argument. - \item There is no automatic activation at the end of the setup. - \item The final argument should not be a keylist (of keys belonging to - \refkeypath{/mmz/auto}) but \hologo{TeX} code adjusting the contents of the - settings macros \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler}, - \refcmd{AdviceOuterHandler}, etc. For the full list of available macros, see - the documentation of their corresponding keys below; the setting macros are - mentioned at the end of each entry. - \end{itemize} -\end{doc} - -\begin{doc}{ - cmd={name=AdviceTracingOn}, - cmd={name=AdviceTracingOff}, - } - Advice tracing is initially off. When it is on, Advice will show (on the - terminal and in the |.log| file) which advice components are executed, and - what arguments and options they have received. -\end{doc} - -\paragraph*{The keys installed into keypath \metabf{namespace}\texttt{/}\metabf{setup key}} -are used to configure advice. They may only occur within the second -argument of the setup key. In the documentation in this subsection, we assume -that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup - key}=\refmmz{auto}. In particular, this also applies to the reference -headers. - -\begingroup -\yadocset{keypath=/mmz/auto} - -\begin{doc}{% - key={name=run conditions, par=\meta{\hologo{TeX} code}, - desc=initially and default: \refcmd{AdviceRuntrue}}, - } - - This key declares the \meta{control sequence} as the run conditions component - of the advice. - - The run conditions macro is executed at the very start of the advice. Its - function is to decide whether we should proceed to advise the command by - executing the outer handler, or execute the original command (after invoking - the bailout handler). - - The run conditions macro should take no arguments. If it determines that the - run conditions are satisfied, it should set the \hologo{TeX} conditional - \docaux{cmd}{ifAdviceRun} to true by executing \docaux{cmd}{AdviceRuntrue}. - There is no need to execute \docaux{cmd}{AdviceRunfalse} when the run - conditions are not satisfied. - - Initially, the run conditions are set to \refcmd{AdviceRuntrue}, translating to - ``always run.'' For two non-trivial examples, see \refmmzauto{run if - memoization is possible} and \refmmzauto{run if memoizing}. Executing this - key without a value restores it to the initial value. - - During advising and advice setup, the run conditions of the advised command - are accessible through \docaux{cmd}{AdviceRunConditions}, a parameterless - macro expanding to the given \meta{\hologo{TeX} code}. -\end{doc} - -\begin{doc}{% - key={name=bailout handler, par=\meta{\hologo{TeX} code}, - desc=initially and default: \cs{relax}}, - } - - This key declares the \meta{\hologo{TeX} code} as the bailout handler - component of the advice. - - The bailout handler is executed when the run conditions are not met, just prior - to executing the original definition of the advised command. The bailout - handler should take no arguments. - - The initial bailout handler, |\relax|, does nothing. Memoize defines and - uses a bailout handler which clears the next-options. Executing this key - without a value restores it to the initial value. - - During advising and advice setup, the bailout handler of the handled command is - accessible through \docaux{cmd}{AdviceBailoutHandler}, a parameterless macro - expanding to the given \meta{\hologo{TeX} code}. -\end{doc} - -\begin{doc}{ - key={name=outer handler,par=\meta{\hologo{TeX} code}, - desc=initially and default: see below}, - } - - This key declares the \meta{\hologo{TeX} code} as the outer handler component - of the advice. - - The outer handler can be safely imagined as the command which replaces the - handled command. This also holds for handled environments, but with a - caveat: for a \hologo{plainTeX} or \hologo{ConTeXt} environment |foo|, the - outer handler replaces |\foo| and |\startfoo|, respectively; in the case of a - \hologo{LaTeX} environment, it replaces |\begin{foo}|.%] - - The outer handler is the first component which has the opportunity to inspect - the arguments given to the handled command. It is invoked just in front of - these arguments (which are, in case \hologo{TeX} hasn't seen them yet, - untokenized), and while it is expected that the advice will consume - the same arguments as the advised command itself would, how precisely that - happens may vary from situation to situation. In particular, the argument - structure of the outer handler is not prescribed. - - In fact, the outer handler has complete control over the remainder of the - advising process. In situations where advising requires knowledge of the - advised command's arguments as a whole, the outer handler executes the - collector, which in turn invokes the inner handler, which does the real work; - see \refmmzauto{memoize} for the usage case which inspired this design. - Sometimes, however, it is the outer handler which does the real work (and - there is thus no inner handler). This is the case in situations when the - arguments of the handled command are irrelevant for the functioning of the - advice, or when the advice needs to inspect some individual argument of the - handled command; for examples of such situations, see \refmmzauto{abort} and - \refmmzauto{ref}. - - To reiterate the argument situation of the outer handler, it sees the - arguments of the handled command as they were given. The arguments are - \emph{not} collected before invoking the outer handler --- in fact, avoiding - the argument collection is the raison d'être of the outer handler! (In the case - of an advised environment, the environment body can be seen as an argument of - \pkg{xparse} type \docref{xparse:+}\docref{xparse:b}.) - - The outer handler (and any other component of the advice it invokes) has - access to the following auxiliary macros, defined by the framework: - \begin{itemize} - \item \tolerance 1000 the macros holding the configuration of the advised - command, as set up by \refmmz{auto}: \refcmd{AdviceRunConditions}, - \refcmd{AdviceBailoutHandler} and \refcmd{AdviceOuterHandler} are probably - useless, as they refer to components already invoked, but the remaining - components (\refcmd{AdviceCollector} and \refcmd{AdviceInnerHandler}) and - their options (\refcmd{AdviceArgs}, \refcmd{AdviceCollectorOptions}, - \refcmd{AdviceRawCollectorOptions} and \refcmd{AdviceOptions}) should be - commonly used. - \item the macros holding information about the namespace and the advised - command or environment: \refcmd{AdviceNamespace}, \refcmd{AdviceName}, - \refcmd{AdviceReplaced} and \refcmd{AdviceOriginal}. (Command - \refcmd{AdviceGetOriginal} might also be useful, although using - \refcmd{AdviceOriginal} will likely be more practical.) - \end{itemize} - - This key is initially set to an internal control sequence which merely - invokes the collector by executing \refcmd{AdviceCollector}; in other words, - the initial outer handler leaves all the work to the collector and the inner - handler. There is no need to specifically set up the outer handler when - using the inner handler. Executing this key without a value restores it to - the initial value. - - During advising and advice setup, the outer handler of the advised command is - accessible through \docaux{cmd}{AdviceOuterHandler}, a parameterless macro - expanding to the given \meta{\hologo{TeX} code}. -\end{doc} - -\begin{doc}{ - key={name=collector,par=\meta{\hologo{TeX} code}, - desc=initially and default: see below}, - } - - This key declares the \meta{\hologo{TeX} code} as the collector component of - the advice. - - The collector, if used, is invoked by the outer handler. It is invoked - immediately in front of the advised command's arguments (which are, in case - \hologo{TeX} hasn't seen them yet, untokenized), and its function is to - collect these arguments and pass them on, as a single argument, to the inner - handler. - - While this manual occasionally states that the initial argument collector is - \refcmd{CollectArguments} of package CollArgs, this is, if we're precise, - incorrect on two counts. For one, the initial collector is not a CollArgs - command, but a macro which acts as the ``bridge'' between Advice - and CollArgs. Second, the initial collector does not really invoke - \refcmd{CollectArguments}, but its cousin, \refcmd{CollectArgumentsRaw}, - which allows Advice (and Memoize) to fine tune its behaviour - using the fast low-level (``programmer interface'') commands rather than the - slower \pkg{pgfkeys} interface; clearly, the latter point also provides - raison d'être for \refmmzauto{raw collector options}. Summing up, this key - is initially set to an internal control sequence which compiles the settings - provided by \refmmzauto{args}, \refmmzauto{collector options} and - \refmmzauto{raw collector options} into an invocation of - \refcmd{CollectArgumentsRaw} of package CollArgs.\footnote{The initial - collector also sets the CollArgs' option \refcollargs{caller} to the - name of the advised command or environment.} Executing this key without a - value restores it to the initial value. - - The above-mentioned collector settings were clearly tailored to suit - \refcmd{CollectArgumentsRaw}. In general, a collector might or might not use - them, and if it does, it may interpret them in any way. For example, Advice - ships with a \refcmd{tikz} collector, \docaux{cmd}{mmzCollectTikZArguments}, which - ignores them completely, as it knows everything about the idiosyncrasies of - that command anyway. Incidentally, \refcmd{mmzCollectTikZArguments} becomes - available upon loading \reffile{advice-tikz.code.tex} (which Memoize does - automatically in the presence of \TikZ). - - The collector has access to the same auxiliary macros as the outer handler. - In particular, it will \emph{have} to use \refcmd{AdviceInnerHandler} (followed - by the braced collected arguments) to invoke the inner handler. - - During advising and advice setup, the collector of the advised command is - accessible through \docaux{cmd}{AdviceCollector}, a parameterless macro - expanding to the given \meta{\hologo{TeX} code}. -\end{doc} - -\begin{doc}{easy, - key={name=args,par=\meta{argument specification}, desc=initially and default: unset}, - } - - This key describes the \meta{argument specification} of the advised command. - - Assuming that the initial value of \refmmzauto{collector} has not been - modified, the given \meta{argument specification} is eventually interpreted - by command \refcmd{CollectArguments} of package CollArgs, which expects an - argument specification in the format specified by package \pkg{xparse}; the - format is summarized in the frame below for convenience, for details, see the - \pkg{xparse} manual. If the specification is not given, the initial collector - assumes that the advised command was defined using \refcmd{NewDocumentCommand} - (or similar) of package \pkg{xparse}, and will attempt to retrieve the argument - specification automatically via \refcmd{GetDocumentCommandArgSpec}. - - \begin{tcolorbox}[float, before float=\hfill, - title={The {\pkg[white]{xparse}} argument specification - (as understood by \refcmd[link color=white]{CollectArguments})}] - \begin{tabularx}{\linewidth}{>{\tt}lX} - \multicolumn{2}{l}{\rm\textbf{Mandatory argument types}}\\ - m&standard (a single token or multiple tokens in braces)\\ - r\meta{token$_1$}\meta{token$_2$}&delimited by - \meta{token$_1$} and \meta{token$_2$} \\ - v&verbatim, in the style of \cs{verb}\\ - b&the body of an environment\\ - [1ex]\multicolumn{2}{l}{\rm\textbf{Optional argument types}}\\ - o&in square brackets\\ - d\meta{token$_1$}\meta{token$_2$}&delimited by - \meta{token$_1$} and \meta{token$_2$}\\ - s&an optional star\\ - t\meta{token}&an optional \meta{token}\\ - e\marg{tokens}&a set of embellishments\\ - [1ex]\multicolumn{2}{l}{\rm\textbf{Weird argument types}}\\ - l&a mandatory argument until the first begin-group token\\ - u\marg{tokens}&\hologo{TeX}'s delimited argument\\ - g&an optional argument inside braces\\ - [1ex]\multicolumn{2}{l}{\rm\textbf{Modifiers}}\\ - +&allow the next argument to be long\\ - !&disallow spaces before arguments of type \docref{xparse:d} and \docref{xparse:t}\\ - >\marg{processor}&process the next argument\\ - [1ex]\multicolumn{2}{l}{\rm\textbf{CollArgs extensions}}\\ - \docref{xparse:b}\marg{name}&set the environment name for this environment\\ - \docref{xparse:amp}\marg{options}&apply CollArgs options to the next argument\\ - \docref{xparse:amp}\docref{xparse:amp}\marg{raw options}&apply raw CollArgs options to the next argument\\ - \end{tabularx} - - \smallskip - - \refcmd{CollectArguments} can grab an argument of any type in the - \refcollargs{verbatim} mode. - - As \refcmd{CollectArguments} does not use the arguments but only collects - them, it does not care about the default values of optional arguments. - Therefore, argument types with defaults (\docref{xparse:O}, - \docref{xparse:D} and \docref{xparse:R}) may be substituted by their - \texttt{-NoValue-} counterparts (\docref{xparse:o}, \docref{xparse:d} and - \docref{xparse:r}) and are therefore not included in the above table. - \end{tcolorbox} - - In general, however, an argument collector may this interpret this setting it - in any way it sees fit --- or not at all. For example, in Memoize the value - of \refmmzauto{args} is ignored for command \refcmd{tikz}, which requires a special - collector (\refcmd{mmzCollectTikZArguments}). - - When setting up advice for a \emph{command}, this key is initially - ``unset,'' i.e.\ it holds a special value indicating that the argument - specification is not provided. Note that this special value is not an empty - string --- \refmmzauto{args}|={}|, or simply \refmmzauto{args}|=|, indicates - a command which takes no arguments. During the execution of the advice, one - may use the \hologo{LaTeX}-style conditional - \docaux{cmd}{AdviceIfArgs}\marg{true branch}\marg{false branch} to test - whether the argument specification was provided. Executing this key without - a value restores it to the initial, unset value. - - When setting up the advice of an \emph{environment}, this key is - initialized to \docref{xparse:+}\docref{xparse:b} (a long environment body), - making it unnecessary to specify this value manually. Note that this holds - even for environments with arguments other that the environment body - ``argument'': those arguments will be caught as the start of the body even if - not explicitly specified. - - During advising and advice setup, the argument specification of the advised - command is accessible through \docaux{cmd}{AdviceArgs}, a parameterless macro - expanding to the given \meta{argument specification}. -\end{doc} - - -\begin{doc}{ - key={name=collector options, par=\marg{keylist}, - desc={cumulative, initially empty, value required}}, - key={name=raw collector options, par=\marg{code}, - desc={cumulative, initially empty, value required}}, - } - These keys append the given value to the list of user-friendly and raw - collector options, respectively. A comma is prefixed to the user-friendly - \meta{keylist} before appending it. - - Both kinds of collector options are intended to be used by the collector, - which may interpret them in any way it sees fit --- or not at all. The - initial collector, which invokes \refcmd{CollectArgumentsRaw} of package - CollArgs, passes both lists to this command, which interprets - \refmmzauto{collector options} as a user-friendly \pkg{pgfkeys} keylist - (which therefore requires a bit of processing) and \refmmzauto{raw collector - options} as plain \hologo{TeX} code (expecting it to contain only the - allowed, ``programmer's interface'' macros).\footnote{Clearly, - \refmmzauto{raw collector options} are why Advice deploys - \refcmd{CollectArgumentsRaw} rather than \refcmd{CollectArguments}. But - how does it then pass the user-friendly \refmmzauto{collector options} to - that command? It embeds them in \refcmd{collargsSet}.} The raw variant is - used internally by both Advice and Memoize, and may be used by a package - deploying the advising framework which wants to save a few processing cycles. - In CollArgs, the two kinds of options are functionally equivalent; both are - documented in section~\ref{sec:ref:collargs}. - - Initially, the list of collector options is empty, and for commands, so is - the list of raw collector options. For environments, however, the latter - list is initialized to set (the raw equivalent of) \refcollargs{environment} - to the environment name, and \refcollargs{end tag} to true. The rationale - for the latter is that the environment body containing the end tag (e.g.\ - |\end{foo}|) is nicely compatible with \refcmd{AdviceReplaced} (which equals -the begin tag, e.g.\ |\begin{foo}|) and \refcmd{AdviceOriginal} (which executes - the original definition of e.g.\ |\begin{foo}|). For example, thanks to - \refcollargs{end tag}, writing \refcmd{AdviceOriginal}|#1| in the inner - handler executes the original environment. Importantly, the original - environment can be executed without explicitly referring to the - environment's name, and with code that works not only for environments of - any \hologo{TeX} format, but is actually the same as the code which invokes - an original \emph{command}. Consequently, the same inner handler works for - both commands and environments, and in all \hologo{TeX} formats. - - Furthermore, the initial collector also sets option \refcollargs{caller} - to the name of the advised command or environment (however, - \refcollargs{caller} never appears in any of the collector options - lists; it is simply prefixed to them while constructing the invocation of - \refcmd{CollectArgumentsRaw}). And in Memoize, using keys - \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} triggers the - addition of the cognominal \refcollargs{verbatim}, \refcollargs{verb} - or \refcollargs{no verbatim} among the collector options. - - Precious few CollArgs' options thus remain to be set by the author. For - memoization, the most likely candidates are \refcollargs{ignore nesting} - and \refcollargs{ignore other tags}, which could help deal with unusual - environments. Overriding the initial \refcollargs{end tag} by - \refcollargs{begin tag}, \refcollargs{end tag} and/or \refcollargs{tags} - might also be useful on occasion. - - During advising and advice setup, the \pkg{pgfkeys} and the raw collector - options of the advised command are accessible through - \docaux{cmd}{AdviceCollectorOptions} and - \docaux{cmd}{AdviceRawCollectorOptions}, both a parameterless macro expanding - to the given \meta{keylist} and \meta{code}, respectively. -\end{doc} - -\begin{doc}{ - key={name=clear collector options}, - key={name=clear raw collector options}, - } - These keys empty the list of user-friendly and raw collector options, - respectively. -\end{doc} - -\begin{doc}{ - key={name=inner handler, par=\meta{\hologo{TeX} code}, long description=0.5, - desc=initially and default: see below}, - } - - This key declares the \meta{\hologo{TeX} code} as the inner handler component - of the advice. - - The inner handler is intended to be used in situations which require - knowledge of the advised command's arguments as a whole. In such situations, - the outer handler will normally invoke the collector, which will in turn - execute the inner handler and provide it with a single (braced) argument, - containing the collected arguments of the advised command. See - \refmmzauto{memoize} for the usage case which inspired this design. - - The simplest example of an inner handler is a (single-parameter) macro which - does nothing. Surprisingly enough, such an inner handler could be useful. - Defining |\def\Gobble#1{}| and setting - \code{\refmmz{auto}=\cs{foo}\bracestt{\refmmzauto{inner handler}=\cs{Gobble}, - \refmmzauto{args}=\bracestt{...}}} with the argument structure - appropriate for |\foo| could be used to eradicate all invocations of |\foo| - from the document. - - The inner handler has access to all the macros available to the outer - handler, but given that most of them have already fulfilled their function, - only the following will likely be useful in the inner handler: - \refcmd{AdviceNamespace}, \refcmd{AdviceName}, \refcmd{AdviceReplaced}, - \refcmd{AdviceOriginal}, and \refcmd{AdviceOptions}. - - Because there is clearly no reasonable default for the inner handler, this key - is initially set to an internal control sequence producing an ``undefined - inner handler'' error. Note that it is not necessary to define a dummy inner - handler when handling is entirely performed by the outer handler, i.e.\ in - cases when the inner handler is not invoked. Executing this key without a - value restores it to the initial value. - - During advising and advice setup, the inner handler of the advised command is - accessible through \docaux{cmd}{AdviceInnerHandler}, a parameterless macro - expanding to the given \meta{\hologo{TeX} code}. -\end{doc} - -\begin{doc}{easy, - key={name=options,par=\marg{keylist}, - desc={cumulative, initially empty, value required}}, - key={name=clear options}, - } - - The first key appends the given \meta{keylist} to the list of advice options - (after prefixing it by a comma), and the second one empties this list. For a - \meta{key} undefined in keypath \refkeypath{/mmz/auto}, - \meta{key}|=|\meta{value} has the same effect as - \refmmzauto{options}|=|\bracestt{\meta{key}=\meta{value}}. - - In Memoize, the options set by this key are known as - \emph{auto-options} --- options which are applied (using \refcmd{mmzset}) at - every invocation of the advised command or environment. For example, the - \env{tcolorbox} environment of package \pkg{tcolorbox} is used extensively - for typesetting this manual, and I have submitted this environment to - automemoization. However, the \env{tcolorbox}es in this manual often include - code listings. To memoize such environments successfully, their bodies must - be grabbed verbatim. I have therefore submitted the \env{tcolorbox} - environment to automemoization like this: - \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize}, - \refmmzauto{options}=\refmmz{verbatim}}; the simpler - \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize}, \refmmz{verbatim}} - would work as well. - - In general, whether to use the options set by this key, and how, remains at - the sole discretion of the advice. Note that they might be used by either - the outer or the inner handler, or perhaps even the collector. - - During advising and advice setup, the options of the advised command are - accessible through \docaux{cmd}{AdviceOptions}, a parameterless macro expanding - to the given \meta{keylist}. -\end{doc} - -\begin{doc}{key={name=reset, desc=style}} - Executing this key restores all \refmmz{auto} keys to their initial values. - - Invoking \refmmz{auto} on the same command or environment again - \emph{updates} the advice configuration. Use this key to start from scratch. -\end{doc} - -\begin{doc}{key={name=after setup, desc={initially empty, cumulative}}} - The code given to this key will be executed after exiting the group opened by - \refmmz{auto}. The same effect may be achieved by appending to macro - \docaux{cmd}{AdviceAfterSetup}. - - For example, \refmmzauto{integrated driver} uses this key to declare a new - conditional. -\end{doc} - -\paragraph{Commands available during the execution of advice} - -With the exception of \refcmd{AdviceGetOriginal}, the commands listed below -only become available in the outer handler, and if that handler does nothing -funky, they should be available in the collector and the inner handler, as -well. However, once the advice yields control to foreign code, these macros -are not guaranteed to hold the expected values anymore, because the foreign -code might trigger another piece of advice. Consequently, these macros -should be expanded, once, before integrating them into arbitrary (non-advice) -code; in particular, this applies to \refcmd{AdviceOriginal}. - -\begin{doc}{cmd={name=AdviceNamespace}} - This macro holds the \meta{namespace}, i.e.\ the keypath which this instance - of the advising framework was installed into. -\end{doc} - -\begin{doc}{cmd={name=AdviceName}} - This macro holds the name of the advised command or environment, i.e.\ the - name which was used as the first argument to \refmmz{auto}. For a command, - this will be a control sequence, e.g.\ |\foo|; for environments (in any - \hologo{TeX} format), their name, e.g.\ |foo|. -\end{doc} - -\begin{doc}{cmd={name=AdviceReplaced}} - This macro holds the code which was replaced by the outer handler. For - commands, this will be the command itself, e.g.\ |\foo|, so - \refcmd{AdviceReplaced} will equal \refcmd{AdviceName}. For an environment - |foo|, \refcmd{AdviceReplaced} is set to |\begin{foo}| in \hologo{LaTeX}, - |\foo| in \hologo{plainTeX} and |\startfoo| in \hologo{ConTeXt}. -\end{doc} - -\begin{doc}{cmd={name=AdviceOriginal}} - This macro executes the original code of the advised command. - - This macro is defined as \refcmd{AdviceGetOriginal}\marg{namespace}\marg{name}, - and therefore acts as a shortcut for an explicit invocation of - \refcmd{AdviceGetOriginal}. When executing the original command directly from - the advice, one may safely write \refcmd{AdviceOriginal}. However, whenever - \refcmd{AdviceOriginal} is embedded in code which might contain other advised - commands, it should be pre-expanded, exactly once. -\end{doc} - -\begin{doc}{cmd={name=AdviceGetOriginal, par=\marg{namespace}\marg{control sequence}}} - This command invokes the original definition of the \meta{control sequence} - advised by the \meta{namespace} instantiation of Advice. It may be safely - used outside the advice, even if the advised command is not activated. - - For example, upon executing key |/ns/|\refmmz{auto}|=\foo{...}|, - |\AdviceGetOriginal{/ns}{\foo}| will recall the original definition of |\foo| - if |\foo| is activated, and simply execute |\foo| otherwise. - - The second argument of this command should \emph{not} be an environment name. - To execute the original environment |foo| in \hologo{TeX} or - \hologo{ConTeXt}, use \refcmd{AdviceGetOriginal} with the appropriate macro: - |\AdviceGetOriginal{/ns}{\foo}| or |\AdviceGetOriginal{/ns}{\startfoo}|. In - \hologo{LaTeX}, one should use |\AdviceGetOriginal{/ns}{\begin}{foo}|, which - executes the original \refcmd{begin} and provides it with the environment name. - - Within the advice, you will probably never have to use this command directly, - but will rather rely on the (plain or pre-expanded) \refcmd{AdviceOriginal}. - However, outside the advice, this command provides the only means to - access the original definition of an advised command. (Unlike the - commands described above, this command is available throughout the - document.) - - \begin{tcolorbox}[warning] - A typo in the invocation of this command may result in an infinite loop. - Assume that the advice for |\foo|, declared in namespace \refkeypath{/mmz}, - executes \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which incorrectly refers - to the non-existing namespace |/zzm|, and that command |\foo| is activated. - Executing |\foo| will eventually execute - \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which won't find the original - definition of |\foo| in the non-existing namespace |/zzm| and will thus - execute macro |\foo| (again), which, being advised, will lead to another - \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, etc. My advice is to define an - abbreviation like |\def\mmzAdviceGetOriginal{\AdviceGetOriginal{/mmz}}|. - And note that the name\-space is a full keypath, which begins with a slash - (|/|), but has no slash at the end. - \end{tcolorbox} -\end{doc} - -\paragraph{Support for specific packages} -At the moment, Advice only implements specific support for \TikZ, by defining a -\refmmzauto{collector} for command \refcmd{tikz}. - -\begin{doc}{ - cmd={name=AdviceCollectTikZArguments}, - } - This command collects the arguments in the format expected by \refcmd{tikz}, and - executes macro \refcmd{AdviceInnerHandler} with the collected arguments given - as a single braced argument. The collector supports both the group and the - semicolor invocation of |\tikz|, i.e.\ both |\tikz{...}| and |\tikz...;|. - - This command is only available upon |\input|ting file - \docaux{file}{advice-tikz.code.tex}. -\end{doc} - -\endgroup % keypath = /mmz/auto - -\subsubsection{Memoization-related additions to the advising framework} -\label{sec:ref:advice:memoization} - -In section~\ref{sec:ref:advice}, we have seen that Memoize installs the -advising framework into keypath \refkeypath{/mmz}, with setup key name -\refmmz{auto}. This populates keypaths \refkeypath{/mmz} and -\refkeypath{/mmz/auto} with various generic advice keys. However, Memoize -installs further advice\slash automemoization-related keys into these keypaths. -It is these keys which are described in this section.\footnote{One such key, - \refmmzauto{integrated driver}, is actually documented in - section~\ref{sec:ref:memoization}.} - -Therefore, in contrast to section~\ref{sec:ref:advice}, \refkeypath{/mmz} and -\refmmz{auto} have no secret generic meaning here, i.e.\ they should \emph{not} -be generalized to \meta{namespace} and \meta{setup key} of -\refkey{/handlers/.install advice}. - -\paragraph{Keys residing in \refkeypath{/mmz}} - -\begingroup -\setlength\textfloatsep{20pt plus 2pt minus 4pt}% back to the default - -\begin{doc}{easy, - key={name=manual, description={preamble-only,{ }}, conditional=false}, - } - When this conditional is set to true, no commands are \refmmz{activate}d at - the beginning of the document. The list of commands and environments advised - and activated out of the box can be found in Table~\ref{tab:advised-commands}. - - The auto-framework allows the activation to be deferred (see - \refmmz{activation} and \refkey{/handlers/.install advice}) but leaves it - open to the specific instance of the framework to use the deferred activation - commands as it sees fit. Normally, Memoize switches to immediate activation - at the end of the preamble (hook \refmmz{begindocument/before}) and issues - \refmmz{activate deferred} at the beginning of the document, more precisely - in hook \refmmz{begindocument/end} (afterwards, \refmmz{activate deferred} is - emptied). However, when \refmmz{manual} is in effect, the deferred - activation is suppressed (though it may be still carried out by the user by - executing \refmmz{activate deferred}). - - In \hologo{LaTeX}, \refmmz{manual} affects the (internal) activation of - \refcmd{begin} as well, which effectively deactivates handling of all environments. -\end{doc} - -\begin{table} - \centering - \begin{tabularx}{\linewidth}{llX} - \toprule - command/environment&handler¬es\\ - \midrule - \refcmd{begin}&custom&Only in \hologo{LaTeX}; declared by Advice.\\ - \refcmd{errmessage}&\refmmzauto{abort}&Not available in \hologo{LuaTeX}, - where better error-detection is implemented.\\ - \refenv{forest}&\refmmzauto{memoize}\\ - \refcmd{Forest}&\refmmzauto{memoize}\\ - \refcmd{index}&\refmmzauto{replicate}&The argument is expanded prior to replication.\\ - \refcmd{label}&custom&Globally appends \refcmd{mmzLabel}\marg{label key}\marg{current label} - to register \refcmd{mmzCCMemo}; \refmmzauto{run if memoizing}.\\ - \refcmd{pageref}&\refmmzauto{ref}\\ - \refcmd[into index=false]{pdfsavepos}&\refmmzauto{abort}&Not available in \hologo{LuaTeX}.\\ - \refcmd{pgfsys@getposition}&\refmmzauto{abort}&Available only in \TikZ is - loaded, it aborts memoization of a picture which gets accidentally marked as - ``remembered''.\\ - \refcmd{ref}&\refmmzauto{ref}\\ - \refcmd[short]{savepos}&\refmmzauto{abort}&Available only in \hologo{LuaTeX}.\\ - \refcmd{tikz}&\refmmzauto{memoize}\\ - \refenv{tikzpicture}&\refmmzauto{memoize}\\ - \bottomrule - \end{tabularx} - \caption{Commands advised by Memoize} - \label{tab:advised-commands} -\end{table} - -\begin{doc}{easy, - key={name=ignore spaces, conditional=false}, - } - Ignore any spaces following \emph{auto}memoized code. This key has no effect - for manual memoization, i.e.\ command \refcmd{mmz} and environment - \refenv{memoize}. - - It is common practice to conclude the definition of a command by \hologo{TeX} - primitive \refcmd{ignorespaces}, which consumes any following spaces, to - prevent unintended blank space after the command's invocation. Automemoizing - such a command disrupts this behaviour.\footnote{It is clear that - \refcmd{ignorespaces} is disrupted during utilization; in this case, the - original command, including the concluding \cs{ignorespaces}, is never even - executed. However, the disruption also occurs during memoization, and even - during regular compilation. In both cases, the memoized code is embedded - in some internal Memoize code. Therefore, the original \cs{ignorespaces} - does not occur directly in front of the rest of the document.} The - workaround is to use this key, normally as an option in the \refmmz{auto} - declaration; it will work both for automemoized macros and environments. -\end{doc} - -\paragraph{Keys residing in \refkeypath{/mmz/auto}} - -\begingroup -\yadocset{keypath=/mmz/auto} - -\begin{doc}{easy,key={name=memoize, desc=style}} - This key sets up advice which triggers memoization of the command or - environment whenever it is encountered; we often refer to such a command as - ``automemoized,'' or say that it was ``submitted to automemoization.'' - - An automemoized command will consume the next-options, whether memoization - actually occurs or not. - - Under the hood, this key declares both an \refmmzauto{outer handler} and an - \refmmzauto{inner handler}. The outer handler opens the memoization group - (so that options can be applied locally), applies the auto-options (given by - \refmmzauto{options} within \refmmz{auto}) and the next-options (given by - \refcmd{mmznext}) by executing \refmmzauto{apply options}, and appends the - verbatim keys to \refmmzauto{collector options} if necessary. The inner - handler invokes \refcmd{Memoize} (which closes the group opened by the outer - handler): the first argument is \refcmd{AdviceReplaced}, expanded once and - followed by the arguments of the handled command; the second argument is - \refcmd{AdviceOriginal}, also expanded once and followed by the arguments of - the handled command. The inner handler also makes sure that \refmmz{ignore - spaces} is respected. -\end{doc} - -\begin{doc}{easy,key={name=nomemoize,desc=style}} - This key installs advice which disables memoization for the space of the - command or environment; we sometimes refer to such commands as - ``autodisabled.'' - - This key is merely an abbreviation for \refmmzauto{noop}|,| - \refmmzauto{options}|=|\refmmz{disable}. See the documentation of - \refmmzauto{noop} for further details. -\end{doc} - -\begin{doc}{key={name=noop,desc=style}} - This key sets up advice which does nothing. - - Ok, not nothing at all. The installed handler applies the auto-options and - the next-options by executing \refmmzauto{apply options}, and makes sure that - \refmmz{verbatim} and \refmmz{ignore spaces} are respected. - - For commands and non-\hologo{LaTeX} environments, this key declares the same - outer handler as \refmmzauto{memoize}, while the inner handler merely - executes the original command (respecting the potential verbatim mode), - closes the group opened by the outer handler, and makes sure that - \refmmz{ignore spaces} is respected. - - For \hologo{LaTeX} environments, which open the group necessary for the local - application of options themselves, this key declares an outer handler which - adds the relevant code into the next hook |env/|\meta{environment - name}|/begin|. There is no need to open a group, collect the environment - body, or make special provisions for the verbatim mode. -\end{doc} - - -\begin{doc}{key={name=apply options,desc=style}} - This style, used by \refmmzauto{memoize}, \refmmzauto{nomemoize} and - \refmmzauto{noop} described above, installs two handlers: - \begin{itemize} - \item an outer handler which opens a group, applies auto-options and - next-options by executing \refcmd{mmzAutoInit}, and executes the collector; - and - \item a bailout handler which clears the next-options. - \end{itemize} -\end{doc} - - -\begin{doc}{cmd={name=mmzAutoInit}} - This macro applies the auto-options and the next-options. - - Additionally, if \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} was - previously executed, this style appends the corresponding CollArgs key - (\refcollargs{verbatim}, \refcollargs{verb} or \refcollargs{no - verbatim}) to \refcmd{AdviceRawCollectorOptions}. In case several of the - verbatim keys were executed, the final one takes effect. -\end{doc} - - -\begin{doc}{easy, key={name=abort,desc=style}} - This key sets up advice which aborts any ongoing memoization. - - Under the hood, the advice merely executes \refcmd{mmzAbort} followed by - \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the - next-options. Out of the box, we submit two control sequences to this - handler: - \begin{itemize} - \item \refcmd{errmessage}: this allows us to detect and abort upon at least some - errors. - \item \refcmd[into index=false]{pdfsavepos} (in \hologo{LuaTeX}, - \refcmd[short]{savepos}): one common effect is that memoization of any - \TikZ picture with \refkey{/tikz/remember picture} set is aborted. - \end{itemize} -\end{doc} - - -\begin{doc}{key={name=unmemoizable,desc=style}} - This key sets up advice which aborts the ongoing memoization and marks the - automemoized code as unmemoizable, so that it will be henceforth compiled - regularly. - - Under the hood, the advice merely executes \refcmd{mmzUnmemoizable} followed - by \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the - next-options. - - Out of the box, we submit no control sequences to this advice, but it might - make sense to submit \refcmd[into index=false]{pdfsavepos}\slash - \refcmd[short]{savepos}. Keys \refmmzauto{abort}|=|\refcmd[short]{savepos} - and \refmmzauto{unmemoizable}|=|\refcmd[short]{savepos} will most often have - the same effect, as far as the author is concerned; the former was chosen as - the default because it does not produce a c-memo; see - \refcmd{mmzUnmemoizable} for a situation where \refmmzauto{unmemoizable} is - preferred. -\end{doc} - - -\begin{doc}{easy, - key={name=ref,desc=style}, - key={name=force ref,desc=style}, - } - - These keys set up advice which adds the reference key to the context - expression. They are intended to be used with cross-referencing commands - such as \refcmd{ref} and \refcmd{pageref}. - - Indeed, \refcmd{ref} and \refcmd{pageref} are submitted to this advice by - Memoize, with the effect that standard cross-referencing inside memoized code - ``just works.'' Note that the stabilization of the document after changing - the reference takes three compilation cycles, i.e.\ one cycle more than - without memoization. - - The advice set up by \refmmzauto{ref} aborts memoization if the reference - key is undefined, the rationale being that the produced memo and extern would - most often be useless, and could even obscure an undefined reference. The - \refmmzauto{force ref} handler produces the memo and the extern even when the - reference is undefined. - - The reference produced by the advised command should be fully expandable - (because it will be expanded as a part of the context expression). - - Typically, a \refcmd{ref} command takes a single argument, the reference key. - However, some packages may define a reference command which takes optional - arguments, as well; in particular, the \pkg{hyperref}'s incarnation of \refcmd{ref} - takes an optional star. This advice does not care: it will accept any number - of any kind of optional arguments, as long as the reference key is the first - braced argument following the advised command; for example, |\ref*{key}|, - |\ref[opt]{key}|, |\ref*[opt]{key}| etc.\ will all be handled correctly, - while |\ref{mand}{key}| will not work. Effectively, it is as if we had set - \refmmzauto{args}|=lm| --- and with the same downside, namely that an - unlikely unbraced single-token reference key, like |\ref k|, will not work. - - Under the hood, these two pieces of advice pass the reference key to macros - \refcmd{mmzNoRef} and \refcmd{mmzForceNoRef}, and it is these commands --- - which may also be used in user-defined advice or the document itself --- - which actually add the reference key to the context expression. -\end{doc} - -\begin{doc}{easy, - key={name=refrange,desc=style}, - key={name=force refrange,desc=style}, - } - These keys have the same function as \refmmzauto{ref} and \refmmzauto{force - ref}, but they operate on reference-range commands, such as \pkg{cleveref}'s - \refcmd{crefrange}, which take two arguments (the starting and the ending reference - key). -\end{doc} - -\begin{doc}{easy, - key={name=multiref,desc=style}, - key={name=force multiref,desc=style}, - } - These keys have the same function as \refmmzauto{ref} and \refmmzauto{force - ref}, but they operate on ``multireference'' commands, such as - \pkg{cleveref}'s \refcmd{cref}, which allow the author to list several - comma-separated reference keys in a single argument. -\end{doc} - -\begin{doc}{ - key={name=replicate, desc=style} - } - This key sets up advice which replicates the invocation of the command in - the cc-memo during memoization. - - When using this key, it is necessary to set \refmmzauto{args} as well. For - \refcmd{index}, Memoize executes - \refmmz{auto}|=\index|\bracestt{\refmmzauto{args}=m, \refmmzauto{replicate}}. - - This key takes an auto-option, \docAux{keypath=/mmz/auto/replicate, - key={name=expanded}}.\footnote{This option is unrelated to Memoize's options, - settable by \refcmd{mmzset}.} If given, the collected arguments will be - expanded before replicating them in the cc-memo; in \hologo{LaTeX}, this - expansion is |\protect|ed. - - In \hologo{LaTeX}, Memoize submits \refcmd{index} to this handler (with - expansion). Therefore, any |\index{key}| in the memoized code gets copied - into the cc-memo. Effectively, indexing from within the memoized code ``just - works.'' - - Note that \refcmd{label}, despite essentially requiring replication, cannot - use this advice, because it needs to replicate not only the label key but - \refcmd{@currentlabel} as well. -\end{doc} - -\begin{doc}{key={name=run if memoization is possible, desc=style}} - Under the run conditions installed by this key, a command is only advised if - Memoize is enabled but we're not already ``within Memoize,'' i.e.\ memoizing - or normally compiling some code submitted to memoization. In code: - \refcmd{ifmemoize}\refcmd{ifinmemoize}|\else|\refcmd{AdviceRuntrue}|\fi\fi|. - - Internally, this key is used by \refmmzauto{memoize} and \refmmzauto{noop}. -\end{doc} - -\begin{doc}{key={name=run if memoizing, desc=style}} - Under the run conditions installed by this key, a command is only advised - during memoization. In code: - \refcmd{ifmemoize}\refcmd{ifmemoizing}\refcmd{AdviceRuntrue}|\fi\fi|. - - Internally, this key is used by \refmmzauto{abort}, \refmmzauto{replicate}, - and \refmmzauto{ref} and friends. -\end{doc} - -\endgroup % keypath = /mmz/auto -\endgroup - - - -\subsubsection{Package CollArgs} -\label{sec:ref:collargs} - -\begingroup -\yadocset{keypath=/collargs} - -\begin{doc}{cmd={ - name=CollectArguments, - par=\oarg{options}\marg{argument specification}\marg{next-code}% - \textcolor{gray}{\meta{tokens}}}} - - This command determines the extent to which the \meta{tokens} following the - the three formal arguments of the command conform to the given \meta{argument - specification}, effectively splitting \meta{tokens} into \meta{argument - tokens} and the \meta{rest} of the tokens, and then executes - \meta{next-code} with the \meta{argument tokens} provided as a single, braced - argument: - \begin{center} - \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}} - \end{center} - If the initial part of \meta{tokens} does not conform to \meta{argument - specification}, \refcmd{CollectArguments} throws an error. (In this case, - \meta{next-code} is not executed, and the \meta{tokens} collected until the - error are thrown away.) - - The optional \meta{options} are processed using the \pkg{pgfkeys} utility of - PGF/\TikZ (see \PGFmanual{87}), with the default path set to - \docaux{key path}{collargs}. The given options apply to all the arguments in - \meta{argument specification}. The recognized keys are listed in the rest of - the section. - - The \meta{argument specification} should be given in the \pkg{xparse} format - (we summarize this format in the documentation for \refmmzauto{args} in - section~\ref{sec:ref:advice}), with several extensions:\footnote{Collargs - internally uses a dot (|.|) to delimit the argument specification from the - following argument tokens. Therefore, the dot really counts as an extra - argument type, in the sense that Collargs will stop working if the dot - becomes an argument type or a modifier in some future release of - \pkg{xparse}.} - \begin{itemize} - \item We introduce modifier \docAux{xparse modifier={name=\&,label=amp,index - annotation/.prefix={additional\ }, index annotation/.append={\ - (options)} }} taking a mandatory argument specifying the options to - apply to the following argument in the specification. Options given here - override the \meta{options} given as the optional argument. - \item The environment body type \docAux{xparse type={name=b,index - annotation/.append={\ (environment body)}}} may be followed by an optional - \emph{braced} argument providing the name of the environment to collect. - The name given here overrides the name given by the - \refcollargs{environment} option. - \item The number of collected ``arguments'' is unlimited. - \end{itemize} - - Also note that the effect of \docref{xparse:O}\marg{default} is the same as - the effect of \docref{xparse:o}, and similarly for other pairs of types with - and without defaults (\docref{xparse:R} and \docref{xparse:r}, - \docref{xparse:D} and \docref{xparse:d}, and \docref{xparse:E} and - \docref{xparse:e}). CollArgs is dedicated to collecting the argument tokens - precisely as they are given: if an optional argument is missing, its default - value is \emph{not} inserted among the collected arguments --- consequently, - \refcmd{CollectArguments} is utterly uninterested in the default value. - - Collection of environments automatically adapts to the format, i.e.\ given - environment body name |foo|, \refcmd{CollectArguments} knows to search for - \refcmd{begin}\bracestt{foo} |...| \refcmd{end}\bracestt{foo} in - \hologo{LaTeX}, \cs{foo} |...| \cs{endfoo} in \hologo{plainTeX}, and - \cs{startfoo} |...| \cs{stopfoo} in \hologo{ConTeXt}. For further - information on environment collection, see keys \refcollargs{ignore - nesting} and \refcollargs{tags}. -\end{doc} - -\begin{doc}{cmd={ - name=CollectArgumentsRaw, - par=\marg{option-setting code}\marg{argument specification}\marg{next-code}% - \textcolor{gray}{\meta{tokens}}}} - - This command is the programmer's interface to CollArgs, intended to be used - instead of \refcmd{CollectArguments} when compilation speed is an issue. The - two commands only differ in how they deal with options. - - One difference is that for \refcmd{CollectArgumentsRaw}, the options form a - mandatory rather than an optional argument. More importantly, however, they - do not take the form of a keylist, but should be composed out of low-level - option-setting commands. Each key documented in this section has a - corresponding low-level macro; these macros are listed in footnotes alongside - the keys. The name of the macro starts with |\collargs| and continues with - the name of the key, without spaces, each word capitalized; if the key is - boolean, this convention applies to the base of the \hologo{TeX} - conditional. For example, - \begin{tcblisting}{listing only, listing options app={escapechar=|}} -\CollectArguments[caller=\foo, tags, verbatim]|\marg{argument specification}\marg{next-code}| - \end{tcblisting} - is equivalent to - \begin{tcblisting}{listing only, listing options app={escapechar=|}} -\CollectArgumentsRaw{% - \collargsCaller{\foo}% - \collargsBeginTagtrue\collargsEndTagtrue - \collargsVerbatim -}|\marg{argument specification}\marg{next-code}| - \end{tcblisting} - - Withing the option-setting code, the programmer may also deploy macro - \docaux{cmd}{collargsSet}, which processes the \meta{options} in the keylist - format. One idea could be to execute this macro at the end of the low-level - options; this would set the ``defaults'' using the fast programmer's - interface, but still allow for user customization. -\end{doc} - - -\begin{doc}[ - pi={\docaux{cmd}{collargsCaller}}, - ]{ - key={name=caller, par=\meta{control sequence (name)}, - desc={no default, initially \cs{CollectArguments}}}, - } - Set the control sequence to refer to in error messages. - - If \meta{tokens} do not match the \meta{argument specification}, - \refcmd{CollectArguments} throws an error. By default, the error message - contains a reference to \refcmd{CollectArguments} itself, for example % - |! Argument of \CollectArguments has an extra }.| However, this might not be -very informative to the author. When \refcollargs{caller}|=\cs| is in effect, -the error messages will refer to the given |\cs| instead. - -If the value of this key is not a control sequence, it is assumed to be an -environment name, but as the caller must be a macro, this name will be -converted into a control sequence. Setting \refcollargs{caller}|=foo| will -result in error messages referencing |\foo| in \hologo{plainTeX}, |\startfoo| -in \hologo{ConTeXt} and |\begin{foo}| (a single control sequence!) in - \hologo{LaTeX}. -\end{doc} - -\begin{doc}[ - pi=\docaux{cmd}{collargsEnvironment}, - ]{key={name=environment, par=\meta{environment name}, - desc={applicable to type \docref{xparse:b}, no default, initially empty}} - } - - Set the name of the environment collected by argument type - \docref{xparse:b}. -\end{doc} - -\begin{doc}[ - pi={\docaux{cmd}{ifcollargsBeginTag}, \docaux{cmd}{ifcollargsEndTag}; - \refcollargs{tags} has no corresponding low-level command} - ]{ - key={name=begin tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false}, - key={name=end tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false}, - key={name=tags, par=\meta{boolean}, % not a conditional - desc={applicable to type \docref{xparse:b}, style, default |true|}}, - } - - When \refcollargs{begin tag}\slash \refcollargs{end tag} is in effect, the - begin\slash end tag will be will be prepended/appended to the collected - environment body. Style \refcollargs{tags} is a shortcut for setting - \refcollargs{begin tag} and \refcollargs{end tag} simultaneously. - - In \hologo{LaTeX}, using \refcollargs{tags} will thus dress up the - collected body in a pair or \refcmd{begin}\marg{environment name} and - \refcmd{end}\marg{environment name}. CollArgs will automatically use the tags - appropriate to the format. - - In the verbatim modes, the added tags are verbatim as well, with the detail - that in \hologo{LaTeX}, there is a slight difference between the full - \refcollargs{verbatim} and the partial \refcollargs{verb} mode. In the - full verbatim mode, the braces surrounding the environment name are verbatim - (the characters used as braces are actually determined by key - \refcollargs{braces}). In the partial verbatim, as well as the - non-verbatim mode, the environment name is surrounded by a pair of actual - braces of category 1 and 2, regardless of which characters are of these - categories in the calling code. -\end{doc} - -\begin{doc}[ - pi=\docaux{cmd}{ifcollargsIgnoreNesting}, - ]{ - key={name=ignore nesting, desc={applicable to type \docref{xparse:b},{ }}, - conditional=false}, - } - When this key is \emph{not} in effect, CollArgs respects the hierarchical - structure created by tag pairs such as \refcmd{begin}\bracestt{foo} and - \refcmd{end}\bracestt{foo}. Given the situation below on the - left, argument type |b{foo}| will collect everything up until the - \emph{second} |\end{foo}|. Now this is what we usually want, because -\hologo{LaTeX} keeps track of environment embedding as well. However, all -\emph{verbatim} environments I know of, starting with the standard -\hologo{LaTeX} \refenv{verbatim}, will ignore the nesting and simply scoop up -everything up to the first \cs{end}|{verbatim}|. In CollArgs, we can replicate -their behaviour by setting \refcollargs{ignore nesting}, as shown below on the -right. (Of course we also need to set \refcollargs{verbatim} if we want to -grab the environment body in the verbatim mode.) - - \begin{tcbraster}[raster columns=2] - \begin{tcblisting}{listing only, mark region={1}{5}, - example title=\texttt{ignore nesting=false}} - ... - \begin{foo} - ... - \end{foo} - ... -\end{foo} - \end{tcblisting} - \ExampleName{_collargs-verbatim} - \makeexample{\examplename.tex.c1} - \tcbinputexample{listing only, no attachment, mark region={1}{3}, - example title=\texttt{ignore nesting=true}} - \end{tcbraster} - - This key applies not only to argument type \docref{xparse:b} (in either - normal or verbatim mode), but also to the verbatim argument type - \docref{xparse:v} and to argument types \docref{xparse:m} and - \docref{xparse:g} in the \refcollargs{verbatim} (but not normal, or - \refcollargs{verb}) mode. With these keys, the relevant structure markers - are braces, |{| and |}|. -\end{doc} - -\begin{doc}[ - pi=\docaux{cmd}{ifcollargsIgnoreOtherTags}, - ]{ - key={name=ignore other tags, desc={applicable to type \docref{xparse:b},{ }}, - conditional=false}, - } - In \hologo{LaTeX}, the environment tags, \refcmd{begin}\marg{name} and - \refcmd{end}\marg{name}, contain braces, which retain their usual category codes - in the non-verbatim and in the partial verbatim mode. Consequently, CollArgs - cannot easily search for the full tags to delimit the - environment. - - When this key is \emph{not} in effect, CollArgs takes the easy path, and - determines the end of the environment only by inspection of \cs{begin}s and - \cs{end}s, without reference to what \meta{name} they begin or end. Only - when this key \emph{is} in effect does CollArgs inspect these \meta{name}s, - effectively ignoring any \cs{begin}s and \cs{end}s not followed by the name of - environment being collected. The effect of absence vs.\ presence of this key - is shown below, where the shaded area marks the code collected into - environment |foo|. - - \begin{tcbraster} - \ExampleName{_collargs-ignore-other-tags} - \makeexample{\examplename.tex.c1 N=2} - \tcbinputexample{listing only, no attachment, mark region={3}{3}, - example title=\texttt{ignore other tags=false}} - \tcbinputexample[.tex][.c2]{listing only, no attachment, mark region={3}{5}, - example title=\texttt{ignore other tags=true}} - \end{tcbraster} - - This key does not have any effect the full verbatim mode, which always - behaves as if this key was set to true, because braces are of category - ``other'' as well. Similarly, it is as if this key was always true in - \hologo{plainTeX} and \hologo{ConTeXt}, simply because environment tags in - these formats don't contain braces. -\end{doc} - -\begin{doc}[ - pi={\raggedright - \docaux{cmd}{collargsAppendPreprocessor}, - \docaux{cmd}{collargsPrependPreprocessor}, - \docaux{cmd}{collargsAppendPostprocessor}, - \docaux{cmd}{collargsPrependPostprocessor}}, - ]{ - key={name=append preprocessor, par=\meta{code}, desc={no default}}, - key={name=prepend preprocessor, par=\meta{code}, desc={no default}}, - key={name=append postprocessor, par=\meta{code}, desc={no default}}, - key={name=prepend postprocessor, par=\meta{code}, desc={no default}}, - } - These keys declare processors which will transform the collected argument - before appending it to the argument list. - - A collected argument undergoes the following transformations: - \begin{itemize} - \item First, the argument is processed by any \emph{pre}processors, in the - order indicated by |append| and |prepend|. - \item Next, the processed argument is dressed up in the delimiters according - to its type. For example, an optional argument of type \docref{xparse:o} - will be surrounded by square brackets. - \item Finally, the delimited argument is processed by any - \emph{post}processors, again in the order indicated by |append| and - |prepend|. - \end{itemize} - - \meta{code} will typically consist of a single control sequence pointing to a - one-argument macro, which will receive the collected argument (possibly - modified by the processors already applied). In general, however, the value - of this key may be any code; Collargs will execute \meta{code}\marg{collected - argument}. - - The processed argument should be returned by storing it into token register - \docaux{cmd}{collargsArg}. - - The following example illustrates how one could go about reimplementing Bruno - Le Floch ingenious package \pkg{cprotect}.\footnote{This example is merely a - proof of concept. For the bells and whistles which would make it useful in - real life, see the documentation of \pkg{cprotect}.} We define processor - |\writetofile| which dumps the argument into a file, replacing it with the - |\input| statement. (Of course, to allow for verbatim content in the - footnote, we also have to mark the argument as \refcollargs{verbatim}. - And we use \refcollargs{no delimiters} to get rid of the braces around the - footnote text.) - - \ExampleName{collargs-processor} - \makeexample{\examplename.tex.c1} - \tcbinputexample{listing and text, listing file=\examplepath.tex.c1} -\end{doc} - -\begin{doc}[ - pi={\docaux{cmd}{collargsClearPreprocessors}, - \docaux{cmd}{collargsClearPostprocessors}}, - ]{ - key={name=clear preprocessors}, - key={name=clear postprocessors}, - } - Clear the list of pre- or post-processors. -\end{doc} - -\begin{doc}[ - pi={\docaux{cmd}{collargsAppendExpandablePreprocessor}, - \docaux{cmd}{collargsPrependExpandablePreprocessor}, - \docaux{cmd}{collargsAppendExpandablePostprocessor}, - \docaux{cmd}{collargsPrependExpandablePostprocessor} - }, - ]{ - key={name=append expandable preprocessor, par=\meta{code}, desc={no default}}, - key={name=prepend expandable preprocessor, par=\meta{code}, desc={no default}}, - key={name=append expandable postprocessor, par=\meta{code}, desc={no default}}, - key={name=prepend expandable postprocessor, par=\meta{code}, desc={no default}}, - } - These keys may be used to add fully expandable processors. A processor added - with one of these keys will end up among the processors declared by - \refcollargs{append preprocessor} et al. - - A processor declared by one of these keys will define the processed argument - as the full expansion (|\edef|) of \meta{code}\marg{collected argument}. - \meta{code} will typically consist of a single control sequence pointing to a - fully expandable one-argument macro. - - For example, |\trim@spaces@noexp| from package \pkg{trimspaces} could be used - as an expandable processor of environment body to remove the spaces around - the grabbed environment body. -\end{doc} - -\begin{doc}[ - pi={\docaux{cmd}{collargsAppendPrewrap}, - \docaux{cmd}{collargsPrependPrewrap}, - \docaux{cmd}{collargsAppendPostwrap}, - \docaux{cmd}{collargsPrependPostwrap}}, - ]{ - key={name=append prewrap, par=\meta{macro definition}, desc={no default}}, - key={name=prepend prewrap, par=\meta{macro definition}, desc={no default}}, - key={name=append postwrap, par=\meta{macro definition}, desc={no default}}, - key={name=prepend postwrap, par=\meta{macro definition}, desc={no default}}, - } - These keys add processors which transform the collected argument in a single - expansion. - - The declared processor will use \meta{macro definition} to define a temporary - one-argument \meta{macro}, and then set the \meta{processed argument} to be - the single expansion of \meta{macro}\marg{collected argument}. - - For example, to add quotes around the collected argument, write - \refcollargs{append prewrap}|={``#1''}| (doubling the hash when executing - \refcmd{CollectArguments} from a macro, of course). Or, perhaps more - usefully, \refcollargs{append prewrap}|={\scantokens{#1}}| can be used to - retokenize a verbatim argument (during the execution of the - \meta{next-code}). -\end{doc} - - -\begin{doc}[ - pi=\docaux{cmd}{ifcollargsNoDelimiters} - ]{ - key={name=no delimiters, conditional=false}, - } - When this key is in effect, the collected argument will not be dressed up - into delimiters that it was dressed up in \meta{argument tokens}. For - example, an optional argument, encountered as [\meta{argument}] inside - \meta{argument tokens}, will be spit out simply as \meta{argument}. - - Any user-specified pre- or post-processing will still be applied. - \ExampleName{collargs-nodelimiters} - \makeexample{\examplename.tex.c1} - \tcbinputexample{comment=\input{\examplepath.tex.c1}} -\end{doc} - - -\begin{doc}[ - pi={\docaux{cmd}{collargsVerbatim}, - \docaux{cmd}{collargsVerb}, - \docaux{cmd}{collargsNoVerbatim}. - To ensure the same effect as with the keys, - place these macros at the end of the option code}, - ]{ - key={name=verbatim, desc=style}, - key={name=verb, desc=style}, - key={name=no verbatim, desc=style}, - } - Select the full verbatim, the partial verbatim, or the non-verbatim mode of - argument collection. - - In the full \refcollargs{verbatim} mode, the arguments are collected under - a category code regime in which all characters are of category 12, ``other''. - The same goes for the partial \refcollargs{verb} mode, except that in this - case, the grouping characters --- usually the braces |{| and |}| --- retain - their usual category codes 1 and 2. Key \refcollargs{no verbatim} selects - the normal, non-verbatim mode. - - The partial \refcollargs{verb} mode can be useful for verbatim collection - of an optional argument. To pass |]| as an optional argument to command -|\foo|, we normally enclose it in braces: ˇ\foo[{~]~}]ˇ. However, if we try to -collect |[{]}]| with \refcmd{CollectArguments}|[|\refmmz{verbatim}|]{o}|, we -will get |{| (and most likely an error, as well), because in the - \refmmz{verbatim} mode, braces do not have their grouping function. Using - the \refmmz{verb} mode solves the problem: occuring within braces, the first - |]| is ``invisible'' to \refcmd{CollectArguments}|[|\refmmz{verb}|]|, so the -optional argument is correctly recognized as ending at the second |]|. - -The partial \refcollargs{verb} mode is also useful for collecting the bodies -of \emph{\hologo{LaTeX}} environments. The full \refcollargs{verbatim} mode -will only correctly collect these bodies when the relevant \refcmd{begin} -and\slash or \refcmd{end} control sequences are followed by the grouped -environment name without any intervening spaces. The partial -\refcollargs{verb} mode has no such restriction. - -In the verbatim modes, modifier \docref{xparse:+} has no effect. The arguments -are always collected as if they were long. - -To correctly collect arguments in the verbatim modes, CollArgs has to mimic the -many details of \hologo{TeX}'s tokenization and argument delineation. These -details depend on the category code regime, and CollArgs automatically adapts -to the ``outside'' category code regime, i.e.\ the regime in effect at the time -of invoking \refcmd{CollectArguments}. In particular, CollArgs remembers which -characters were of category codes 0, 1, 2, 5, 10 and 11, and adapts the -argument collection accordingly. For example, it will correctly pick up a -control sequence as a single-token \docref{xparse:m}-type (\hologo{TeX}'s undelimited) -argument even when it begins with a non-standard character of category code 0. -The single caveat is that only a single pair of characters can function as the -grouping characters in the full verbatim mode; to compensate for the -deficiency, this character pair is customizable via key -\refcollargs{braces}. -\end{doc} - -\begin{doc}[ - pi={\docaux{cmd}{collargsFixFromVerbatim}, - \docaux{cmd}{collargsFixFromVerb}, - \docaux{cmd}{collargsFixFromNoVerbatim}}, - ]{ - key={name=fix from verbatim, desc=style}, - key={name=fix from verb, desc=style}, - key={name=fix from no verbatim, desc=style}, - } - Key \refcollargs{fix from no verbatim} should be used when the first - argument should be collected in a verbatim mode, but the outside code has - already tokenized the first character of the subsequent input stream (most - probably by a |\futurelet|) in the non-verbatim category code regime. Using - this key will trigger a CollArgs' ``mode transition'' (described below) which - will fix the situation. (This key is used in the implementation of - \refcmd{mmz}.) - - The other two keys should be used in the unlikely reverse situation, where - the outside code has tokenized the following character in a verb(atim) mode, - while CollArgs is requested to collect the first argument in the non-verbatim - mode. -\end{doc} - -\begin{doc}[ - pi=\docaux{cmd}{collargsBraces} - ]{ - key={name=braces, par={\meta{begin-group char}\meta{end-group char}}, - desc={no default, for details see below}}, - }% here - This key sets the verbatim begin-group and end-group characters. The setting - affects collection of argument types \docref{xparse:m}, \docref{xparse:g}, - \docref{xparse:v}, \docref{xparse:l}, \docref{xparse:e} and (in - \hologo{LaTeX}) \docref{xparse:b} in the \emph{full} verbatim - mode.\footnote{The choice of the verbatim grouping characters also affects - the effect of \refcollargs{begin tag} and\slash or \refcollargs{end tag}; - see the documentation of these keys for details.} - - For example, in the non-verbatim and the partial verbatim mode, an - \docref{xparse:m}-type argument may be delimited by any characters of - category code 1 (``begin-group'') and 2 (``end-group''). In the full - verbatim mode, there are of course no characters of these categories, so - CollArgs internally assigns the grouping function to some pair of characters. - When entering the full verbatim mode, CollArgs automatically sets the - verbatim grouping characters to characters which were of categories 1 and 2 - in the ``outside'' category code regime, i.e.\ the regime in effect at the - time of invoking \refcmd{CollectArguments}. However, in contrast to - \hologo{TeX}'s internal argument parser, only one pair of characters may - serve as the begin-group and the end-group character in CollArgs' full - verbatim mode. In case multiple characters were of category 1 or 2 on the - outside, CollArgs therefore has to make a choice, and it chooses the - candidate with the lowest character code. This choice may be overridden by - the user by invoking key \refcollargs{braces}; the user may even choose - characters which did not belong to categories 1 and 2 in the outside regime. - - When \meta{begin-group char} and \meta{end-group char} are of categories 1 - and 2 in the outside category regime, they must be enclosed in a triple - group. For example, if both |()| and |{}| have the grouping function on the - outside, and the user wants to select |{}| as the verbatim grouping - characters (CollArgs would go for |()|, as this pair has lower character - codes), the correct way to invoke this key is |braces={{{{}}}}| or - |braces=((({})))|. \footnote{This complication is due to the details of - \pkg{pgfkeys}' keylist processing, and does not apply to - \refcmd{collargsBraces}.} -\end{doc} - -\begin{doc}[ - pi=\docaux{cmd}{collargsVerbatimRanges} - ]{ - key={name=verbatim ranges, - par={\bracestt{\meta{from}\texttt{-}\meta{to}(\texttt{,} \meta{from}\texttt{-}\meta{to})*}}, - desc={no default, initially \texttt{0-255}}}, - } - If run under the \hologo{pdfTeX} or \hologo{XeTeX} engine, this key - determines which characters will be assigned category code 12 in the verbatim - mode. In \hologo{pdfTeX}, the range should remain at the initial |0-255|, - but in \hologo{XeTeX}, some rare situations might require extending this - range (don't attempt to set the full range of |0-1114111|, as this would be - very slow and you would most likely run out of save stack). - - In \hologo{LuaTeX}, we switch the category code regime using category code - tables, so this key has another meaning: it determines the range in which - CollArgs will scan for characters of category codes 1, 2 and 14, whose - identity it needs to know, for internal reasons. -\end{doc} - - - -\paragraph{Mode transition limitations} - -\refcmd{CollectArguments} has some minor limitations regarding the transition -from a verbatim into non-verbatim mode, or vice versa. The gist of the issue -is best illustrated with the optional argument type \docref{xparse:o} collected -in the verbatim mode. CollArgs determines whether an argument of this type is -present by peeking ahead (using \hologo{TeX}'s |\futurelet| primitive) into the -input stream. If the argument is present (i.e.\ if the input stream continues -with an open bracket, |[|%] -), all is well. But when the optional argument is absent, the peek-ahead will -tokenize the following character, which presents a problem when no more -arguments are present in the input stream, like in the example below, where the -verbatim \docref{xparse:o} is the (only and) final type in the argument -specification. In this case, the peek-ahead ``incorrectly'' assigns category -code 12 (``other'') to the first |$|. %$ -This character was intended to be tokenized as the math shift character of -category 3, to start the math mode after \refcmd{CollectArguments} is finished, -but having been assigned category code 12, it cannot perform this function, -resulting in error |! Missing $ inserted| %$ -once \hologo{TeX} encounters the superscript character |^|. - -\ExampleName{collargs-transition-ok} -\makeexample{\examplename.tex.c1} -\tcbinputexample{sidebyside, lefthand ratio=0.6, - comment=\input{\examplepath.tex.c1}} - -Well --- this is what \emph{would} happen if CollArgs didn't address the -transition issue described above. In fact, the above example compiles just -fine, because CollArgs \emph{does} address this issue, but unfortunately, -certain transition problems simply cannot be resolved --- read on to learn what -can go wrong. - -For example, you can typeset the name of the document author via -\hologo{LaTeX}'s internal command |\@author|, but to use this command in the -document, you have to precede it by |\makeatletter|. As shown by the first -line of the example below, this works rather nicely: |\makeatletter| sets the -category code of |@| to 11 (``letter''), so |@| may help form the control word -|\@author| --- importantly, |\makeatletter| sets the category code of |@| -\emph{before} control sequence |\@author| is constructed, even if it precedes -it immediately. - -\ExampleName{collargs-transition-cs} -\makeexample{\examplename.tex.c1} -\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''} -\tcbinputexample{comment=\input{\examplepath.tex.c1}} - -In the second line of the example, our clever invocation of |\@author| is -immediately preceded by a call to \refcmd{CollectArguments}, which tries to -collect a verbatim argument of type \docref{xparse:o}. It doesn't find it, which results in -the wrong, verbatim tokenization of the escape character of |\makeletter|. -CollArgs realizes the problem and tries to fix it. But while it is searching -for the end of control sequence |\makeletter| (which it successfully -constructs), it triggers the tokenization of what follows --- which, as |@| is -at that point of category 12 (``other''), yields the control symbol |\@| (later -followed by word ``author'', typeset in the example). - -In short, the solution has created another, delayed instance of the problem --- -an instance which cannot be addressed any further. But we're nevertheless -better off, as this particular issue will bite only in the case when the -``corrupted'' control sequence immediately following the invocation -\refcmd{CollectArguments} changes category codes in a way that affects the -tokenization of what immediately follows it. - -This was an example of what can go wrong in the transition from the -\meta{argument tokens} to the \meta{rest} of the tokens following an invocation -of \refcmd{CollectArguments}. As the ``outside world'' is non-verbatim, this -transition can only be problematic if the argument which corrupted the first of -the \meta{rest} tokens was verbatim, so if the transition was from the verbatim -to the non-verbatim mode. Such a transition can also occur within the -\meta{argument tokens}, but the good news here is that CollArgs successfully -solves any problems that occur there, so you should only worry about the -end-of-arguments situation. - -The other direction of the transition, from the non-verbatim to the verbatim -mode, however, can affect both the internal and the external transitions. Let -us illustrate the problem with the internal transition. Say you want to -collect an optional argument (\docref{xparse:o}) in the non-verbatim mode, and -then a mandatory argument (\docref{xparse:m}) in (either full or partial) -verbatim mode. In the first invocation of \refcmd{CollectArguments} below, the -optional argument is present, and we get what we expect: the percent sign is -collected, verbatim, into the mandatory argument. In the second invocation, -however, the percent character retains its usual commenting function --- -despite the fact that we have requested verbatim mode for the mandatory -argument --- which results in the group in the second line being picked up as -the mandatory argument. Again, this happens because CollArgs has to peek ahead -in the input stream when determining whether the optional argument is present. -Having requested non-verbatim mode for the optional argument, the peeking is -performed in the non-verbatim mode, and as the optional argument is not -present, it finds the comment character, which fulfills its regular function of -disappearing along with the rest of the line. Once CollArgs sets to find the -(verbatim) mandatory argument, the rest of the line is already gone, so it -searches for, and finds, this argument in the next line. - -\ExampleName{collargs-transition-comment} -\makeexample{\examplename.tex.c1} -\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''} -\tcbinputexample{listing and text} - -Nothing can be done here --- commenting deletes information, irrevocably --- -and in a similar fashion, nothing can be done to catch a verbatim end-of-line -character when preceded by an absent optional argument in the non-verbatim mode -(because it was already tokenized into a space token). - -Finally, the transition issues are not limited to transits from argument type -\docref{xparse:o}. The full list of argument types which give rise to -transition problems (when transiting \emph{from} arguments of these types) is -as follows: \docref{xparse:o}, \docref{xparse:d}, \docref{xparse:s}, -\docref{xparse:t}, \docref{xparse:g}, \docref{xparse:e}. - - -\endgroup % /collargs - - -\section{Varia} - -\subsectionclearpagefalse - -\subsection{Known issues} -\label{sec:known-issues} - -\paragraph*{Bitmap graphics export} is coming up in the next release of the -package. - - -\paragraph{Disappearing errors} -If a non-fatal internal \hologo{TeX} error occurs during memoization, the memos -and externs may be nevertheless produced and utilized in subsequent -compilations. In such a case, the erroneous code won't be compiled again, and -therefore won't yield any errors, giving the mistaken impression that the code -is error-free. - -This problem does not apply to errors which trigger \refcmd{errmessage}, -because that control sequence is advised by \refmmzauto{abort}. -Note that internal \hologo{TeX} errors like |Undefined control sequence| are -\emph{not} reported through \refcmd{errmessage}, and will therefore cause the -issue. - -This problem does not affect \hologo{LuaTeX}, because this engine allows -Memoize to detect errors and abort memoization if it encounters any. - - -\paragraph{A minimal issue with \hologo{XeLaTeX}} -If the very first page of a document of class |minimal|, compiled by -\hologo{XeLaTeX}, happens to be an extern page, we have a problem: all the -regular pages of the document will be of the same size as that extern page. -\hologo{pdfLaTeX} and \hologo{LuaLaTeX} do not exhibit this behaviour, nor do -\hologo{LaTeX} classes other than |minimal|, even when compiled with -\hologo{XeLaTeX}. - - -\paragraph{CollArgs} - -Due to an unfortunate design decision, CollArgs does not accept a dot |.| as -the \meta{token} argument of types, \docref{xparse:r}, \docref{xparse:R}, -\docref{xparse:d}, \docref{xparse:D}, and \docref{xparse:t}. Furthermore, -\refcollargs{append prewrap} and friends (and their macro counterparts) do not -accept a parameter symbol consistently. These issues will be fixed in the next -release. - - -\subsection{Troubleshooting} -\label{sec:troubleshooting} - -\paragraph{Extern extraction does not work} -\label{sec:fadings-problem} -\label{sec:loading-order} - -If the internal extern extraction does not work, you should have gotten one of -the following warnings: -\begin{enumerate} -\item \code{Package memoize Warning: Extraction of externs from document - "\meta{jobname}.pdf" using method "perl" was unsuccessful. Have you set the - shell escape mode as suggested in chapter 1 of the manual?} - - This warning is thrown when the extraction log \meta{jobname}|.mmz.log|, - which should be produced by the extraction script, is incomplete, i.e.\ it - does not finish by |\endinput|.\footnote{The extraction log will never be - absent, because Memoize clears it before executing the extraction script - --- and clearing it creates it if it doesn't exist.} The most likely - reason is that the script was not executed at all, so the question asked in - the warning message is pertinent. However, if all is well with the - shell-mode, you have a mystery at your hands. To gather information, I - suggest inspecting: - \begin{itemize} - \item \meta{jobname}|.log| --- search for \code{runsystem(memoize-extract.pl - ...)}, it will tell you whether the script was executed; - \item the terminal output --- if the script was executed, it should've - announced itself by \code{Extracting externs from \meta{jobname}.pdf}; are - there any further messages between this header and the warning message? - \item \meta{the path to the extern}|.log|, if you are using the - \hologo{TeX}-based extraction (\code{\refmmz{extract}=\refmmz{extract=tex}}). - \end{itemize} - \hologo{TeX}-based extraction will yield this error even in the case of a - missing or corrupted PDF, i.e.\ in - case~\ref{item:warning:extract:cannot-read} below. - -\item\label{item:warning:extract:cannot-read} - \code{Package memoize (perl-based extraction) Warning: Cannot read file - "\meta{jobname}.pdf". Perhaps you have to load Memoize earlier in the - preamble?} - - Embedded extern extraction requires an intact document PDF from the previous - compilation, so Memoize must be loaded before the document PDF is opened for - writing the results of the ongoing compilation. In particular, the PDF is - opened by \PGF library |fadings|, included by \TikZ's libraries |fadings| - and |shadows|, so Memoize must be loaded before any of these libraries. - - With \pkg{beamer}, the problem is particularly acute because the PDF is - opened while loading the class. In this case, simply moving - |\usepackage{memoize}| up the preamble, as suggested, won't help: you have to - write |\RequirePackage{memoize}| \emph{before} |\documentclass{beamer}|! - -\item\label{item:warning:extract:unexpected-size} \code{Package memoize - (perl-based extraction) Warning: I refuse to extract page $n$ from - "\meta{jobname}.pdf", because its size ($\meta{width}\times\meta{height}$) - is not what I expected\\ ($\meta{expected width}\times\meta{expected - height}$)} - - If the compilation which produced the offending extern pages yielded any - errors, you should probably disregard this warning, fix the errors, and - compile again. Otherwise, you have somehow winded up with mismatched - \meta{jobname}|.pdf| and \meta{jobname}\dmmz (the latter file contains - instructions on which pages to extract, complete with the expected - dimensions). Are you sure that they were produced by the same compilation, - and have remained untouched since? Are you perhaps trying to perform the - extraction the second time, after the first extraction - \refscript{memoize-extract.pl--prune}d the PDF? - - If the warning stubbornly persists, but you are sure that the page the script - is refusing to extract is correct, you can force the extraction by adding - option \refscript{memoize-extract.pl--force} to the script invocation, which - can be set by \refmmz{perl extraction options}. However, as such a situation - probably indicates a bug in Memoize, please let me know about it. -\item\code{\myfontexpansionon Invalid dictionary key at - /.../perl5/site\_perl/PDF/API2/Basic/PDF/File.pm line $N$\myfontexpansionoff} - - This can sometimes happen when extracting with the Perl-based script. The - issue might be related to PDF version --- by default, \hologo{TeX} produces - PDFs of version 1.5, while the PDF library - \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} officially only - supports versions up to 1.4 --- but I'm afraid I haven't identified the exact - circumstances yet. Possibly, the externalizing a picture containing an - embedded PDF file might be the culprit. The workaround is to use Python- or - \hologo{TeX}-based extraction, i.e.\ load Memoize with package option - \refmmz{extract}|=|\refmmz{extract=python} or - \refmmz{extract}|=|\refmmz{extract=tex}. -\end{enumerate} - -The failure to extract externs intentionally doesn't result in an error, -because what if you deleted the document PDF on purpose? At any rate, the -compilation will succeed, it's just that as Memoize cannot extract the externs, -they will be produced, and dumped into your document, at each and every -compilation. - -\paragraph{An extern won't be included} - -Did you receive a warning or error message? -\begin{enumerate} -\item \code{Package memoize Warning: Unexpected size of extern "\meta{extern - path}.pdf"; expected\\ $\meta{expected width}\times\meta{expected - height}$, got $\meta{width}\times\meta{height}$} - - This warning is related to warning~\ref{item:warning:extract:unexpected-size} - above, only that it occurs once the extern is extracted. The same - investigative methods apply. -\item \code{!pdfTeX error: pdflatex (file \meta{extern path}.pdf): reading - image file failed} - - This is a fatal error. The extern file got corrupted, somehow --- - inexistent, or even empty, extern files merely trigger recompilation. -\end{enumerate} - -If there was no warning or error --- are you certain that Memoize is -\refmmz{enable}d, and that it is not in the \refmmz{recompile} mode? Remember -that these settings can also apply only to a part of the document; search for -any stray \refcmd{mmzset} or \refcmd{mmznext} commands. - - -\paragraph*{Warnings about duplicate labels, indices, etc.} may be safely -disregarded. - -Externalization causes any (non-immediate) |\write| commands in the extern to -be executed twice, once upon the shipout of the regular page, and once upon the -shipout of the extern page. This results in warnings about doubly defined -labels, hyperreferences, indices, etc. For example, you might get -\texttt{LaTeX Warning: Label `' multiply defined} or \texttt{warning (pdf - backend): ignoring duplicate destination with the name ''}. You can -safely disregard these warnings; they will disappear once the extern is -utilized. - - -\paragraph{Memoization was aborted} - -This warning means that either: -\begin{itemize} -\item you are trying to (auto)memoize a \refenv{tikzpicture} with - \refkey{/tikz/remember picture} set, or more generally, some code which - contains \refcmd{savepos} --- this can't be done, see - section~\ref{sec:tut:good-to-know}; or -\item an error occurred during memoization --- in this case, Memoize cowardly - refuses to proceed with memoization, see section~\ref{sec:known-issues} for - details. -\end{itemize} - - -\subsection{License} - -Copyright \textcopyright\ 2020- Sašo Živanović. - -This work may be distributed and/or modified under the conditions of the -\hologo{LaTeX} Project Public License, either version 1.3c of this license or -(at your option) any later version. The latest version of this license is -in \url{https://www.latex-project.org/lppl.txt} and version 1.3 or later is -part of all distributions of \hologo{LaTeX} version 2008 or later. - -This work comprises of the sources, generated files, accompanying scripts, -auxiliary files and scripts, documentation, documentation sources, examples and -example sources of packages |memoize|, |nomemoize|, |memoizable|, |auto| and -|collargs|. The files belonging to this work and covered by LPPL are listed in -\meta{texmf}|/doc/generic/memoize/FILES|. - -This work has the LPPL maintenance status `maintained.' The Current Maintainer -of this work is Sašo Živanović. The work is available on CTAN -at \url{https://ctan.org/pkg/memoize} and on GitHub -at \url{https://github.com/sasozivanovic/memoize}. - - -\subsection{Changelog} - -\newcommand\githubrelease[2]{% - \href{https://github.com/sasozivanovic/memoize/releases/tag/#2}{#1 #2}% -} -\begin{description} -\item[\githubrelease{2020/07/17}{v0.1}] - The proof-of-concept, available only at GitHub. -\item[\githubrelease{2023/10/10}{v1.0.0}] - A complete, fully documented reimplementation. -\end{description} - -\subsection{Acknowledgments} - -First and foremost, my gratitude goes to Stefan Müller of Language Science -Press for his encouragement and patience --- without him, this package might've -remained an idea forever! - -But there were others as well. Shunsaku Hirata of the \hologo{XeTeX} team -promptly resolved an issue with forward references in |xdvipdfmx|, and thereby -made Memoize work with \hologo{XeTeX}. Joseph Wright kindly didn't object to -me misappropriating \pkg{etoolbox} for other formats. Petra Rübe-Pugliese -(CTAN) and Karl Berry (\hologo{TeX}Live) offered good advice on packaging the -package; I would never get to call Advice Advice without their advice. Karl -also performed a security review of the extraction scripts, providing some very -useful comments along the way, and agreed to include the scripts among the -shell escape commands allowed in \hologo{TeX}Live. - -Thank you all! - - -\printindex - -\end{document} - - -% Local Variables: -% TeX-engine: luatex -% TeX-master: t -% End: diff --git a/macros/generic/memoize/memoize-clean.pl b/macros/generic/memoize/memoize-clean.pl index 4e1334c695..7d7e4f5b43 100755 --- a/macros/generic/memoize/memoize-clean.pl +++ b/macros/generic/memoize/memoize-clean.pl @@ -18,18 +18,19 @@ # The files belonging to this work and covered by LPPL are listed in # /doc/generic/memoize/FILES. +my $PROG = 'memoize-clean.pl'; +my $VERSION = '2024/01/02 v1.1.0'; + use strict; use Getopt::Long; -# I intend to rewrite this script using Path::Class. use Cwd 'realpath'; use File::Spec; use File::Basename; -my $VERSION = '2023/10/10 v1.0.0'; - -my $usage = "usage: memoize-clean.py [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] [mmz ...]\n"; +my $usage = "usage: $PROG [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] " . + "[mmz ...]\n"; my $Help = <catfile(($dir),$fn); - if ($fn =~ /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(\.memo|(?:-[0-9]+)?\.pdf|\.log)/ and ($all || !exists($keep{$path}))) { - push @tbdeleted, $path; + if ($fn =~ + /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?# + (\.memo|(?:-[0-9]+)?\.pdf|\.log)/x + and ($all || !exists($keep{$path}))) { + push @tbdeleted, $path; } } closedir(MD); @@ -161,3 +166,7 @@ if (scalar(@tbdeleted) != 0) { } elsif (!$quiet) { print "Nothing to do, the directory seems clean.\n"; } + +# Local Variables: +# after-save-hook: pl2dtx +# End: diff --git a/macros/generic/memoize/memoize-clean.py b/macros/generic/memoize/memoize-clean.py index dd543327cb..db758bbd35 100755 --- a/macros/generic/memoize/memoize-clean.py +++ b/macros/generic/memoize/memoize-clean.py @@ -18,13 +18,14 @@ # The files belonging to this work and covered by LPPL are listed in # /doc/generic/memoize/FILES. -__version__ = '2023/10/10 v1.0.0' +__version__ = '2024/01/02 v1.1.0' import argparse, re, sys, pathlib, os parser = argparse.ArgumentParser( description="Remove (stale) memo and extern files.", - epilog = "For details, see the man page or the Memoize documentation." + epilog = "For details, see the man page or the Memoize documentation " + "(https://ctan.org/pkg/memoize)." ) parser.add_argument('--yes', '-y', action = 'store_true', help = 'Do not ask for confirmation.') @@ -63,7 +64,8 @@ for mmz_fn in args.mmz: elif endinput: raise RuntimeError( - r'Bailing out, \endinput is not the last line of file $mmz_fn.') + rf'Bailing out, ' + rf'\endinput is not the last line of file {mmz_fn}.') elif m := re_prefix.match(line): prefix = m[1] @@ -109,7 +111,8 @@ tbdeleted = [] def populate_tbdeleted(folder, basename_prefix): re_aux = re.compile( re.escape(basename_prefix) + - '[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$') + '[0-9A-F]{32}(?:-[0-9A-F]{32})?' + '(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$') try: for f in folder.iterdir(): if re_aux.match(f.name) and (args.all or f not in keep): @@ -157,3 +160,8 @@ if tbdeleted: print("Bailing out.") elif not args.quiet: print('Nothing to do, the directory seems clean.') + +# Local Variables: +# fill-column: 79 +# after-save-hook: py2dtx +# End: diff --git a/macros/generic/memoize/memoize-extract.pl b/macros/generic/memoize/memoize-extract.pl index 8fba3d356f..96eec29b97 100755 --- a/macros/generic/memoize/memoize-extract.pl +++ b/macros/generic/memoize/memoize-extract.pl @@ -18,188 +18,757 @@ # The files belonging to this work and covered by LPPL are listed in # /doc/generic/memoize/FILES. +my $PROG = 'memoize-extract.pl'; +my $VERSION = '2024/01/02 v1.1.0'; + use strict; +use File::Basename qw/basename/; use Getopt::Long; -use Path::Class; -use File::Spec; -use File::Basename; -use PDF::API2; +use File::Spec::Functions + qw/splitpath catpath splitdir rootdir file_name_is_absolute/; +use File::Path qw(make_path); +# We will only try to import the PDF processing library once we set up the error +# log. + +# Declare variables for command-line arguments and for |kpathsea| +# variables. They are defined here so that they are global in the subs which use +# them. +our ($pdf_file, $prune, $keep, $format, $force, $quiet, + $pdf_library, $print_version, $mkdir, $help, + $openin_any, $openout_any, $texmfoutput, $texmf_output_directory); + +# \paragraph{Messages} + +# The messages are written both to the extraction log and the terminal (we +# output to stdout rather than stderr so that messages on the TeX terminal and +# document |.log| appear in chronological order). Messages are automatically +# adapted to the TeX |--format|. + +# The format of the messages. It depends on the given |--format|; the last +# entry is for t the terminal output. + +my %ERROR = ( + latex => '\PackageError{memoize (perl-based extraction)}{$short}{$long}', + plain => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}', + context => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}', + '' => '$header$short. $long'); + +my %WARNING = ( + latex => '\PackageWarning{memoize (perl-based extraction)}{$texindent$text}', + plain => '\message{memoize (perl-based extraction) Warning: $texindent$text}', + context => '\message{memoize (perl-based extraction) Warning: $texindent$text}', + '' => '$header$indent$text.'); + +my %INFO = ( + latex => '\PackageInfo{memoize (perl-based extraction)}{$texindent$text}', + plain => '\message{memoize (perl-based extraction): $texindent$text}', + context => '\message{memoize (perl-based extraction): $texindent$text}', + '' => '$header$indent$text.'); -my $VERSION = '2023/10/10 v1.0.0'; +# Some variables used in the message routines; note that |header| will be +# redefined once we parse the arguments. + +my $exit_code = 0; +my $log; +my $header = ''; +my $indent = ''; +my $texindent = ''; + +# The message routines. + +sub error { + my ($short, $long) = @_; + if (! $quiet) { + $_ = $ERROR{''}; + s/\$header/$header/; + s/\$short/$short/; + s/\$long/$long/; + print(STDOUT "$_\n"); + } + if ($log) { + $long =~ s/\\/\\string\\/g; + $_ = $ERROR{$format}; + s/\$short/$short/; + s/\$long/$long/; + print(LOG "$_\n"); + } + $exit_code = 11; + endinput(); +} + +sub warning { + my $text = shift; + if ($log) { + $_ = $WARNING{$format}; + s/\$texindent/$texindent/; + s/\$text/$text/; + print(LOG "$_\n"); + } + if (! $quiet) { + $_ = $WARNING{''}; + s/\$header/$header/; + s/\$indent/$indent/; + s/\$text/$text/; + print(STDOUT "$_\n"); + } + $exit_code = 10; +} + +sub info { + my $text = shift; + if ($text && ! $quiet) { + $_ = $INFO{''}; + s/\$header/$header/; + s/\$indent/$indent/; + s/\$text/$text/; + print(STDOUT "$_\n"); + if ($log) { + $_ = $INFO{$format}; + s/\$texindent/$texindent/; + s/\$text/$text/; + print(LOG "$_\n"); + } + } +} + +# Mark the log as complete and exit. +sub endinput { + if ($log) { + print(LOG "\\endinput\n"); + close(LOG); + } + exit $exit_code; +} + +sub die_handler { + stderr_to_warning(); + my $text = shift; + chomp($text); + error("Perl error: $text", ''); +} + +sub warn_handler { + my $text = shift; + chomp($text); + warning("Perl warning: $text"); +} + +# This is used to print warning messages from PDF::Builder, which are output to +# STDERR. +my $stderr; +sub stderr_to_warning { + if ($stderr) { + my $w = ' Perl info: '; + my $nl = ''; + for (split(/\n/, $stderr)) { + /(^\s*)(.*?)(\s*)$/; + $w .= ($1 ? ' ' : $nl) . $2; + $nl = "\n"; + } + warning("$w"); + $stderr = ''; + } +} + +# \paragraph{Permission-related functions} + +# We will need these variables below. Note that we only support Unix and +# Windows. +my $on_windows = $^O eq 'MSWin32'; +my $dirsep = $on_windows ? '\\' : '/'; + +# |paranoia_in|/|out| should work exactly as +# |kpsewhich -safe-in-name|/|-safe-out-name|. +sub paranoia_in { + my ($f, $remark) = @_; + error("I'm not allowed to read from '$f' (openin_any = $openin_any)", + $remark) unless _paranoia($f, $openin_any); +} + +sub paranoia_out { + my ($f, $remark) = @_; + error("I'm not allowed to write to '$f' (openin_any = $openout_any)", + $remark) unless _paranoia($f, $openout_any); +} + +sub _paranoia { + # |f| is the path to the file (it should not be empty), and |mode| is the + # value of |openin_any| or |openout_any|. + my ($f, $mode) = @_; + return if (! $f); + # We split the filename into the directory and the basename part, and the + # directory into components. + my ($volume, $dir, $basename) = splitpath($f); + my @dir = splitdir($dir); + return ( + # In mode `any' (|a|, |y| or |1|), we may access any file. + $mode =~ /^[ay1]$/ + || ( + # Otherwise, we are at least in the restricted mode, so we should + # not open dot files on Unix-like systems (except file called + # |.tex|). + ! (!$on_windows && $basename =~ /^\./ && !($basename =~ /^\.tex$/)) + && ( + # If we are precisely in the restricted mode (|r|, |n|, |0|), + # then there are no further restrictions. + $mode =~ /^[rn0]$/ + # Otherwise, we are in the paranoid mode (officially |p|, but + # any other value is interpreted as |p| as well). There are + # two further restrictions in the paranoid mode. + || ( + # We're not allowed to go to a parent directory. + ! grep(/^\.\.$/, @dir) && $basename ne '..' + && + # If the given path is absolute, is should be a descendant + # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|. + (!file_name_is_absolute($f) + || + is_ancestor($texmf_output_directory, $f) + || + is_ancestor($texmfoutput, $f) + ))))); +} + +# Only removes final "/"s. This is unlike |File::Spec|'s |canonpath|, which also +# removes |.| components, collapses multiple |/| --- and unfortunately also goes +# up for |..| on Windows. +sub normalize_path { + my $path = shift; + my ($v, $d, $n) = splitpath($path); + if ($n eq '' && $d =~ /[^\Q$dirsep\E]\Q$dirsep\E+$/) { + $path =~ s/\Q$dirsep\E+$//; + } + return $path; +} + +# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with the +# |\| but lacking the drive. |File::Spec|'s function |file_name_is_absolute| +# returns 2 if the path is absolute with a volume, 1 if it's absolute with no +# volume, and 0 otherwise. After a path was sanitized using this function, +# |file_name_is_absolute| will work as we want it to. +sub sanitize_path { + my $f = normalize_path(shift); + my ($v, $d, $n) = splitpath($f); + if ($on_windows) { + my $a = file_name_is_absolute($f); + if ($a == 1 || ($a == 0 && $v) ) { + error("\"Semi-absolute\" paths are disallowed: " . $f, + "The path must either both contain the drive letter and " . + "start with '\\', or none of these; paths like 'C:foo\\bar' " . + "and '\\foo\\bar' are disallowed"); + } + } +} + +sub access_in { + return -r shift; +} + +sub access_out { + my $f = shift; + my $exists; + eval { $exists = -e $f }; + # Presumably, we get this error when the parent directory is not executable. + return if ($@); + if ($exists) { + # An existing file should be writable, and if it's a directory, it + # should also be executable. + my $rw = -w $f; my $rd = -d $f; my $rx = -x $f; + return -w $f && (! -d $f || -x $f); + } else { + # For a non-existing file, the parent directory should be writable. + # (This is the only place where function |parent| is used, so it's ok + # that it returns the logical parent.) + my $p = parent($f); + return -w $p; + } +} + +# This function finds the location for an input file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it tries +# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always), and +# |TEXMFOUTPUT| directory (if defined), in this order. The first readable file +# found is returned; if no readable file is found, the file in the current +# directory is returned. +sub find_in { + my $f = shift; + sanitize_path($f); + return $f if file_name_is_absolute($f); + for my $df ( + $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef, + $f, + $texmfoutput ? join_paths($texmfoutput, $f) : undef) { + return $df if $df && -r $df; + } + return $f; +} + +# This function finds the location for an output file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it tries +# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless +# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if +# defined), in this order. The first writable file found is returned; if no +# writable file is found, the file in either the current or the output directory +# is returned. +sub find_out { + my $f = shift; + sanitize_path($f); + return $f if file_name_is_absolute($f); + for my $df ( + $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef, + $texmf_output_directory ? undef : $f, + $texmfoutput ? join_paths($texmfoutput, $f) : undef) { + return $df if $df && access_out($df); + } + return $texmf_output_directory ? join_paths($texmf_output_directory, $f) : $f; +} + +# We next define some filename-related utilities matching what Python offers out +# of the box. We avoid using |File::Spec|'s |canonpath|, because on Windows, +# which has no concept of symlinks, this function resolves |..| to the parent. + +sub name { + my $path = shift; + my ($volume, $dir, $filename) = splitpath($path); + return $filename; +} + +sub suffix { + my $path = shift; + my ($volume, $dir, $filename) = splitpath($path); + $filename =~ /\.[^.]*$/; + return $&; +} -my $usage = "usage: memoize-extract.pl [-h] [--pdf PDF] [--prune] [--keep] [--force] [--log LOG] [--warning-template WARNING_TEMPLATE] [--quiet] [--mkdir] mmz\n"; +sub with_suffix { + my ($path, $suffix) = @_; + my ($volume, $dir, $filename) = splitpath($path); + if ($filename =~ s/\.[^.]*$/$suffix/) { + return catpath($volume, $dir, $filename); + } else { + return catpath($volume, $dir, $filename . $suffix); + } +} + +sub with_name { + my ($path, $name) = @_; + my ($volume, $dir, $filename) = splitpath($path); + my ($v,$d,$f) = splitpath($name); + die "Runtime error in with_name: " . + "'$name' should not contain the directory component" + unless $v eq '' && $d eq '' && $f eq $name; + return catpath($volume, $dir, $name); +} + +sub join_paths { + my $path1 = normalize_path(shift); + my $path2 = normalize_path(shift); + return $path2 if !$path1 || file_name_is_absolute($path2); + my ($volume1, $dir1, $filename1) = splitpath($path1, 'no_file'); + my ($volume2, $dir2, $filename2) = splitpath($path2); + die if $volume2; + return catpath($volume1, + join($dirsep, ($dir1 eq $dirsep ? '' : $dir1, $dir2)), + $filename2); +} + +# The logical parent. The same as |pathlib.parent| in Python. +sub parent { + my $f = normalize_path(shift); + my ($v, $dn, $_dummy) = splitpath($f, 1); + my $p_dn = $dn =~ s/[^\Q$dirsep\E]+$//r; + if ($p_dn eq '') { + $p_dn = $dn =~ /^\Q$dirsep\E/ ? $dirsep : '.'; + } + my $p = catpath($v, $p_dn, ''); + $p = normalize_path($p); + return $p; +} + +# This function assumes that both paths are absolute; ancestor may be '', +# signaling a non-path. +sub is_ancestor { + my $ancestor = normalize_path(shift); + my $descendant = normalize_path(shift); + return if ! $ancestor; + $ancestor .= $dirsep unless $ancestor =~ /\Q$dirsep\E$/; + return $descendant =~ /^\Q$ancestor/; +} + +# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|. +sub make_directory { + my $folder = find_out(shift); + if (! -d $folder) { + paranoia_out($folder); + # Using |make_path| is fine because we know that + # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that + # ``folder'' contains no |..|. + make_path($folder); + # This does not get logged when the function is invoked via |--mkdir|, + # as it is not clear what the log name should be. + info("Created directory $folder"); + } +} + +sub unquote { + shift =~ s/"(.*?)"/\1/rg; +} + +# \paragraph{Kpathsea} + +# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and +# |TEXMF_OUTPUT_DIRECTORY|. + +my $maybe_backslash = $on_windows ? '' : '\\'; +my $query = 'kpsewhich -expand-var=' . + "openin_any=$maybe_backslash\$openin_any," . + "openout_any=$maybe_backslash\$openout_any," . + "TEXMFOUTPUT=$maybe_backslash\$TEXMFOUTPUT"; +my $kpsewhich_output = `$query`; +if (! $kpsewhich_output) { + # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In + # absence of |kpathsea| information, we get very paranoid. + ($openin_any, $openout_any) = ('p', 'p'); + ($texmfoutput, $texmf_output_directory) = ('', ''); + # Unfortunately, this warning can't make it into the log. But then again, + # the chances of a missing |kpsewhich| are very slim, and its absence would + # show all over the place anyway. + warning('I failed to execute "kpsewhich", is there no TeX system installed? ' . + 'Assuming openin_any = openout_any = "p" ' . + '(i.e. restricting all file operations to non-hidden files ' . + 'in the current directory of its subdirectories).'); +} else { + $kpsewhich_output =~ /^openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)/; + ($openin_any, $openout_any, $texmfoutput) = @{^CAPTURE}; + $texmf_output_directory = $ENV{'TEXMF_OUTPUT_DIRECTORY'}; + if ($openin_any =~ '^\$openin_any') { + # When the |open*_any| variables are not expanded, we assume we're + # running MiKTeX. The two config settings below correspond to TeXLive's + # |openin_any| and |openout_any|; afaik, there is no analogue to + # |TEXMFOUTPUT|. + $query = 'initexmf --show-config-value=[Core]AllowUnsafeInputFiles ' . + '--show-config-value=[Core]AllowUnsafeOutputFiles'; + my $initexmf_output = `$query`; + $initexmf_output =~ /^(.*)\n(.*)\n/m; + $openin_any = $1 eq 'true' ? 'a' : 'p'; + $openout_any = $2 eq 'true' ? 'a' : 'p'; + $texmfoutput = ''; + $texmf_output_directory = ''; + } +} + +# An output directory should exist, and may not point to the root on Linux. On +# Windows, it may point to the root, because being absolute also implies +# containing the drive; see |sanitize_filename|. +sub sanitize_output_dir { + return unless my $d = shift; + sanitize_path($d); + # On Windows, |rootdir| returns |\|, so it cannot possibly match |$d|. + return $d if -d $d && $d ne rootdir(); +} + +$texmfoutput = sanitize_output_dir($texmfoutput); +$texmf_output_directory = sanitize_output_dir($texmf_output_directory); + +# We don't delve into the real script when loaded from the testing code. +return 1 if caller; + +# \paragraph{Arguments} + +my $usage = "usage: $PROG [-h] [-P PDF] [-p] [-k] [-F {latex,plain,context}] [-f] " . + "[-L {PDF::API2,PDF::Builder}] [-q] [-m] [-V] mmz\n"; my $Help = < \$pdf_arg, - "keep|k" => \$keep, + "pdf|P=s" => \$pdf_file, "prune|p" => \$prune, - "log|l=s" => \$log, - "quiet|q" => \$quiet, - "embedded|e" => \$embedded, + "keep|k" => \$keep, + "format|F=s" => \$format, "force|f" => \$force, - "warning-template|w=s" => \$warning_template, + "quiet|q" => \$quiet, + "library|L=s" => \$pdf_library, "mkdir|m" => \$mkdir, - "help|h|?" => \$help, "version|V" => \$print_version, + "help|h|?" => \$help, ) or die $usage; + if ($help) {print("$usage\n$Help"); exit 0} -if ($print_version) { print("memoize-extract.pl of Memoize $VERSION\n"); exit 0 } -die $usage unless @ARGV == 1; -my $message_prefix = $embedded ? basename($0) . ': ' : ''; -print("\n") if ($embedded); +if ($print_version) { print("$PROG of Memoize $VERSION\n"); exit 0 } -my @output_paths = (dir()->absolute->resolve); -my $texmfoutput = `kpsewhich --var-value=TEXMFOUTPUT`; -$texmfoutput =~ s/^\s+|\s+$//g; -if ($texmfoutput) { - my $texmfoutput_dir = dir($texmfoutput)->absolute->resolve; - push(@output_paths, $texmfoutput_dir) unless $texmfoutput_dir->dir_list == 1 && ! $texmfoutput_dir->volume; -} +die "${usage}$PROG: error: the following arguments are required: mmz\n" + unless @ARGV == 1; -sub paranoia { - my $file = $_[0]; - die "${message_prefix}Cannot create a hidden file or dir: $file" - if $file->basename =~ /^\./; - my $parent = $file->parent->absolute->resolve; - die "${message_prefix}Cannot write outside the current working or output directory tree: $file" - unless grep($_->contains($parent), @output_paths); -} +die "${usage}$PROG: error: argument -F/--format: invalid choice: '$format' " . + "(choose from 'latex', 'plain', 'context')\n" + unless grep $_ eq $format, ('', 'latex', 'plain', 'context'); + +die "${usage}$PROG: error: argument -L/--library: invalid choice: '$pdf_library' " . + "(choose from " . join(", ", @valid_libraries) . ")\n" + if $pdf_library && ! grep $_ eq $pdf_library, @valid_libraries; -my $mmz_arg = $ARGV[0]; -my $mmz_file = file($mmz_arg); -my $mmz_dir = $mmz_file->parent; +$header = $format ? basename($0) . ': ' : ''; +# start a new line in the TeX terminal output +print("\n") if $format; + +# \paragraph{Initialization} + +# With |--mkdir|, argument |mmz| is interpreted as the directory to create. if ($mkdir) { - my $dir = dir($mmz_arg)->cleanup; - my $current = dir(File::Spec->catpath($dir->volume, - $dir->is_absolute ? File::Spec->rootdir : File::Spec->curdir, - ''))->absolute; - for my $c ($dir->components) { - $current = $current->subdir($c); - if (-d $current) { - $current = $current->resolve; - } else { - paranoia($current); - mkdir($current); - print("${message_prefix}Created directory $current\n") unless $quiet; - } - } - exit; -} else { - die "${message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '$mmz_file'\n" unless $mmz_file->basename =~ /\.mmz$/; + make_directory($ARGV[0]); + exit 0; } -# Enable in-place editing (of the .mmz file). -paranoia($mmz_file) unless $keep; -$^I = "" unless $keep; - -my $pdf_file = $pdf_arg ? file($pdf_arg) : $mmz_dir->file($mmz_file->basename =~ s/\.mmz$/\.pdf/r)->cleanup; -paranoia($pdf_file) if $prune; +# Normalize the |mmz| argument into a |.mmz| filename. +my $mmz_file = $ARGV[0]; +$mmz_file = with_suffix($mmz_file, '.mmz') + if suffix($mmz_file) eq '.tex'; +$mmz_file = with_name($mmz_file, name($mmz_file) . '.mmz') + if suffix($mmz_file) ne '.mmz'; -if ($log) { - paranoia(file($log)); +# Once we have the |.mmz| filename, we can open the log. +if ($format) { + my $_log = find_out(with_suffix($mmz_file, '.mmz.log')); + paranoia_out($_log); + info("Logging to '$_log'"); + $log = $_log; open LOG, ">$log"; +} + +# Now that we have opened the log file, we can try loading the PDF processing +# library. +if ($pdf_library) { + eval "use $pdf_library"; + error("Perl module '$pdf_library' was not found", + 'Have you followed the instructions is section 1.1 of the manual?') + if ($@); } else { - *LOG = *STDERR; + for (@valid_libraries) { + eval "use $_"; + if (!$@) { + $pdf_library = $_; + last; + } + } + if (!$pdf_library) { + error("No suitable Perl module for PDF processing was found, options are " . + join(", ", @valid_libraries), + 'Have you followed the instructions is section 1.1 of the manual?'); + } +} + +# Catch any errors in the script and output them to the log. +$SIG{__DIE__} = \&die_handler; +$SIG{__WARN__} = \&warn_handler; +close(STDERR); +open(STDERR, ">", \$stderr); + +# Find the |.mmz| file we will read, but retain the original filename in +# |$given_mmz_file|, as we will still need it. +my $given_mmz_file = $mmz_file; +$mmz_file = find_in($mmz_file, 1); +if (! -e $mmz_file) { + info("File '$given_mmz_file' does not exist, assuming there's nothing to do"); + endinput(); } +paranoia_in($mmz_file); +paranoia_out($mmz_file, + 'I would have to rewrite this file unless option --keep is given.') + unless $keep; -my ($pdf, %extern_pages); -print "${message_prefix}Extracting externs from $pdf_file:\n" unless $quiet; +# Determine the PDF filename: it is either given via |--pdf|, or constructed +# from the |.mmz| filename. +$pdf_file = with_suffix($given_mmz_file, '.pdf') if !$pdf_file; +$pdf_file = find_in($pdf_file); +paranoia_in($pdf_file); +paranoia_out($pdf_file, + 'I would have to rewrite this file because option --prune was given.') + if $prune; +# Various initializations. +my $pdf; +my %extern_pages; +my $new_mmz; my $tolerance = 0.01; -my $done_message = "${message_prefix}Done (there was nothing to extract).\n"; - -while (<>) { - if (/^\\mmzNewExtern *{(?P(?P.*?)(?P[0-9A-F]{32})-(?P[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P[0-9]+)}{(?P[0-9.]*)pt}{(?P[0-9.]*)pt}/) { - my $extern_file = file($+{extern_path}); - if (! $extern_file->is_absolute) { - $extern_file = $mmz_dir->file($+{extern_path}); - } - paranoia($extern_file); - my $page = $+{page_n}; - my $expected_width_pt = $+{expected_width}; - my $expected_height_pt = $+{expected_height}; - my $c_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '.memo'); - my $cc_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '-' . $+{context_md5sum} . '.memo'); - if (!(-e $c_memo_file && -e $cc_memo_file)) { - print LOG ($warning_template =~ s/\\warningtext/Not extracting page $page into extern $extern_file, because the associated (c)c-memo does not exist/gr), "\n\\endinput\n"; - last; - } - eval { $pdf = PDF::API2->open($pdf_file->stringify) unless $pdf; }; - if ($@) { - print LOG ($warning_template =~ s/\\warningtext/Cannot read file "$pdf_file". Perhaps you have to load Memoize earlier in the preamble?/gr), "\n\\endinput\n"; - close LOG; - die "${message_prefix}File '$pdf_file' cannot be read, bailing out.\n"; - } - my $extern = PDF::API2->new(); - $extern->version($pdf->version); - $extern->import_page($pdf, $page); - my $extern_page = $extern->open_page(1); - my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox(); - my $width_pt = ($x1 - $x0) / 72 * 72.27; - my $height_pt = ($y1 - $y0) / 72 * 72.27; - my $warning = ''; - if (abs($width_pt - $expected_width_pt) > $tolerance - || abs($height_pt - $expected_height_pt) > $tolerance) - { - $warning = "I refuse to extract page $page from $pdf_file, because its size (${width_pt}pt x ${height_pt}pt) is not what I expected (${expected_width_pt}pt x ${expected_height_pt}pt)"; - print LOG ($warning_template =~ s/\\warningtext/$warning/gr), "\n"; - } - if ($warning && !$force) { - unlink $extern_file; - } else { - $extern->saveas($extern_file->stringify); - $done_message = "${message_prefix}Done.\n"; - print STDOUT "${message_prefix} Page $page --> $extern_file\n" unless $quiet; - $extern_pages{$page} = 1 if $prune; - print("%") unless $keep; - } - } - print unless $keep; -} - -print $done_message unless $quiet; - -if ($pdf and $prune) { - paranoia($pdf_file); - my $pruned_pdf = PDF::API2->new(); - for (my $n = 1; $n <= $pdf->page_count(); $n++) { - if (! $extern_pages{$n}) { - $pruned_pdf->import_page($pdf, $n); - } +info("Extracting new externs listed in '$mmz_file' " . + "from '$pdf_file' using Perl module $pdf_library"); +my $done_message = "Done (there was nothing to extract)"; +$indent = ' '; +$texindent = '\space\space '; +my $dir_to_make; + +# \paragraph{Process \texttt{.mmz}} +# +# We cannot process the .mmz file using in-place editing. It would fail when +# the file is writable but its parent directory is not. + +open (MMZ, $mmz_file); +while () { + my $mmz_line = $_; + if (/^\\mmzPrefix *{(?P)}/) { + # Found |\mmzPrefix|: create the extern directory, but only later, if an + # extern file is actually produced. We parse the prefix in two steps + # because we have to unquote the entire prefix. + my $prefix = unquote($+{prefix}); + warning("Cannot parse line '$mmz_line'") unless + $prefix =~ /(?P.*\/)?(?P.*?)/; + $dir_to_make = $+{dir_prefix}; + } elsif (/^\\mmzNewExtern\ *{(?P.*?)}{(?P[0-9]+)}# + {(?P[0-9.]*)pt}{(?P[0-9.]*)pt}/x) { + # Found |\mmzNewExtern|: extract the extern page into an extern file. + $done_message = "Done"; + my $ok = 1; + my %m_ne = %+; + # The extern filename, as specified in |.mmz|: + my $extern_file = unquote($m_ne{extern_path}); + # We parse the extern filename in a separate step because we have to + # unquote the entire path. + warning("Cannot parse line '$mmz_line'") unless + $extern_file =~ /(?P.*\/)?(?P.*?)# + (?P[0-9A-F]{32})-# + (?P[0-9A-F]{32})(?:-[0-9]+)?.pdf/x; + # The actual extern filename: + my $extern_file_out = find_out($extern_file); + paranoia_out($extern_file_out); + my $page = $m_ne{page_n}; + # Check whether c-memo and cc-memo exist (in any input directory). + my $c_memo = with_name($extern_file, + $+{name_prefix} . $+{code_md5sum} . '.memo'); + my $cc_memo = with_name($extern_file, + $+{name_prefix} . $+{code_md5sum} . + '-' . $+{context_md5sum} . '.memo'); + my $c_memo_in = find_in($c_memo); + my $cc_memo_in = find_in($cc_memo); + if ((! access_in($c_memo_in) || ! access_in($cc_memo_in)) && !$force) { + warning("I refuse to extract page $page into extern '$extern_file', " . + "because the associated c-memo '$c_memo' and/or " . + "cc-memo '$cc_memo' does not exist"); + $ok = ''; + } + # Load the PDF. We only do this now so that we don't load it if there + # is nothing to extract. + if ($ok && ! $pdf) { + if (!access_in($pdf_file)) { + warning("Cannot open '$pdf_file'", ''); + endinput(); + } + # Temporarily disable error handling, so that we can catch the error + # ourselves. + $SIG{__DIE__} = undef; $SIG{__WARN__} = undef; + # All safe, |paranoia_in| was already called above. + eval { $pdf = $pdf_library->open($pdf_file, msgver => 0) }; + $SIG{__DIE__} = \&die_handler; $SIG{__WARN__} = \&warn_handler; + error("File '$pdf_file' seems corrupted. " . + "Perhaps you have to load Memoize earlier in the preamble", + "In particular, Memoize must be loaded before TikZ library " . + "'fadings' and any package deploying it, and in Beamer, " . + "load Memoize by writing \\RequirePackage{memoize} before " . + "\\documentclass{beamer}. " . + "This was the error thrown by Perl:" . "\n$@") if $@; + } + # Does the page exist? + if ($ok && $page > (my $n_pages = $pdf->page_count())) { + error("I cannot extract page $page from '$pdf_file', " . + "as it contains only $n_pages page" . + ($n_pages > 1 ? 's' : ''), ''); + } + if ($ok) { + # Import the page into the extern PDF (no disk access yet). + my $extern = $pdf_library->new(outver => $pdf->version); + $extern->import_page($pdf, $page); + my $extern_page = $extern->open_page(1); + # Check whether the page size matches the |.mmz| expectations. + my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox(); + my $width_pt = ($x1 - $x0) / 72 * 72.27; + my $height_pt = ($y1 - $y0) / 72 * 72.27; + my $expected_width_pt = $m_ne{expected_width}; + my $expected_height_pt = $m_ne{expected_height}; + if ((abs($width_pt - $expected_width_pt) > $tolerance + || abs($height_pt - $expected_height_pt) > $tolerance) && !$force) { + warning("I refuse to extract page $page from $pdf_file, " . + "because its size (${width_pt}pt x ${height_pt}pt) " . + "is not what I expected " . + "(${expected_width_pt}pt x ${expected_height_pt}pt)"); + } else { + # All tests were successful, let's create the extern file. + # First, the containing directory, if necessary. + if ($dir_to_make) { + make_directory($dir_to_make); + $dir_to_make = undef; + } + # Now the extern file. Note that |paranoia_out| was already + # called above. + info("Page $page --> $extern_file_out"); + $extern->saveas($extern_file_out); + # This page will get pruned. + $extern_pages{$page} = 1 if $prune; + # Comment out this |\mmzNewExtern|. + $new_mmz .= '%' unless $keep; + } + } } - $pruned_pdf->save($pdf_file->stringify); - print("${message_prefix}The following extern pages were pruned out of the PDF: ", - join(",", sort(keys(%extern_pages))) . "\n") unless $quiet; + $new_mmz .= $mmz_line unless $keep; + stderr_to_warning(); } +close(MMZ); +$indent = ''; +$texindent = ''; +info($done_message); -if ($log) { - print LOG "\\endinput\n"; - close LOG; +# Write out the |.mmz| file with |\mmzNewExtern| lines commented out. (All safe, +# |paranoia_out| was already called above.) +if (!$keep) { + open(MMZ, ">", $mmz_file); + print MMZ $new_mmz; + close(MMZ); } + +# Remove the extracted pages from the original PDF. (All safe, |paranoia_out| +# was already called above.) +if ($prune and keys(%extern_pages) != 0) { + my $pruned_pdf = $pdf_library->new(); + for (my $n = 1; $n <= $pdf->page_count(); $n++) { + if (! $extern_pages{$n}) { + $pruned_pdf->import_page($pdf, $n); + } + } + $pruned_pdf->save($pdf_file); + info("The following extern pages were pruned out of the PDF: " . + join(",", sort(keys(%extern_pages)))); +} + +endinput(); + +# Local Variables: +# fill-column: 79 +# after-save-hook: pl2dtx +# End: diff --git a/macros/generic/memoize/memoize-extract.py b/macros/generic/memoize/memoize-extract.py index 54d8870e7b..29ea58192c 100755 --- a/macros/generic/memoize/memoize-extract.py +++ b/macros/generic/memoize/memoize-extract.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 - +# # This file is a part of Memoize, a TeX package for externalization of # graphics and memoization of compilation results in general, available at # https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize. @@ -18,166 +18,552 @@ # The files belonging to this work and covered by LPPL are listed in # /doc/generic/memoize/FILES. -__version__ = '2023/10/10 v1.0.0' - -import argparse, re, sys, os, pdfrw, subprocess, itertools -from pathlib import Path - -parser = argparse.ArgumentParser( - description = "Extract extern pages out of the document PDF.", - epilog = "For details, see the man page or the Memoize documentation.", - prog = 'memoize-extract.py', -) -parser.add_argument('--pdf', '-P', help = 'extract from file PDF') -parser.add_argument('--prune', '-p', action = 'store_true', - help = 'remove the extern pages after extraction') -parser.add_argument('--keep', '-k', action = 'store_true', - help = 'do not mark externs as extracted') -parser.add_argument('--force', '-f', action = 'store_true', - help = 'extract even if the size-check fails') -parser.add_argument('--log', '-l', default = os.devnull, help = 'the log file') -parser.add_argument('--warning-template', '-w', default = '\warningtext', - help = '\warningtext in the template will be replaced by the warning message') -parser.add_argument('--quiet', '-q', action = 'store_true', - help = "describe what's happening") -parser.add_argument('--embedded', '-e', action = 'store_true', - help = "prefix all messages to the standard output with the script name") -parser.add_argument('--mkdir', '-m', action = 'store_true', - help = 'create a directory (and exit)') -parser.add_argument('mmz', - help = 'the record file produced by Memoize: doc.mmz when compiling doc.tex') -parser.add_argument('--version', '-V', action = 'version', - version = f"%(prog)s of Memoize " + __version__) - -args = parser.parse_args() - -message_prefix = parser.prog + ': ' if args.embedded else '' -if args.embedded: - print() - -# Even a bit more paranoid than required: -# (a) disallowing TEXMFOUTPUT=/ (while allowing C:\ on Windows) -# (b) waiting for access to '-output-directory'. -output_paths = [Path.cwd()] -if texmfoutput := subprocess.run( - ['kpsewhich', '--var-value=TEXMFOUTPUT'], - capture_output = True).stdout.decode().strip(): - texmfoutput_dir = Path(texmfoutput).resolve() - if len(texmfoutput_dir.parts) != 1 or texmfoutput_dir.drive: - output_paths.append(texmfoutput_dir) - -def paranoia(f): - p = Path(f).resolve() - if p.stem.startswith('.'): - raise RuntimeError(f"{message_prefix}Cannot create a hidden file or dir: {f}") - if not any(p.is_relative_to(op) for op in output_paths): - raise RuntimeError(f"{message_prefix}Cannot write outside the current working or output directory tree: {f}") - -mmz_file = Path(args.mmz) +__version__ = '2024/01/02 v1.1.0' + +import argparse, re, sys, os, subprocess, itertools, traceback, platform +from pathlib import Path, PurePath +# We will only try to import the PDF processing library once we set up the +# error log. + +# \paragraph{Messages} + +# The messages are written both to the extraction log and the terminal (we +# output to stdout rather than stderr so that messages on the TeX terminal and +# document |.log| appear in chronological order). Messages are automatically +# adapted to the TeX |--format|. + +# The format of the messages. It depends on the given |--format|; the last +# entry is for t the terminal output. + +ERROR = { + 'latex': r'\PackageError{{{package_name}}}{{{short}}}{{{long}}}', + 'plain': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}', + 'context': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}', + None: '{header}{short}.\n{long}', +} + +WARNING = { + 'latex': r'\PackageWarning{{{package_name}}}{{{texindent}{text}}}', + 'plain': r'\message{{{package_name}: {texindent}{text}}}', + 'context': r'\message{{{package_name}: {texindent}{text}}}', + None: r'{header}{indent}{text}.', +} + +INFO = { + 'latex': r'\PackageInfo{{{package_name}}}{{{texindent}{text}}}', + 'plain': r'\message{{{package_name}: {texindent}{text}}}', + 'context': r'\message{{{package_name}: {texindent}{text}}}', + None: r'{header}{indent}{text}.', +} + +# Some variables used in the message routines; note that |header| will be +# redefined once we parse the arguments. + +package_name = 'memoize (python-based extraction)' +exit_code = 0 +log = None +header = '' +indent = '' +texindent = '' + +# The message routines. + +def error(short, long): + if not args.quiet: + print(ERROR[None].format(short = short, long = long, header = header)) + if log: + long = long.replace('\\', '\\string\\') + print( + ERROR[args.format].format( + short = short, long = long, package_name = package_name), + file = log) + global exit_code + exit_code = 11 + endinput() + +def warning(text): + if log: + print( + WARNING[args.format].format( + text = text, texindent = texindent, package_name = package_name), + file = log) + if not args.quiet: + print(WARNING[None].format(text = text, header = header, indent = indent)) + global exit_code + exit_code = 10 -if args.mkdir: # Here, argument "mmz" is interpreted as the directory to create. - # We cannot simply say - # paranoia(mmz_file) - # mmz_file.mkdir(parents = True, exist_ok = True) - # because have be paranoid about the intermediate directories as well: - # memoize-extract.py -m a/../../BAD/..//b - # Note that paranoia might kick in only after a part of the given path was - # already created. This is in accord to how "mkdir" behaves wrt existing - # files. - for folder in itertools.chain(reversed(mmz_file.parents), (mmz_file,)): - if not folder.is_dir(): - paranoia(folder) - folder.mkdir(exist_ok = True) - if not args.quiet: - print(f"{message_prefix}Created directory {folder}") - sys.exit() -elif mmz_file.suffix != '.mmz': - raise RuntimeError(f"{message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '{mmz_file}'") - -mmz_dir = mmz_file.parent -pdf_file = Path(args.pdf) if args.pdf else mmz_file.with_suffix('.pdf') -paranoia(pdf_file) -pdf = None -extern_pages = [] -new_mmz = [] -args.log is os.devnull or paranoia(Path(args.log)) -re_newextern = re.compile(r'\\mmzNewExtern *{(?P(?P.*?)(?P[0-9A-F]{32})-(?P[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P[0-9]+)}{(?P[0-9.]*)pt}{(?P[0-9.]*)pt}') -tolerance = 0.01 -done_message = f"{message_prefix}Done (there was nothing to extract)." - -# Complication: I want to use 'with', but don't have to open stderr. -with open(args.log, 'w') as log: - log = sys.stderr if args.log is os.devnull else log +def info(text): + if text and not args.quiet: + print(INFO[None].format(text = text, header = header, indent = indent)) + if log: + print( + INFO[args.format].format( + text = text, texindent = texindent, package_name = package_name), + file = log) + +# Mark the log as complete and exit. +def endinput(): + if log: + print(r'\endinput', file = log) + log.close() + sys.exit(exit_code) + +# \paragraph{Permission-related functions} + +# |paranoia_in|/|out| should work exactly as +# |kpsewhich -safe-in-name|/|-safe-out-name|. +def paranoia_in(f, remark = ''): + if f and not _paranoia(f, openin_any): + error(f"I'm not allowed to read from '{f}' (openin_any = {openin_any})", + remark) + +def paranoia_out(f, remark = ''): + if f and not _paranoia(f, openout_any): + error(f"I'm not allowed to write to '{f}' (openout_any = {openout_any})", + remark) + +def _paranoia(f, mode): + # |mode| is the value of |openin_any| or |openout_any|. |f| is a + # |pathlib.Path| object. + return ( + # In mode `any' (|a|, |y| or |1|), we may access any file. + mode in 'ay1' + or ( + # Otherwise, we are at least in the restricted mode, so we should not + # open dot files on Unix-like systems (except file called |.tex|). + not (os.name == 'posix' and f.stem.startswith('.') and f.stem != '.tex') + and ( + # If we are precisely in the restricted mode (|r|, |n|, |0|), + # then there are no further restrictions. + mode in 'rn0' + # Otherwise, we are in the paranoid mode (officially |p|, but + # any other value is interpreted as |p| as well). There are + # two further restrictions in the paranoid mode. + or ( + # We're not allowed to go to a parent directory. + '..' not in f.parts + and + # If the given path is absolute, is should be a descendant + # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|. + (not f.is_absolute() + or + is_ancestor(texmf_output_directory, f) + or + is_ancestor(texmfoutput, f) + ))))) + +# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with +# the |\| but lacking the drive. On Windows, |pathlib|'s |is_absolute| returns +# |True| only for paths starting with |\| and containing the drive. +def sanitize_filename(f): + if f and platform.system() == 'Windows' and not (f.is_absolute() or not f.drive): + error(f"\"Semi-absolute\" paths are disallowed: '{f}'", r"The path must " + r"either contain both the drive letter and start with '\', " + r"or none of these; paths like 'C:foo' and '\foo' are disallowed") + +def access_in(f): + return os.access(f, os.R_OK) + +# This function can fail on Windows, reporting a non-writable file or dir as +# writable, because |os.access| does not work with Windows' |icacls| +# permissions. Consequence: we might try to write to a read-only current or +# output directory instead of switching to the temporary directory. Paranoia +# is unaffected, as it doesn't use |access_*| functions. +def access_out(f): try: - mmz = mmz_file.open() - except FileNotFoundError: - print(f'''{message_prefix}File "{mmz_file}" does not exist, assuming there's nothing to do.''', - file = sys.stderr) + exists = f.exists() + # Presumably, we get this error when the parent directory is not + # executable. + except PermissionError: + return + if exists: + # An existing file should be writable, and if it's a directory, it + # should also be executable. + return os.access(f, os.W_OK) and (not f.is_dir() or os.access(f, os.X_OK)) else: - if not args.quiet: - print(f"{message_prefix}Extracting externs from {pdf_file}") + # For a non-existing file, the parent directory should be writable. + # (This is the only place where function |pathlib.parent| is used, so + # it's ok that it returns the logical parent.) + return os.access(f.parent, os.W_OK) + +# This function finds the location for an input file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it +# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always), +# and |TEXMFOUTPUT| directory (if defined), in this order. The first readable +# file found is returned; if no readable file is found, the file in the current +# directory is returned. +def find_in(f): + sanitize_filename(f) + if f.is_absolute(): + return f + for df in (texmf_output_directory / f if texmf_output_directory else None, + f, + texmfoutput / f if texmfoutput else None): + if df and access_in(df): + return df + return f + +# This function finds the location for an output file, respecting +# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the +# filesystem. It returns an absolute file as-is. For a relative file, it +# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless +# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if +# defined), in this order. The first writable file found is returned; if no +# writable file is found, the file in either the current or the output +# directory is returned. +def find_out(f): + sanitize_filename(f) + if f.is_absolute(): + return f + for df in (texmf_output_directory / f if texmf_output_directory else None, + f if not texmf_output_directory else None, + texmfoutput / f if texmfoutput else None): + if df and access_out(df): + return df + return texmf_output_directory / f if texmf_output_directory else f + +# This function assumes that both paths are absolute; ancestor may be |None|, +# signaling a non-path. +def is_ancestor(ancestor, descendant): + if not ancestor: + return + a = ancestor.parts + d = descendant.parts + return len(a) < len(d) and a == d[0:len(a)] + +# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|. +def mkdir(folder): + folder = find_out(Path(folder)) + if not folder.exists(): + paranoia_out(folder) + # Using |folder.mkdir| is fine because we know that + # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that + # ``folder'' contains no |..|. + folder.mkdir(parents = True, exist_ok = True) + # This does not get logged when the function is invoked via |--mkdir|, + # as it is not clear what the log name should be. + info(f"Created directory {folder}") + +_re_unquote = re.compile(r'"(.*?)"') +def unquote(fn): + return _re_unquote.sub(r'\1', fn) + +# \paragraph{Kpathsea} + +# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and +# |TEXMF_OUTPUT_DIRECTORY|. + +kpsewhich_output = subprocess.run(['kpsewhich', + f'-expand-var=' + f'openin_any=$openin_any,' + f'openout_any=$openout_any,' + f'TEXMFOUTPUT=$TEXMFOUTPUT'], + capture_output = True + ).stdout.decode().strip() +if not kpsewhich_output: + # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In + # absence of |kpathsea| information, we get very paranoid, but still try to + # get |TEXMFOUTPUT| from an environment variable. + openin_any, openout_any = 'p', 'p' + texmfoutput, texmf_output_directory = None, None + # Unfortunately, this warning can't make it into the log. But then again, + # the chances of a missing |kpsewhich| are very slim, and its absence would + # show all over the place anyway. + warning('I failed to execute "kpsewhich"; , is there no TeX system installed? ' + 'Assuming openin_any = openout_any = "p" ' + '(i.e. restricting all file operations to non-hidden files ' + 'in the current directory of its subdirectories).') +else: + m = re.fullmatch(r'openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)', + kpsewhich_output) + openin_any, openout_any, texmfoutput = m.groups() + texmf_output_directory = os.environ.get('TEXMF_OUTPUT_DIRECTORY', None) + if openin_any == '$openin_any': + # When the |open*_any| variables are not expanded, we assume we're + # running MiKTeX. The two config settings below correspond to TeXLive's + # |openin_any| and |openout_any|; afaik, there is no analogue to + # |TEXMFOUTPUT|. + initexmf_output = subprocess.run( + ['initexmf', '--show-config-value=[Core]AllowUnsafeInputFiles', + '--show-config-value=[Core]AllowUnsafeOutputFiles'], + capture_output = True).stdout.decode().strip() + openin_any, openout_any = initexmf_output.split() + openin_any = 'a' if openin_any == 'true' else 'p' + openout_any = 'a' if openout_any == 'true' else 'p' + texmfoutput = None + texmf_output_directory = None + +# An output directory should exist, and may not point to the root on Linux. On +# Windows, it may point to the root, because we only allow absolute filenames +# containing the drive, e.g.\ |F:\|; see |is_absolute|. +def sanitize_output_dir(d_str): + d = Path(d_str) if d_str else None + sanitize_filename(d) + return d if d and d.is_dir() and \ + (not d.is_absolute() or len(d.parts) != 1 or d.drive) else None + +texmfoutput = sanitize_output_dir(texmfoutput) +texmf_output_directory = sanitize_output_dir(texmf_output_directory) + +class NotExtracted(UserWarning): + pass + +# We don't delve into the real script when loaded from the testing code. +if __name__ == '__main__': + + # \paragraph{Arguments} + + parser = argparse.ArgumentParser( + description = "Extract extern pages produced by package Memoize " + "out of the document PDF.", + epilog = "For details, see the man page or the Memoize documentation.", + prog = 'memoize-extract.py', + ) + parser.add_argument('-P', '--pdf', help = 'extract from file PDF') + parser.add_argument('-p', '--prune', action = 'store_true', + help = 'remove the extern pages after extraction') + parser.add_argument('-k', '--keep', action = 'store_true', + help = 'do not mark externs as extracted') + parser.add_argument('-F', '--format', choices = ['latex', 'plain', 'context'], + help = 'the format of the TeX document invoking extraction') + parser.add_argument('-f', '--force', action = 'store_true', + help = 'extract even if the size-check fails') + parser.add_argument('-q', '--quiet', action = 'store_true', + help = "describe what's happening") + parser.add_argument('-m', '--mkdir', action = 'store_true', + help = 'create a directory (and exit); ' + 'mmz argument is interpreted as directory name') + parser.add_argument('-V', '--version', action = 'version', + version = f"%(prog)s of Memoize " + __version__) + parser.add_argument('mmz', help = 'the record file produced by Memoize: ' + 'doc.mmz when compiling doc.tex ' + '(doc and doc.tex are accepted as well)') + + args = parser.parse_args() + + header = parser.prog + ': ' if args.format else '' + + # Start a new line in the TeX terminal output. + if args.format: + print() + + # \paragraph{Initialization} + + # With |--mkdir|, argument |mmz| is interpreted as the directory to create. + if args.mkdir: + mkdir(args.mmz) + sys.exit() + + # Normalize the |mmz| argument into a |.mmz| filename. + mmz_file = Path(args.mmz) + if mmz_file.suffix == '.tex': + mmz_file = mmz_file.with_suffix('.mmz') + elif mmz_file.suffix != '.mmz': + mmz_file = mmz_file.with_name(mmz_file.name + '.mmz') + + # Once we have the |.mmz| filename, we can open the log. + if args.format: + log_file = find_out(mmz_file.with_suffix('.mmz.log')) + paranoia_out(log_file) + info(f"Logging to '{log_file}'"); + log = open(log_file, 'w') + + # Now that we have opened the log file, we can try loading the PDF + # processing library. + try: + import pdfrw + except ModuleNotFoundError: + error("Python module 'pdfrw' was not found", + 'Have you followed the instructions is section 1.1 of the manual?') + + # Catch any errors in the script and output them to the log. + try: + + # Find the |.mmz| file we will read, but retain the original filename + # in |given_mmz_file|, as we will still need it. + given_mmz_file = mmz_file + mmz_file = find_in(mmz_file) + paranoia_in(mmz_file) + if not args.keep: + paranoia_out(mmz_file, + remark = 'This file is rewritten unless option --keep is given.') + try: + mmz = open(mmz_file) + except FileNotFoundError: + info(f"File '{given_mmz_file}' does not exist, " + f"assuming there's nothing to do") + endinput() + + # Determine the PDF filename: it is either given via |--pdf|, or + # constructed from the |.mmz| filename. + pdf_file = find_in(Path(args.pdf) + if args.pdf else given_mmz_file.with_suffix('.pdf')) + paranoia_in(pdf_file) + if args.prune: + paranoia_out(pdf_file, + remark = 'I would have to rewrite this file ' + 'because option --prune was given.') + + # Various initializations. + + re_prefix = re.compile(r'\\mmzPrefix *{(?P.*?)}') + re_split_prefix = re.compile(r'(?P.*/)?(?P.*?)') + re_newextern = re.compile( + r'\\mmzNewExtern *{(?P.*?)}{(?P[0-9]+)}' + r'{(?P[0-9.]*)pt}{(?P[0-9.]*)pt}') + re_extern_path = re.compile( + r'(?P.*/)?(?P.*?)' + r'(?P[0-9A-F]{32})-' + r'(?P[0-9A-F]{32})(?:-[0-9]+)?.pdf') + pdf = None + extern_pages = [] + new_mmz = [] + tolerance = 0.01 + dir_to_make = None + info(f"Extracting new externs listed in '{mmz_file}' from '{pdf_file}'") + done_message = "Done (there was nothing to extract)" + indent = ' ' + texindent = '\space\space ' + + # \paragraph{Process \texttt{.mmz}} + for line in mmz: - if m := re_newextern.match(line): - extern_file = mmz_dir / m['extern_fn'] - paranoia(extern_file) - page_n = int(m['page_n'])-1 - c_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '.memo') - cc_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '-' + m['context_md5sum'] + '.memo') - if not (c_memo.exists() and cc_memo.exists()): - print(args.warning_template.replace('\warningtext', f'Not extracting page {page_n} into extern {extern_file}, because the associated (c)c-memo does not exist'), file = log) - continue - if not pdf: - try: - pdf = pdfrw.PdfReader(pdf_file) - except pdfrw.errors.PdfParseError: - print(f'{message_prefix}File "{pdf_file}" cannot be read, bailing out.', file = sys.stderr) - print(args.warning_template.replace('\warningtext', f'Cannot read file "{pdf_file}". Perhaps you have to load Memoize earlier in the preamble?'), file = log) - args.keep = True - break - extern = pdfrw.PdfWriter(extern_file) - page = pdf.pages[page_n] - expected_width_pt, expected_height_pt = float(m['expected_width']), float(m['expected_height']) - mb = page['/MediaBox'] - width_bp, height_bp = float(mb[2]) - float(mb[0]), float(mb[3]) - float(mb[1]) - width_pt = width_bp / 72 * 72.27 - height_pt = height_bp / 72 * 72.27 - warning = None - if abs(width_pt - expected_width_pt) > tolerance \ - or abs(height_pt - expected_height_pt) > tolerance: - warning = ( - f'I refuse to extract page {page_n+1} from "{pdf_file}", ' - f'because its size ({width_pt}pt x {height_pt}pt) is not ' - f'what I expected ({expected_width_pt}pt x {expected_height_pt}pt)') - print(args.warning_template.replace('\warningtext', warning), file = log) - if warning and not args.force: - extern_file.unlink(missing_ok = True) - else: + try: + if m_p := re_prefix.match(line): + # Found |\mmzPrefix|: create the extern directory, but only + # later, if an extern file is actually produced. We parse + # the prefix in two steps because we have to unquote the + # entire prefix. + prefix = unquote(m_p['prefix']) + if not (m_sp := re_split_prefix.match(prefix)): + warning(f"Cannot parse line {line.strip()}") + dir_to_make = m_sp['dir_prefix'] + elif m_ne := re_newextern.match(line): + # Found |\mmzNewExtern|: extract the extern page into an + # extern file. + done_message = "Done" + # The extern filename, as specified in |.mmz|: + unquoted_extern_path = unquote(m_ne['extern_path']) + extern_file = Path(unquoted_extern_path) + # We parse the extern filename in a separate step because + # we have to unquote the entire path. + if not (m_ep := re_extern_path.match(unquoted_extern_path)): + warning(f"Cannot parse line {line.strip()}") + # The actual extern filename: + extern_file_out = find_out(extern_file) + paranoia_out(extern_file_out) + page_n = int(m_ne['page_n'])-1 + # Check whether c-memo and cc-memo exist (in any input + # directory). + c_memo = extern_file.with_name( + m_ep['name_prefix'] + m_ep['code_md5sum'] + '.memo') + cc_memo = extern_file.with_name( + m_ep['name_prefix'] + m_ep['code_md5sum'] + + '-' + m_ep['context_md5sum'] + '.memo') + c_memo_in = find_in(c_memo) + cc_memo_in = find_in(cc_memo) + if not (access_in(c_memo_in) and access_in(cc_memo_in)) \ + and not args.force: + warning(f"I refuse to extract page {page_n+1} into extern " + f"'{extern_file}', because the associated c-memo " + f"'{c_memo}' and/or cc-memo '{cc_memo}' " + f"does not exist") + raise NotExtracted() + # Load the PDF. We only do this now so that we don't load + # it if there is nothing to extract. + if not pdf: + if not access_in(pdf_file): + warning(f"Cannot open '{pdf_file}'") + endinput() + try: + # All safe, |paranoia_in| was already called above. + pdf = pdfrw.PdfReader(pdf_file) + except pdfrw.errors.PdfParseError as err: + error(rf"File '{pdf_file}' seems corrupted. Perhaps you " + rf"have to load Memoize earlier in the preamble", + f"In particular, Memoize must be loaded before " + f"TikZ library 'fadings' and any package " + f"deploying it, and in Beamer, load Memoize " + f"by writing \RequirePackage{{memoize}} before " + f"\documentclass{{beamer}}. " + f"This was the error thrown by Python: \n{err}") + # Does the page exist? + if page_n >= len(pdf.pages): + error(rf"I cannot extract page {page_n} from '{pdf_file}', " + rf"as it contains only {len(pdf.pages)} page" + + ('s' if len(pdf.pages) > 1 else ''), '') + # Check whether the page size matches the |.mmz| + # expectations. + page = pdf.pages[page_n] + expected_width_pt = float(m_ne['expected_width']) + expected_height_pt = float(m_ne['expected_height']) + mb = page['/MediaBox'] + width_bp = float(mb[2]) - float(mb[0]) + height_bp = float(mb[3]) - float(mb[1]) + width_pt = width_bp / 72 * 72.27 + height_pt = height_bp / 72 * 72.27 + if (abs(width_pt - expected_width_pt) > tolerance + or abs(height_pt - expected_height_pt) > tolerance) \ + and not args.force: + warning( + f"I refuse to extract page {page_n+1} from '{pdf_file}' " + f"because its size ({width_pt}pt x {height_pt}pt) " + f"is not what I expected " + f"({expected_width_pt}pt x {expected_height_pt}pt)") + raise NotExtracted() + # All tests were successful, let's create the extern file. + # First, the containing directory, if necessary. + if dir_to_make: + mkdir(dir_to_make) + dir_to_make = None + # Now the extern file. Note that |paranoia_out| was + # already called above. + info(f"Page {page_n+1} --> {extern_file_out}") + extern = pdfrw.PdfWriter(extern_file_out) extern.addpage(page) - if not args.quiet: - print(f"{message_prefix} Page {page_n+1} --> {extern_file}", file = sys.__stdout__) extern.write() - done_message = f"{message_prefix}Done." + # This page will get pruned. if args.prune: extern_pages.append(page_n) + # Comment out this |\mmzNewExtern|. if not args.keep: line = '%' + line - if not args.keep: - new_mmz.append(line) + except NotExtracted: + pass + finally: + if not args.keep: + new_mmz.append(line) mmz.close() - if not args.quiet: - print(done_message) + indent = '' + texindent = '' + info(done_message) + + # Write out the |.mmz| file with |\mmzNewExtern| lines commented + # out. (All safe, |paranoia_out| was already called above.) if not args.keep: - paranoia(mmz_file) with open(mmz_file, 'w') as mmz: for line in new_mmz: print(line, file = mmz, end = '') + + # Remove the extracted pages from the original PDF. (All safe, + # |paranoia_out| was already called above.) if args.prune and extern_pages: pruned_pdf = pdfrw.PdfWriter(pdf_file) pruned_pdf.addpages( page for n, page in enumerate(pdf.pages) if n not in extern_pages) pruned_pdf.write() - if not args.quiet: - print(f"{message_prefix}The following extern pages were pruned out of the PDF:", - ",".join(str(page+1) for page in extern_pages)) - if args.log is not os.devnull: - print(r'\endinput', file = log) + info(f"The following extern pages were pruned out of the PDF: " + + ",".join(str(page+1) for page in extern_pages)) + + # Report that extraction was successful. + endinput() + + # Catch any errors in the script and output them to the log. + except Exception as err: + error(f'Python error: {err}', traceback.format_exc()) + +# Local Variables: +# fill-column: 79 +# after-save-hook: py2dtx +# End: diff --git a/macros/generic/memoize/memoize.edtx b/macros/generic/memoize/memoize.edtx index 10794c7a3b..86ed2b6197 100644 --- a/macros/generic/memoize/memoize.edtx +++ b/macros/generic/memoize/memoize.edtx @@ -1,5 +1,3 @@ -% \iffalse -% % memoize.edtx (this is not a .dtx file; to produce a .dtx, process it with edtx2dtx) % %% This file is a part of Memoize, a TeX package for externalization of @@ -21,8 +19,6 @@ %% The files belonging to this work and covered by LPPL are listed in %% (/doc/generic/memoize/)FILES. % -% \fi -% % \begin{macrocode} % % \relax @@ -68,7 +64,7 @@ % listing options app={basicstyle=\ttfamily\scriptsize}}} % % \begin{tcbraster}[raster columns=100] -% \tcbinputlisting{raster multicolumn=55, ins listing, top=1mm, bottom=1mm, title=memoize.ins,listing file=../memoize.ins, linerange={27-38}, leftupper=1mm} +% \tcbinputlisting{raster multicolumn=55, ins listing, top=1mm, bottom=1mm, title=memoize.ins,listing file=../memoize.ins, linerange={27-39}, leftupper=1mm} % \begin{tcboxedraster}[raster columns=1]{blankest, raster multicolumn=45} % \tcbinputlisting{ins listing, title=advice.ins, listing file=../advice.ins, linerange=28-31} % \tcbinputlisting{ins listing, title=collar\raisebox{0pt}[\height][0pt]{g}s.ins, listing file=../collargs.ins, linerange=29-31} @@ -77,8 +73,7 @@ % % Memoize also contains two scripts, |memoize-extract| and |memoize-clean|. % Both come in two functionally equivalent implementations: Perl (|.pl|) and a -% Python (|.py|). The code is listed in \S\ref{sec:code:scripts}, I believe it -% is self-explanatory enough to lack more than a occasional comment. +% Python (|.py|). Their code is listed in \S\ref{sec:code:scripts}. % % % \thispagestyle{empty} @@ -95,55 +90,55 @@ % % \paragraph{Identification} of |memoize|, |memoizable| and |nomemoize|. %<*mmz> -%\ProvidesPackage{memoize}[2023/10/10 v1.0.0 Fast and flexible externalization] +%\ProvidesPackage{memoize}[2024/1/02 v1.1.0 Fast and flexible externalization] %%D \module[ %%D file=t-memoize.tex, -%%D version=1.0.0, +%%D version=1.1.0, %%D title=Memoize, %%D subtitle=Fast and flexible externalization, %%D author=Saso Zivanovic, -%%D date=2023-10-10, +%%D date=2024-01-02, %%D copyright=Saso Zivanovic, %%D license=LPPL, %%D ] %\writestatus{loading}{ConTeXt User Module / memoize} %\unprotect %\startmodule[memoize] -%% Package memoize 2023/10/10 v1.0.0 +%% Package memoize 2024/01/02 v1.1.0 % %<*mmzable> -%\ProvidesPackage{memoizable}[2023/10/10 v1.0.0 A programmer's stub for Memoize] +%\ProvidesPackage{memoizable}[2024/1/02 v1.1.0 A programmer's stub for Memoize] %%D \module[ %%D file=t-memoizable.tex, -%%D version=1.0.0, +%%D version=1.1.0, %%D title=Memoizable, %%D subtitle=A programmer's stub for Memoize, %%D author=Saso Zivanovic, -%%D date=2023-10-10, +%%D date=2024-01-02, %%D copyright=Saso Zivanovic, %%D license=LPPL, %%D ] %\writestatus{loading}{ConTeXt User Module / memoizable} %\unprotect %\startmodule[memoizable] -%% Package memoizable 2023/10/10 v1.0.0 +%% Package memoizable 2024/01/02 v1.1.0 % %<*nommz> -%\ProvidesPackage{nomemoize}[2023/10/10 v1.0.0 A no-op stub for Memoize] +%\ProvidesPackage{nomemoize}[2024/1/02 v1.1.0 A no-op stub for Memoize] %%D \module[ %%D file=t-nomemoize.tex, -%%D version=1.0.0, +%%D version=1.1.0, %%D title=Memoize, %%D subtitle=A no-op stub for Memoize, %%D author=Saso Zivanovic, -%%D date=2023-10-10, +%%D date=2024-01-02, %%D copyright=Saso Zivanovic, %%D license=LPPL, %%D ] %\writestatus{loading}{ConTeXt User Module / nomemoize} %\unprotect %\startmodule[nomemoize] -%% Package nomemoize 2023/10/10 v1.0.0 +%% Package nomemoize 2024/01/02 v1.1.0 % % % \paragraph{Required packages} and \hologo{LaTeX}ization of \hologo{plainTeX} @@ -163,13 +158,6 @@ \def\newtoks{\alloc@5\toks\toksdef\@cclvi} \def\newwrite{\alloc@7\write\chardef\sixt@@n} % -% |Nomemoize| has to load |pgfopts| as well, and process package options (right -% away, why not), otherwise \hologo{LaTeX} will complain. -%<*latex> -%\RequirePackage{pgfopts} % pgfkeys-based package options -%\pgfkeys{/memoize/package options/.unknown/.code={}} -%\ProcessPgfPackageOptions{/memoize/package options} -% % I can't really write any code without |etoolbox| \dots %<*mmz> %\RequirePackage{etoolbox} @@ -238,6 +226,7 @@ }% }% } +% % % In \hologo{ConTeXt}, |\unexpanded| means |\protected|, and the usual % |\unexpanded| is available as |\normalunexpanded|. Option one: use dtx @@ -249,6 +238,18 @@ % |\unexpanded| in the |.dtx|, and |sed| through the generated \hologo{ConTeXt} % files to replace all its occurrences by |\normalunexpanded|. Oh yeah! % +% Load |pgfkeys| in |nomemoize| and |memoizable|. Not necessary in |memoize|, +% as it is already loaded by CollArgs. +%<*nommz,mmzable> +%\RequirePackage{pgfkeys} +%\input pgfkeys +%\input t-pgfkey +% +% +% Different formats of |memoizable| merely load |memoizable.code.tex|, which +% exists so that |memoizable| can be easily loaded by generic code, like a +% |tikz| library. +%\input memoizable.code.tex % % \paragraph.{Shipout} % We will next load our own auxiliary package, CollArgs, but before we do @@ -260,6 +261,7 @@ % section~\ref{sec:code:extern}. Below, we first check that the current % meaning of |\shipout| is primitive, and then redefine it. % +%<*mmz> %<*plain> \def\mmz@regular@shipout{% \global\advance\mmzRegularPages1\relax @@ -284,7 +286,7 @@ % \paragraph.{Loading order} |memoize| and |nomemoize| are mutually exclusive, % and |memoizable| must be loaded before either of them. |\mmz@loadstatus|: 1 % = memoize, 2 = memoizable, 3 = nomemoize. -%<*(plain,context)&(mmz,nommz,mmzable)> +%<*mmz,nommz> \def\ifmmz@loadstatus#1{% \ifnum#1=0\csname mmz@loadstatus\endcsname\relax \expandafter\@firstoftwo @@ -292,11 +294,9 @@ \expandafter\@secondoftwo \fi } -% +% %<*mmz> -%\@ifpackageloaded{nomemoize}% -%\ifmmz@loadstatus{3}% -{% +\ifmmz@loadstatus{3}{% \PackageError{memoize}{Cannot load the package, as "nomemoize" is already loaded. Memoization will NOT be in effect}{Packages "memoize" and "nomemoize" are mutually exclusive, please load either one or the other.}% @@ -304,9 +304,7 @@ %\ProcessPgfPackageOptions{/memoize/package options} \endinput }{}% -%\@ifpackageloaded{memoizable}% -%\ifmmz@loadstatus{2}% -{% +\ifmmz@loadstatus{2}{% \PackageError{memoize}{Cannot load the package, as "memoizable" is already loaded}{Package "memoizable" is loaded by packages which support memoization. Memoize must be loaded before all such packages. The @@ -324,30 +322,20 @@ %\ProcessPgfPackageOptions{/memoize/package options} \endinput }{}% -%\ifmmz@loadstatus{1}{\endinput}{}% -%\def\mmz@loadstatus{1}% +\ifmmz@loadstatus{1}{\endinput}{}% +\def\mmz@loadstatus{1}% % -%<*mmzable> -%\@ifpackageloaded{memoize}% -%\ifmmz@loadstatus{1}% -{\endinput}{} -%\@ifpackageloaded{nomemoize}% -%\ifmmz@loadstatus{3}% -{\endinput}{}% -%\ifmmz@loadstatus{2}{\endinput}{}% -%\def\mmz@loadstatus{2}% -% +%<*mmzable&generic> +\ifcsname mmz@loadstatus\endcsname\endinput\fi +\def\mmz@loadstatus{2}% +% %<*nommz> -%\@ifpackageloaded{memoize} -%\ifmmz@loadstatus{1}% -{% +\ifmmz@loadstatus{1}{% \PackageError{nomemoize}{Cannot load the package, as "memoize" is already loaded; memoization will remain in effect}{Packages "memoize" and "nomemoize" are mutually exclusive, please load either one or the other.}% \endinput }{}% -%\@ifpackageloaded{memoizable}% -%\ifmmz@loadstatus{2}% -{% +\ifmmz@loadstatus{2}{% \PackageError{nomemoize}{Cannot load the package, as "memoizable" is already loaded}{Package "memoizable" is loaded by packages which support memoization. (No)Memoize must be loaded before all such packages. The @@ -363,8 +351,8 @@ of that package.}% \endinput }{}% -%\ifmmz@loadstatus{3}{\endinput}{}% -%\def\mmz@loadstatus{3}% +\ifmmz@loadstatus{3}{\endinput}{}% +\def\mmz@loadstatus{3}% % % %<*mmz> @@ -373,7 +361,7 @@ % Read \hologo{TeX} file |#2| into token register |#1| (under the current % category code regime); |\toksapp| is defined in CollArgs. \def\filetotoks#1#2{% - \immediate\openin0{#2}% + \immediate\mmz@openin0{#2}% #1={}% \loop \unless\ifeof0 @@ -385,6 +373,12 @@ } % \end{macro} % +% \begin{macro}{\mmz@openin,\mmz@openout} +% A workaround for |morewrites|. +\let\mmz@openin\openin +\let\mmz@openout\openout +% \end{macro} +% % \paragraph{Other} little things. \newif\ifmmz@temp \newtoks\mmz@temptoks @@ -395,22 +389,24 @@ % \label{sec:code:configuration} % % \begin{macro}{\mmzset} -% The user primarily interacts with |Memoize| through the |pgfkeys|-based +% The user primarily interacts with Memoize through the |pgfkeys|-based % configuration macro |\mmzset|, which executes keys in path |/mmz|. In % |nomemoize| and |memoizable|, is exists as a no-op. \def\mmzset#1{\pgfqkeys{/mmz}{#1}\ignorespaces} % -%\def\mmzset#1{\ignorespaces} +%<*nommz,mmzable&generic> +\def\mmzset#1{\ignorespaces} +% % \end{macro} -% -% \begin{macro}{\mmzset} +% +% \begin{macro}{\nommzkeys} % Any |/mmz| keys used outside of |\mmzset| must be declared by this macro % for |nomemoize| package to work. %\def\nommzkeys#1{} -%<*nommz,mmzable> +%<*nommz,mmzable&generic> \def\nommzkeys{\pgfqkeys{/mmz}} \pgfqkeys{/mmz}{.unknown/.code={\pgfkeysdef{\pgfkeyscurrentkey}{}}} -% +% % \end{macro} % % \begin{key}{enable, disable} @@ -427,7 +423,9 @@ % % |Nomemoize| does not need the keys themselves, but it does need the % underlying conditional --- which will be always false. -%\newif\ifmemoize +%<*mmz,nommz,mmzable&generic> +\newif\ifmemoize +% %<*mmz> \mmzset{% enable/.style={begindocument/.append code=\memoizetrue}, @@ -439,10 +437,15 @@ % Memoize is enabled at the beginning of the document, unless explicitly % disabled by the user in the preamble. enable, -} % \end{macro} % \end{key} % +% \begin{key}{options} +% Execute the given value as a keylist of Memoize settings. + options/.style={#1}, +} +% \end{key} +% % \begin{key}{normal,readonly,recompile} % When Memoize is enabled, it can be in one of three modes % (\MS\ref{sec:tut:working-on-a-picture}): normal, readonly, and recompile. @@ -461,7 +464,7 @@ } % \end{key} % -% \begin{key}{path, path/relative, path/dir, path/prefix} +% \begin{key}{prefix} % Key |path| executes the given keylist in path |/mmz/path|, to determine the % full \emph{path prefix} to memo and extern files % (\MS\ref{sec:tut:memodir},\ref{sec:memos}): |relative|, true by default, @@ -471,56 +474,77 @@ % under user control, and neither is the suffix. These subkeys will be % initialized a bit later, via |no memo dir|. \mmzset{% - path/.code={\pgfqkeys{/mmz/path}{#1}}, - path/.cd, - relative/.is if=mmz@relativepath, - dir/.store in=\mmz@dir, - dir/.value required, - prefix/.store in=\mmz@prefix, - prefix/.value required, -} -\newif\ifmmz@relativepath -% Key |path| concludes by performing two post-path-setting actions: creating -% the given directory if |mkdir| is in effect, and noting the new path prefix -% in record files (by eventually executing |record/prefix|, which typically -% puts a |\mmzPrefix| line in the |.mmz| file). These actions are the reason -% for having the path setting keys grouped under |path| --- we don't want them -% to be triggered by changes of the individual components of the path. -% Similarly, we don't want them triggered by multiple invocations of |path| in -% the preamble; only the final setting matters, so |path| is only equipped with -% the action-triggering code at the beginning of the document. + prefix/.code={\mmz@parse@prefix{#1}}, +} +% \begin{macro}{\mmz@split@prefix} +% This macro stores the detokenized expansion of |#1| into |\mmz@prefix|, +% which it then splits into |\mmz@prefix@dir| and |\mmz@prefix@name| at the +% final |/|. The slash goes into |\mmz@prefix@dir|. If there is no slash, +% |\mmz@prefix@dir| is empty. +\begingroup +\catcode`\/=12 +\gdef\mmz@parse@prefix#1{% + \edef\mmz@prefix{\detokenize\expandafter{\expanded{#1}}}% + \def\mmz@prefix@dir{}% + \def\mmz@prefix@name{}% + \expandafter\mmz@parse@prefix@i\mmz@prefix/\mmz@eov +} +\gdef\mmz@parse@prefix@i#1/#2{% + \ifx\mmzeov#2% + \def\mmz@prefix@name{#1}% + \else + \appto\mmz@prefix@dir{#1/}% + \expandafter\mmz@parse@prefix@i\expandafter#2% + \fi +} +\endgroup +% \end{macro} +% +% Key |prefix| concludes by performing two actions: it creates the given +% directory if |mkdir| is in effect, and notes the new prefix in record files +% (by eventually executing |record/prefix|, which typically puts a |\mmzPrefix| +% line in the |.mmz| file). In the preamble, only the final setting of +% |prefix| matters, so this key is only equipped with the action-triggering +% code at the beginning of the document. \mmzset{% begindocument/.append style={ - path/.append code=\mmz@maybe@mkmemodir\mmz@record@prefix, + prefix/.append code=\mmz@maybe@mkmemodir\mmz@record@prefix, }, - % Consequently, the post-path-setting actions must be triggered manually at + % Consequently, the post-prefix-setting actions must be triggered manually at % the beginning of the document. Below, we trigger directory creation; % |record/prefix| will be called from |record/begin|, which is executed at % the beginning of the document, so it shouldn't be mentioned here. begindocument/.append code=\mmz@maybe@mkmemodir, } -% Define the paths to the memo directory and the prefix. -\def\mmz@dir@path{\ifmmz@relativepath.\fi/\mmz@dir} -\def\mmz@prefix@path{\mmz@dir@path/\mmz@prefix} % \end{key} % % \begin{key}{mkdir, mkdir command} % Should we create the memo/extern directory if it doesn't exist? And which -% command should we use to create it? Of course, shell escape must be -% properly configured for this to work (\MS\ref{sec:shell-escape}). -% ^^A todo: What about quoting the paths containing spaces on Windows? Will this work? +% command should we use to create it? There is no initial value for the +% latter, because |mkdir| cannot be executed out of the box, but note that +% |extract=perl| and |extract=python| will set the extraction script with +% option |--mkdir| as the value of |mkdir command|. \mmzset{ mkdir/.is if=mmz@mkdir, - mkdir command/.code={\def\mmz@mkdir@command##1{#1}}, - mkdir command={mkdir "#1"}, + mkdir command/.store in=\mmz@mkdir@command, + mkdir command={}, } % The underlying conditional \cs{ifmmz@mkdir} is only ever used in % |\mmz@maybe@mkmemodir| below, which is itself only executed at the end of -% |path| and in |begindocument|. +% |prefix| and in |begindocument|. \newif\ifmmz@mkdir +\mmz@mkdirtrue +% We only attempt to create the memo directory if |\ifmmz@mkdir| is in effect +% and if both |\mmz@mkdir@command| and |\mmz@prefix@dir| are specified (i.e.\ +% non-empty). \def\mmz@maybe@mkmemodir{% \ifmmz@mkdir - \pdf@system{\mmz@mkdir@command{\mmzOutputDirectory\mmz@dir@path}}% + \ifdefempty\mmz@mkdir@command{}{% + \ifdefempty\mmz@prefix@dir{}{% + \mmz@remove@quotes{\mmz@prefix@dir}\mmz@temp + \pdf@system{\mmz@mkdir@command\space"\mmz@temp"}% + }% + }% \fi } % \end{key} @@ -533,51 +557,49 @@ % filenames themselves have no prefix. Furthermore, |memo dir| triggers the % creation of the directory. \mmzset{% - memo dir/.style={ - mkdir, - path={ - relative, - dir={#1.memo.dir}, - prefix={}, - }, - }, - memo dir/.default=\mmzUnquote\jobname, - no memo dir/.style={ - mkdir=false, - path={ - relative, - dir={}, - prefix={#1.}, - }, - }, - no memo dir/.default=\mmzUnquote\jobname, + memo dir/.style={prefix={#1.memo.dir/}}, + memo dir/.default=\jobname, + no memo dir/.style={prefix={#1.}}, + no memo dir/.default=\jobname, no memo dir, } % \end{key} % -% \begin{macro}{\mmzUnquote} -% If the expanded argument is surrounded by double quotes, remove them. This -% relies on |#1| containing no quotes other than the potential surrounding -% quotes, which should be the case when applying the macro to |\jobname|. An -% empty |#1| is dealt with correctly, even if |\jobname| can hardly ever be -% empty (needs |openout_any=a|). +% \begin{macro}{\mmz@remove@quotes} +% This macro removes fully expands |#1|, detokenizes the expansion and then +% removes all double quotes the string. The result is stored in the control +% sequence given in |#2|. % % We use this macro when we are passing a filename constructed from % |\jobname| to external programs. -\def\mmzUnquote#1{\expanded{\noexpand\mmz@unquote#1}\mmz@unquote@end} -\def\mmz@unquote#1{% - \ifx\mmz@unquote@end#1% +\def\mmz@remove@quotes#1#2{% + \def\mmz@remove@quotes@end{\let#2\mmz@temp}% + \def\mmz@temp{}% + \expanded{% + \noexpand\mmz@remove@quotes@i + \detokenize\expandafter{\expanded{#1}}% + "\noexpand\mmz@eov + }% +} +\def\mmz@remove@quotes@i{% + \CollectArgumentsRaw + {\collargsBraceCollectedfalse + \collargsNoDelimiterstrue + \collargsAppendPostwrap{{##1}}% + }% + {u"u\mmz@eov}% + \mmz@remove@quotes@ii +} +\def\mmz@remove@quotes@ii#1#2{% + \appto\mmz@temp{#1}% + \ifx&% + \mmz@remove@quotes@end + \expandafter\@gobble \else - \ifx"#1% - \expandafter\expandafter\expandafter\mmz@unquote@quotes - \else - \expandafter\expandafter\expandafter\mmz@unquote@noquotes - \expandafter\expandafter\expandafter#1% - \fi + \expandafter\@firstofone \fi + {\mmz@remove@quotes@i#2\mmz@eov}% } -\def\mmz@unquote@quotes#1"\mmz@unquote@end{#1} -\def\mmz@unquote@noquotes#1\mmz@unquote@end{#1} % \end{macro} % % \begin{key}{ignore spaces} @@ -832,6 +854,7 @@ %\let\startmemoize\startnomemoize %\let\stopmemoize\stopnomemoize % +% % \end{environment} % % \subsection{The memoization process} @@ -843,6 +866,7 @@ % anywhere else. It is checked by |\Memoize| to prevent nested % memoizations, deployed in advice run conditions set by |run only if % memoizing|, etc. +%<*mmz,nommz,mmzable&generic> \newif\ifmemoizing % \end{macro} % @@ -859,7 +883,7 @@ % An auxiliary macro which rescans the given % code using |\scantokens| if the verbatim mode is active. We also need it % in NoMemoize, to properly grab verbatim manually memoized code. -% +% %<*mmz> \def\mmz@maybe@scantokens{% \ifmmz@verbatim @@ -1091,15 +1115,16 @@ \gtoksapp\mmzCMemo{\global\mmzUnmemoizabletrue}% \mmz@write@cmemo \mmz@trace@endmemoize@unmemoizable - \PackageWarning{memoize}{Marking this code as unmemoizable}% + \PackageInfo{memoize}{Marking this code as unmemoizable}% \else \ifmmz@abort % If memoization was aborted, we create an empty c-memo, to make sure that % no leftover c-memo tricks Memoize into thinking that the code was % successfully memoized. \mmz@trace@endmemoize@aborted - \PackageWarning{memoize}{Memoization was aborted}% - \mmz@write@empty@cmemo + \PackageInfo{memoize}{Memoization was aborted}% + \mmz@compute@context@mdfivesum + \mmz@write@cmemo \else % If memoization was not aborted, we compute the \meta{context md5sum}, % open and write out the memos, and shipout the externs (as pages into the @@ -1320,7 +1345,7 @@ % % \paragraph.{Tracing} We populate the hooks which send the tracing info to the % terminal. -\def\mmz@trace#1{\immediate\write16{[tracing memoize] #1}} +\def\mmz@trace#1{\advice@typeout{[tracing memoize] #1}} \def\mmz@trace@context{\mmz@trace{\space\space Context: "\expandonce{\mmz@context@key}" --> \mmz@context@mdfivesum}} \def\mmz@trace@Memoize@on{% @@ -1424,9 +1449,6 @@ % \end{macro} % \end{key} % -% -%\newcommand\IfMemoizing[2][]{\@secondoftwo} -%<*mmz> % % \subsection{Context} % \label{sec:code:context} @@ -1465,24 +1487,27 @@ % context, key value to context, /handlers/.meaning to context, % /handlers/.value to context} % Utilities to put the meaning of various stuff into |context|. + % \indentmacrocode meaning to context/.code={\forcsvlist\mmz@mtoc{#1}}, - csname meaning to context/.code={\mmz@mtoc@cs{#1}}, - key meaning to context/.code={\forcsvlist\mmz@mtoc\mmz@mtoc@keycmd{#1}}, + csname meaning to context/.code={\mmz@mtoc@csname{#1}}, + key meaning to context/.code={% + \forcsvlist\mmz@mtoc\mmz@mtoc@keycmd{#1}}, key value to context/.code={\forcsvlist\mmz@mtoc@key{#1}}, - /handlers/.meaning to context/.code={% - \expanded{\noexpand\mmz@mtoc@cs{pgfk@\pgfkeyscurrentpath/.@cmd}}}, + /handlers/.meaning to context/.code={\expanded{% + \noexpand\mmz@mtoc@csname{pgfk@\pgfkeyscurrentpath/.@cmd}}}, /handlers/.value to context/.code={% - \expanded{\noexpand\mmz@mtoc@cs{pgfk@\pgfkeyscurrentpath}}}, + \expanded{\noexpand\mmz@mtoc@csname{pgfk@\pgfkeyscurrentpath}}}, } +% \noindentmacrocode \def\mmz@mtoc#1{% \collargs@cs@cases{#1}% {\mmz@mtoc@cmd{#1}}% {\mmz@mtoc@error@notcsorenv{#1}}% {% - \mmz@mtoc@cs{% + \mmz@mtoc@csname{% %start% #1}% - \mmz@mtoc@cs{% + \mmz@mtoc@csname{% %end% %stop% #1}% @@ -1492,17 +1517,18 @@ \begingroup \escapechar=-1 \expandafter\endgroup - \expandafter\mmz@mtoc@cs\expandafter{\string#1}% + \expandafter\mmz@mtoc@csname\expandafter{\string#1}% } -\def\mmz@mtoc@cs#1{% +\def\mmz@mtoc@csname#1{% \pgfkeysvalueof{/mmz/context/.@cmd}% - \expandafter\string\csname#1\endcsname={\expandafter\meaning\csname#1\endcsname}% - \pgfeov + \detokenize{#1}={\expandafter\meaning\csname#1\endcsname}% + \pgfeov } -\def\mmz@mtoc@key#1{\mmz@mtoc@cs{pgfk@#1}} -\def\mmz@mtoc@key#1{\mmz@mtoc@cs{pgfk@#1/.@cmd}} +\def\mmz@mtoc@key#1{\mmz@mtoc@csname{pgfk@#1}} +\def\mmz@mtoc@keycmd#1{\mmz@mtoc@csname{pgfk@#1/.@cmd}} \def\mmz@mtoc@error@notcsorenv#1{% - \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context' is neither a command nor an environment}{}% + \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context' + is neither a command nor an environment}{}% } % \end{key} % @@ -1512,7 +1538,7 @@ % % \paragraph{The path} to a c-memo consists of the path prefix, the MD5 sum of % the memoized code, and suffix |.memo|. -\def\mmz@cmemo@path{\mmz@prefix@path\mmz@code@mdfivesum.memo} +\def\mmz@cmemo@path{\mmz@prefix\mmz@code@mdfivesum.memo} % \begin{macro}{\mmzCMemo} % The additional, free-form content of the c-memo is collected in this token % register. @@ -1536,7 +1562,7 @@ % |\mmzCMemo|. \def\mmz@write@cmemo{% % Open the file for writing. - \immediate\openout\mmz@out{\mmz@cmemo@path}% + \immediate\mmz@openout\mmz@out{\mmz@cmemo@path}% % The memo starts with the |\mmzMemo| marker (a signal that the memo is valid). \immediate\write\mmz@out{\noexpand\mmzMemo}% % We store the content of |\mmzContextExtra| by writing out a command that @@ -1559,17 +1585,6 @@ } % \end{macro} % -% \begin{macro}{\mmz@write@empty@cmemo} -% This macro is used to create an empty c-memo on aborted memoization, to -% make sure that no leftover c-memo tricks Memoize into thinking that the -% code was successfully memoized. -\def\mmz@write@empty@cmemo{% - \immediate\openout\mmz@out{\mmz@cmemo@path}% - \immediate\closeout\mmz@out -} -% \end{macro} -% -% % \begin{macro}{\mmzSource} % The c-memo memoized code marker. This macro is synonymous with |\endinput|, % so the source following it is ignored when inputting the c-memo. @@ -1644,7 +1659,7 @@ % hyphen-separated MD5 sums of the memoized code and the (evaluated) context, % and suffix |.memo|. \def\mmz@ccmemo@path{% - \mmz@prefix@path\mmz@code@mdfivesum-\mmz@context@mdfivesum.memo} + \mmz@prefix\mmz@code@mdfivesum-\mmz@context@mdfivesum.memo} % % \paragraph{The structure} of a cc-memo: % \begin{itemize} @@ -1699,7 +1714,7 @@ % memoized code can update the context. This is one of the two reasons why % we couldn't write the cc-memo directly into the file, but had to collect % its contents into token register |\mmzCCMemo|. - \immediate\openout\mmz@out{\mmz@ccmemo@path}% + \immediate\mmz@openout\mmz@out{\mmz@ccmemo@path}% % Token register |\mmz@ccmemo@resources| consists of calls to % |\mmz@ccmemo@append@resource|, so the following code writes down the list % of created externs into the cc-memo. Wanting to have this list at the top @@ -1751,7 +1766,7 @@ % might be produced by a failed \hologo{TeX}-based extraction, should count % as no file. The |0| behind |\ifnum| is there because |\pdffilesize| % returns an empty string when the file does not exist. - \ifnum0\pdf@filesize{\mmz@dir@path/#1}=0 + \ifnum0\pdf@filesize{\mmz@prefix@dir/#1}=0 \ifmmz@direct@ccmemo@input \let\mmzMemo\endinput \else @@ -1768,8 +1783,10 @@ % % \begin{macro}{\mmz@process@ccmemo,\mmzThisContext,\mmzEndMemo} % This macro processes the cc-memo. +% \indentmacrocode \def\mmz@process@ccmemo{% \mmz@trace@process@ccmemo + % \noindentmacrocode % The following conditional signals whether cc-memo was successfully % utilized. If the cc-memo file does not exist, |\ifmmz@abort| will remain % true. If it exists, it is headed by the list of resources. If a @@ -1865,11 +1882,11 @@ % full path. \newcount\mmz@seq \def\mmz@extern@basename{% - \mmz@prefix\mmz@code@mdfivesum-\mmz@context@mdfivesum + \mmz@prefix@name\mmz@code@mdfivesum-\mmz@context@mdfivesum \ifnum\mmz@seq>0 -\the\mmz@seq\fi } \def\mmz@extern@name{\mmz@extern@basename.pdf} -\def\mmz@extern@basepath{\mmz@dir@path/\mmz@extern@basename} +\def\mmz@extern@basepath{\mmz@prefix@dir\mmz@extern@basename} \def\mmz@extern@path{\mmz@extern@basepath.pdf} % % \begin{key}{padding left, padding right, padding top, padding bottom} @@ -2124,12 +2141,14 @@ % |\mmzExternPages| holds the number of shipped-out extern pages; and % |\mmzExtraPages| holds, or at least should hold, the number of pages % shipped out using any other means. - \edef\pagenumber{\the\numexpr\mmzRegularPages + \edef\pagenumber{% + \the\numexpr\mmzRegularPages % In \hologo{LaTeX}, the |\mmzRegularPages| holds to number of pages % already shipped out. In \hologo{ConTeXt}, the counter is already % increased while processing the page, so we need to subtract $1$. % -1% - +\mmzExternPages+\mmzExtraPages}% + +\mmzExternPages+\mmzExtraPages + }% % Record the creation of the new extern. We do this after shipping out the % extern page, so that the recording mechanism can serve as an after-shipout % hook, for the unlikely situation that some package really needs to do @@ -2212,7 +2231,7 @@ enddocument/afterlastpage/.append code={% \ifnum\mmzExternPages>0 \PackageWarning{memoize}{The compilation produced \the\mmzExternPages\space - new extern\ifnum\mmzExternPages>1 s\fi.}% + new extern\ifnum\mmzExternPages>1 s\fi}% \fi }, } @@ -2304,13 +2323,13 @@ % % \begin{macro}{\mmz@include@extern@from@tbe@box} % Include the extern number |#1| residing in |\mmz@tbe@box| into the -% document. This helper macro makes it possible for a complex memoization -% driver to process the cc-memo right after memoization --- by using the -% |\mmzkeepexternstrue\xtoksapp\mmzAfterMemoizationExtra{\the\mmzCCMemo}| -% trick --- to ensure that the result of the memoizing compilation matches -% the result of inputting the cc-memo. The rest of the arguments are -% gobbled, as we don't have to do any size adjustment or checking here, and -% the box is of the correct type. +% document. It may be called as |\mmzIncludeExtern| from |after memoization| +% hook if |\ifmmzkeepexterns| was set to true during memoization. The macro +% takes the same arguments as |\mmzIncludeExtern| but disregards all but +% the first one, the extern sequential number. Using this macro, a complex +% memoization driver can process the cc-memo right after memoization, by +% issuing +% |\global\mmzkeepexternstrue\xtoksapp\mmzAfterMemoizationExtra{\the\mmzCCMemo}|. \def\mmz@include@extern@from@tbe@box#1#2#3#4#5#6#7#8#9{% \setbox0\vbox{% \@tempcnta#1\relax @@ -2344,7 +2363,7 @@ \mmzset{ extract/.estore in=\mmz@extraction@method, extract/.value required, - begindocument/.append style={extract/.code=\mmz@preamble@only@warning}, + begindocument/.append style={extract/.code=\mmz@preamble@only@error}, % \end{key} % % \begin{key}{extract/perl, extract/python} @@ -2352,10 +2371,10 @@ % Memoize ships with two extraction scripts, a Perl script and a Python % script, which are selected by |extract=perl| (the default) and % |extract=python|, respectively. We run the scripts in verbose mode - % (without |-q|), and keep the |.mmz| file as is, i.e.\ we're not commenting - % out the |\mmzNewExtern| lines, because we're about to overwrite it - % anyway. We also request the log file, which will contain - % |\mmzExtractionSuccessful| if extraction was successful. + % (without |-q|), and keep the |.mmz| file as is (without |-k|), i.e.\ + % we're not commenting out the |\mmzNewExtern| lines, because we're about + % to overwrite it anyway. We inform the script about the format of the + % document (|-F|). extract/perl/.code={% \mmz@clear@extraction@log \pdf@system{% @@ -2363,17 +2382,14 @@ \mmzvalueof{perl extraction options}% }% \mmz@check@extraction@log{perl}% - \def\mmz@mkdir@command##1{\mmzvalueof{perl extraction command}\space --mkdir "##1"}% + \def\mmz@mkdir@command{\mmzvalueof{perl extraction command} --mkdir}% }, perl extraction command/.initial=memoize-extract.pl, - perl extraction options/.initial={% -% \begin{listingregion}{perl-extraction-options.tex} - -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w - %"\string\PackageWarning{memoize (perl-based extraction)}{\string\warningtext}" - %"\string\warning{memoize (perl-based extraction): \string\warningtext}" - %"\string\warning{memoize (perl-based extraction): \string\warningtext}" - "\mmzOutputDirectory\mmzUnquote\jobname.mmz" -% \end{listingregion} + perl extraction options/.initial={\space + %-F latex + %-F plain + %-F context + \jobname\space }, extract=perl, extract/python/.code={% @@ -2383,21 +2399,20 @@ \mmzvalueof{python extraction options}% }% \mmz@check@extraction@log{python}% - \def\mmz@mkdir@command##1{\mmzvalueof{python extraction command}\space --mkdir "##1"}% + \def\mmz@mkdir@command{\mmzvalueof{python extraction command} --mkdir}% }, python extraction command/.initial=memoize-extract.py, - python extraction options/.initial={% - -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w - %"\string\PackageWarning{memoize (python-based extraction)}{\string\warningtext}" - %"\string\warning{memoize (python-based extraction): \string\warningtext}" - %"\string\warning{memoize (python-based extraction): \string\warningtext}" - "\mmzOutputDirectory\mmzUnquote\jobname.mmz" + python extraction options/.initial={\space + %-F latex + %-F plain + %-F context + \jobname\space }, } -\def\mmz@preamble@only@warning{% - \PackageWarning{memoize}{% +\def\mmz@preamble@only@error{% + \PackageError{memoize}{% Ignoring the invocation of "\pgfkeyscurrentkey". - This key may only be executed in the preamble}% + This key may only be executed in the preamble}{}% } % \end{key} % @@ -2406,15 +2421,14 @@ % communicate with the system command via the ``extraction log file,'' produced % by both \hologo{TeX}-based extraction and the Perl and Python extraction % script. This file signals whether the embedded extraction was successful --- -% if it is, the file contains |\mmzExtractionSuccessful| --- and also contains -% any size-mismatch warnings (these are currently only thrown by the -% \hologo{TeX}-based extraction). As the log is really a \hologo{TeX} file, -% the idea is to simply input it after extracting each extern (for +% if it is, the file ends if |\endinput| --- and also contains any warnings and +% errors thrown by the script. As the log is really a \hologo{TeX} file, the +% idea is to simply input it after extracting each extern (for % \hologo{TeX}-based extraction) or after the extraction of all externs (for % the external scripts). \def\mmz@clear@extraction@log{% \begingroup - \immediate\openout0{\mmzUnquote\jobname.mmz.log"}% + \immediate\mmz@openout0{\jobname.mmz.log}% \immediate\closeout0 \endgroup } @@ -2426,9 +2440,11 @@ \@input{\jobname.mmz.log}% \ifmmz@temp \else \mmz@extraction@error \fi \endgroup } \def\mmz@extraction@error{% - \PackageWarning{memoize}{Extraction of externs from document "\mmzUnquote\jobname.pdf" - using method "\extractionmethod" was unsuccessful. Have you set the - shell escape mode as suggested in chapter 1 of the manual?}{}} + \PackageError{memoize}{Extraction of externs from document + "\jobname.pdf" using method "\extractionmethod" was + unsuccessful}{The extraction script "\mmzvalueof{\extractionmethod\space + extraction command}" wasn't executed or didn't finish execution + properly.}} % % \subsection{The record files} % \label{sec:code:record} @@ -2451,7 +2467,7 @@ % surely occurs at the top of the |.mmz| file. Listing each prefix type % separately in this hook ensures that |prefix| of a certain type is % executed after that type's |begin|. - /mmz/record/#1/prefix/.try/.expanded=\mmz@prefix@path, + /mmz/record/#1/prefix/.try/.expanded=\mmz@prefix, }, record/prefix/.append style={/mmz/record/#1/prefix/.try={##1}}, record/new extern/.append style={/mmz/record/#1/new extern/.try={##1}}, @@ -2495,7 +2511,7 @@ % We define this macro because |\aftergroup|, used in |record/prefix|, only % accepts a token. \def\mmz@record@prefix{% - \mmzset{/mmz/record/prefix/.expanded=\mmz@prefix@path}% + \mmzset{/mmz/record/prefix/.expanded=\mmz@prefix}% } % \paragraph{Initialize} the hook keys, preactivate |mmz| record type, and % execute hooks |begin| and |end| at the edges of the document. @@ -2516,12 +2532,11 @@ % contains are intentionally as simple as possible (just a macro plus braced % arguments), to facilitate parsing by the external scripts. % -% These hooks simply put the calls of the corresponding macros into the file. -% All but hooks but |begin| and |end| receive the full path to the relevant -% file as the only argument (ok, |prefix| receives the full path prefix, as set -% by key |path|). -% % \begin{key}{record/mmz/...} +% These hooks simply put the calls of the corresponding macros into the file. +% All but hooks but |begin| and |end| receive the full path to the relevant +% file as the only argument (ok, |prefix| receives the full path prefix, as +% set by key |path|). \mmzset{ record/mmz/begin/.code={% \newwrite\mmz@mmzout @@ -2529,7 +2544,7 @@ % location (the current directory, i.e.\ the directory where \hologo{TeX} % is executed from; usually, this will be the directory containing the % \hologo{TeX} source). - \immediate\openout\mmz@mmzout{\jobname.mmz}% + \immediate\mmz@openout\mmz@mmzout{\jobname.mmz}% }, % The |\mmzPrefix| is used by the clean-up script, which will remove all % files with the given path prefix but (unless called with |--all|) those @@ -2592,7 +2607,7 @@ % \begin{key}{record/sh/...} Define the Linux shell script record type. record/sh/begin/.code={% \newwrite\mmz@shout - \immediate\openout\mmz@shout{\mmz@shname}% + \immediate\mmz@openout\mmz@shout{\mmz@shname}% }, record/sh/new extern/.code={% \begingroup @@ -2611,7 +2626,7 @@ % Rinse and repeat for Windows. record/bat/begin/.code={% \newwrite\mmz@batout - \immediate\openout\mmz@batout{\mmz@batname}% + \immediate\mmz@openout\mmz@batout{\mmz@batname}% }, record/bat/new extern/.code={% \begingroup @@ -2649,7 +2664,7 @@ % variables |.DEFAULT_GOAL| and |.PHONY|. \newwrite\mmz@makefileout \newtoks\mmz@makefile@externs - \immediate\openout\mmz@makefileout{\mmz@makefilename}% + \immediate\mmz@openout\mmz@makefileout{\mmz@makefilename}% \immediate\write\mmz@makefileout{.DEFAULT_GOAL = externs}% \immediate\write\mmz@makefileout{.PHONY: externs}% }, @@ -2693,11 +2708,10 @@ } % \end{key} % -% \begingroup -% \setlength\MacroIndent{2em} % \begin{macro}{\mmzUsedCMemo,\mmzUsedCCMemo,\mmzUsedExtern,\mmzNewCMemo,\mmzNewCCMemo,\mmzPrefix} % We can ignore everything but |\mmzNewExtern|s. All these macros receive a % single argument. +% \indentmacrocode \def\mmzUsedCMemo#1{} \def\mmzUsedCCMemo#1{} \def\mmzUsedExtern#1{} @@ -2705,7 +2719,6 @@ \def\mmzNewCCMemo#1{} \def\mmzPrefix#1{} % \end{macro} -% \endgroup % % \begin{macro}{\mmzNewExtern} % Command |\mmzNewExtern| takes four arguments. It instructs us to extract @@ -2744,11 +2757,11 @@ % \end{macro} \def\mmz@pageextraction@error{% \PackageError{memoize}{Extraction of extern page \pagenumber\space from - document "\mmzUnquote\jobname.pdf" using method "\extractionmethod" was unsuccessful. - Have you set the shell escape mode as suggested in chapter 1 of the - manual?}{If "\mmzvalueof{tex extraction command}" was executed, - shell escape mode is not the problem, and inspecting "\externbasepath.log" - might give you a clue what's wrong}} + document "jobname.pdf" using method "\extractionmethod" was + unsuccessful.}{Check the log file to see if the extraction script was + executed at all, and if it finished successfully. You might also want to + inspect "\externbasepath.log", the log file of the embedded TeX compilation + which ran the extraction script}} % % \begin{key}{tex extraction command, tex extraction options, tex extraction script} % Using these keys, we set the system call @@ -2786,12 +2799,11 @@ -halt-on-error -interaction=batchmode -jobname "\externbasepath" - \ifdefempty\mmzOutputDirectory{}{-output-directory "\mmzOutputDirectory"} % \end{listingregion} }, tex extraction script/.initial={% % \begin{listingregion}{tex-extraction-script.tex} ^^A todo: context - \def\noexpand\fromdocument{"\mmzOutputDirectory"\jobname.pdf}% + \def\noexpand\fromdocument{\jobname.pdf}% \def\noexpand\pagenumber{\pagenumber}% \def\noexpand\expectedwidth{\expectedwidth}% \def\noexpand\expectedheight{\expectedheight}% @@ -3008,6 +3020,7 @@ %noop/.style={inner handler=\mmz@auto@noop}, nomemoize/.style={noop, options=disable}, replicate/.style={run if memoizing, inner handler=\mmz@auto@replicate}, + to context/.style={run if memoizing, outer handler=\mmz@auto@tocontext}, } % % \paragraph{Abortion} @@ -3221,6 +3234,27 @@ } % \end{macro} % \end{mmzautokey} +% +% \begin{mmzautokey}{to context} +% \begin{macro}{\mmz@auto@tocontext} +% This outer handler appends the original definition of the handled command +% to the context. The |\expandafter| are there to expand |\AdviceName| +% once before fully expanding |\AdviceGetOriginalCsname|. +\def\mmz@auto@tocontext{% + \expanded{% + \noexpand\pgfkeysvalueof{/mmz/context/.@cmd}% + original "\AdviceNamespace" csname "\AdviceCsname"={% + \noexpand\expanded{% + \noexpand\noexpand\noexpand\meaning + \noexpand\AdviceCsnameGetOriginal{\AdviceNamespace}{\AdviceCsname}% + }% + }% + }% + \pgfeov + \AdviceOriginal +} +% \end{macro} +% \end{mmzautokey} % % \subsection{\hologo{LaTeX}-specific handlers} % @@ -3277,12 +3311,7 @@ % appends the reference key to the context. |\mmzNoRef| only does that if % the reference is defined, otherwise it aborts the memoization. \def\mmzForceNoRef#1{% - \ifmemoizing - \expandafter\gtoksapp\expandafter\mmzContextExtra - \else - \expandafter\toksapp\expandafter\mmzContext - \fi - {r@#1={\csname r@#1\endcsname}}% + \mmz@mtoc@csname{r@#1}% \ignorespaces } \def\mmzNoRef#1{% @@ -3360,11 +3389,17 @@ \endgroup } % -% % % \end{macro} % % \section{Support for various classes and packages} +% +%<*latex> +\AddToHook{shipout/before}[memoize]{\global\advance\mmzExtraPages-1\relax} +\AddToHook{shipout/after}[memoize]{\global\advance\mmzExtraPages1\relax} +\mmzset{auto=\DiscardShipoutBox{ + outer handler=\global\advance\mmzExtraPages1\relax\AdviceOriginal}} +% % % \subsection{\TikZ;} % \label{sec:code:mmz:tikz} @@ -3373,7 +3408,6 @@ % Advice). All the action happens at the end of the preamble, so that we can % detect whether \TikZ; was loaded (regardless of whether Memoize was loaded % before \TikZ;, or vice versa), but still input the definitions. -%<*mmz> \mmzset{ begindocument/before/.append code={% %\@ifpackageloaded{tikz}{% @@ -3384,32 +3418,23 @@ % We define and activate the automemoization handlers for the \TikZ; command % and environment. \mmzset{% - % \begin{listingregion}{_auto-tikz-collector.tex} - auto=\tikz{memoize, collector=\AdviceCollectTikZArguments}, - % \end{listingregion} - %/utils/exec={\tracingall}, - auto={tikzpicture}{memoize}, - % A hack to prevent memoizing pictures which are accidentally marked as - % remembered --- accidentally in the sense that because the document - % changed, the |.aux| file contains a |\pgfsyspdfmark| command which - % erroneously refers to the picture being memoized. We know that - % memoizing a remembered picture can't be right, as we always abort on - % |\pdfsavepos|. This is implemented by hacking into PGF's - % |\pgfsys@getposition|, and aborting memoization if the mark does not - % equal |\relax|. (We have to duplicate |#| because of |.append code|.) - auto=\pgfsys@getposition{ - run if memoizing, outer handler=\mmz@pgfsys@getposition}, - }% - \def\mmz@pgfsys@getposition##1{% - \expandafter\ifx\csname pgf@sys@pdf@mark@pos@##1\endcsname\relax - \else - \mmzAbort - \fi - \AdviceOriginal{##1}% + auto/memoize tikz/.style={ + memoize, + at begin memoization=\edef\mmz@pgfpictureid{% + \the\pgf@picture@serial@count + }, + at end memoization=\xtoksapp\mmzCCMemo{% + \unexpanded{% + \global\expandafter\advance\csname pgf@picture@serial@count\endcsname + }% + \the\numexpr\pgf@picture@serial@count-\mmz@pgfpictureid\relax\relax + }, + }, + auto=\tikz{memoize tikz, collector=\AdviceCollectTikZArguments}, + auto={tikzpicture}{memoize tikz}, }% }, } -% % % % \subsection{Forest} @@ -3417,7 +3442,6 @@ % % Forest will soon feature extensive memoization support, but for now, let's % just enable the basic, single extern externalization. -%<*mmz> %<*latex> \mmzset{ begindocument/before/.append code={% @@ -3478,6 +3502,98 @@ }{}} % % +% \subsection{Morewrites} +% \label{sec:code:morewrites} +% +% Use the old grammar for |\openin| and |\openout| as a temporary workaround. +% |prefix|es containing spaces must be quoted manually. +%<*latex> +\AddToHook{begindocument/before}{% + \@ifpackageloaded{morewrites}{% + \def\mmz@openin#1#2{\openin#1=#2\relax}% + \def\mmz@openout#1#2{\openout#1=#2\relax}% + }{}% +} +% +% +% \subsection{Biblatex} +% \label{sec:biblatex} +% +%<*latex> +\mmzset{ + begindocument/before/.append style={% + auto=\blx@bbl@entry{outer handler=\mmz@biblatex@entry}, + auto/cite/.style={run if memoizing, outer handler=\mmz@biblatex@cite}, + auto/cites/.style={run if memoizing, outer handler=\mmz@biblatex@cites}, + auto=\cite{cite}, + auto=\cites{cites}, + }% +} +% \begin{macro}{\mmz@biblatex@entry} +% This macro stores the MD5 sum of the |\entry| when reading the |.bbl| file. +\def\mmz@biblatex@entry#1#2\endentry{% + \csxdef{mmz@bbl@#1}{\pdf@mdfivesum{#2}}% + \AdviceOriginal{#1}#2\endentry +} +% \end{macro} +% \begin{macro}{\mmz@biblatex@cite} +% This macro puts the cites reference keys into the context, and adds the +% handled |\cite| command to the cc-memo. +\def\mmz@biblatex@cite#1#{\mmz@biblatex@cite@i{#1}} +\def\mmz@biblatex@cite@i#1#2{% + \forcsvlist\mmz@biblatex@cite@do@key{#2}% + \xtoksapp\mmzCCMemo{% + \noexpand\setbox0\noexpand\hbox{% + \expandonce\AdviceOriginal\unexpanded{#1}{#2}% + }}% + \AdviceOriginal#1{#2}% +} +\def\mmz@biblatex@cite@do@key#1{% + \mmz@mtoc@csname{mmz@bbl@#1}% + \ifcsdef{mmz@bbl@#1}{}{\mmzAbort}% +} +% \end{macro} +% \begin{macro}{\mmz@biblatex@cites} +% This macro puts the cites reference keys into the context, and adds the +% handled |\cites| command to the cc-memo. +\def\mmz@biblatex@cites{% + \mmz@temptoks{}% + \mmz@biblatex@cites@i +} +\def\mmz@biblatex@cites@i{% + \futurelet\mmz@temp\mmz@biblatex@cites@ii +} +\def\mmz@biblatex@cites@ii{% + \mmz@tempfalse + \ifx\mmz@temp\bgroup + \mmz@temptrue + \else + \ifx\mmz@temp[%] + \mmz@temptrue + \fi + \fi + \ifmmz@temp + \expandafter\mmz@biblatex@cites@iii + \else + \expandafter\mmz@biblatex@cites@z + \fi +} +\def\mmz@biblatex@cites@iii#1#{\mmz@biblatex@cites@iv{#1}} +\def\mmz@biblatex@cites@iv#1#2{% + \forcsvlist\mmz@biblatex@cite@do@key{#2}% + \toksapp\mmz@temptoks{#1{#2}}% + \mmz@biblatex@cites@i +} +\def\mmz@biblatex@cites@z{% + \xtoksapp\mmzCCMemo{% + \noexpand\setbox0\noexpand\hbox{% + \expandonce\AdviceOriginal\the\mmz@temptoks + }}% + \expandafter\AdviceOriginal\the\mmz@temptoks +} +% +% \end{macro} +% % % \section{Initialization} % \label{sec:code:initialization} @@ -3512,6 +3628,10 @@ % % \end{key} % +% Formats other than \hologo{plainTeX} need a way to prevent extraction during +% package-loading. +%\mmzset{extract/no/.code={}} +% % \paragraph{\texttt{memoize.cfg}} Load the configuration file. Note that % |nomemoize| must input this file as well, because any special % memoization-related macros defined by the user should be available; for @@ -3521,63 +3641,91 @@ %\InputIfFileExists{memoize.cfg}{}{} %<*mmz> % -% Formats other than \hologo{plainTeX} need a way to prevent extraction during -% package-loading. -\mmzset{ - %extract/no/.code={}, -% \begin{key}{output directory} -% \begin{macro}{mmzOutputDirectory} -% Set the |-output-directory| --- manually, as there is no other way. -% \indentmacrocode - output-directory/.store in=\mmzOutputDirectory, -} -% \end{macro} -% \end{key} +% For formats other than \hologo{plainTeX}, we also save the current (initial +% or |memoize.cfg|-set) value of |extract|, so that we can restore it when +% package options include |extract=no|. Then, |extract| can be called without +% an argument in the preamble, triggering extraction using this method; this is +% useful e.g.\ if Memoize is compiled into a format. +%\let\mmz@initial@extraction@method\mmz@extraction@method % % \paragraph{Process} the package options (except in \hologo{plainTeX}). -%\ProcessPgfPackageOptions{/mmz} +%<*latex> +\DeclareUnknownKeyHandler[mmz]{% + \expanded{\noexpand\pgfqkeys{/mmz}{#1\IfBlankF{#2}{={#2}}}}} +\ProcessKeyOptions[mmz] +% %\expandafter\mmzset\expandafter{\currentmoduleparameters} -% -% Define |\mmzOutputDirectory| if |output-directory| was not given. -\ifdefined\mmzOutputDirectory -\else - \def\mmzOutputDirectory{}% -\fi -\mmzset{output directory/.code={\PackageError{memoize}{Key "output-directory" - may only be used as a package option}{}}} +% +% In \hologo{LaTeX}, |nomemoize| has to process package options as well, +% otherwise \hologo{LaTeX} will complain. +% +%<*latex&nommz> +\DeclareUnknownKeyHandler[mmz]{} +\ProcessKeyOptions[mmz] +% +% \paragraph{Extern extraction} +% We redefine |extract| to immediately trigger extraction. This is crucial in +% \hologo{plainTeX}, where extraction must be invoked after loading the +% package, but also potentially useful in other formats when package options +% include |extract=no|. +%<*mmz> \mmzset{ - % \paragraph{Extract} the externs using the method given by |memoize.cfg| or - % the package options --- unless we're running \hologo{plainTeX}. - %<*!plain> - extract/\mmz@extraction@method, - % In non-\hologo{plainTeX} formats, also disable |extract| in the preamble. - extract/.code={\PackageError{memoize}{Key "extract" is only allowed as a - package option.}{If you really want to extract in the preamble, execute - "extract/".}}, - % - %<*plain> - % In \hologo{plainTeX}, where extraction must be invoked after - % loading the package, we now have to redefine |extract|, so that it will - % immediately trigger extraction. extract/.is choice, extract/.default=\mmz@extraction@method, % But only once: extract/.append style={ - extract/.code={\PackageError{memoize}{Key "extract" is only allowed to - be used once.}{If you really want to extract again, execute - "extract/".}}, + extract/.code={\PackageError{memoize}{Key "extract" was invoked twice.}{In + principle, externs should be extracted only once. If you really want + to extract again, execute "extract/".}}, }, - % -} + % In formats other than \hologo{plainTeX}, we remember the current |extract| + % code and then trigger the extraction. + %/utils/exec={\pgfkeysgetvalue{/mmz/extract/.@cmd}\mmz@temp@extract}, + %extract=\mmz@extraction@method, +} +% Option |extract=no| (which only exists in formats other than +% \hologo{plainTeX}) should allow for an explicit invocation of |extract| in +% the preamble. +%<*!plain> +\def\mmz@temp{no} +\ifx\mmz@extraction@method\mmz@temp + \pgfkeyslet{/mmz/extract/.@cmd}\mmz@temp@extract + \let\mmz@extraction@method\mmz@initial@extraction@method +\fi +\let\mmz@temp@extract\relax +% % % Memoize was not really born for the draft mode, as it cannot produce new -% externs there. But we don't want to disable the package, as utilization is -% still perfectly valid in this mode, so let's just warn the user. +% externs there. But we don't want to disable the package, as utilization and +% pure memoization are still perfectly valid in this mode, so let's just warn +% the user. \ifnum\pdf@draftmode=1 - \PackageWarning{memoize}{No memoization will be performed in the draft mode}% + \PackageWarning{memoize}{No externalization will be performed in the draft mode}% \fi % % +% Several further things which need to be defined as dummies in +% |nomemoize|\slash|memoizable|. +%<*nommz,mmzable&generic> +\pgfkeys{% + /handlers/.meaning to context/.code={}, + /handlers/.value to context/.code={}, +} +\let\mmzAbort\relax +\let\mmzUnmemoizable\relax +\newcommand\IfMemoizing[2][]{\@secondoftwo} +\let\mmzNoRef\@gobble +\let\mmzForceNoRef\@gobble +\newtoks\mmzContext +\newtoks\mmzContextExtra +\newtoks\mmzCMemo +\newtoks\mmzCCMemo +\newcount\mmzExternPages +\newcount\mmzExtraPages +\let\mmzTracingOn\relax +\let\mmzTracingOff\relax +% +% % \paragraph{The end} of |memoize|, |nomemoize| and |memoizable|. %<*mmz,nommz,mmzable> %\resetatcatcode diff --git a/macros/generic/memoize/memoize.ins b/macros/generic/memoize/memoize.ins index f288e8e555..2c24ac368b 100644 --- a/macros/generic/memoize/memoize.ins +++ b/macros/generic/memoize/memoize.ins @@ -34,6 +34,7 @@ \file{memoizable.sty}{\from{memoize.dtx}{mmzable,latex}}% \file{memoizable.tex}{\from{memoize.dtx}{mmzable,plain}}% \file{t-memoizable.tex}{\from{memoize.dtx}{mmzable,context}}% + \file{memoizable.code.tex}{\from{memoize.dtx}{mmzable,generic}}% \file{memoize-extract-one.tex}{\from{memoize.dtx}{extract-one}}% } \endbatchfile diff --git a/macros/latex/contrib/changebar/CATALOG b/macros/latex/contrib/changebar/CATALOG index 4878681f57..74e408ac34 100644 --- a/macros/latex/contrib/changebar/CATALOG +++ b/macros/latex/contrib/changebar/CATALOG @@ -1,7 +1,7 @@ Name: ChangeBar Problems-To: johannes.braams(at)texniek.nl Author: J.L. Braams -Version: 3.7b +Version: 3.7c Description: diff --git a/macros/latex/contrib/changebar/README b/macros/latex/contrib/changebar/README index e5ef309494..847be179da 100644 --- a/macros/latex/contrib/changebar/README +++ b/macros/latex/contrib/changebar/README @@ -1,6 +1,6 @@ changebar bundle - December 30, 2023 + January 1, 2024 This bundle contains a package that I maintain to add changebars to a LaTeX document. @@ -14,7 +14,7 @@ be used to compare two versions of a document and automatically add the changebars. The file chbar.1, also contributed, documents the workings of chbar.sh -Copyright (C) 2023 Johannes L. Braams johannes.braams(at)texniek.nl +Copyright (C) 2024 Johannes L. Braams johannes.braams(at)texniek.nl This program can be redistributed and/or modified under the terms of the LaTeX Project Public License Distributed from CTAN diff --git a/macros/latex/contrib/changebar/changebar.bug b/macros/latex/contrib/changebar/changebar.bug index aed1dc80a7..4d2b9fa0a7 100644 --- a/macros/latex/contrib/changebar/changebar.bug +++ b/macros/latex/contrib/changebar/changebar.bug @@ -6,6 +6,7 @@ % VTeX (Michael Vulis) % PDFTeX (Piet van Oostrum) % XeTeX (Apostolos Syropoulos) +% luaTeX % % change history % @@ -205,6 +206,16 @@ % % v3.7a JLB: Added support for LuaTeX based on the pdfTEX and XeTeX code % +% v3.7b JLB: In creating the support for luaTeX a XeTeX error message got +% inadvertantly remove, reinserted. +% After adding those four lines the production of changebar.dtx +% led to a runaway argument error. It turned out that the comma's, +% used in were in one instance replaced by \TeX-code. Replaced +% the comma's with ";" to solve this. +% +% v3.7c JLB: Arash Esbati reported inconsistent spacing whit or without the +% optional argumenty to \cbstart or \begin{changebar}. +% % Known problems % - changebars for marginal paragraphs collide with changebars % for the main body of text diff --git a/macros/latex/contrib/changebar/changebar.dtx b/macros/latex/contrib/changebar/changebar.dtx index e4d71b5c39..2c67180adb 100644 --- a/macros/latex/contrib/changebar/changebar.dtx +++ b/macros/latex/contrib/changebar/changebar.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% Copyright 1994-2023 (C) Johannes Braams. All rights reserved. +%% Copyright 1994-2024 (C) Johannes Braams. All rights reserved. % % This file is part of the changebar package. % ------------------------------------------- @@ -24,7 +24,7 @@ % and covered by LPPL is defined by the unpacking scripts (with % extension .ins) which are part of the distribution. % \fi -% \CheckSum{2952} +% \CheckSum{2953} % %\iffalse % This file is built for \LaTeXe, so we make sure an error is @@ -39,7 +39,7 @@ \ProvidesFile{changebar.dtx} % %<+package>\ProvidesPackage{changebar} - [2023/12/30 v3.7b Indicate changes with marginal bars] + [2024/01/01 v3.7c Indicate changes with marginal bars] % \end{macrocode} % % \section{A driver for this document} @@ -162,7 +162,7 @@ % commands supported by `dvi drivers'. Currently six different % \changes{v3.5a}{2005/05/23}{PDF\TeX{} support added} % \changes{v3.6a}{2011/04/30}{Xe\TeX{} support added} -% drivers are supported, plus pdftex and Xe\TeX +% drivers are supported, plus pdftex, Xe\TeX\ and lua\TeX\ % support. More can easily be added. % \end{abstract} % @@ -752,7 +752,6 @@ % \item V\TeX % \item PDF\TeX % \item Xe\TeX -% % \cbstart\item luaTeX\cbend % \end{itemize} % @@ -2236,7 +2235,7 @@ % grouping errors.} % \changes{v3.4e}{2002/10/28}{Reverted the change from version 3.3i} % \begin{macrocode} -\newcommand*{\cbstart}{\@ifnextchar [%] +\newcommand*{\cbstart}{\@ifnextchar[%] {\cb@start}% {\cb@start[\changebarwidth]}} % \end{macrocode} @@ -2257,7 +2256,7 @@ % The default width (and length) of it are stored in % |\deletebarwidth|. % \begin{macrocode} -\newcommand*{\cbdelete}{\@ifnextchar [%] +\newcommand*{\cbdelete}{\@ifnextchar[%] {\cb@delete}% {\cb@delete[\deletebarwidth]}} % \end{macrocode} @@ -2280,8 +2279,8 @@ % environments. % \begin{macrocode} \newenvironment{changebar}% - {\@ifnextchar [{\cb@start}% - {\cb@start[\changebarwidth]}}% + {\@ifnextchar[{\cb@start}%] + {\cb@start[\changebarwidth]}}% {\cb@end} % \end{macrocode} % \end{macro} @@ -2373,7 +2372,8 @@ % depends on the model. % \begin{macrocode} \DeclareRobustCommand\cbcolor{% - \@ifnextchar[\@undeclaredcbcolor\@declaredcbcolor} + \@ifnextchar[%] + \@undeclaredcbcolor\@declaredcbcolor} % \end{macrocode} % \end{macro} % @@ -2471,9 +2471,11 @@ \fi % \end{macrocode} % Restore |\cb@pagecount|. +% \changes{v3.7c}{2024/01/01}{Added \cs{ignorespaces} at the end to +% prevent a spcae token from creaping into the output} % \begin{macrocode} \cb@pagecount\cb@cnta - \cb@advancePoint} + \cb@advancePoint\ignorespaces} % \end{macrocode} % \end{macro} % @@ -2523,6 +2525,8 @@ \cb@checkpage\thr@@ % \end{macrocode} % Again, we need to temporarily overwrite |\cb@pagecount|. +% \changes{v3.7c}{2024/01/01}{We need \cs{ignorespacesafterend} here, +% not \cs{ignorespaces}} % \begin{macrocode} \cb@cnta\cb@pagecount \cb@pagecount\cb@page\advance\cb@pagecount\m@ne @@ -2533,7 +2537,7 @@ \fi \cb@pagecount\cb@cnta \fi - \ignorespaces} + \ignorespacesafterend} % \end{macrocode} %\end{macro} % diff --git a/macros/latex/contrib/changebar/changebar.pdf b/macros/latex/contrib/changebar/changebar.pdf index 549d3e8883..ef79e6b5dc 100644 Binary files a/macros/latex/contrib/changebar/changebar.pdf and b/macros/latex/contrib/changebar/changebar.pdf differ diff --git a/macros/latex/contrib/eolang/eolang.dtx b/macros/latex/contrib/eolang/eolang.dtx index 9407f46eb6..9e1bd88e9d 100644 --- a/macros/latex/contrib/eolang/eolang.dtx +++ b/macros/latex/contrib/eolang/eolang.dtx @@ -50,7 +50,7 @@ %\NeedsTeXFormat{LaTeX2e} %\ProvidesPackage{eolang} %<*package> -[2023-12-28 0.17.0 Formulas and Graphs for EO Programming Language] +[2024-01-02 0.17.1 Formulas and Graphs for EO Programming Language] % %<*driver> \documentclass{ltxdoc} @@ -84,16 +84,19 @@ % \maketitle % % \textbf{\color{red}NB!} -% You must run \TeX{} processor with |--shell-escape| option +% You must run \TeX{} processor with |-shell-escape| option % and you must have \href{https://www.perl.org}{Perl} installed. -% If you omit the |--shell-escape| option, the package will try +% If you omit the |-shell-escape| option, the package will try % to use cached files, if they exist. If they don't, compilation will -% crash. Thus, when you must prepare your document for a compilation -% without the |--shell-escape| option, run it locally with the option -% and then package all files (including the files in the |_eolang| directory) -% into a single ZIP archive. +% fail. Thus, when you must prepare your document for a compilation +% without the |-shell-escape| option, run it locally with the option provided +% and then package all files (including the files in the |_eolang-*| directories) +% into a single ZIP archive. It is advised to use |tmpdir| package option +% in this case, in order to make the directory name not depend on the +% \LaTeX{} engine. % -% If |--shell-escape| is set, this package doesn't work on Windows. +% If |-shell-escape| is set, this package won't work on Windows, because +% it uses POSIX command line interface. % \section{Introduction} % @@ -778,12 +781,21 @@ % Then, we make a directory where all temporary files will be kept: % \begin{macrocode} -\ifnum\ShellEscapeStatus=1% - \iexec[null]{mkdir -p "\eolang@tmpdir/\jobname"}% -\else% - \message{eolang: Temporary directory "\eolang@tmpdir/\jobname" - is not created, because --shell-escape is not set^^J}% -\fi% +\RequirePackage{shellesc} +\IfFileExists + {\eolang@tmpdir/\jobname} + {\message{eolang: Temporary directory "\eolang@tmpdir/\jobname" + already exists^^J}} + {% + \ifnum\ShellEscapeStatus=1% + \iexec[null]{mkdir -p "\eolang@tmpdir/\jobname"}% + \else% + \message{eolang: Temporary directory "\eolang@tmpdir/\jobname" + is not created, because -shell-escape is not set, and + it doesn't exist, most probably the compilation + will fail later^^J}% + \fi% + } % \end{macrocode} % \begin{macro}{\eolang@lineno} @@ -805,7 +817,7 @@ % \end{macrocode} % \end{macro} -% \begin{macro}{eolang-phi.pl} +% \begin{macro}{-phi.pl} % \changes{0.0.2}{2022/10/21}{The symbols ``\texttt{[}'' and ``\texttt{]}'' replaced with ``\texttt{[[}'' and ``\texttt{]]}'' for abstract object brackets, because they conflicted with normal square brackets} % \changes{0.0.2}{2022/10/21}{New symbol added for basket slots} % \changes{0.0.2}{2022/10/21}{Parsing of the symbols ``\texttt{@},'' ``\texttt{\^{}},'' and ``\texttt{\&}'' enabled (\texttt{\char`\\varphi}, \texttt{\char`\\rho}, and \texttt{\char`\\sigma})} @@ -828,11 +840,17 @@ % \href{https://ctan.org/pkg/fancyvrb}{fancyvrb}: % \begin{macrocode} \makeatletter -\begin{VerbatimOut}{\eolang@tmpdir/eolang-phi.pl} +\openin 15=\eolang@tmpdir/\jobname-phi.pl +\ifeof 15 +\message{eolang: Perl script is going to be created, + because it is absent at "\eolang@tmpdir/\jobname-phi.pl", + but if -shell-escape is not set, the compilation will + most likely fail now^^J} +\begin{VerbatimOut}{\eolang@tmpdir/\jobname-phi.pl} $macro = $ARGV[0]; open(my $fh, '<', $ARGV[1]); my $tex; { local $/; $tex = <$fh>; } -print "% This file is auto-generated by 0.17.0\n"; +print "% This file is auto-generated by 0.17.1\n"; print '% There are ', length($tex), ' chars in the input: ', $ARGV[1], "\n"; print '% ---', "\n"; @@ -909,9 +927,9 @@ $tex =~ s/([^\\{a-z0-9]|^)D>/\1\\Delta{}..>/g; $tex =~ s/([^\\{a-z0-9]|^)L>/\1\\lambda{}..>/g; $tex =~ s/"([^"]+)"/|"\1"|/g; $tex =~ s/(^|(?<=[\s)(\]\[,.>\/]))([a-zA-Z][a-z0-9]+)(?=[\s)(\]\[,.-]|$)/|\2|/g; -$tex =~ s/([^^_]|^)([0-9]+|\*)\/(\\?[a-z]+|\|[a-z]+\|) +$tex =~ s/([^_^]|^)([0-9]+|\*)\/(\\?[a-z]+|\|[a-z]+\|) (->|\.\.>|~>|:=|!->)/\1\\alpha_{\2}\\vert{}\3\\space{}\4/xg; -$tex =~ s/([^^_]|^)([0-9]+|\*) +$tex =~ s/([^_^]|^)([0-9]+|\*) (->|\.\.>|~>|:=|!->)/\1\\alpha_{\2}\\space{}\3/xg; if ($macro ne 'phiq') { if (not $splitting) { @@ -999,7 +1017,12 @@ if ($macro eq 'phiq') { print '\endinput'; \end{VerbatimOut} \message{eolang: File with Perl script - '\eolang@tmpdir/eolang-phi.pl' saved^^J}% + '\eolang@tmpdir/\jobname-phi.pl' saved^^J} +\else + \message{eolang: Perl script already exists at + "\eolang@tmpdir/\jobname-phi.pl"^^J} +\fi +\closein 15 \makeatother % \end{macrocode} % \end{macro} @@ -1027,7 +1050,7 @@ print '\endinput'; \input{#1}} {% \ifnum\ShellEscapeStatus=1\else% - \message{eolang: The --shell-escape command line + \message{eolang: The -shell-escape command line option is not provided, most probably compilation will fail now:^^J}% \fi% @@ -1053,7 +1076,7 @@ print '\endinput'; "\eolang@tmpdir/\jobname/\hash.tex"}% \message{Start parsing 'phi' at line no. \the\inputlineno^^J} \iexec[trace,stdout=\eolang@tmpdir/\jobname/\hash-post.tex]{ - perl "\eolang@tmpdir/eolang-phi.pl" + perl "\eolang@tmpdir/\jobname-phi.pl" '#1' "\eolang@tmpdir/\jobname/\hash.tex" \ifdefined\eolang@nocomments | perl -pe 's/\%.*(\\n|$)//g'\fi @@ -1097,7 +1120,7 @@ print '\endinput'; \iexec[quiet,null]{cp "\eolang@tmpdir/\jobname/phiq.tex" "\eolang@tmpdir/\jobname/\hash-phiq.tex"}% \iexec[trace,stdout=\eolang@tmpdir/\jobname/\hash-phiq-post.tex]{ - perl \eolang@tmpdir/eolang-phi.pl 'phiq' + perl \eolang@tmpdir/\jobname-phi.pl 'phiq' "\eolang@tmpdir/\jobname/\hash-phiq.tex" \ifdefined\eolang@nocomments | perl -pe 's/\%.*(\\n|$)//g' \fi}% }% @@ -1120,7 +1143,7 @@ print '\endinput'; % \end{macrocode} % \end{macro} -% \begin{macro}{eolang-sodg.pl} +% \begin{macro}{-sodg.pl} % \changes{0.0.2}{2022/10/24}{The Perl file now has a fixed name, which doesn't depend on the name of the TeX job. This file may be shared among jobs, no need to make it uniquely named.} % \changes{0.1.0}{2022/10/26}{There are two Perl scripts now: one for \texttt{phiquation}, another one for \texttt{sodg}.} % \changes{0.2.0}{2022/10/28}{The content of the \texttt{atom} and the \texttt{data} boxes is parsed automatically as formulas and numbers, respectively.} @@ -1140,7 +1163,13 @@ print '\endinput'; % \href{https://ctan.org/pkg/fancyvrb}{fancyvrb}: % \begin{macrocode} \makeatletter -\begin{VerbatimOut}{\eolang@tmpdir/eolang-sodg.pl} +\openin 15=\eolang@tmpdir/\jobname-sodg.pl +\ifeof 15 +\message{eolang: Perl script is going to be created, + because it is absent at "\eolang@tmpdir/\jobname-sodg.pl", + but if -shell-escape is not set, the compilation will + most likely fail now^^J} +\begin{VerbatimOut}{\eolang@tmpdir/\jobname-sodg.pl} sub num { my ($i) = @_; $i =~ s/(\+|-)\./\10./g; @@ -1413,7 +1442,12 @@ print '% --- (' . (0+@cmds) . " lines)\n"; print '\endinput'; \end{VerbatimOut} \message{eolang: File with Perl script - '\eolang@tmpdir/eolang-sodg.pl' saved^^J}% + '\eolang@tmpdir/\jobname-sodg.pl' saved^^J} +\else + \message{eolang: Perl script already exists at + "\eolang@tmpdir/\jobname-sodg.pl"^^J} +\fi +\closein 15 \makeatother % \end{macrocode} % \end{macro} @@ -1559,7 +1593,7 @@ print '\endinput'; "\eolang@tmpdir/\jobname/\hash.tex"}% \message{eolang: Start parsing `sodg' at line no. \the\inputlineno^^J} \iexec[trace,stdout=\eolang@tmpdir/\jobname/\hash-sodg-post.tex]{ - perl "\eolang@tmpdir/eolang-sodg.pl" + perl "\eolang@tmpdir/\jobname-sodg.pl" "\eolang@tmpdir/\jobname/\hash.tex" \ifdefined\eolang@nocomments | perl -pe 's/\%.*(\\n|$)//g'\fi \ifdefined\eolang@sodgSaveTo > \eolang@sodgSaveTo\fi}% diff --git a/macros/latex/contrib/eolang/eolang.pdf b/macros/latex/contrib/eolang/eolang.pdf index c970e3b189..759f2c3ac6 100644 Binary files a/macros/latex/contrib/eolang/eolang.pdf and b/macros/latex/contrib/eolang/eolang.pdf differ diff --git a/macros/latex/contrib/nicematrix/nicematrix-code.dtx b/macros/latex/contrib/nicematrix/nicematrix-code.dtx index cfb0eb00d0..dea9c0ecd7 100644 --- a/macros/latex/contrib/nicematrix/nicematrix-code.dtx +++ b/macros/latex/contrib/nicematrix/nicematrix-code.dtx @@ -2,7 +2,7 @@ % % This file should be compiled with $xelatex$. % -% Copyright (C) 2018-2023 by F. Pantigny +% Copyright (C) 2018-2024 by F. Pantigny % ------------------------------------------ % % This file may be distributed and/or modified under the @@ -17,8 +17,8 @@ % % \fi % \iffalse -\def\myfileversion{6.26b} -\def\myfiledate{2023/12/13} +\def\myfileversion{6.26c} +\def\myfiledate{2024/01/02} % % %<*batchfile> @@ -29,7 +29,7 @@ \preamble -Copyright (C) 2018-2023 by F. Pantigny +Copyright (C) 2018-2024 by F. Pantigny ----------------------------------- This file may be distributed and/or modified under the @@ -121,7 +121,7 @@ version 2005/12/01 or later. % \pkg{nicematrix} relies upon the fact that the package |{array}| uses % |\ialign| to begin the |\halign|. % -% +% % \bigskip % \section{Declaration of the package and packages loaded} % @@ -1821,15 +1821,15 @@ version 2005/12/01 or later. % argument of |\tabularnote|.} % \item During the composition of the main tabular, the tabular notes will be % numbered from |\g_@@_notes_caption_int|+1 and the notes will be stored in -% |\g_@@_notes_seq|. Each composant of |\g_@@_notes_seq| will be a kind of +% |\g_@@_notes_seq|. Each component of |\g_@@_notes_seq| will be a kind of % couple of the form : \texttt{\{\textsl{label}\}\{\textsl{text of the -% tabularnote}\}}. The first composante is the optional argument (between square +% tabularnote}\}}. The first component is the optional argument (between square % brackets) of the command |\tabularnote| (if the optional argument is not used, % the value will be the special marker |\c_novalue_tl|). % \item During the composition of the caption (value of |\l_@@_caption_tl|), the % tabular notes will be numbered from $1$ to |\g_@@_notes_caption_int| and the % notes themselves will be stored in |\g_@@_notes_in_caption_seq|. The structure -% of the composantes of that sequence will be the same as for |\g_@@_notes_seq|. +% of the components of that sequence will be the same as for |\g_@@_notes_seq|. % \item After the composition of the main tabular and after the composition of % the caption, the sequences |\g_@@_notes_in_caption_seq| and |\g_@@_notes_seq| % will be merged (in that order) and the notes will be composed. @@ -1973,6 +1973,11 @@ version 2005/12/01 or later. } % \end{macrocode} % +% \begin{macrocode} +\cs_new_protected:Npn \@@_test_first_novalue:nnn #1 #2 #3 + { \tl_if_novalue:nT { #1 } { #3 } } +% \end{macrocode} +% % \bigskip % For the version in normal conditions, that is to say not in the |caption|. % |#1| is the optional argument of |\tabularnote| (maybe equal to the special @@ -1998,12 +2003,20 @@ version 2005/12/01 or later. % \end{center} % If the user have used |\tabularnote| without the optional argument, the % \texttt{\textsl{label}} will be the special marker |\c_novalue_tl|. +% +% When we will go through the sequence |\g_@@_notes_seq|, we will count in +% |\l_tmpb_int| the notes without explicit label in order to have the +% ``current'' value of the counter |\c@tabularnote|. % \begin{macrocode} + \int_zero:N \l_tmpb_int \seq_map_indexed_inline:Nn \g_@@_notes_seq { + \@@_test_first_novalue:nnn ##2 { \int_incr:N \l_tmpb_int } \tl_if_eq:nnT { { #1 } { #2 } } { ##2 } - { - \int_set:Nn \l_tmpa_int { ##1 } + { + \tl_if_novalue:nTF { #1 } + { \int_set_eq:NN \l_tmpa_int \l_tmpb_int } + { \int_set:Nn \l_tmpa_int { ##1 } } \seq_map_break: } } @@ -4536,7 +4549,7 @@ version 2005/12/01 or later. % Now, we prepare the token list for the instructions that we will have to write % on the |aux| file at the end of the environment. % \begin{macrocode} - \tl_build_gbegin:N \g_@@_aux_tl + \tl_gclear:N \g_@@_aux_tl % \end{macrocode} % % \begin{macrocode} @@ -4609,7 +4622,7 @@ version 2005/12/01 or later. % \begin{macrocode} \int_compare:nNnT \g_@@_total_X_weight_int > \c_zero_int { - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \bool_set_true:N \l_@@_X_columns_aux_bool \dim_set:Nn \l_@@_X_columns_dim @@ -4804,7 +4817,6 @@ version 2005/12/01 or later. % We write on the |aux| file all the informations corresponding to the % current environment. % \begin{macrocode} - \tl_build_gend:N \g_@@_aux_tl \iow_now:Nn \@mainaux { \ExplSyntaxOn } \iow_now:Nn \@mainaux { \char_set_catcode_space:n { 32 } } \iow_now:Nx \@mainaux @@ -6410,7 +6422,7 @@ version 2005/12/01 or later. % \begin{macrocode} \int_compare:nNnT \g_@@_notes_caption_int > \c_zero_int { - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \tl_set:Nn \exp_not:N \l_@@_note_in_caption_tl { \int_use:N \g_@@_notes_caption_int } @@ -7637,7 +7649,7 @@ version 2005/12/01 or later. % % \medskip % \begin{macrocode} - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \seq_gset_from_clist:Nn \exp_not:N \g_@@_size_seq { @@ -7656,7 +7668,7 @@ version 2005/12/01 or later. % \begin{macrocode} \seq_if_empty:NF \g_@@_pos_of_blocks_seq { - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \seq_gset_from_clist:Nn \exp_not:N \g_@@_pos_of_blocks_seq { \seq_use:Nnnn \g_@@_pos_of_blocks_seq , , , } @@ -7664,7 +7676,7 @@ version 2005/12/01 or later. } \seq_if_empty:NF \g_@@_multicolumn_cells_seq { - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \seq_gset_from_clist:Nn \exp_not:N \g_@@_multicolumn_cells_seq { \seq_use:Nnnn \g_@@_multicolumn_cells_seq , , , } @@ -7872,7 +7884,7 @@ version 2005/12/01 or later. \seq_if_empty:NF \g_@@_rowlistcolors_seq { \@@_clear_rowlistcolors_seq: } \tl_if_empty:NF \g_@@_pre_code_before_tl { - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \tl_gset:Nn \exp_not:N \g_@@_pre_code_before_tl { \exp_not:o \g_@@_pre_code_before_tl } @@ -7881,7 +7893,7 @@ version 2005/12/01 or later. } \tl_if_empty:NF \g_nicematrix_code_before_tl { - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \tl_gset:Nn \exp_not:N \g_@@_code_before_tl { \exp_not:o \g_nicematrix_code_before_tl } @@ -10060,10 +10072,8 @@ version 2005/12/01 or later. \keys_define:nn { NiceMatrix / RowStyle } { cell-space-top-limit .dim_set:N = \l_tmpa_dim , - cell-space-top-limit .initial:n = \c_zero_dim , cell-space-top-limit .value_required:n = true , cell-space-bottom-limit .dim_set:N = \l_tmpb_dim , - cell-space-bottom-limit .initial:n = \c_zero_dim , cell-space-bottom-limit .value_required:n = true , cell-space-limits .meta:n = { @@ -10095,6 +10105,8 @@ version 2005/12/01 or later. \tl_clear:N \l_tmpa_tl % value of \rowcolor \tl_clear:N \l_@@_color_tl \int_set_eq:NN \l_@@_key_nb_rows_int \c_one_int + \dim_zero:N \l_tmpa_dim + \dim_zero:N \l_tmpb_dim \keys_set:nn { NiceMatrix / RowStyle } { #1 } % \end{macrocode} % If the key |rowcolor| has been used. @@ -10135,15 +10147,16 @@ version 2005/12/01 or later. % \begin{macrocode} \dim_compare:nNnT \l_tmpa_dim > \c_zero_dim { - \@@_put_in_row_style:n + \exp_args:Nx \@@_put_in_row_style:n { - \exp_not:n - { - \tl_gput_right:Nn \g_@@_cell_after_hook_tl - { - \dim_set:Nn \l_@@_cell_space_top_limit_dim - { \dim_use:N \l_tmpa_dim } - } + \tl_gput_right:Nn \exp_not:N \g_@@_cell_after_hook_tl + { +% \end{macrocode} +% It's not possible to chanage the following code by using |\dim_set_eq:NN| +% (because of expansion). +% \begin{macrocode} + \dim_set:Nn \l_@@_cell_space_top_limit_dim + { \dim_use:N \l_tmpa_dim } } } } @@ -10152,15 +10165,12 @@ version 2005/12/01 or later. % \begin{macrocode} \dim_compare:nNnT \l_tmpb_dim > \c_zero_dim { - \@@_put_in_row_style:n + \exp_args:Nx \@@_put_in_row_style:n { - \exp_not:n - { - \tl_gput_right:Nn \g_@@_cell_after_hook_tl - { - \dim_set:Nn \l_@@_cell_space_bottom_limit_dim - { \dim_use:N \l_tmpb_dim } - } + \tl_gput_right:Nn \exp_not:N \g_@@_cell_after_hook_tl + { + \dim_set:Nn \l_@@_cell_space_bottom_limit_dim + { \dim_use:N \l_tmpb_dim } } } } @@ -12345,7 +12355,7 @@ version 2005/12/01 or later. % commands which color the |rows|, |columns| and |cells| must not color the % cells in the corners. % \begin{macrocode} - \tl_build_gput_right:Nx \g_@@_aux_tl + \tl_gput_right:Nx \g_@@_aux_tl { \seq_set_from_clist:Nn \exp_not:N \l_@@_corners_cells_seq { \seq_use:Nnnn \l_@@_corners_cells_seq , , , } @@ -17156,6 +17166,7 @@ version 2005/12/01 or later. left-margin,~ light-syntax,~ matrix/columns-type,~ + no-cell-nodes,~ notes~(several~subkeys),~ nullify-dots,~ pgf-node-code,~ @@ -17214,6 +17225,7 @@ version 2005/12/01 or later. left-margin,~ light-syntax,~ name,~ + no-cell-nodes,~ nullify-dots,~ pgf-node-code,~ renew-dots,~ @@ -17278,6 +17290,7 @@ version 2005/12/01 or later. left-margin,~ light-syntax,~ name,~ + no-cell-nodes,~ nullify-dots,~ pgf-node-code,~ r,~ @@ -17340,6 +17353,7 @@ version 2005/12/01 or later. left-margin,~ light-syntax,~ name,~ + no-cell-nodes,~ notes~(several~subkeys),~ nullify-dots,~ pgf-node-code,~ diff --git a/macros/latex/contrib/nicematrix/nicematrix-code.pdf b/macros/latex/contrib/nicematrix/nicematrix-code.pdf new file mode 100644 index 0000000000..f05f5fc311 Binary files /dev/null and b/macros/latex/contrib/nicematrix/nicematrix-code.pdf differ diff --git a/macros/latex/contrib/nicematrix/nicematrix-french.pdf b/macros/latex/contrib/nicematrix/nicematrix-french.pdf index 456c820b75..5245e44dce 100644 Binary files a/macros/latex/contrib/nicematrix/nicematrix-french.pdf and b/macros/latex/contrib/nicematrix/nicematrix-french.pdf differ diff --git a/macros/latex/contrib/nicematrix/nicematrix-french.tex b/macros/latex/contrib/nicematrix/nicematrix-french.tex index 64ac62e240..3ba956b397 100644 --- a/macros/latex/contrib/nicematrix/nicematrix-french.tex +++ b/macros/latex/contrib/nicematrix/nicematrix-french.tex @@ -316,6 +316,10 @@ $\begin{pNiceMatrix} \end{pNiceMatrix}$ \end{scope} +\bigskip +Il est également possible de changer ces paramètres pour certaines lignes +seulement grâce à la commande |\RowStyle| (cf.~p.~\pageref{RowStyle}). + \medskip \section{La clé baseline} @@ -1779,10 +1783,6 @@ Rappelons que l'extension \pkg{colortbl} peut être chargée directement par |\usepackage{colortbl}| ou en chargeant l'extension \pkg{xcolor} avec l'option \texttt{table} : |\usepackage[table]{xcolor}|. -\medskip -Comme l'extension \pkg{nicematrix} est fondée sur \pkg{array}, il est possible -d'utiliser \pkg{colortbl} avec \pkg{nicematrix}. - \medskip Il y a néanmoins deux inconvénients : \begin{itemize} @@ -5199,6 +5199,7 @@ environnement de \pkg{nicematrix}, elle est sans effet. Pour un exemple d'utilisation, voir \url{tex.stackexchange.com/questions/488566} + \section{Utilisation de TikZ avec nicematrix} \label{name}\label{PGF-nodes} @@ -5207,6 +5208,8 @@ Pour un exemple d'utilisation, voir \url{tex.stackexchange.com/questions/488566} \subsection{Les nœuds correspondant aux contenus des cases} +\index{no-cell-nodes} + L'extension \pkg{nicematrix} crée un nœud PGF-TikZ\footnote{On rappelle que TikZ est une sur-couche de PGF. L'extension \pkg{nicematrix} charge PGF et ne charge pas TikZ. On parle de «nœud PGF-TikZ» pour rappeler que, en fait, les @@ -5224,6 +5227,11 @@ Néanmoins, on peut forcer la création d'un nœud avec la commande |\NotEmpty|. lignes pointillées (cf.~p.~\pageref{Cdots}) et la détermination des «coins» (cf. p.~\pageref{corners}).} +\medskip +Comme la création de ces nœuds requiert du temps et de la mémoire, il est +possible de la désactiver ponctuellement avec la clé |no-cell-nodes| pour +accéler les compilations. + \medskip Tous les nœuds du document doivent avoir des noms deux à deux distincts et le nom de ces nœuds doit donc faire intervenir le numéro de l'environnement diff --git a/macros/latex/contrib/nicematrix/nicematrix.ins b/macros/latex/contrib/nicematrix/nicematrix.ins index 171d3f7449..88bb4d6754 100644 --- a/macros/latex/contrib/nicematrix/nicematrix.ins +++ b/macros/latex/contrib/nicematrix/nicematrix.ins @@ -1,5 +1,5 @@ %% -%% Copyright (C) 2018-2023 by F. Pantigny +%% Copyright (C) 2018-2024 by F. Pantigny %% %% %% This file may be distributed and/or modified under the @@ -17,7 +17,7 @@ \usedir{tex/latex/nicematrix} \preamble -Copyright (C) 2018-2023 by F. Pantigny +Copyright (C) 2018-2024 by F. Pantigny This file may be distributed and/or modified under the conditions of the LaTeX Project Public License, either diff --git a/macros/latex/contrib/nicematrix/nicematrix.pdf b/macros/latex/contrib/nicematrix/nicematrix.pdf index cc1afcb859..e1f5e455dd 100644 Binary files a/macros/latex/contrib/nicematrix/nicematrix.pdf and b/macros/latex/contrib/nicematrix/nicematrix.pdf differ diff --git a/macros/latex/contrib/nicematrix/nicematrix.tex b/macros/latex/contrib/nicematrix/nicematrix.tex index 10dd6c789a..e1cd2af742 100644 --- a/macros/latex/contrib/nicematrix/nicematrix.tex +++ b/macros/latex/contrib/nicematrix/nicematrix.tex @@ -288,7 +288,9 @@ $\begin{pNiceMatrix} \end{pNiceMatrix}$ \end{scope} - +\bigskip +It's also possible to change these parameters for only a few rows by using the +command |\RowStyle| provided by \pkg{nicematrix} (cf.~p.~\pageref{RowStyle}). @@ -1727,10 +1729,6 @@ We recall that the package \pkg{colortbl} can be loaded directly with |\usepackage{colortbl}| or by loading \pkg{xcolor} with the key |table|: |\usepackage[table]{xcolor}|. -\medskip -Since the package \pkg{nicematrix} is based on \pkg{array}, it's possible to -use \pkg{colortbl} with \pkg{nicematrix}. - \medskip However, there is two drawbacks: \begin{itemize} @@ -5095,10 +5093,12 @@ For an example of utilisation, see \url{tex.stackexchange.com/questions/488566} \label{name}\label{PGF-nodes} \index{tikza@TikZ (utilisation with \pkg{nicematrix})} -\index{node@\textbf{Nodes of PGF/TikZ|}(} +\index{node@\textbf{Nodes of PGF/TikZ}|(} \subsection{The nodes corresponding to the contents of the cells} +\index{no-cell-nodes} + The package \pkg{nicematrix} creates a PGF/TikZ node\footnote{We recall that TikZ is a layer over PGF. The extension \pkg{nicematrix} loads PGF but does not load TikZ. We speak of PGF/TikZ nodes to emphase the fact that the PGF nodes @@ -5118,6 +5118,11 @@ considered as non-empty, which has consequencies for the continuous dotted lines (cf. p.~\pageref{Cdots}) and the computation of the ``corners'' (cf.~p.~\pageref{corners}).} +\medskip +The creation of those nodes needs time and memory. It's possible to desactive +ponctually the creation of those nodes with the key |no-cell-nodes| in order to +speed up the compilations. + \medskip The nodes of a document must have distinct names. That's why the names of the nodes created by \pkg{nicematrix} contains the number of the current diff --git a/macros/latex/contrib/proflycee/doc/ProfLycee-doc.pdf b/macros/latex/contrib/proflycee/doc/ProfLycee-doc.pdf index 21a32e22f8..7928e78fef 100644 Binary files a/macros/latex/contrib/proflycee/doc/ProfLycee-doc.pdf and b/macros/latex/contrib/proflycee/doc/ProfLycee-doc.pdf differ diff --git a/macros/latex/contrib/proflycee/doc/ProfLycee-doc.tex b/macros/latex/contrib/proflycee/doc/ProfLycee-doc.tex index 0dfda9ed87..806a31ab3b 100644 --- a/macros/latex/contrib/proflycee/doc/ProfLycee-doc.tex +++ b/macros/latex/contrib/proflycee/doc/ProfLycee-doc.tex @@ -5,8 +5,8 @@ % arara: lualatex: {shell: yes, synctex: no, interaction: batchmode} if found('log', '(undefined references|Please rerun|Rerun to get)') \documentclass[a4paper,french,11pt]{article} -\def\PLversion{3.01a} -\def\PLdate{6 décembre 2023} +\def\PLversion{3.01b} +\def\PLdate{2 janvier 2024} \usepackage{amsfonts} \usepackage{ProfLycee} \useproflyclib{piton,minted,pythontex,ecritures} diff --git a/macros/latex/contrib/proflycee/tex/ProfLycee.sty b/macros/latex/contrib/proflycee/tex/ProfLycee.sty index 6d2d836957..d1c9f5a086 100644 --- a/macros/latex/contrib/proflycee/tex/ProfLycee.sty +++ b/macros/latex/contrib/proflycee/tex/ProfLycee.sty @@ -3,7 +3,8 @@ % or later, see http://www.latex-project.org/lppl.txtf \NeedsTeXFormat{LaTeX2e} -\ProvidesPackage{ProfLycee}[2023/12/06 3.01a Aide pour l'utilisation de LaTeX en lycee] +\ProvidesPackage{ProfLycee}[2024/01/02 3.01b Aide pour l'utilisation de LaTeX en lycee] +% 3.01b Ajout de la division euclidienne + correction du bugs mineurs % 3.01a Ajout des courbes ECC/FCC + paramètres par interpolation % 3.00g Amélioration de la marge Gauche dans le PseudoCode % 3.00f Conversion d'une écriture décimale périodique en fraction (avec rédaction éventuelle) @@ -187,6 +188,8 @@ \input{proflycee-tools-aleatoire.tex} %%------Suites \input{proflycee-tools-suites.tex} +%%------Complexes +\input{proflycee-tools-complexes.tex} %%------Géométrie \input{proflycee-tools-geom.tex} %%------Récréations diverses diff --git a/macros/latex/contrib/proflycee/tex/proflycee-tools-analyse.tex b/macros/latex/contrib/proflycee/tex/proflycee-tools-analyse.tex index 00f6f0206a..90672aaf1d 100644 --- a/macros/latex/contrib/proflycee/tex/proflycee-tools-analyse.tex +++ b/macros/latex/contrib/proflycee/tex/proflycee-tools-analyse.tex @@ -2,6 +2,8 @@ % Copyright 2023 Cédric Pierquet % Released under the LaTeX Project Public License v1.3c or later, see http://www.latex-project.org/lppl.txt +%Correction d'un bug sur la raacine carrée... le cas où ça fait 1 ! + %%------SimplFrac nouvelle version \NewDocumentCommand\ConversionFraction{ s O{} m }{% %*=moins sur le numérateur @@ -295,43 +297,49 @@ %%------SIMPLIFRACINES \DeclareDocumentCommand\SimplificationRacine{ m }{% argument mandataire {calcul ou fraction} \xdef\calculargument{\xintIrr{\xinteval{#1}}}% - \IfEndWith{\calculargument}{/1}% - {%si c'est un entier ! - \xdef\calculargument{\xintiieval{#1}}% - \xdef\ExtractRacStop{\xintiFloor{\xintfloateval{sqrt(\calculargument)}}}% - \xintFor* ##1 in {\xintSeq{1}{\ExtractRacStop}}\do{% - \xintifboolexpr{ \xintiiRem{\calculargument}{\xintiieval{##1*##1}} == 0}%si le carré divise - {\xdef\ExtractRacID{##1}}% - {}% - }% - \xdef\ExtracReste{\xintiieval{\calculargument/(\ExtractRacID*\ExtractRacID)}}% - \ensuremath{\xintifboolexpr{\ExtractRacID == 1 && \ExtracReste == 1}% - {1}% - { \xintifboolexpr{\ExtractRacID == 1}{}{\num{\xintiieval{\ExtractRacID}}}% - \xintifboolexpr{\ExtracReste == 1}{}{\sqrt{\num{\ExtracReste}}} }}% -% }% - }% - {%on va mettre sous forme a\sqrt(d)/b := buggue avec des millièmes... - \StrBefore{\calculargument}{/}[\numerateur]%on extrait le numérateur - \StrBehind{\calculargument}{/}[\denominateur]%on extrait le dénominateur - \xdef\ExtractRacNNum{\xintiieval{\numerateur*\denominateur}}% - \xdef\ExtractRacStop{\xintiFloor{\xintfloateval{sqrt(\ExtractRacNNum)}}}% - \xintFor* ##1 in {\xintSeq{1}{\ExtractRacStop}}\do{% - \xintifboolexpr{ \xintiiRem{\ExtractRacNNum}{\xintiieval{##1*##1}} == 0}%si le carré divise - {\xdef\ExtractRacID{##1}}% - {} + \xintifboolexpr{\calculargument == 1}% + {% + \ensuremath{1}% }% - \xdef\ExtractRacGCD{\xintiiGCD{\ExtractRacID}{\denominateur}}% - \xdef\RacNumSimpl{\xintiieval{\ExtractRacID/\ExtractRacGCD}}%simpl num - \xdef\RacDenomSimpl{\xintiieval{\denominateur/\ExtractRacGCD}}% - \xdef\RacRacSimpl{\xintiieval{\ExtractRacNNum/(\ExtractRacID*\ExtractRacID)}}% - \ensuremath{\frac{% - \xintifboolexpr{\RacNumSimpl == 1 && \RacRacSimpl == 1}% - {1}% - { \xintifboolexpr{\RacNumSimpl == 1}{}{\RacNumSimpl} \xintifboolexpr{\RacRacSimpl == 1}{}{\sqrt{\num{\RacRacSimpl}}} }% + {% + \IfEndWith{\calculargument}{/1}% + {%si c'est un entier ! + \xdef\calculargument{\xintiieval{#1}}% + \xdef\ExtractRacStop{\xintiFloor{\xintfloateval{sqrt(\calculargument)}}}% + \xintFor* ##1 in {\xintSeq{1}{\ExtractRacStop}}\do{% + \xintifboolexpr{ \xintiiRem{\calculargument}{\xintiieval{##1*##1}} == 0}%si le carré divise + {\xdef\ExtractRacID{##1}}% + {}% + }% + \xdef\ExtracReste{\xintiieval{\calculargument/(\ExtractRacID*\ExtractRacID)}}% + \ensuremath{\xintifboolexpr{\ExtractRacID == 1 && \ExtracReste == 1}% + {1}% + { \xintifboolexpr{\ExtractRacID == 1}{}{\num{\xintiieval{\ExtractRacID}}}% + \xintifboolexpr{\ExtracReste == 1}{}{\sqrt{\num{\ExtracReste}}} }}% + % }% + }% + {%on va mettre sous forme a\sqrt(d)/b := buggue avec des millièmes... + \StrBefore{\calculargument}{/}[\numerateur]%on extrait le numérateur + \StrBehind{\calculargument}{/}[\denominateur]%on extrait le dénominateur + \xdef\ExtractRacNNum{\xintiieval{\numerateur*\denominateur}}% + \xdef\ExtractRacStop{\xintiFloor{\xintfloateval{sqrt(\ExtractRacNNum)}}}% + \xintFor* ##1 in {\xintSeq{1}{\ExtractRacStop}}\do{% + \xintifboolexpr{ \xintiiRem{\ExtractRacNNum}{\xintiieval{##1*##1}} == 0}%si le carré divise + {\xdef\ExtractRacID{##1}}% + {} }% - { \RacDenomSimpl }}% - }% + \xdef\ExtractRacGCD{\xintiiGCD{\ExtractRacID}{\denominateur}}% + \xdef\RacNumSimpl{\xintiieval{\ExtractRacID/\ExtractRacGCD}}%simpl num + \xdef\RacDenomSimpl{\xintiieval{\denominateur/\ExtractRacGCD}}% + \xdef\RacRacSimpl{\xintiieval{\ExtractRacNNum/(\ExtractRacID*\ExtractRacID)}}% + \ensuremath{\frac{% + \xintifboolexpr{\RacNumSimpl == 1 && \RacRacSimpl == 1}% + {1}% + { \xintifboolexpr{\RacNumSimpl == 1}{}{\RacNumSimpl} \xintifboolexpr{\RacRacSimpl == 1}{}{\sqrt{\num{\RacRacSimpl}}} }% + }% + { \RacDenomSimpl }}% + }% + }% } %%------DICHOTOMIE @@ -631,6 +639,8 @@ %clés \restoreKV[FracPeriod]% revenir au valeurs par défaut \setKV[FracPeriod]{#1}% lit les arguments optionnels + %vmode ? + \ifvmode\leavevmode\fi% %les variables utiles \IfEndWith{#2}{.}% {% diff --git a/macros/latex/contrib/proflycee/tex/proflycee-tools-arithm.tex b/macros/latex/contrib/proflycee/tex/proflycee-tools-arithm.tex index cabba276fb..9b2d9eb653 100644 --- a/macros/latex/contrib/proflycee/tex/proflycee-tools-arithm.tex +++ b/macros/latex/contrib/proflycee/tex/proflycee-tools-arithm.tex @@ -502,11 +502,11 @@ \xintFor* ##1 in {\xintSeq{2}{\argcal}}\do{% \xintifboolexpr{ \xintiiRem{\argcal}{##1} == 0 }% {% - \xdef\tmplistdiv{\tmplistdiv /\num{##1}} + \xdef\tmplistdiv{\tmplistdiv /\num{##1}}% }% {}% }% - \ensuremath{\ifboolKV[listdiv]{AffNom}{\IfBooleanTF{#1}{\mathscr{D}}{\mathcal{D}}_{\num{\argcal}}=}{}\EcritureEnsemble[\strut]{\tmplistdiv}} + \ensuremath{\ifboolKV[listdiv]{AffNom}{\IfBooleanTF{#1}{\mathscr{D}}{\mathcal{D}}_{\num{\argcal}}=}{}\EcritureEnsemble[\strut]{\tmplistdiv}}% } %arbre diviseurs @@ -913,4 +913,24 @@ }% } +%====DIV EUCL (OK) +\setKVdefault[diveucl]{% + Quotient=true,% + Reste=true,% + Vide=false,% + Pointilles=\ldots +} +\NewDocumentCommand\DivEucl{ s O{} m m }{% + \restoreKV[diveucl]% + \setKV[diveucl]{#2}% + \ifboolKV[diveucl]{Vide}% + {% + \setKV[diveucl]{Quotient=false,Reste=false}% + }% + {}% + \xdef\tmp@quotient{\xintiiQuo{#3}{#4}}\xdef\tmp@reste{\xintiiRem{#3}{#4}}% + \ensuremath{\num{#3}=\num{#4}\times\ifboolKV[diveucl]{Quotient}{\xintifboolexpr{\tmp@quotient < 0}{(\num{\tmp@quotient})}{\num{\tmp@quotient}}}{\useKV[diveucl]{Pointilles}}+\ifboolKV[diveucl]{Reste}{\num{\tmp@reste}}{\useKV[diveucl]{Pointilles}}}% + \IfBooleanT{#1}{~avec $0 \leqslant \num{\xintiiRem{#3}{#4}} < \xintifboolexpr{#4 < 0}{\lvert\num{#4}\rvert}{\num{#4}}$}% +} + \endinput \ No newline at end of file diff --git a/macros/latex/contrib/proflycee/tex/proflycee-tools-complexes.tex b/macros/latex/contrib/proflycee/tex/proflycee-tools-complexes.tex new file mode 100644 index 0000000000..ca64c0ac40 --- /dev/null +++ b/macros/latex/contrib/proflycee/tex/proflycee-tools-complexes.tex @@ -0,0 +1,609 @@ +% proflycee-tools-complexes.tex +% Copyright 2023 Cédric Pierquet +% Released under the LaTeX Project Public License v1.3c or later, see http://www.latex-project.org/lppl.txt + +\NewDocumentCommand\PartieReelle{ D<>{} m O{\PartReRes} }{% + \StrSubstitute{#2}{I}{0}[#3]% + \IfEq{#1}{n}% + {\ensuremath{\num{\xinteval{#3}}}}{}% + \IfEq{#1}{f}% + {\ensuremath{\ConversionFraction{#3}}}{}% + \IfEq{#1}{df}% + {\ensuremath{\ConversionFraction[d]{#3}}}{}% +} +\NewDocumentCommand\PartieImaginaire{ D<>{} m O{\PartImRes} }{% + \StrSubstitute{#2}{I}{0}[\TmpPartReCplxA]% + \StrSubstitute{(#2)-(\TmpPartReCplxA)}{I}{1}[#3]% + \IfEq{#1}{n}% + {\ensuremath{\num{\xinteval{#3}}}}{}% + \IfEq{#1}{f}% + {\ensuremath{\ConversionFraction{#3}}}{}% + \IfEq{#1}{df}% + {\ensuremath{\ConversionFraction[d]{#3}}}{}% +} + +\NewDocumentCommand\AffComplexe{ O{} }{% + \ensuremath{% + \xintifboolexpr{\xinttmpreCalc == 0 'and' \xinttmpimCalc == 0}{0}{}% + \xintifboolexpr{\xinttmpreCalc == 0 'and' \xinttmpimCalc == 1}{\i}{}% + \xintifboolexpr{\xinttmpreCalc == 0 'and' \xinttmpimCalc == -1}{-\i}{}% + \xintifboolexpr{\xinttmpreCalc == 0 'and' abs(\xinttmpimCalc) != 1 'and' \xinttmpimCalc != 0}{\ConversionFraction[#1]{\tmpimCalc}\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc < 0 'and' \xinttmpimCalc != -1}{\ConversionFraction[#1]{\tmpreCalc}-\ConversionFraction[#1]{-(\tmpimCalc)}\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc == -1}{\ConversionFraction[#1]{\tmpreCalc}-\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc == 1}{\ConversionFraction[#1]{\tmpreCalc}+\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc > 0 'and' \xinttmpimCalc != 1}{\ConversionFraction[#1]{\tmpreCalc}+\ConversionFraction[#1]{\tmpimCalc}\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc == 0}{\ConversionFraction[#1]{\tmpreCalc}}{}% + }% +} + +\NewDocumentCommand\Complexe{ O{} m }{% + \PartieReelle{#2}[\tmpreA]% + \PartieImaginaire{#2}[\tmpimA]% + \xdef\xinttmpreCalc{\xinteval{\tmpreA}}% + \xdef\xinttmpimCalc{\xinteval{\tmpimA}}% + \ensuremath{% + \xintifboolexpr{\xinttmpreCalc == 0 'and' \xinttmpimCalc == 0}{0}{}% + \xintifboolexpr{\xinttmpreCalc == 0 'and' \xinttmpimCalc == 1}{\i}{}% + \xintifboolexpr{\xinttmpreCalc == 0 'and' \xinttmpimCalc == -1}{-\i}{}% + \xintifboolexpr{\xinttmpreCalc == 0 'and' abs(\xinttmpimCalc) != 1 'and' \xinttmpimCalc != 0}{\ConversionFraction[#1]{\tmpimA}\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc < 0 'and' \xinttmpimCalc != -1}{\ConversionFraction[#1]{\tmpreA}-\ConversionFraction[#1]{-(\tmpimA)}\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc == -1}{\ConversionFraction[#1]{\tmpreA}-\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc == 1}{\ConversionFraction[#1]{\tmpreA}+\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc > 0 'and' \xinttmpimCalc != 1}{\ConversionFraction[#1]{\tmpreA}+\ConversionFraction[#1]{\tmpimA}\i}{}% + \xintifboolexpr{\xinttmpreCalc != 0 'and' \xinttmpimCalc == 0}{\ConversionFraction[#1]{\tmpreA}}{}% + }% +} + +\NewDocumentCommand\SommeComplexes{ O{} m m }{% + \PartieReelle{#2}[\tmpreA]% + \PartieReelle{#3}[\tmpreB]% + \PartieImaginaire{#2}[\tmpimA]% + \PartieImaginaire{#3}[\tmpimB]% + \xdef\tmpreCalc{(\tmpreA)+(\tmpreB)}% + \xdef\xinttmpreCalc{\xinteval{(\tmpreA)+(\tmpreB)}}% + \xdef\tmpimCalc{(\tmpimA)+(\tmpimB)}% + \xdef\xinttmpimCalc{\xinteval{(\tmpimA)+(\tmpimB)}}% + %\xinttmpreCalc\text{ et }\xinttmpimCalc. + \AffComplexe[#1]% +} + +\NewDocumentCommand\ProduitComplexes{ O{} m m }{% + \PartieReelle{#2}[\tmpreA]% + \PartieReelle{#3}[\tmpreB]% + \PartieImaginaire{#2}[\tmpimA]% + \PartieImaginaire{#3}[\tmpimB]% + \xdef\tmpreCalc{(\tmpreA)*(\tmpreB)-(\tmpimA)*(\tmpimB)}% + \xdef\xinttmpreCalc{\xinteval{\tmpreCalc}}% + \xdef\tmpimCalc{(\tmpreA)*(\tmpimB)+(\tmpimA)*(\tmpreB)}% + \xdef\xinttmpimCalc{\xinteval{\tmpimCalc}}% + %\xinttmpreCalc\text{ et }\xinttmpimCalc. + \AffComplexe[#1]% +} + +\NewDocumentCommand\QuotientComplexes{ O{} m m }{% + \PartieReelle{#2}[\tmpreA]% + \PartieReelle{#3}[\tmpreB]% + \PartieImaginaire{#2}[\tmpimA]% + \PartieImaginaire{#3}[\tmpimB]% + \xdef\tmpreCalc{((\tmpreA)*(\tmpreB)+(\tmpimA)*(\tmpimB))/((\tmpreB)*(\tmpreB)+(\tmpimB)*(\tmpimB))}% + \xdef\xinttmpreCalc{\xinteval{\tmpreCalc}}% + \xdef\tmpimCalc{(-(\tmpreA)*(\tmpimB)+(\tmpimA)*(\tmpreB))/((\tmpreB)*(\tmpreB)+(\tmpimB)*(\tmpimB))}% + \xdef\xinttmpimCalc{\xinteval{\tmpimCalc}}% + %\xinttmpreCalc\text{ et }\xinttmpimCalc. + \AffComplexe[#1]% +} + +\NewDocumentCommand\CarreComplexe{ O{} m }{% + \PartieReelle{#2}[\tmpreA]% + \PartieImaginaire{#2}[\tmpimA]% + \xdef\tmpreCalc{(\tmpreA)*(\tmpreA)-(\tmpimA)*(\tmpimA)}% + \xdef\xinttmpreCalc{\xinteval{\tmpreCalc}}% + \xdef\tmpimCalc{2*(\tmpreA)*(\tmpimA)}% + \xdef\xinttmpimCalc{\xinteval{\tmpimCalc}}% + %\xinttmpreCalc\text{ et }\xinttmpimCalc. + \AffComplexe[#1]% +} + +\NewDocumentCommand\CubeComplexe{ O{} m }{% + \PartieReelle{#2}[\tmpreA]% + \PartieImaginaire{#2}[\tmpimA]% + \xdef\tmpreCalc{(\tmpreA)*(\tmpreA)*(\tmpreA)-3*(\tmpreA)*(\tmpimA)*(\tmpimA)}% + \xdef\xinttmpreCalc{\xinteval{\tmpreCalc}}% + \xdef\tmpimCalc{3*(\tmpreA)*(\tmpreA)*(\tmpimA)-(\tmpimA)*(\tmpimA)*(\tmpimA)}% + \xdef\xinttmpimCalc{\xinteval{\tmpimCalc}}% + %\xinttmpreCalc\text{ et }\xinttmpimCalc. + \AffComplexe[#1]% +} + +\NewDocumentCommand\ModuleComplexe{ m }{% + \PartieReelle{#1}[\tmpreA]% + \PartieImaginaire{#1}[\tmpimA]% + \IfSubStr{\tmpreA}{sqrt}% + {% + \StrDel{\tmpreA}{sqrt}[\tmpretmpA]% + }% + {% + \xdef\tmpretmpA{\tmpreA}% + }% + \IfSubStr{\tmpimA}{sqrt}% + {% + \StrDel{\tmpimA}{sqrt}[\tmpimtmpA]% + }% + {% + \xdef\tmpimtmpA{\tmpimA}% + }% + \IfSubStr{\tmpreA}{sqrt}% + {% + \IfSubStr{\tmpimA}{sqrt}% + {% + \xdef\tmpCarreModule{abs(\tmpretmpA)+abs(\tmpimtmpA)}% + }% + {% + \xdef\tmpCarreModule{abs(\tmpretmpA)+(\tmpimtmpA)*(\tmpimtmpA)}% + }% + }% + {% + \xdef\tmpCarreModule{(\tmpretmpA)*(\tmpretmpA)+(\tmpimtmpA)*(\tmpimtmpA)}% + }% + \ensuremath{\SimplificationRacine{\tmpCarreModule}}% +} + +\NewDocumentCommand\ArgumentComplexe{ O{} m }{% + \PartieReelle{#2}[\tmpreA]% + \PartieImaginaire{#2}[\tmpimA]% + \xdef\tmpCarreModule{(\tmpreA)*(\tmpreA)+(\tmpimA)*(\tmpimA)}% + \xdef\tmpModUnRe{(\tmpreA)/(sqrt(\tmpCarreModule))}% + \xdef\tmpModUnIm{(\tmpimA)/(sqrt(\tmpCarreModule))}% + \ensuremath{% + \xintifboolexpr{\tmpModUnRe == 1 'and' \tmpModUnIm == 0}{0}{}% + \xintifboolexpr{\tmpModUnRe == -1 'and' \tmpModUnIm == 0}{\pi}{}% + \xintifboolexpr{\tmpModUnRe == 0.5 'and' \tmpModUnIm > 0}{\IfEq{#1}{d}{\dfrac{\pi}{3}}{\frac{\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe == 0.5 'and' \tmpModUnIm < 0}{\IfEq{#1}{d}{\dfrac{-\pi}{3}}{\frac{-\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe == -0.5 'and' \tmpModUnIm > 0}{\IfEq{#1}{d}{\dfrac{2\pi}{3}}{\frac{2\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe == -0.5 'and' \tmpModUnIm < 0}{\IfEq{#1}{d}{\dfrac{-2\pi}{3}}{\frac{-2\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe > 0 'and' \tmpModUnIm == 0.5}{\IfEq{#1}{d}{\dfrac{\pi}{6}}{\frac{\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe < 0 'and' \tmpModUnIm == 0.5}{\IfEq{#1}{d}{\dfrac{5\pi}{6}}{\frac{5\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe > 0 'and' \tmpModUnIm == -0.5}{\IfEq{#1}{d}{\dfrac{-\pi}{6}}{\frac{-\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe < 0 'and' \tmpModUnIm == -0.5}{\IfEq{#1}{d}{\dfrac{-5\pi}{6}}{\frac{-5\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe == \tmpModUnIm 'and' \tmpModUnRe > 0}{\IfEq{#1}{d}{\dfrac{\pi}{4}}{\frac{\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == \tmpModUnIm 'and' \tmpModUnRe < 0}{\IfEq{#1}{d}{\dfrac{-3\pi}{4}}{\frac{-3\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == -\tmpModUnIm 'and' \tmpModUnRe > 0}{\IfEq{#1}{d}{\dfrac{-\pi}{4}}{\frac{-\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == -\tmpModUnIm 'and' \tmpModUnRe < 0}{\IfEq{#1}{d}{\dfrac{3\pi}{4}}{\frac{3\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == 0 'and' \tmpModUnIm == 1}{\IfEq{#1}{d}{\dfrac{\pi}{2}}{\frac{\pi}{2}}}{}% + \xintifboolexpr{\tmpModUnRe == 0 'and' \tmpModUnIm == -1}{\IfEq{#1}{d}{\dfrac{-\pi}{2}}{\frac{-\pi}{2}}}{}% + }% +} + +\NewDocumentCommand\FormeExpoComplexe{ m }{% + \PartieReelle{#1}[\tmpreA]% + \PartieImaginaire{#1}[\tmpimA]% + \IfSubStr{\tmpreA}{sqrt}% + {% + \StrDel{\tmpreA}{sqrt}[\tmpretmpA]% + }% + {% + \xdef\tmpretmpA{\tmpreA}% + }% + \IfSubStr{\tmpimA}{sqrt}% + {% + \StrDel{\tmpimA}{sqrt}[\tmpimtmpA]% + }% + {% + \xdef\tmpimtmpA{\tmpimA}% + }% + \IfSubStr{\tmpreA}{sqrt}% + {% + \IfSubStr{\tmpimA}{sqrt}% + {% + \xdef\tmpCarreModule{abs(\tmpretmpA)+abs(\tmpimtmpA)}% + }% + {% + \xdef\tmpCarreModule{abs(\tmpretmpA)+(\tmpimtmpA)*(\tmpimtmpA)}% + }% + }% + {% + \xdef\tmpCarreModule{(\tmpretmpA)*(\tmpretmpA)+(\tmpimtmpA)*(\tmpimtmpA)}% + }% + \xdef\tmpModUnRe{(\tmpreA)/(sqrt(\tmpCarreModule))}% + \xdef\tmpModUnIm{(\tmpimA)/(sqrt(\tmpCarreModule))}% + \ensuremath{% + \xintifboolexpr{\tmpCarreModule == 1}{}{\SimplificationRacine{\tmpCarreModule}}% + \e^{% + \xintifboolexpr{\tmpModUnRe == 1 'and' \tmpModUnIm == 0}{0}{}% + \xintifboolexpr{\tmpModUnRe == -1 'and' \tmpModUnIm == 0}{\i\pi}{}% + \xintifboolexpr{\tmpModUnRe == 0.5 'and' \tmpModUnIm > 0}{\IfEq{#1}{d}{\dfrac{\i\pi}{3}}{\frac{\i\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe == 0.5 'and' \tmpModUnIm < 0}{\IfEq{#1}{d}{\dfrac{-\i\pi}{3}}{\frac{-\i\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe == -0.5 'and' \tmpModUnIm > 0}{\IfEq{#1}{d}{\dfrac{2\i\pi}{3}}{\frac{2\i\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe == -0.5 'and' \tmpModUnIm < 0}{\IfEq{#1}{d}{\dfrac{-2\i\pi}{3}}{\frac{-2\i\pi}{3}}}{}% + \xintifboolexpr{\tmpModUnRe > 0 'and' \tmpModUnIm == 0.5}{\IfEq{#1}{d}{\dfrac{\i\pi}{6}}{\frac{\i\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe < 0 'and' \tmpModUnIm == 0.5}{\IfEq{#1}{d}{\dfrac{5\i\pi}{6}}{\frac{5\i\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe > 0 'and' \tmpModUnIm == -0.5}{\IfEq{#1}{d}{\dfrac{-\i\pi}{6}}{\frac{-\i\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe < 0 'and' \tmpModUnIm == -0.5}{\IfEq{#1}{d}{\dfrac{-5\i\pi}{6}}{\frac{-5\i\pi}{6}}}{}% + \xintifboolexpr{\tmpModUnRe == \tmpModUnIm 'and' \tmpModUnRe > 0}{\IfEq{#1}{d}{\dfrac{\i\pi}{4}}{\frac{\i\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == \tmpModUnIm 'and' \tmpModUnRe < 0}{\IfEq{#1}{d}{\dfrac{-3\i\pi}{4}}{\frac{-3\i\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == -\tmpModUnIm 'and' \tmpModUnRe > 0}{\IfEq{#1}{d}{\dfrac{-\i\pi}{4}}{\frac{-\i\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == -\tmpModUnIm 'and' \tmpModUnRe < 0}{\IfEq{#1}{d}{\dfrac{3\i\pi}{4}}{\frac{3\i\pi}{4}}}{}% + \xintifboolexpr{\tmpModUnRe == 0 'and' \tmpModUnIm == 1}{\IfEq{#1}{d}{\dfrac{\i\pi}{2}}{\frac{\i\pi}{2}}}{}% + \xintifboolexpr{\tmpModUnRe == 0 'and' \tmpModUnIm == -1}{\IfEq{#1}{d}{\dfrac{-\i\pi}{2}}{\frac{-\i\pi}{2}}}{}% + }% + }% +} + +%====commandes alternatives +\NewDocumentCommand\ExtractionCoeffExprRacines{ m O{\tmpCoeffA} O{\tmpCoeffB} O{\tmpCoeffC} O{\tmpCoeffD} }{%a*rac(b)+c*rac(d) + \IfSubStr{#1}{+} + {% + \StrCut{#1}{+}{\exprtestG}{\exprtestD}% + \IfSubStr{\exprtestG}{*sqrt}% + {% + \StrBefore{\exprtestG}{*}[#2]% + \StrBetween{\exprtestG}{sqrt(}{)}[#3]% + }% + {% + \xdef#2{\exprtestG}\xdef#3{1}% + }% + \IfSubStr{\exprtestD}{*sqrt}% + {% + \StrBefore{\exprtestD}{*}[#4]% + \StrBetween{\exprtestD}{sqrt(}{)}[#5]% + }% + {% + \xdef#4{\exprtestD}\xdef#5{1}% + }% + }% + {% + %si 2 moins... + \StrCount{#1}{-}[\tmpNbmoins]% + \xintifboolexpr{\tmpNbmoins == 2}% + {% + \StrCut[2]{#1}{-}{\exprtestG}{\exprtestD}% + }% + {% + \StrCut{#1}{-}{\exprtestG}{\exprtestD}% + }% + \IfSubStr{\exprtestG}{*sqrt}% + {% + \StrBefore{\exprtestG}{*}[#2]% + \StrBetween{\exprtestG}{sqrt(}{)}[#3]% + }% + {% + \xdef#2{\exprtestG}\xdef#3{1}% + }% + \IfSubStr{\exprtestD}{*sqrt}% + {% + \StrBefore{\exprtestD}{*}[#4]% + \xdef#4{-#4}% + \StrBetween{\exprtestD}{sqrt(}{)}[#5]% + }% + {% + \xdef#4{-#4}\xdef#5{1}% + }% + }% +} + +\NewDocumentCommand\SimplifCarreExprRacine{ O{} m }{% + \ExtractionCoeffExprRacines{#2}% + \xintifboolexpr{\tmpCoeffA > 0 'and' \tmpCoeffC > 0}% + {% + \ensuremath{% + \ConversionFraction[#1]{(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffB)+(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffD)}+% + \IfEq{#1}{d}{\displaystyle}{}\SimplificationRacine{4*(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffB)*(\tmpCoeffD)}% + }% + }% + {}% + \xintifboolexpr{\tmpCoeffA > 0 'and' \tmpCoeffC < 0}% + {% + \ensuremath{% + \ConversionFraction[#1]{(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffB)+(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffD)}-% + \IfEq{#1}{d}{\displaystyle}{}\SimplificationRacine{4*(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffB)*(\tmpCoeffD)}% + }% + }% + {}% + \xintifboolexpr{\tmpCoeffA < 0 'and' \tmpCoeffC > 0}% + {% + \ensuremath{% + \ConversionFraction[#1]{(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffB)+(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffD)}-% + \IfEq{#1}{d}{\displaystyle}{}\SimplificationRacine{4*(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffB)*(\tmpCoeffD)}% + }% + }% + {}% + \xintifboolexpr{\tmpCoeffA < 0 'and' \tmpCoeffC < 0}% + {% + \ensuremath{% + \ConversionFraction[#1]{(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffB)+(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffD)}+% + \IfEq{#1}{d}{\displaystyle}{}\SimplificationRacine{4*(\tmpCoeffA)*(\tmpCoeffA)*(\tmpCoeffC)*(\tmpCoeffC)*(\tmpCoeffB)*(\tmpCoeffD)}% + }% + }% + {}% +} + +\NewDocumentCommand\CalculModuleCplx{ O{} m m }{% + \ExtractionCoeffExprRacines{#2}[\Crea][\Creb][\Crec][\Cred]% + \ExtractionCoeffExprRacines{#3}[\Cima][\Cimb][\Cimc][\Cimd]% + \xdef\TmpCoeffsDebut{(\Crea)*(\Crea)*(\Creb)+(\Crec)*(\Crec)*(\Cred)+(\Cima)*(\Cima)*(\Cimb)+(\Cimc)*(\Cimc)*(\Cimd)}% + \xdef\TmpCoeffsRacineA{4*(\Crea)*(\Crea)*(\Crec)*(\Crec)*(\Creb)*(\Cred)}% + \xdef\TmpCoeffsRacineB{4*(\Cima)*(\Cima)*(\Cimc)*(\Cimc)*(\Cimb)*(\Cimd)}% + %\xinteval{\TmpCoeffsDebut}/\xinteval{\TmpCoeffsRacineA}/\xinteval{\TmpCoeffsRacineB}/\xinteval{(\Crea)*(\Crec)}/\xinteval{(\Cima)*(\Cimc)}= + \xintifboolexpr{\TmpCoeffsRacineA == 0 'and' \TmpCoeffsRacineB == 0}% + {% + \ensuremath{\IfEq{#1}{d}{\displaystyle}{}\SimplificationRacine{\TmpCoeffsDebut}}% + }% + {}% + \xintifboolexpr{\TmpCoeffsRacineA == 0 'and' \TmpCoeffsRacineB != 0}% + {% + \ensuremath{\IfEq{#1}{d}{\displaystyle}{}\sqrt{\SimplificationRacine{(\TmpCoeffsDebut)*(\TmpCoeffsDebut)}\xintifboolexpr{(\Cima)*(\Cimc) < 0}{-}{+}\SimplificationRacine{\TmpCoeffsRacineB}}}% + }% + {}% + \xintifboolexpr{\TmpCoeffsRacineA != 0 'and' \TmpCoeffsRacineB == 0}% + {% + \ensuremath{\IfEq{#1}{d}{\displaystyle}{}\sqrt{\SimplificationRacine{(\TmpCoeffsDebut)*(\TmpCoeffsDebut)}\xintifboolexpr{(\Crea)*(\Crec) < 0}{-}{+}\SimplificationRacine{\TmpCoeffsRacineA}}}% + }% + {}% + \xintifboolexpr{\TmpCoeffsRacineA != 0 'and' \TmpCoeffsRacineB != 0 'and' (\Crea)*(\Crec) < 0 'and' (\Cima)*(\Cimc) > 0 'and' \TmpCoeffsRacineA == \TmpCoeffsRacineB}% + {% + \ensuremath{\IfEq{#1}{d}{\displaystyle}{}\SimplificationRacine{\TmpCoeffsDebut}}% + }% + {}% + \xintifboolexpr{\TmpCoeffsRacineA != 0 'and' \TmpCoeffsRacineB != 0 'and' (\Crea)*(\Crec) > 0 'and' (\Cima)*(\Cimc) < 0 'and' \TmpCoeffsRacineA == \TmpCoeffsRacineB}% + {% + \ensuremath{\IfEq{#1}{d}{\displaystyle}{}\SimplificationRacine{\TmpCoeffsDebut}}% + }% + {}% + \xintifboolexpr{\TmpCoeffsRacineA != \TmpCoeffsRacineB}% + {% + \ensuremath{\IfEq{#1}{d}{\displaystyle}{}\sqrt{\SimplificationRacine{(\TmpCoeffsDebut)*(\TmpCoeffsDebut)}\xintifboolexpr{(\Crea)*(\Crec) < 0}{-}{+}\SimplificationRacine{\TmpCoeffsRacineA}\xintifboolexpr{(\Cima)*(\Cimc) < 0}{-}{+}\SimplificationRacine{\TmpCoeffsRacineB}}}% + }% + {}% +} + +\NewDocumentCommand\TestArgumentComplexe{ O{} m m m }{% + \xintifboolexpr{\TmpArg == #2 'or' \TmpArg == #3}{\ensuremath{\IfEq{#1}{d}{\displaystyle}{}#4}}{}% +} + +\NewDocumentCommand\CalculArgumentCplx{ s O{} m m }{% + \xdef\TmpArg{\xintfloateval{trunc(Argd(#3,#4),1)}}%\TmpArg% + \IfBooleanTF{#1}% + {% + %les pi/2 + \TestArgumentComplexe[#2]{0}{0.0}{0}% + \TestArgumentComplexe[#2]{90}{90.0}{\frac{\pi}{2}}% + \TestArgumentComplexe[#2]{-90}{-90.0}{\frac{3\pi}{2}}% + \TestArgumentComplexe[#2]{180}{180.0}{\pi}% + %les pi/3 + \TestArgumentComplexe[#2]{60}{60.0}{\frac{\pi}{3}}% + \TestArgumentComplexe[#2]{120}{120.0}{\frac{2\pi}{3}}% + \TestArgumentComplexe[#2]{-60}{-60.0}{\frac{5\pi}{3}}% + \TestArgumentComplexe[#2]{-120}{-120.0}{\frac{4\pi}{3}}% + %les pi/4 + \TestArgumentComplexe[#2]{45}{45.0}{\frac{\pi}{4}}% + \TestArgumentComplexe[#2]{135}{135.0}{\frac{3\pi}{4}}% + \TestArgumentComplexe[#2]{-45}{-45.0}{\frac{7\pi}{4}}% + \TestArgumentComplexe[#2]{-135}{-135.0}{\frac{5\pi}{4}}% + %les pi/5 + \TestArgumentComplexe[#2]{36}{36.0}{\frac{\pi}{5}}% + \TestArgumentComplexe[#2]{72}{72.0}{\frac{2\pi}{5}}% + \TestArgumentComplexe[#2]{108}{108.0}{\frac{3\pi}{5}}% + \TestArgumentComplexe[#2]{144}{144.0}{\frac{4\pi}{5}}% + \TestArgumentComplexe[#2]{-36}{-36.0}{\frac{9\pi}{5}}% + \TestArgumentComplexe[#2]{-72}{-72.0}{\frac{8\pi}{5}}% + \TestArgumentComplexe[#2]{-108}{-108.0}{\frac{7\pi}{5}}% + \TestArgumentComplexe[#2]{-144}{-144.0}{\frac{6\pi}{5}}% + %les pi/6 + \TestArgumentComplexe[#2]{30}{30.0}{\frac{\pi}{6}}% + \TestArgumentComplexe[#2]{150}{150.0}{\frac{5\pi}{6}}% + \TestArgumentComplexe[#2]{-30}{-30.0}{\frac{11\pi}{6}}% + \TestArgumentComplexe[#2]{-150}{-150.0}{\frac{7\pi}{6}}% + %les pi/8 + \TestArgumentComplexe[#2]{22.5}{22.5}{\frac{\pi}{8}}% + \TestArgumentComplexe[#2]{67.5}{67.5}{\frac{3\pi}{8}}% + \TestArgumentComplexe[#2]{112.5}{112.5}{\frac{5\pi}{8}}% + \TestArgumentComplexe[#2]{157.5}{157.5}{\frac{7\pi}{8}}% + \TestArgumentComplexe[#2]{-22.5}{-22.5}{\frac{15\pi}{8}}% + \TestArgumentComplexe[#2]{-67.5}{-67.5}{\frac{13\pi}{8}}% + \TestArgumentComplexe[#2]{-112.5}{-112.5}{\frac{11\pi}{8}}% + \TestArgumentComplexe[#2]{-157.5}{-157.5}{\frac{9\pi}{8}}% + %les pi/12 + \TestArgumentComplexe[#2]{15}{15.0}{\frac{\pi}{12}}% + \TestArgumentComplexe[#2]{75}{75.0}{\frac{5\pi}{12}}% + \TestArgumentComplexe[#2]{105}{105.0}{\frac{7\pi}{12}}% + \TestArgumentComplexe[#2]{165}{165.0}{\frac{11\pi}{12}}% + \TestArgumentComplexe[#2]{-15}{-15.0}{\frac{23\pi}{12}}% + \TestArgumentComplexe[#2]{-75}{-75.0}{\frac{19\pi}{12}}% + \TestArgumentComplexe[#2]{-105}{-105.0}{\frac{17\pi}{12}}% + \TestArgumentComplexe[#2]{-165}{-165.0}{\frac{13\pi}{12}}% + %les pi/10 + \TestArgumentComplexe[#2]{18}{18.0}{\frac{\pi}{10}}% + \TestArgumentComplexe[#2]{54}{54.0}{\frac{3\pi}{10}}% + \TestArgumentComplexe[#2]{126}{126.0}{\frac{7\pi}{10}}% + \TestArgumentComplexe[#2]{162}{162.0}{\frac{9\pi}{10}}% + \TestArgumentComplexe[#2]{-18}{-18.0}{\frac{19\pi}{10}}% + \TestArgumentComplexe[#2]{-54}{-54.0}{\frac{17\pi}{10}}% + \TestArgumentComplexe[#2]{-126}{-126.0}{\frac{13\pi}{10}}% + \TestArgumentComplexe[#2]{-162}{-162.0}{\frac{11\pi}{10}}% + }% + {% + %les pi/2 + \TestArgumentComplexe[#2]{0}{0.0}{0}% + \TestArgumentComplexe[#2]{90}{90.0}{\frac{\pi}{2}}% + \TestArgumentComplexe[#2]{-90}{-90.0}{\frac{-\pi}{2}}% + \TestArgumentComplexe[#2]{180}{180.0}{\pi}% + %les pi/3 + \TestArgumentComplexe[#2]{60}{60.0}{\frac{\pi}{3}}% + \TestArgumentComplexe[#2]{120}{120.0}{\frac{2\pi}{3}}% + \TestArgumentComplexe[#2]{-60}{-60.0}{\frac{-\pi}{3}}% + \TestArgumentComplexe[#2]{-120}{-120.0}{\frac{-2\pi}{3}}% + %les pi/4 + \TestArgumentComplexe[#2]{45}{45.0}{\frac{\pi}{4}}% + \TestArgumentComplexe[#2]{135}{135.0}{\frac{3\pi}{4}}% + \TestArgumentComplexe[#2]{-45}{-45.0}{\frac{-\pi}{4}}% + \TestArgumentComplexe[#2]{-135}{-135.0}{\frac{-3\pi}{4}}% + %les pi/5 + \TestArgumentComplexe[#2]{36}{36.0}{\frac{\pi}{5}}% + \TestArgumentComplexe[#2]{72}{72.0}{\frac{2\pi}{5}}% + \TestArgumentComplexe[#2]{108}{108.0}{\frac{3\pi}{5}}% + \TestArgumentComplexe[#2]{144}{144.0}{\frac{4\pi}{5}}% + \TestArgumentComplexe[#2]{-36}{-36.0}{\frac{-\pi}{5}}% + \TestArgumentComplexe[#2]{-72}{-72.0}{\frac{-2\pi}{5}}% + \TestArgumentComplexe[#2]{-108}{-108.0}{\frac{-3\pi}{5}}% + \TestArgumentComplexe[#2]{-144}{-144.0}{\frac{-4\pi}{5}}% + %les pi/6 + \TestArgumentComplexe[#2]{30}{30.0}{\frac{\pi}{6}}% + \TestArgumentComplexe[#2]{150}{150.0}{\frac{5\pi}{6}}% + \TestArgumentComplexe[#2]{-30}{-30.0}{\frac{-\pi}{6}}% + \TestArgumentComplexe[#2]{-150}{-150.0}{\frac{-5\pi}{6}}% + %les pi/8 + \TestArgumentComplexe[#2]{22.5}{22.5}{\frac{\pi}{8}}% + \TestArgumentComplexe[#2]{67.5}{67.5}{\frac{3\pi}{8}}% + \TestArgumentComplexe[#2]{112.5}{112.5}{\frac{5\pi}{8}}% + \TestArgumentComplexe[#2]{157.5}{157.5}{\frac{7\pi}{8}}% + \TestArgumentComplexe[#2]{-22.5}{-22.5}{\frac{-\pi}{8}}% + \TestArgumentComplexe[#2]{-67.5}{-67.5}{\frac{-3\pi}{8}}% + \TestArgumentComplexe[#2]{-112.5}{-112.5}{\frac{-5\pi}{8}}% + \TestArgumentComplexe[#2]{-157.5}{-157.5}{\frac{-7\pi}{8}}% + %les pi/12 + \TestArgumentComplexe[#2]{15}{15.0}{\frac{\pi}{12}}% + \TestArgumentComplexe[#2]{75}{75.0}{\frac{5\pi}{12}}% + \TestArgumentComplexe[#2]{105}{105.0}{\frac{7\pi}{12}}% + \TestArgumentComplexe[#2]{165}{165.0}{\frac{11\pi}{12}}% + \TestArgumentComplexe[#2]{-15}{-15.0}{\frac{-\pi}{12}}% + \TestArgumentComplexe[#2]{-75}{-75.0}{\frac{-5\pi}{12}}% + \TestArgumentComplexe[#2]{-105}{-105.0}{\frac{-7\pi}{12}}% + \TestArgumentComplexe[#2]{-165}{-165.0}{\frac{-11\pi}{12}}% + %les pi/10 + \TestArgumentComplexe[#2]{18}{18.0}{\frac{\pi}{10}}% + \TestArgumentComplexe[#2]{54}{54.0}{\frac{3\pi}{10}}% + \TestArgumentComplexe[#2]{126}{126.0}{\frac{7\pi}{10}}% + \TestArgumentComplexe[#2]{162}{162.0}{\frac{9\pi}{10}}% + \TestArgumentComplexe[#2]{-18}{-18.0}{\frac{-\pi}{10}}% + \TestArgumentComplexe[#2]{-54}{-54.0}{\frac{-3\pi}{10}}% + \TestArgumentComplexe[#2]{-126}{-126.0}{\frac{-7\pi}{10}}% + \TestArgumentComplexe[#2]{-162}{-162.0}{\frac{-9\pi}{10}}% + }% +} + +\NewDocumentCommand\CalculFormeExpoCplx{ s O{} m m }{% + \xdef\TmpArg{\xintfloateval{trunc(Argd(#3,#4),1)}}%\TmpArg% + \ensuremath{% + \xintifboolexpr{(#3)**2+(#4)**2 == 1 'and' \TmpArg == 0}{1}{}% + \xintifboolexpr{(#3)**2+(#4)**2 == 1 'and' \TmpArg != 0}{}{\CalculModuleCplx[#2]{#3}{#4}}% + \IfBooleanTF{#1}% + {% + %les pi/2 + \TestArgumentComplexe[#2]{0}{0.0}{}% + \TestArgumentComplexe[#2]{90}{90.0}{\e^{\frac{\i\pi}{2}}}% + \TestArgumentComplexe[#2]{-90}{-90.0}{\e^{\frac{3\i\pi}{2}}}% + \TestArgumentComplexe[#2]{180}{180.0}{\e^{\i\pi}}% + %les pi/3 + \TestArgumentComplexe[#2]{60}{60.0}{\e^{\frac{\i\pi}{3}}}% + \TestArgumentComplexe[#2]{120}{120.0}{\e^{\frac{2\i\pi}{3}}}% + \TestArgumentComplexe[#2]{-60}{-60.0}{\e^{\frac{5\i\pi}{3}}}% + \TestArgumentComplexe[#2]{-120}{-120.0}{\e^{\frac{4\i\pi}{3}}}% + %les pi/4 + \TestArgumentComplexe[#2]{45}{45.0}{\e^{\frac{\i\pi}{4}}}% + \TestArgumentComplexe[#2]{135}{135.0}{\e^{\frac{3\i\pi}{4}}}% + \TestArgumentComplexe[#2]{-45}{-45.0}{\e^{\frac{7\i\pi}{4}}}% + \TestArgumentComplexe[#2]{-135}{-135.0}{\e^{\frac{5\i\pi}{4}}}% + %les pi/5 + \TestArgumentComplexe[#2]{36}{36.0}{\e^{\frac{\i\pi}{5}}}% + \TestArgumentComplexe[#2]{72}{72.0}{\e^{\frac{2\i\pi}{5}}}% + \TestArgumentComplexe[#2]{108}{108.0}{\e^{\frac{3\i\pi}{5}}}% + \TestArgumentComplexe[#2]{144}{144.0}{\e^{\frac{4\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-36}{-36.0}{\e^{\frac{9\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-72}{-72.0}{\e^{\frac{8\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-108}{-108.0}{\e^{\frac{7\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-144}{-144.0}{\e^{\frac{6\i\pi}{5}}}% + %les pi/6 + \TestArgumentComplexe[#2]{30}{30.0}{\e^{\frac{\i\pi}{6}}}% + \TestArgumentComplexe[#2]{150}{150.0}{\e^{\frac{5\i\pi}{6}}}% + \TestArgumentComplexe[#2]{-30}{-30.0}{\e^{\frac{11\i\pi}{6}}}% + \TestArgumentComplexe[#2]{-150}{-150.0}{\e^{\frac{7\i\pi}{6}}}% + %les pi/8 + \TestArgumentComplexe[#2]{22.5}{22.5}{\e^{\frac{\i\pi}{8}}}% + \TestArgumentComplexe[#2]{67.5}{67.5}{\e^{\frac{3\i\pi}{8}}}% + \TestArgumentComplexe[#2]{112.5}{112.5}{\e^{\frac{5\i\pi}{8}}}% + \TestArgumentComplexe[#2]{157.5}{157.5}{\e^{\frac{7\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-22.5}{-22.5}{\e^{\frac{15\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-67.5}{-67.5}{\e^{\frac{13\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-112.5}{-112.5}{\e^{\frac{11\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-157.5}{-157.5}{\e^{\frac{9\i\pi}{8}}}% + %les pi/12 + \TestArgumentComplexe[#2]{15}{15.0}{\e^{\frac{\i\pi}{12}}}% + \TestArgumentComplexe[#2]{75}{75.0}{\e^{\frac{5\i\pi}{12}}}% + \TestArgumentComplexe[#2]{105}{105.0}{\e^{\frac{7\i\pi}{12}}}% + \TestArgumentComplexe[#2]{165}{165.0}{\e^{\frac{11\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-15}{-15.0}{\e^{\frac{23\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-75}{-75.0}{\e^{\frac{19\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-105}{-105.0}{\e^{\frac{17\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-165}{-165.0}{\e^{\frac{13\i\pi}{12}}}% + %les pi/10 + \TestArgumentComplexe[#2]{18}{18.0}{\e^{\frac{\i\pi}{10}}}% + \TestArgumentComplexe[#2]{54}{54.0}{\e^{\frac{3\i\pi}{10}}}% + \TestArgumentComplexe[#2]{126}{126.0}{\e^{\frac{7\i\pi}{10}}}% + \TestArgumentComplexe[#2]{162}{162.0}{\e^{\frac{9\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-18}{-18.0}{\e^{\frac{19\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-54}{-54.0}{\e^{\frac{17\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-126}{-126.0}{\e^{\frac{13\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-162}{-162.0}{\e^{\frac{11\i\pi}{10}}}% + }% + {% + %les pi/2 + \TestArgumentComplexe[#2]{0}{0.0}{}% + \TestArgumentComplexe[#2]{90}{90.0}{\e^{\frac{\i\pi}{2}}}% + \TestArgumentComplexe[#2]{-90}{-90.0}{\e^{\frac{-\i\pi}{2}}}% + \TestArgumentComplexe[#2]{180}{180.0}{\e^{\i\pi}}% + %les pi/3 + \TestArgumentComplexe[#2]{60}{60.0}{\e^{\frac{\i\pi}{3}}}% + \TestArgumentComplexe[#2]{120}{120.0}{\e^{\frac{2\i\pi}{3}}}% + \TestArgumentComplexe[#2]{-60}{-60.0}{\e^{\frac{-\i\pi}{3}}}% + \TestArgumentComplexe[#2]{-120}{-120.0}{\e^{\frac{-2\i\pi}{3}}}% + %les pi/4 + \TestArgumentComplexe[#2]{45}{45.0}{\e^{\frac{\i\pi}{4}}}% + \TestArgumentComplexe[#2]{135}{135.0}{\e^{\frac{3\i\pi}{4}}}% + \TestArgumentComplexe[#2]{-45}{-45.0}{\e^{\frac{-\i\pi}{4}}}% + \TestArgumentComplexe[#2]{-135}{-135.0}{\e^{\frac{-3\i\pi}{4}}}% + %les pi/5 + \TestArgumentComplexe[#2]{36}{36.0}{\e^{\frac{\i\pi}{5}}}% + \TestArgumentComplexe[#2]{72}{72.0}{\e^{\frac{2\i\pi}{5}}}% + \TestArgumentComplexe[#2]{108}{108.0}{\e^{\frac{3\i\pi}{5}}}% + \TestArgumentComplexe[#2]{144}{144.0}{\e^{\frac{4\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-36}{-36.0}{\e^{\frac{-\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-72}{-72.0}{\e^{\frac{-2\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-108}{-108.0}{\e^{\frac{-3\i\pi}{5}}}% + \TestArgumentComplexe[#2]{-144}{-144.0}{\e^{\frac{-4\i\pi}{5}}}% + %les pi/6 + \TestArgumentComplexe[#2]{30}{30.0}{\e^{\frac{\i\pi}{6}}}% + \TestArgumentComplexe[#2]{150}{150.0}{\e^{\frac{5\i\pi}{6}}}% + \TestArgumentComplexe[#2]{-30}{-30.0}{\e^{\frac{-\i\pi}{6}}}% + \TestArgumentComplexe[#2]{-150}{-150.0}{\e^{\frac{-5\i\pi}{6}}}% + %les pi/8 + \TestArgumentComplexe[#2]{22.5}{22.5}{\e^{\frac{\i\pi}{8}}}% + \TestArgumentComplexe[#2]{67.5}{67.5}{\e^{\frac{3\i\pi}{8}}}% + \TestArgumentComplexe[#2]{112.5}{112.5}{\e^{\frac{5\i\pi}{8}}}% + \TestArgumentComplexe[#2]{157.5}{157.5}{\e^{\frac{7\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-22.5}{-22.5}{\e^{\frac{-\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-67.5}{-67.5}{\e^{\frac{-3\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-112.5}{-112.5}{\e^{\frac{-5\i\pi}{8}}}% + \TestArgumentComplexe[#2]{-157.5}{-157.5}{\e^{\frac{-7\i\pi}{8}}}% + %les pi/12 + \TestArgumentComplexe[#2]{15}{15.0}{\e^{\frac{\i\pi}{12}}}% + \TestArgumentComplexe[#2]{75}{75.0}{\e^{\frac{5\i\pi}{12}}}% + \TestArgumentComplexe[#2]{105}{105.0}{\e^{\frac{7\i\pi}{12}}}% + \TestArgumentComplexe[#2]{165}{165.0}{\e^{\frac{11\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-15}{-15.0}{\e^{\frac{-\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-75}{-75.0}{\e^{\frac{-5\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-105}{-105.0}{\e^{\frac{-7\i\pi}{12}}}% + \TestArgumentComplexe[#2]{-165}{-165.0}{\e^{\frac{-11\i\pi}{12}}}% + %les pi/10 + \TestArgumentComplexe[#2]{18}{18.0}{\e^{\frac{\i\pi}{10}}}% + \TestArgumentComplexe[#2]{54}{54.0}{\e^{\frac{3\i\pi}{10}}}% + \TestArgumentComplexe[#2]{126}{126.0}{\e^{\frac{7\i\pi}{10}}}% + \TestArgumentComplexe[#2]{162}{162.0}{\e^{\frac{9\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-18}{-18.0}{\e^{\frac{-\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-54}{-54.0}{\e^{\frac{-3\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-126}{-126.0}{\e^{\frac{-7\i\pi}{10}}}% + \TestArgumentComplexe[#2]{-162}{-162.0}{\e^{\frac{-9\i\pi}{10}}}% + }% + }% +} + +\endinput \ No newline at end of file -- cgit v1.2.3