diff options
Diffstat (limited to 'Master/texmf-dist/source')
| -rw-r--r-- | Master/texmf-dist/source/lualatex/piton/piton.dtx | 1712 |
1 files changed, 1187 insertions, 525 deletions
diff --git a/Master/texmf-dist/source/lualatex/piton/piton.dtx b/Master/texmf-dist/source/lualatex/piton/piton.dtx index 7d7a94fa1d4..c381b9eb27c 100644 --- a/Master/texmf-dist/source/lualatex/piton/piton.dtx +++ b/Master/texmf-dist/source/lualatex/piton/piton.dtx @@ -15,8 +15,8 @@ % % \fi % \iffalse -\def\myfileversion{0.8} -\def\myfiledate{2022/10/23} +\def\myfileversion{0.9} +\def\myfiledate{2022/10/31} % % %<*batchfile> @@ -58,6 +58,7 @@ version 2005/12/01 or later. \usepackage{ragged2e} \usepackage[footnotehyper,escape-inside=$$]{piton} % $$ + \def\interitem{\vskip 7mm plus 2 mm minus 3mm} \parindent 0pt \skip\footins = 2\bigskipamount @@ -72,6 +73,9 @@ version 2005/12/01 or later. %</driver> % \fi +% +% \catcode`\" = 11 +% % \title{The package \pkg{piton}\thanks{This document corresponds to the % version~\myfileversion\space of \pkg{piton}, at the date of~\myfiledate.}} % \author{F. Pantigny \\ \texttt{fpantigny@wanadoo.fr}} @@ -84,6 +88,7 @@ version 2005/12/01 or later. % \end{abstract} % % +% % \section{Presentation} % % @@ -105,8 +110,10 @@ version 2005/12/01 or later. % from math import pi % % def arctan(x,n=10): -% '''Compute the value of arctan(x) -% n is the number of terms of the sum''' +% """Compute the mathematical value of arctan(x) +% +% n is the number of terms in the sum +% """ % if x < 0: % return -arctan(-x) # recursive call % elif x > 1: @@ -179,47 +186,62 @@ version 2005/12/01 or later. % \begin{itemize} % \item The key |gobble| takes in as value a positive integer~$n$: the first $n$ % characters are discarded (before the process of highlightning of the code) for -% each line of the environment |{Piton}|. +% each line of the environment |{Piton}|. These characters are not necessarily +% spaces. % -% \item Then the key |auto-gobble| is in force, the extension \pkg{piton} computes -% the minimal value $n$ of the number of consecutives space beginning each (non +% \item When the key |auto-gobble| is in force, the extension \pkg{piton} computes +% the minimal value $n$ of the number of consecutive spaces beginning each (non % empty) line of the environment |{Piton}| and applies |gobble| with that value % of~$n$. % -% \item When the key |env-gobble| is in force, \pkg{piton} applies |gobble| with -% a value of $n$ equal to the number of spaces before |\end{Piton}| on the last -% line (if that line contains only spaces). The name of that key comes from -% \emph{environment gobble}: the effect of gobble is set by the position of the -% commands |\begin{Piton}| and |\end{Piton}| which delimit the current environment. -% -% \item With the key |line-numbers|, the \emph{non empty} lines are numbered in -% the environments |{Piton}| and in the listings resulting from the use of -% |\PitonInputFile|. +% \item When the key |env-gobble| is in force, \pkg{piton} analyzes the last +% line of the environment |{Piton}|, that is to say the line which contains +% |\end{Piton}| and determines whether that line contains only spaces followed +% by the |\end{Piton}|. If we are in that situation, \pkg{piton} computes the +% number~$n$ of spaces on that line and applies |gobble| with that value of~$n$. +% The name of that key comes from \emph{environment gobble}: the effect of +% gobble is set by the position of the commands |\begin{Piton}| and +% |\end{Piton}| which delimit the current environment. +% +% \item With the key |line-numbers|, the \emph{non empty} lines are numbered (on +% the left) in the environments |{Piton}| and in the listings resulting from the +% use of |\PitonInputFile|. % % \item With the key |all-line-numbers|, \emph{all} the lines are numbered, % including the empty ones. % -% \item By default, the counter of lines is set to zero at the beginning of each -% environment |{Piton}| or use of |\PitonInputFile|. With the key |resume|, that -% reinitialisation is not done. +% \item With the key |resume| the counter of lines is not set to zero at the +% beginning of each environment |{Piton}| or use |\PitonInputFile| as it is +% otherwise. That's allows a numbering of the lines accross several +% environments. See an example part \ref{example-numbering} on +% page~\pageref{example-numbering}. +% +% \item The key |left-margin| corresponds to a margin on the left. That key may +% be useful in conjonction with the key |line-numbers| or the key +% |line-all-numbers| if one does not want the numbers in an overlapping position +% on the left. % % \item The key |splittable| allows pages breaks within the environments % |{Piton}| and the listings produced by % |\PitonInputFile|. % +% \smallskip +% \colorbox{yellow!50}{\textbf{New 0.9}}\enskip It's possible to give as value to +% the key |splittable| a positive integer~$n$. With that value, the environments +% |{Piton}| and the listings produced by |\PitonInputFile| are splittable but no +% page break can occur within the first $n$ lines and within the last $n$ lines. +% The default value of the key |splittable| is, in fact, $1$, which allows pages +% breaks everywhere. +% % \item The key |background-color| sets the background color of the environments % |{Piton}| and the listings produced by |\PitonInputFile| (that background has % a width of |\linewidth|). Even with a background color, the pages breaks are -% allowed, as soon as the key |splittable| is in force.\footnote{The -% environments \texttt{\{Piton\}} are breakable, even within a breakable -% environment of \pkg{tcolorbox}. Remind that an environment of \pkg{tcolorbox} -% included in another environment of \pkg{tcolorbox} is \emph{not} breakable, -% even when both environments use the key |breakable| of \pkg{tcolorbox}.} -% -% \item \colorbox{yellow!50}{\textbf{New 0.8}}\enskip The key |left-margin| -% corresponds to a margin on the left. That key may be useful in conjonction -% with the key |line-numbers| or the key |line-all-numbers| if one does not want -% the numbers in an overlapping position on the left. +% allowed, as soon as the key |splittable| is in force.\footnote{With the key +% |splittable|, the environments \texttt{\{Piton\}} are breakable, even within a +% (breakable) environment of \pkg{tcolorbox}. Remind that an environment of +% \pkg{tcolorbox} included in another environment of \pkg{tcolorbox} is +% \emph{not} breakable, even when both environments use the key |breakable| of +% \pkg{tcolorbox}.} % \end{itemize} % % \bigskip @@ -230,10 +252,11 @@ version 2005/12/01 or later. % ~emphase&\PitonOptions{line-numbers,auto-gobble,background-color = gray!15}@ % \begin{Piton} % from math import pi -% % def arctan(x,n=10): -% '''Compute the value of arctan(x) -% n is the number of terms of the sum''' +% """Compute the mathematical value of arctan(x) +% +% n is the number of terms in the sum +% """ % if x < 0: % return -arctan(-x) # recursive call % elif x > 1: @@ -254,8 +277,10 @@ version 2005/12/01 or later. % from math import pi % % def arctan(x,n=10): -% '''Compute the value of arctan(x) -% n is the number of terms of the sum''' +% """Compute the mathematical value of arctan(x) +% +% n is the number of terms in the sum +% """ % if x < 0: % return -arctan(-x) # recursive call % elif x > 1: @@ -323,7 +348,7 @@ version 2005/12/01 or later. % The package \pkg{piton} provides the command |\SetPitonStyle| to customize the % different styles used to format the syntactic elements of the Python listings. % The customizations done by that command are limited to the current TeX -% group.\footnote{We remind that an LaTeX environment is, in particuilar, a TeX group.} +% group.\footnote{We remind that an LaTeX environment is, in particular, a TeX group.} % % \bigskip % The command |\SetPitonStyle| takes in as argument a comma-separated list of @@ -379,25 +404,27 @@ version 2005/12/01 or later. % String.Long & the long strings (between \texttt{'''} or \verb|"""|) except the % documentation strings \\ % String & that keys sets both |String.Short| and |String.Long| \\ -% String.Doc & the documentation strings \\ +% String.Doc & the documentation strings (only between \texttt{"""} following +% PEP~257) \\ % String.Interpol & the syntactic elements of the fields of the f-strings (that -% is to say the characters \texttt{\{}, \texttt{\}} and \texttt{:}) \\ +% is to say the characters \texttt{\{} and \texttt{\}}) \\ % Operator & the following operators : \texttt{!= == << >> - \~{} + / * \% = < > \& .} \verb+|+ \verb|@|\\ % Operator.Word & the following operators : \texttt{in}, \texttt{is}, % \texttt{and}, \texttt{or} and \texttt{not} \\ % Name.Builtin & the predefined functions of Python \\ % Name.Function & the name of the functions defined by the user, at the point of % their definition (that is to say after the keyword |def|) \\ -% Name.Decorator & the decorators (instructions beginning by \verb|@| in the classes) \\ +% Name.Decorator & the decorators (instructions beginning by \verb|@|) \\ % Name.Namespace & the name of the modules (= external libraries) \\ -% Name.Class & the name of the classes at the point of their definition \\ +% Name.Class & the name of the classes at the point of their definition (that is +% to say after the keyword |class|) \\ % Exception & the names of the exceptions (eg: \texttt{SyntaxError}) \\ % Comment & the comments beginning with \verb|#| \\ -% Comment.LaTeX & the comments beginning by \verb|##| which are composed in LaTeX by -% \pkg{piton} (and called merely ``LaTeX comments'' in this document) \\ +% Comment.LaTeX & the comments beginning by \verb|##|, which are composed in LaTeX by +% \pkg{piton} (and simply called ``LaTeX comments'' in this document) \\ % Keyword.Constant & |True|, |False| and |None| \\ % Keyword & the following keywords : -% \ttfamily assert, break, case, continue, del, +% \ttfamily as, assert, break, case, continue, def, del, % elif, else, except, exec, finally, for, from, % global, if, import, lambda, non local, % pass, raise, return, try, while, @@ -458,8 +485,8 @@ version 2005/12/01 or later. % pair |\footnotemark|--|\footnotetext|. % % \smallskip -% It's also possible to extract the footnotes with the help of the package -% \pkg{footnote} or the package \pkg{footnotehyper}. +% However, it's also possible to extract the footnotes with the help of the +% package \pkg{footnote} or the package \pkg{footnotehyper}. % % \smallskip % If \pkg{piton} is loaded with the option |footnote| (with @@ -479,17 +506,30 @@ version 2005/12/01 or later. % in particular: it must be loaded after the package \pkg{xcolor} and it is not % perfectly compatible with \pkg{hyperref}. % -% \medskip -% In this document, the package \pkg{piton} has been loaded with the option -% |footnotehyper|: see the first page of this document for an example of a -% footnote in an environment |{Piton}|. +% \medskip +% In this document, the package \pkg{piton} has been loaded with the +% option |footnotehyper|. For examples of notes, cf. \ref{notes-examples}, +% p.~\pageref{notes-examples}. % +% \subsection{Tabulations} % -% \section{Examples} +% \smallskip +% \colorbox{yellow!50}{\textbf{New 0.9}}\enskip Even though it's recommended to +% indent the Python listings with spaces (see PEP~8), \pkg{piton} accepts the +% characters of tabulation (that is to say the characters U+0009) at the +% beginning of the lines. Each character U+0009 is replaced by $n$~spaces. The +% initial value of $n$ is $4$ but it's possible to change it with the key +% |tab-size| of |\PitonOptions|. % +% \section{Examples} % % \subsection{Line numbering} % +% \label{example-numbering} +% +% We remind that it's possible to have an automatic numbering of the lines in +% the Python listings by using the key |line-numbers| or the key |all-line-numbers|. +% % By default, the numbers of the lines are composed by \pkg{piton} in an % overlapping position on the left (by using internally the command |\llap| of LaTeX). % @@ -625,8 +665,10 @@ version 2005/12/01 or later. % % \subsection{Notes in the listings} % +% \label{notes-examples} +% % In order to be able to extract the notes (which are typeset with the command -% |\footnote|, the extension \pkg{piton} must be loaded with the key |footnote| +% |\footnote|), the extension \pkg{piton} must be loaded with the key |footnote| % or the key |footenotehyper| as explained in the section \ref{footnote} % p.~\pageref{footnote}. In this document, the extension \pkg{piton} has been % loaded with the key |footnotehyper|. @@ -709,7 +751,7 @@ version 2005/12/01 or later. % % % \vspace{1cm} -% If one embed an environment |{Piton}| in an environment |{minipage}| +% If we embed an environment |{Piton}| in an environment |{minipage}| % (typically in order to limit the width of a colored background), it's % necessary to embed the whole environment |{minipage}| in an environment % |{savenotes}| (of \pkg{footnote} or \pkg{footnotehyper}) in order to have the @@ -778,7 +820,7 @@ version 2005/12/01 or later. % Name.Builtin = , % Name.Function = \bfseries \colorbox{gray!20} , % Comment = \color{gray} , -% Comment.LaTeX = \color{gray}, +% Comment.LaTeX = \normalfont \color{gray}, % Keyword = \bfseries , % Name.Namespace = , % Name.Class = , @@ -792,19 +834,19 @@ version 2005/12/01 or later. % % \setmonofont[Scale=0.85]{DejaVu Sans Mono} % -% \PitonOptions{splittable} +% \PitonOptions{splittable} % % \SetPitonStyle % { % Number = , % String = \itshape , % String.Doc = \color{gray} \slshape , -% Operator = , % Operator.Word = \bfseries , +% Operator = , % Name.Builtin = , % Name.Function = \bfseries \colorbox{gray!20} , % Comment = \color{gray} , -% Comment.LaTeX = \color{gray} , +% Comment.LaTeX = \normalfont \color{gray} , % Keyword = \bfseries , % Name.Namespace = , % Name.Class = , @@ -819,8 +861,10 @@ version 2005/12/01 or later. % from math import pi % % def arctan(x,n=10): -% '''Compute the value of arctan(x) -% n is the number of terms if the sum''' +% """Compute the mathematical value of arctan(x) +% +% n is the number of terms in the sum +% """ % if x < 0: % return -arctan(-x) # appel récursif % elif x > 1: @@ -837,12 +881,137 @@ version 2005/12/01 or later. % % % \bigskip -% The previous example has been composed while the key |splittable| (of -% |\PitonOptions|) is in force. That's why a page break may have occurred. % \clearpage % % \section{Implementation} % +% \subsection{Introduction} +% +% The main job of the package \pkg{piton} is to take in as input a Python +% listing and to send back to LaTeX as output that code \emph{with interlaced LaTeX +% instructions of formatting}. +% +% In fact, all that job is done by a \textsc{lpeg} called |SyntaxPython|. That +% \textsc{lpeg}, when matched against the string of a Python listing, +% returns as capture a Lua table containing data to send to LaTeX. +% The only thing to do after will be to apply |tex.tprint| to each element of +% that table.\footnote{Recall that |tex.tprint| takes in as argument a Lua table whose +% first component is a ``catcode table'' and the second element a string. The +% string will be sent to LaTeX with the regime of catcodes specified by the +% catcode table. If no catcode table is provided, the standard catcodes of LaTeX +% will be used.} +% +% \bigskip +% Consider, for example, the following Python code: +% +% \begin{Piton} +% def parity(x): +% return x%2 +% \end{Piton} +% +% The capture returned by the \pkg{lpeg} |SyntaxPython| against that code is the +% Lua table containing the following elements : +% +% \bigskip +% \begin{minipage}{\linewidth} +% \color{gray} +% +% |{ "\\__piton_begin_line:" }|\footnote{Each line of the Python listings will +% be encapsulated in a pair: \texttt{\textbackslash_@@_begin_line:} -- +% \texttt{\textbackslash@@_end_line:}. The token +% \texttt{\textbackslash@@_end_line:} must be explicit because it will be used as +% marker in order to delimit the argument of the command \texttt{\textbackslash +% @@\_begin\_line:}. Both tokens \texttt{\textbackslash_@@_begin_line:} and +% \texttt{\textbackslash@@_end_line:} will be nullified in the command +% \texttt{\textbackslash piton} (since there can't be lines breaks in the +% argument of a command \texttt{\textbackslash piton}).} +% +% \texttt{\{ "\{\textbackslash PitonStyle\{Keyword\}\{" \}}\footnote{The +% lexical elements of Python for which we have a \pkg{piton} style will be +% formatted via the use of the command \texttt{\textbackslash PitonStyle}. +% Such an element is typeset in LaTeX via the syntax \texttt{\{\textbackslash +% PitonStyle\{\textsl{style}\}\{...\}\}} because the instructions inside an \texttt{\textbackslash +% PitonStyle} may be both semi-global declarations like +% \texttt{\textbackslash bfseries} and commands with one argument like +% \texttt{\textbackslash fbox}.} +% +% \texttt{\{ +% luatexbase.catcodetables.CatcodeTableOther\footnote{\texttt{luatexbase.catcodetables.CatcodeTableOther} is a mere number which corresponds to the ``catcode table'' whose all characters have the catcode ``other'' (which means that they will be typeset by LaTeX verbatim).}, "def" \} } +% +% |{ "}}" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, " " }| +% +% |{ "{\PitonStyle{Name.Function}{" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, "parity" }| +% +% |{ "}}" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, "(" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, "x" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, ")" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, ":" }| +% +% |{ "\\__piton_end_line: \\__piton_newline: \\__piton_begin_line:" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, " " }| +% +% |{ "{\PitonStyle{Keyword}{" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, "return" }| +% +% |{ "}}" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, " " }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, "x" }| +% +% |{ "{\PitonStyle{Operator}{" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, "&" }| +% +% |{ "}}" }| +% +% |{ "{\PitonStyle{Number}{" }| +% +% |{ luatexbase.catcodetables.CatcodeTableOther, "2" }| +% +% |{ "}}" }| +% +% |{ "\\__piton_end_line:" }| +% +% \end{minipage} +% +% \bigskip +% We give now the LaTeX code which is sent back by Lua to TeX (we have written +% on several lines for legibility but no character |\r| will be sent to LaTeX). The +% characters which are greyed-out are sent to LaTeX with the catcode ``other'' +% (=12). All the others characters are sent with the regime of catcodes of L3 +% (as set by |\ExplSyntaxOn|) +% +% +% \begingroup +% \def\gbox#1{\colorbox{gray!20}{\strut #1}} +% \setlength{\fboxsep}{1pt} +% +% \begin{Verbatim*}[formatcom = \color{black}] +% \__piton_begin_line:{\PitonStyle{Keyword}{~gbox#def@}} +% ~gbox# @{\PitonStyle{Name.Function}{~gbox#parity@}}~gbox#(x):@\__piton_end_line:\__piton_newline: +% \__piton_begin_line:~gbox# @{\PitonStyle{Keyword}{~gbox#return@}} +% ~gbox# x@{\PitonStyle{Operator}{~gbox#%@}}{\PitonStyle{Number}{~gbox#2@}}\__piton_end_line: +% \end{Verbatim*} +% \endgroup +% +% +% +% +% \subsection{The L3 part of the implementation} +% +% \subsubsection{Declaration of the package} % \begin{macrocode} \NeedsTeXFormat{LaTeX2e} \RequirePackage{l3keys2e} @@ -1010,9 +1179,23 @@ version 2005/12/01 or later. % |\c_@@_footnote_bool| in order to know if we have to insert an environment % |{savenotes}|. % -% \medskip -% \subsection{Parameters} +% \bigskip +% \subsubsection{Parameters and technical definitions} % +% We will compute (with Lua) the numbers of lines of the Python code and store +% it in the following counter. +% \begin{macrocode} +\int_new:N \l_@@_nb_lines_int +% \end{macrocode} +% +% The following counter will be used to count the lines during the composition. +% It will count all the lines, empty or not empty. It won't be used to print the +% numbers of the lines. +% \begin{macrocode} +\int_new:N \g_@@_line_int +% \end{macrocode} +% +% \medskip % The following token list will contains the (potential) informations to write % on the |aux| (to be used in the next compilation). % \begin{macrocode} @@ -1020,9 +1203,18 @@ version 2005/12/01 or later. % \end{macrocode} % % \medskip -% The following flag corresponds to the key |splittable| of |\PitonOptions|. +% The following counter corresponds to the key |splittable| of |\PitonOptions|. +% If the value of |\l_@@_splittable_int| is equal to $n$, then no line break can +% occur within the first $n$~lines or the last $n$~lines of the listings. % \begin{macrocode} -\bool_new:N \l_@@_splittable_bool +\int_new:N \l_@@_splittable_int +% \end{macrocode} +% +% \medskip +% An initial value of |splittable| equal to 100 is equivalent to say that the +% environments |{Piton}| are unbreakable. +% \begin{macrocode} +\int_set:Nn \l_@@_splittable_int { 100 } % \end{macrocode} % % \medskip @@ -1033,9 +1225,12 @@ version 2005/12/01 or later. % % \medskip % We will compute the maximal width of the lines of an environment |{Piton}| in -% |\l_@@_width_dim|. +% |\g_@@_width_dim|. We need a global variable because when the key |footnote| +% is in force, each line when be composed in an environment |{savenotes}| and +% (when |slim| is in force) we need to exit |\g_@@_width_dim| from that +% environment. % \begin{macrocode} -\dim_new:N \l_@@_width_dim +\dim_new:N \g_@@_width_dim % \end{macrocode} % The value of that dimension as written on the |aux| file will be stored in % |\l_@@_width_on_aux_dim|. @@ -1061,24 +1256,52 @@ version 2005/12/01 or later. % \begin{macrocode} \dim_new:N \l_@@_left_margin_dim % \end{macrocode} -% -% \subsection{The gobbling mechanism} % % \medskip -% The following integer is the number of characters to gobble on the left side -% of the Python listings. Of course, the initial value is $0$. +% The tabulators will be replaced by the content of the following token list. % \begin{macrocode} -\int_new:N \l_@@_gobble_int +\tl_new:N \l_@@_tab_tl % \end{macrocode} % % \medskip % \begin{macrocode} -\cs_new_protected:Npn \@@_define_gobble_syntax:n #1 - { \lua_now:n { define_gobble_syntax(#1) } } +\cs_new_protected:Npn \@@_set_tab_tl:n #1 + { + \tl_clear:N \l_@@_tab_tl + \prg_replicate:nn { #1 } + { \tl_put_right:Nn \l_@@_tab_tl { ~ } } + } +\@@_set_tab_tl:n { 4 } % \end{macrocode} -% +% +% \medskip +% \begin{macrocode} +\cs_new_protected:Npn \@@_newline: + { + \int_gincr:N \g_@@_line_int + \int_compare:nNnT \g_@@_line_int > { \l_@@_splittable_int - 1 } + { + \int_compare:nNnT + { \l_@@_nb_lines_int - \g_@@_line_int } > \l_@@_splittable_int + { + \egroup + \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } + \newline + \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } + \vtop \bgroup + } + } + } +% \end{macrocode} +% % \medskip -% \subsection{Treatment of a line of code} +% The following integer corresponds to the key |gobble|. +% \begin{macrocode} +\int_new:N \l_@@_gobble_int +% \end{macrocode} +% +% \bigskip +% \subsubsection{Treatment of a line of code} % % In the contents provided by Lua, each line of the Python code will be % surrounded by |\@@_begin_line:| and |\@@_end_line:|. @@ -1086,8 +1309,6 @@ version 2005/12/01 or later. % \begin{macrocode} \cs_set_protected:Npn \@@_begin_line: #1 \@@_end_line: { - \bool_lazy_and:nnT \l_@@_splittable_bool \c_@@_footnote_bool - { \begin { savenotes } } % \end{macrocode} % % Be careful: there is curryfication in the following lines. @@ -1115,11 +1336,11 @@ version 2005/12/01 or later. #1 \hfil } % \end{macrocode} -% We compute in |\l_@@_width_dim| the maximal width of the lines of the +% We compute in |\g_@@_width_dim| the maximal width of the lines of the % environments. % \begin{macrocode} - \dim_compare:nNnT { \box_wd:N \l_tmpa_box } > \l_@@_width_dim - { \dim_set:Nn \l_@@_width_dim { \box_wd:N \l_tmpa_box } } + \dim_compare:nNnT { \box_wd:N \l_tmpa_box } > \g_@@_width_dim + { \dim_gset:Nn \g_@@_width_dim { \box_wd:N \l_tmpa_box } } \box_set_dp:Nn \l_tmpa_box { \box_dp:N \l_tmpa_box + 1.25 pt } \box_set_ht:Nn \l_tmpa_box { \box_ht:N \l_tmpa_box + 1.25 pt } \tl_if_empty:NTF \l_@@_background_color_str @@ -1139,15 +1360,13 @@ version 2005/12/01 or later. \box_use_drop:N \l_tmpa_box } } - \bool_lazy_and:nnT \l_@@_splittable_bool \c_@@_footnote_bool - { \end { savenotes } } \vspace { - 2.5 pt } } % \end{macrocode} % % -% \medskip -% \subsection{PitonOptions} +% \bigskip +% \subsubsection{PitonOptions} % % \medskip % The following parameters correspond to the keys |line-numbers| and @@ -1184,19 +1403,22 @@ version 2005/12/01 or later. all-line-numbers .value_forbidden:n = true , resume .bool_set:N = \l_@@_resume_bool , resume .value_forbidden:n = true , - splittable .bool_set:N = \l_@@_splittable_bool , - splittable .default:n = true , + splittable .int_set:N = \l_@@_splittable_int , + splittable .default:n = 1 , background-color .str_set:N = \l_@@_background_color_str , background-color .value_required:n = true , slim .bool_set:N = \l_@@_slim_bool , slim .default:n = true , left-margin .dim_set:N = \l_@@_left_margin_dim , left-margin .value_required:n = true , + tab-size .code:n = \@@_set_tab_tl:n { #1 } , + tab-size .value_required:n = true , unknown .code:n = \msg_error:nn { piton } { Unknown~key~for~PitonOptions } } % \end{macrocode} % +% % \bigskip % \begin{macrocode} \msg_new:nnn { piton } { Unknown~key~for~PitonOptions } @@ -1204,7 +1426,7 @@ version 2005/12/01 or later. Unknown~key. \\ The~key~'\l_keys_key_str'~is~unknown~for~\token_to_str:N \PitonOptions.~The~ available~keys~are:~all-line-numbers,~auto-gobble,~env-gobble,~gobble,~ - left-margin,~line-numbers,~resume,~slim~and~splittable.\\ + left-margin,~line-numbers,~resume,~slim~splittable~and~tab-size.\\ If~you~go~on,~that~key~will~be~ignored. } % \end{macrocode} @@ -1215,32 +1437,33 @@ version 2005/12/01 or later. \NewDocumentCommand \PitonOptions { } { \keys_set:nn { PitonOptions } } % \end{macrocode} % -% \medskip -% \subsection{The numbers of the lines} +% \bigskip +% \subsubsection{The numbers of the lines} % % \medskip % The following counter will be used to count the lines in the code when the -% user requires the numbers of the lines to be printed. +% user requires the numbers of the lines to be printed (with |line-numbers| or +% |all-line-numbers|). % \begin{macrocode} -\int_new:N \g_@@_line_int +\int_new:N \g_@@_visual_line_int % \end{macrocode} % % % \begin{macrocode} \cs_new_protected:Npn \@@_print_number: { - \int_gincr:N \g_@@_line_int + \int_gincr:N \g_@@_visual_line_int \hbox_overlap_left:n { - { \color { gray } \footnotesize \int_to_arabic:n \g_@@_line_int } + { \color { gray } \footnotesize \int_to_arabic:n \g_@@_visual_line_int } \skip_horizontal:n { 0.4 em } } } % \end{macrocode} % % -% \medskip -% \subsection{The command to write on the aux file} +% \bigskip +% \subsubsection{The command to write on the aux file} % % % \begin{macrocode} @@ -1270,15 +1493,15 @@ version 2005/12/01 or later. \tl_gput_right:Nx \g_@@_aux_tl { \dim_set:Nn \l_@@_width_on_aux_dim - { \dim_use:N \l_@@_width_dim + 0.5 em } + { \dim_eval:n { \g_@@_width_dim + 0.5 em } } } } } } % \end{macrocode} % -% \medskip -% \subsection{The main commands and environments for the final user} +% \bigskip +% \subsubsection{The main commands and environments for the final user} % % \begin{macrocode} \NewDocumentCommand { \piton } { v } @@ -1305,29 +1528,48 @@ version 2005/12/01 or later. } % \end{macrocode} % -% % \bigskip +% Despite its name, |\@@_pre_env:| will be used both in |\PitonInputFile| dans +% in the environments such as |{Piton}|. % \begin{macrocode} -\NewDocumentCommand { \PitonInputFile } { m } +\cs_new:Npn \@@_pre_env: { \int_gincr:N \g_@@_env_int \tl_gclear:N \g_@@_aux_tl \tl_if_exist:cT { c_@@ _ \int_use:N \g_@@_env_int _ tl } { \use:c { c_@@ _ \int_use:N \g_@@_env_int _ tl } } - \bool_if:NF \l_@@_splittable_bool - { \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } } \dim_compare:nNnT \l_@@_width_on_aux_dim = \c_zero_dim { \dim_set_eq:NN \l_@@_width_on_aux_dim \linewidth } - \bool_if:NF \l_@@_resume_bool { \int_gzero:N \g_@@_line_int } + \bool_if:NF \l_@@_resume_bool { \int_gzero:N \g_@@_visual_line_int } + \dim_gzero:N \g_@@_width_dim + \int_gzero:N \g_@@_line_int + \dim_zero:N \parindent + \dim_zero:N \lineskip + } +% \end{macrocode} +% +% \bigskip +% \begin{macrocode} +\NewDocumentCommand { \PitonInputFile } { m } + { \group_begin: - \dim_set_eq:NN \parindent \c_zero_dim + \@@_pre_env: + \mode_if_vertical:TF \mode_leave_vertical: \newline +% \end{macrocode} +% We count with Lua the number of lines of the argument. The result will be +% stored by Lua in |\l_@@_nb_lines_int|. That information will be used to allow +% or disallow page breaks. +% \begin{macrocode} + \lua_now:n { CountLinesFile(token.scan_argument()) } { #1 } \ttfamily - \lua_now:e { ParseFile(token.scan_argument()) } { #1 } - \@@_width_to_aux: + \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } + \vtop \bgroup + \lua_now:n { ParseFile(token.scan_argument()) } { #1 } + \egroup + \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } + \@@_width_to_aux: \group_end: \@@_write_aux: - \bool_if:NF \l_@@_splittable_bool - { \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } } } % \end{macrocode} % @@ -1353,42 +1595,26 @@ version 2005/12/01 or later. } { \group_end: - \mode_if_vertical:TF - { \mode_leave_vertical: } - \newline + \mode_if_vertical:TF \mode_leave_vertical: \newline % \end{macrocode} -% Be careful: there is curryfication in the following code. +% We count with Lua the number of lines of the argument. The result will be +% stored by Lua in |\l_@@_nb_lines_int|. That information will be used to allow +% or disallow page breaks. % \begin{macrocode} - \bool_if:NF \l_@@_splittable_bool + \lua_now:n { CountLines(token.scan_argument()) } { ##1 } + \ttfamily + \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } + \vtop \bgroup + \lua_now:e { - \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } - \vbox_top:n - } - { - \ttfamily - \dim_zero:N \lineskip - \int_case:nnF \l_@@_gobble_int - { - 0 -% \end{macrocode} -% Be careful: the last argument is provided by curryfication. -% \begin{macrocode} - { \lua_now:n { Parse(token.scan_argument()) } } - { -1 } - { \lua_now:n { AutoGobbleParse(token.scan_argument()) } } - { -2 } - { \lua_now:n { EnvGobbleParse(token.scan_argument()) } } - } - { - \exp_args:NV \@@_define_gobble_syntax:n \l_@@_gobble_int - \lua_now:n { GobbleParse(token.scan_argument()) } - } - { ##1 } - \vspace { 2.5 pt } - \@@_width_to_aux: - } - \bool_if:NF \l_@@_splittable_bool - { \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } } + GobbleParse + ( \int_use:N \l_@@_gobble_int , token.scan_argument() ) + } + { ##1 } + \vspace { 2.5 pt } + \egroup + \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } + \@@_width_to_aux: % \end{macrocode} % The following |\end{#1}| is only for the groups and the stack of % environments of LaTeX. @@ -1407,17 +1633,10 @@ version 2005/12/01 or later. \NewDocumentEnvironment { #1 } { #2 } { #3 - \int_gincr:N \g_@@_env_int - \tl_gclear:N \g_@@_aux_tl - \tl_if_exist:cT { c_@@ _ \int_use:N \g_@@_env_int _ tl } - { \use:c { c_@@ _ \int_use:N \g_@@_env_int _ tl } } - \dim_compare:nNnT \l_@@_width_on_aux_dim = \c_zero_dim - { \dim_set_eq:NN \l_@@_width_on_aux_dim \linewidth } - \bool_if:NF \l_@@_resume_bool { \int_gzero:N \g_@@_line_int } + \@@_pre_env: \group_begin: - \box_clear:N \l_tmpa_box \tl_map_function:nN - { \ \\ \{ \} \$ \& \# \^ \_ \% \~ } + { \ \\ \{ \} \$ \& \# \^ \_ \% \~ \^^I } \char_set_catcode_other:N \use:c { _@@_collect_ #1 :w } } @@ -1443,7 +1662,7 @@ version 2005/12/01 or later. % \end{macrocode} % % \bigskip -% \subsection{The styles} +% \subsubsection{The styles} % % \medskip % The following command is fundamental: it will be used by the Lua code. @@ -1553,8 +1772,8 @@ version 2005/12/01 or later. } % \end{macrocode} % -% -% \subsection{The initial style} +% \bigskip +% \subsubsection{The initial style} % % The initial style is inspired by the style ``manni'' of Pygments. % @@ -1581,12 +1800,15 @@ version 2005/12/01 or later. Name.Type = \color[HTML]{336666} , InitialValues = \@@_piton:n , Dict.Value = \@@_piton:n , - Post.Function = \@@_piton:n , Interpol.Inside = \color{black}\@@_piton:n , + Post.Function = \@@_piton:n , } % \end{macrocode} +% The last style |Post.Function| should be considered as an ``internal style'' +% (not available for the final user). % -% \subsection{Security} +% \bigskip +% \subsubsection{Security} % % \begin{macrocode} \AddToHook { env / piton / begin } @@ -1603,371 +1825,739 @@ version 2005/12/01 or later. % % % \bigskip -% \subsection{The Lua code} +% \subsection{The Lua part of the implementation} % -% \medskip +% \bigskip % \begin{macrocode} \ExplSyntaxOff \RequirePackage{luacode} % \end{macrocode} % -% \vspace{1cm} +% \medskip % \begin{macrocode} \begin{luacode*} -local P, S, V , C , Ct, Cc, Cf = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cf - - ---[[ By convention, a capture which provides as value a table (and not a string), provides, in fact, -a string (the first element of the table) which is a formatting LaTeX instruction (it will be -thrown back to TeX with normal catcodes (ant not ``other'' catcode for everybody).]] - -local function L(string) - return Cc ( { string } ) +% \end{macrocode} +% +% \bigskip +% \subsubsection{Special functions dealing with LPEG} +% +% \medskip +% We will use the Lua library \pkg{lpeg} which is built in LuaTeX. That's why we +% define first aliases for several functions of that library. +% \begin{macrocode} +local P, S, V, C, Ct, Cc, Cf = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cf +% \end{macrocode} +% +% +% \bigskip +% The function |Q| takes in as argument a pattern and returns a \textsc{lpeg} +% \emph{which does a capture} of the pattern. That capture will be sent to LaTeX +% with the catcode ``other'' for all the characters: it's suitable for elements +% of the Python listings that \pkg{piton} will typeset verbatim (thanks to the +% catcode ``other''). That function will be widely used. +% \begin{macrocode} +local function Q(pattern) + return Ct ( Cc ( luatexbase.catcodetables.CatcodeTableOther ) * C ( pattern ) ) end - +% \end{macrocode} +% +% \bigskip +% The function |L| takes in as argument a pattern and returns a \textsc{lpeg} +% \emph{which does a capture} of the pattern. That capture will be sent to LaTeX +% with standard LaTeX catcodes for all the characters: the elements captured +% will be formatted as normal LaTeX codes. It's suitable for the ``comment +% LateX'' in the environments |{Piton}| and the elements beetween +% ``|escape-inside|''. That function won't be much used. +% \begin{macrocode} +local function L(pattern) + return Ct ( C ( pattern ) ) +end +% \end{macrocode} +% +% \bigskip +% The function |Lc| (the c is for \emph{constant}) takes in as argument a string +% and returns a \textsc{lpeg} \emph{with does a constant capture} which returns +% that string. The elements captured will be formatted as L3 code. It will be +% used to send to LaTeX all the formatting LaTeX instructions we have to insert +% in order to do the syntactic highlighting (that's the main job of +% \pkg{piton}). That function will be widely used. +% \begin{macrocode} +local function Lc(string) + return Cc ( { luatexbase.catcodetables.expl , string } ) +end +% \end{macrocode} +% +% \bigskip +% The function |K| function creates a \textsc{lpeg} which will return as capture +% the whole LaTeX code corresponding to a Python chunk (that is to say with the +% LaTeX formatting instructions corresponding to the syntactic nature of that +% Python chunk). The first argument is a pattern (that is to say a \textsc{lpeg} +% without capture) and the second element is a Lua string corresponding to the +% name of a \pkg{piton} style. +% \begin{macrocode} local function K(pattern, style) return - L ( "{\\PitonStyle{" .. style .. "}{" ) - * C(pattern) - * L ( "}}" ) + Lc ( "{\\PitonStyle{" .. style .. "}{" ) + * Q ( pattern ) + * Lc ( "}}" ) end - - ---[[ The text in "escape" (between begin_escape and end_escape) is captured -and put in a table (with only one component). Indeed, we have decided that a capture -which is encapsulated in a table must be transmitted to TeX with the normal TeX catcodes.]] - -local Escape = P(begin_escape) - * Ct ( C ( ( 1 - P(end_escape) ) ^ 1 ) ) - * P(end_escape) - - -lpeg.locale(lpeg) -- mandatory - -local alpha , digit , space , punct = lpeg.alpha , lpeg.digit , lpeg.space , lpeg.punct - --- Remember that à, â, ç, etc. are strings of length 2 (2 bytes) +% \end{macrocode} +% The formatting commands in a given \pkg{piton} style (eg. the style |Keyword|) +% may be semi-global declarations (such as |\bfseries| or |\slshape|) or LaTeX +% macros with an argument (such as |\fbox| or |\colorbox{yellow}|). In order to +% deal with both syntaxes, we have used two pairs of braces: +% |{\PitonStyle{Keyword}{|\texttt{\slshape text to format}|}}|. +% +% \bigskip +% The following \textsc{lpeg} catches the Python chunks which are in LaTeX +% escapes (and that chunks will be considered as normal LaTeX constructions). We +% recall that |begin_espace| and |end_escape| are Lua strings corresponding to +% the key |escape-inside|\footnote{The \pkg{piton} key |escape-inside| is +% available at load-time only.}. Since the elements that will be catched must be +% sent to LaTeX with standard LaTeX catcodes, we put the capture (done by the +% function |C|) in a table (by using |Ct|, which is an alias for |lpeg.Ct|) +% without number of catcode table at the first component of the table. +% \begin{macrocode} +local Escape = + P(begin_escape) + * L ( ( 1 - P(end_escape) ) ^ 1 ) + * P(end_escape) +% \end{macrocode} +% +% \vspace{1cm} +% The following line is mandatory. +% \begin{macrocode} +lpeg.locale(lpeg) +% \end{macrocode} +% +% \bigskip +% \subsubsection{The LPEG SyntaxPython} +% +% \begin{macrocode} +local alpha, digit, space = lpeg.alpha, lpeg.digit, lpeg.space +% \end{macrocode} +% +% Remember that, for \textsc{lpeg}, the Unicode characters such as |à|, |â|, +% |ç|, etc. are in fact strings of length 2 (2 bytes) because \pkg{lpeg} is not +% Unicode-aware. +% \begin{macrocode} local letter = alpha + P "_" - + P "â" + P "à" + P "ç" + P "é" + P "è" + P "ê" + P "ë" + P "ï" + P "î" + P "ô" + P "û" + P "ü" + - P "Â" + P "À" + P "Ç" + P "É" + P "È" + P "Ê" + P "Ë" + P "Ï" + P "Î" + P "Ô" + P "Û" + P "Ü" + + P "â" + P "à" + P "ç" + P "é" + P "è" + P "ê" + P "ë" + P "ï" + P "î" + + P "ô" + P "û" + P "ü" + P "Â" + P "À" + P "Ç" + P "É" + P "È" + P "Ê" + + P "Ë" + P "Ï" + P "Î" + P "Ô" + P "Û" + P "Ü" local alphanum = letter + digit - +% \end{macrocode} +% +% \bigskip +% The following \textsc{lpeg} |identifier| is a mere pattern (that is to say +% more or less a regular expression) which matches the Python identifiers (hence +% the name). +% \begin{macrocode} local identifier = letter * alphanum ^ 0 - -local Identifier = C ( identifier ) - -local Space = C ( ( space - P "\r" ) ^ 1 ) - -local SkipSpace = C ( ( space - P "\r" ) ^ 0 ) - -local Punct = C ( punct ) - - -local EOL = ( P "\r" ) - * - ( - ( space^0 * -1 ) - + - Cc ( - { - luatexbase.catcodetables.expl , - '\\__piton_end_line: \\bool_if:NT \\l__piton_splittable_bool \\newline \\__piton_begin_line:' - } - ) - ) - - +% \end{macrocode} +% +% \medskip +% On the other hand, the \textsc{lpeg} |Identifier| (with a capital) also return +% a \emph{capture}. Since no special LaTeX formatting will be applied to the +% Python identifiers, we use the function~|Q| and not the function~|K|. For +% elements which require formatting, we will usually use our function~|K| +% instead of the function~|C|. See just below for an example of use of the +% function~|K|. +% \begin{macrocode} +local Identifier = Q ( identifier ) +% \end{macrocode} +% +% \bigskip +% By convention, we will use names with an initial capital for \textsc{lpeg} +% which return captures. +% +% +% \bigskip +% Here is the first use of our function~|K|. That function will be used to +% construct \textsc{lpeg} which capture Python chunks for which we have a +% dedicated \pkg{piton} style. For example, for the numbers, \pkg{piton} +% provides a style which is called |Number|. The name of the style is provided +% as a Lua string in the second argument of the function~|K|. By convention, we +% use single quotes for delimiting the Lua strings which are names of +% \pkg{piton} styles (but this is only a convention). +% \begin{macrocode} local Number = - K ( - ( digit^1 * P "." * digit^0 + digit^0 * P "." * digit^1 + digit^1 ) - * ( S "eE" * S "+-" ^ -1 * digit^1 ) ^ -1 - + digit^1 - , 'Number' ) - - -local Word = C ( ( ( 1 - space ) - S "'\"\r[()]" - digit ) ^ 1 ) - + K ( + ( digit^1 * P "." * digit^0 + digit^0 * P "." * digit^1 + digit^1 ) + * ( S "eE" * S "+-" ^ -1 * digit^1 ) ^ -1 + + digit^1 , + 'Number' + ) +% \end{macrocode} +% +% \bigskip +% We recall that |begin_espace| and |end_escape| are Lua strings +% corresponding to the key |escape-inside|\footnote{The \pkg{piton} key +% |escape-inside| is available at load-time only.}. Of course, if the final user +% has not used the key |escape-inside|, these strings are empty. +% \begin{macrocode} if begin_escape ~= '' -then Word = C ( ( ( 1 - space - P(begin_escape) - P(end_escape) ) - S "'\"\r[()]" - digit ) ^ 1 ) +then Word = Q ( ( ( 1 - space - P(begin_escape) - P(end_escape) ) + - S "'\"\r[()]" - digit ) ^ 1 ) +else Word = Q ( ( ( 1 - space ) - S "'\"\r[()]" - digit ) ^ 1 ) end +% \end{macrocode} +% +% \bigskip +% \begin{macrocode} +local Space = Q ( ( space - P "\r" ) ^ 1 ) + +local SkipSpace = Q ( ( space - P "\r" ) ^ 0 ) + +local Punct = Q ( S ".,:;!" ) +% \end{macrocode} +% +% \bigskip +% \begin{macrocode} +local Tab = P "\t" * Lc ( '\\l_@@_tab_tl' ) +% \end{macrocode} +% +% \bigskip +% The following \textsc{lpeg} |EOL| is for the end of lines. +% \begin{macrocode} +local EOL = + P "\r" + * + ( + ( space^0 * -1 ) + + +% \end{macrocode} +% We recall that each line in the Python code we have to parse will be sent +% back to LaTeX between a pair |\@@_begin_line:| -- +% |\@@_end_line:|\footnote{Remember that the \texttt{\textbackslash +% @@\_end\_line:} must be explicit because it will be used as marker in order to +% delimit the argument of the command \texttt{\textbackslash @@\_begin\_line:}}. +% \begin{macrocode} + Lc ( '\\@@_end_line: \\@@_newline: \\@@_begin_line:' ) + ) +% \end{macrocode} +% +% \bigskip +% \begin{macrocode} +local Delim = Q ( S "[()]" ) -local Delim = C ( S "[()]" ) +local Operator = + K ( P "!=" + P "==" + P "<<" + P ">>" + S "-~+/*%=<>&.@|" , 'Operator') +local OperatorWord = + K ( P "in" + P "is" + P "and" + P "or" + P "not" , 'Operator.Word') local Keyword = - K ( P "assert" + P "break" + P "case" + P "continue" + P "del" - + P "elif" + P "else" + P "except" + P "exec" + P "finally" + P "for" + P "from" - + P "global" + P "if" + P "import" + P "lambda" + P "non local" - + P "pass" + P "return" + P "try" + P "while" - + P "with" + P "yield" + P "yield from" , - 'Keyword' ) - + K ( P "True" + P "False" + P "None" , 'Keyword.Constant' ) - + K ( P "as" + P "assert" + P "break" + P "case" + P "class" + P "continue" + + P "def" + P "del" + P "elif" + P "else" + P "except" + P "exec" + + P "finally" + P "for" + P "from" + P "global" + P "if" + P "import" + + P "lambda" + P "non local" + P "pass" + P "return" + P "try" + + P "while" + P "with" + P "yield" + P "yield from" , + 'Keyword' ) + + K ( P "True" + P "False" + P "None" , 'Keyword.Constant' ) local Builtin = - K ( P "__import__" + P "abs" + P "all" + P "any" + P "bin" + P "bool" + P "bytearray" - + P "bytes" + P "chr" + P "classmethod" + P "compile" + P "complex" + P "delattr" - + P "dict" + P "dir" + P "divmod" + P "enumerate" + P "eval" + P "filter" - + P "float" + P "format" + P "frozenset" + P "getattr" + P "globals" + P "hasattr" - + P "hash" + P "hex" + P "id" + P "input" + P "int" + P "isinstance" + P "issubclass" - + P "iter" + P "len" + P "list" + P "locals" + P "map" + P "max" + P "memoryview" + P "min" - + P "next" + P "object" + P "oct" + P "open" + P "ord" + P "pow" + P "print" + P "property" - + P "range" + P "repr" + P "reversed" + P "round" + P "set" + P "setattr" + P "slice" - + P "sorted" + P "staticmethod" + P "str" + P "sum" + P "super" + P "tuple" + P "type" - + P "vars" + P "zip" , - 'Name.Builtin' ) - + K ( P "__import__" + P "abs" + P "all" + P "any" + P "bin" + P "bool" + + P "bytearray" + P "bytes" + P "chr" + P "classmethod" + P "compile" + + P "complex" + P "delattr" + P "dict" + P "dir" + P "divmod" + + P "enumerate" + P "eval" + P "filter" + P "float" + P "format" + + P "frozenset" + P "getattr" + P "globals" + P "hasattr" + P "hash" + + P "hex" + P "id" + P "input" + P "int" + P "isinstance" + P "issubclass" + + P "iter" + P "len" + P "list" + P "locals" + P "map" + P "max" + + P "memoryview" + P "min" + P "next" + P "object" + P "oct" + P "open" + + P "ord" + P "pow" + P "print" + P "property" + P "range" + P "repr" + + P "reversed" + P "round" + P "set" + P "setattr" + P "slice" + P "sorted" + + P "staticmethod" + P "str" + P "sum" + P "super" + P "tuple" + P "type" + + P "vars" + P "zip" , + 'Name.Builtin' ) local Exception = - K ( "ArithmeticError" + P "AssertionError" + P "AttributeError" - + P "BaseException" + P "BufferError" + P "BytesWarning" + P "DeprecationWarning" - + P "EOFError" + P "EnvironmentError" + P "Exception" + P "FloatingPointError" - + P "FutureWarning" + P "GeneratorExit" + P "IOError" + P "ImportError" - + P "ImportWarning" + P "IndentationError" + P "IndexError" + P "KeyError" - + P "KeyboardInterrupt" + P "LookupError" + P "MemoryError" + P "NameError" - + P "NotImplementedError" + P "OSError" + P "OverflowError" - + P "PendingDeprecationWarning" + P "ReferenceError" + P "ResourceWarning" - + P "RuntimeError" + P "RuntimeWarning" + P "StopIteration" - + P "SyntaxError" + P "SyntaxWarning" + P "SystemError" + P "SystemExit" - + P "TabError" + P "TypeError" + P "UnboundLocalError" + P "UnicodeDecodeError" - + P "UnicodeEncodeError" + P "UnicodeError" + P "UnicodeTranslateError" - + P "UnicodeWarning" + P "UserWarning" + P "ValueError" + P "VMSError" - + P "Warning" + P "WindowsError" + P "ZeroDivisionError" - + P "BlockingIOError" + P "ChildProcessError" + P "ConnectionError" - + P "BrokenPipeError" + P "ConnectionAbortedError" + P "ConnectionRefusedError" - + P "ConnectionResetError" + P "FileExistsError" + P "FileNotFoundError" - + P "InterruptedError" + P "IsADirectoryError" + P "NotADirectoryError" - + P "PermissionError" + P "ProcessLookupError" + P "TimeoutError" - + P "StopAsyncIteration" + P "ModuleNotFoundError" + P "RecursionError" , - 'Exception' ) - -local RaiseException = K ( P "raise" , 'Keyword' ) * SkipSpace * Exception * C ( P "(" ) - -local ExceptionInConsole = Exception * C ( ( 1 - P "\r" ) ^ 0 ) * EOL - - -local Namespace = - K ( P "from" , 'Keyword' ) * Space * K ( alphanum^1 , 'Name.Namespace' ) - * ( Space * K ( P "import" , 'Keyword' ) ) ^ -1 - - -local ImportAs = K ( P "import" , 'Keyword' ) - * Space - * K ( identifier , 'Name.Namespace' ) - * ( SkipSpace * C ( P "," ) * SkipSpace * K ( identifier , 'Name.Namespace' ) ) ^ 0 - * ( - Space * K ( P "as" , 'Keyword' ) * Space * K ( identifier , 'Name.Namespace' ) - ) ^ 0 - -local Class = K ( P "class" , 'Keyword' ) - * ( Space * K ( identifier , 'Name.Class' ) ) ^ -1 - + K ( "ArithmeticError" + P "AssertionError" + P "AttributeError" + + P "BaseException" + P "BufferError" + P "BytesWarning" + P "DeprecationWarning" + + P "EOFError" + P "EnvironmentError" + P "Exception" + P "FloatingPointError" + + P "FutureWarning" + P "GeneratorExit" + P "IOError" + P "ImportError" + + P "ImportWarning" + P "IndentationError" + P "IndexError" + P "KeyError" + + P "KeyboardInterrupt" + P "LookupError" + P "MemoryError" + P "NameError" + + P "NotImplementedError" + P "OSError" + P "OverflowError" + + P "PendingDeprecationWarning" + P "ReferenceError" + P "ResourceWarning" + + P "RuntimeError" + P "RuntimeWarning" + P "StopIteration" + + P "SyntaxError" + P "SyntaxWarning" + P "SystemError" + P "SystemExit" + + P "TabError" + P "TypeError" + P "UnboundLocalError" + P "UnicodeDecodeError" + + P "UnicodeEncodeError" + P "UnicodeError" + P "UnicodeTranslateError" + + P "UnicodeWarning" + P "UserWarning" + P "ValueError" + P "VMSError" + + P "Warning" + P "WindowsError" + P "ZeroDivisionError" + + P "BlockingIOError" + P "ChildProcessError" + P "ConnectionError" + + P "BrokenPipeError" + P "ConnectionAbortedError" + P "ConnectionRefusedError" + + P "ConnectionResetError" + P "FileExistsError" + P "FileNotFoundError" + + P "InterruptedError" + P "IsADirectoryError" + P "NotADirectoryError" + + P "PermissionError" + P "ProcessLookupError" + P "TimeoutError" + + P "StopAsyncIteration" + P "ModuleNotFoundError" + P "RecursionError" , + 'Exception' ) + +local RaiseException = K ( P "raise" , 'Keyword' ) * SkipSpace * Exception * Q ( P "(" ) + +local ExceptionInConsole = Exception * Q ( ( 1 - P "\r" ) ^ 0 ) * EOL +% \end{macrocode} +% +% \bigskip +% In Python, a ``decorator'' is a statement whose begins by |@| which patches +% the function defined in the following statement. +% \begin{macrocode} local Decorator = K ( P "@" * letter^1 , 'Name.Decorator' ) - +% \end{macrocode} +% +% \bigskip +% The following \textsc{lpeg} |DefClass| will be used to detect the definition of a +% new class (the name of that new class will be formatted with the \pkg{piton} +% style |Name.Class|). +% +% \smallskip +% Example:\enskip \piton|class myclass:| +% \begin{macrocode} +local DefClass = + K ( P "class" , 'Keyword' ) * Space * K ( identifier , 'Name.Class' ) +% \end{macrocode} +% +% If the word |class| is not followed by a identifier, it will be catched as +% keyword by the \textsc{lpeg} |Keyword| (useful if we want to type a +% list of keywords). +% +% +% \bigskip +% The following \textsc{lpeg} |ImportAs| is used for the lines beginning by |import|. +% % We have to detect the potential keyword |as| because both the name of the +% module and its alias must be formatted with the \pkg{piton} style |Name.Namespace|. +% +% \smallskip +% Example:\enskip \piton|import numpy as np| +% +% \smallskip +% Moreover, after the keyword |import|, it's possible to have a comma-separated +% list of modules (if the keyword |as| is not used). +% +% \smallskip +% Example:\enskip \piton|import math, numpy| +% \begin{macrocode} +local ImportAs = + K ( P "import" , 'Keyword' ) + * Space + * K ( identifier * ( P "." * identifier ) ^ 0 , + 'Name.Namespace' + ) + * ( + ( Space * K ( P "as" , 'Keyword' ) * Space * K ( identifier , 'Name.Namespace' ) ) + + + ( SkipSpace * Q ( P "," ) * SkipSpace * K ( identifier , 'Name.Namespace' ) ) ^ 0 + ) +% \end{macrocode} +% Be careful: there is no commutativity of |+| in the previous expression. +% +% \bigskip +% The \textsc{lpeg} |FromImport| is used for the lines beginning by |from|. We +% need a special treatment because the identifier following the keyword |from| +% must be formatted with the \pkg{piton} style |Name.Namespace| and the +% following keyword |import| must be formatted with the \pkg{piton} style +% |Keyword| and must \emph{not} be catched by the \textsc{lpeg} |ImportAs|. +% +% \smallskip +% Example:\enskip \piton|from math import pi| +% +% \smallskip +% \begin{macrocode} +local FromImport = + K ( P "from" , 'Keyword' ) + * Space * K ( identifier , 'Name.Namespace' ) + * Space * K ( P "import" , 'Keyword' ) +% \end{macrocode} +% +% \bigskip +% \paragraph{The strings of Python} +% +% For the strings in Python, there are four categories of delimiters (without +% counting the prefixes for f-strings and raw strings). We will use, in the +% names of our \textsc{lpeg}, prefixes to distinguish the \textsc{lpeg} dealing +% with that categories of strings, as presented in the following tabular. +% \begin{center} +% \begin{tabular}{ccc} +% \toprule +% & |Single| & |Double| \\ +% \midrule +% |Short| & |'text'| & |"text"| \\ +% |Long| & |'''test'''| & |"""text"""| \\ +% \bottomrule +% \end{tabular} +% \end{center} +% +% +% \bigskip +% First, we define \textsc{lpeg} for the interpolations in the f-strings. Here +% is an example of a f-string with an interpolation and a format +% instruction\footnote{There is no special \pkg{piton} style for the formatting +% instruction (after the comma): the style which will be applied will be the +% style of the encompassing string, that is to say |String.Short| or +% |String.Long|.} in that interpolation: +% +% \piton|f'Total price: {total+1:.2f} €'| +% +% \medskip +% The following \textsc{lpeg} |SingleShortInterpol| (and the three variants) +% will catch the whole interpolation, included the braces, that is to say, in +% the previous example:\enskip |{total+1:.2f}| +% +% \medskip +% \begin{macrocode} local SingleShortInterpol = K ( P "{" , 'String.Interpol') * K ( ( 1 - S "}':" ) ^ 0 , 'Interpol.Inside' ) - * C ( P ":" * (1 - S "}:'") ^ 0 ) ^ -1 + * Q ( P ":" * (1 - S "}:'") ^ 0 ) ^ -1 * K ( P "}" , 'String.Interpol' ) local DoubleShortInterpol = K ( P "{" , 'String.Interpol' ) * K ( ( 1 - S "}\":" ) ^ 0 , 'Interpol.Inside' ) - * ( K ( P ":" , 'String.Interpol' ) * C ( (1 - S "}:\"") ^ 0 ) ) ^ -1 + * ( K ( P ":" , 'String.Interpol' ) * Q ( (1 - S "}:\"") ^ 0 ) ) ^ -1 * K ( P "}" , 'String.Interpol' ) local SingleLongInterpol = K ( P "{" , 'String.Interpol' ) * K ( ( 1 - S "}:\r" - P "'''" ) ^ 0 , 'Interpol.Inside' ) - * C ( P ":" * (1 - S "}:\r" - P "'''" ) ^ 0 ) ^ -1 + * Q ( P ":" * (1 - S "}:\r" - P "'''" ) ^ 0 ) ^ -1 * K ( P "}" , 'String.Interpol' ) local DoubleLongInterpol = K ( P "{" , 'String.Interpol' ) * K ( ( 1 - S "}:\r" - P "\"\"\"" ) ^ 0 , 'Interpol.Inside' ) - * C ( P ":" * (1 - S "}:\r" - P "\"\"\"" ) ^ 0 ) ^ -1 + * Q ( P ":" * (1 - S "}:\r" - P "\"\"\"" ) ^ 0 ) ^ -1 * K ( P "}" , 'String.Interpol' ) +% \end{macrocode} +% +% \bigskip +% Now, we define \textsc{lpeg} for the parts of the strings which are \emph{not} +% in the interpolations. +% \begin{macrocode} +local SingleShortPureString = + Q ( ( P "\\'" + P "{{" + P "}}" + 1 - S "{}'" ) ^ 1 ) -local SingleShortPureString = C ( ( P "\\'" + P "{{" + P "}}" + 1 - S "{}'" ) ^ 1 ) - -local DoubleShortPureString = C ( ( P "\\\"" + P "{{" + P "}}" + 1 - S "{}\"" ) ^ 1 ) - -local SingleLongPureString = C ( ( 1 - P "'''" - S "{}'\r" ) ^ 1 ) +local DoubleShortPureString = + Q ( ( P "\\\"" + P "{{" + P "}}" + 1 - S "{}\"" ) ^ 1 ) -local DoubleLongPureString = C ( ( 1 - P "\"\"\"" - S "{}\"\r" ) ^ 1 ) +local SingleLongPureString = + Q ( ( 1 - P "'''" - S "{}'\r" ) ^ 1 ) +local DoubleLongPureString = + Q ( ( 1 - P "\"\"\"" - S "{}\"\r" ) ^ 1 ) +% \end{macrocode} +% +% \bigskip +% We can now define the \textsc{lpeg} for the four kinds of strings. It's not +% possible to use our function~|K| because of the interpolations which must be +% formatted with another \pkg{piton} style that the rest of the +% string.\footnote{The interpolations are formatted with the \pkg{piton} style +% |Interpol.Inside|. The initial value of that style is \texttt{\textbackslash +% @@\_piton:n} wich means that the interpolations are parsed once again by \pkg{piton}.} +% \begin{macrocode} local SingleShortString = - L ( "{\\PitonStyle{String.Short}{" ) - * ( - C ( P "f'" + P "F'" ) - * ( SingleShortInterpol + SingleShortPureString ) ^ 0 - * C ( P "'" ) - + - C ( ( P "'" + P "r'" + P "R'" ) * ( P "\\'" + 1 - S "'\r" ) ^ 0 * P "'" ) - ) - * L ( "}}" ) + Lc ( "{\\PitonStyle{String.Short}{" ) + * ( +% \end{macrocode} +% First, we deal with the f-strings of Python, which are prefixed by |f| or |F|. +% \begin{macrocode} + Q ( P "f'" + P "F'" ) + * ( SingleShortInterpol + SingleShortPureString ) ^ 0 + * Q ( P "'" ) + + +% \end{macrocode} +% Now, we deal with the standard strings of Python, but also the ``raw strings''. +% \begin{macrocode} + Q ( ( P "'" + P "r'" + P "R'" ) + * ( P "\\'" + 1 - S "'\r" ) ^ 0 * P "'" ) + ) + * Lc ( "}}" ) local DoubleShortString = - L ( "{\\PitonStyle{String.Short}{" ) - * ( - C ( P "f\"" + P "F\"" ) - * ( DoubleShortInterpol + DoubleShortPureString ) ^ 0 - * C ( P "\"" ) - + - C ( ( P "\"" + P "r\"" + P "R\"" ) * ( P "\\\"" + 1 - S "\"\r" ) ^ 0 * P "\"" ) - ) - * L ( "}}" ) + Lc ( "{\\PitonStyle{String.Short}{" ) + * ( + Q ( P "f\"" + P "F\"" ) + * ( DoubleShortInterpol + DoubleShortPureString ) ^ 0 + * Q ( P "\"" ) + + + Q ( ( P "\"" + P "r\"" + P "R\"" ) + * ( P "\\\"" + 1 - S "\"\r" ) ^ 0 * P "\"" ) + ) + * Lc ( "}}" ) local ShortString = SingleShortString + DoubleShortString - - +% \end{macrocode} +% +% \bigskip +% Of course, it's more complicated for ``longs strings'' because, by definition, +% in Python, those strings may be broken by an end on line (which is catched by +% the \textsc{lpeg} |EOL|). +% \begin{macrocode} local SingleLongString = - L "{\\PitonStyle{String.Long}{" - * ( - C ( S "fF" * P "'''" ) - * ( SingleLongInterpol + SingleLongPureString ) ^ 0 - * L "}}" - * ( - EOL - + - L "{\\PitonStyle{String.Long}{" - * ( SingleLongInterpol + SingleLongPureString ) ^ 0 - * L "}}" - * EOL - ) ^ 0 - * L "{\\PitonStyle{String.Long}{" + Lc "{\\PitonStyle{String.Long}{" + * ( + Q ( S "fF" * P "'''" ) + * ( SingleLongInterpol + SingleLongPureString ) ^ 0 + * Lc "}}" + * ( + EOL + + + Lc "{\\PitonStyle{String.Long}{" * ( SingleLongInterpol + SingleLongPureString ) ^ 0 - + - C ( ( S "rR" ) ^ -1 * P "'''" * ( 1 - P "'''" - P "\r" ) ^ 0 ) - * L "}}" - * ( - L "{\\PitonStyle{String.Long}{" - * C ( ( 1 - P "'''" - P "\r" ) ^ 0 ) - * L "}}" - * EOL - ) ^ 0 - * L "{\\PitonStyle{String.Long}{" - * C ( ( 1 - P "'''" - P "\r" ) ^ 0 ) - ) - * C ( P "'''" ) - * L "}}" + * Lc "}}" + * EOL + ) ^ 0 + * Lc "{\\PitonStyle{String.Long}{" + * ( SingleLongInterpol + SingleLongPureString ) ^ 0 + + + Q ( ( S "rR" ) ^ -1 * P "'''" + * ( 1 - P "'''" - P "\r" ) ^ 0 ) + * Lc "}}" + * ( + Lc "{\\PitonStyle{String.Long}{" + * Q ( ( 1 - P "'''" - P "\r" ) ^ 0 ) + * Lc "}}" + * EOL + ) ^ 0 + * Lc "{\\PitonStyle{String.Long}{" + * Q ( ( 1 - P "'''" - P "\r" ) ^ 0 ) + ) + * Q ( P "'''" ) + * Lc "}}" local DoubleLongString = - L "{\\PitonStyle{String.Long}{" - * ( - C ( S "fF" * P "\"\"\"" ) - * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0 - * L "}}" - * ( - EOL - + - L "{\\PitonStyle{String.Long}{" - * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0 - * L "}}" - * EOL - ) ^ 0 - * L "{\\PitonStyle{String.Long}{" + Lc "{\\PitonStyle{String.Long}{" + * ( + Q ( S "fF" * P "\"\"\"" ) + * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0 + * Lc "}}" + * ( + EOL + + + Lc "{\\PitonStyle{String.Long}{" * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0 - + - C ( ( S "rR" ) ^ -1 * P "\"\"\"" * ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) - * L "}}" - * ( - L "{\\PitonStyle{String.Long}{" - * C ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) - * L "}}" - * EOL - ) ^ 0 - * L "{\\PitonStyle{String.Long}{" - * C ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) - ) - * C ( P "\"\"\"" ) - * L "}}" - - - + * Lc "}}" + * EOL + ) ^ 0 + * Lc "{\\PitonStyle{String.Long}{" + * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0 + + + Q ( ( S "rR" ) ^ -1 * P "\"\"\"" + * ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) + * Lc "}}" + * ( + Lc "{\\PitonStyle{String.Long}{" + * Q ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) + * Lc "}}" + * EOL + ) ^ 0 + * Lc "{\\PitonStyle{String.Long}{" + * Q ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) + ) + * Q ( P "\"\"\"" ) + * Lc "}}" +% \end{macrocode} +% +% \begin{macrocode} local LongString = SingleLongString + DoubleLongString +% \end{macrocode} +% +% \bigskip +% We have a \textsc{lpeg} for the Python docstrings. That \textsc{lpeg} will +% be used in the \textsc{lpeg} |DefFunction| which deals with the whole preamble +% of a function definition (which begins with |def|). +% \begin{macrocode} +local StringDoc = + K ( P "\"\"\"" , 'String.Doc' ) + * ( K ( (1 - P "\"\"\"" - P "\r" ) ^ 0 , 'String.Doc' ) * EOL * Tab ^0 ) ^ 0 + * K ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 * P "\"\"\"" , 'String.Doc' ) +% \end{macrocode} +% +% \bigskip +% \paragraph{The comments in the Python listings} +% +% We define different \textsc{lpeg} dealing with comments in the Python +% listings. +% \begin{macrocode} +local CommentMath = + P "$" * K ( ( 1 - S "$\r" ) ^ 1 , 'Comment.Math' ) * P "$" + +local Comment = + Lc ( "{\\PitonStyle{Comment}{" ) + * Q ( P "#" ) + * ( CommentMath + Q ( ( 1 - S "$\r" ) ^ 1 ) ) ^ 0 + * Lc ( "}}" ) + * ( EOL + -1 ) +% \end{macrocode} +% -local Expression = - P { "E" , - E = ( 1 - S "{}()[]\r," ) ^ 0 - * ( - ( P "{" * V "F" * P "}" - + P "(" * V "F" * P ")" - + P "[" * V "F" * P "]" ) * ( 1 - S "{}()[]\r," ) ^ 0 - ) ^ 0 , - F = ( 1 - S "{}()[]\r\"'" ) ^ 0 - * ( ( - P "'" * (P "\\'" + 1 - S"'\r" )^0 * P "'" - + P "\"" * (P "\\\"" + 1 - S"\"\r" )^0 * P "\"" - + P "{" * V "F" * P "}" - + P "(" * V "F" * P ")" - + P "[" * V "F" * P "]" - ) * ( 1 - S "{}()[]\r\"'" ) ^ 0 ) ^ 0 , - } - -local Param = SkipSpace * K ( identifier , '' ) * SkipSpace - * ( K ( P "=" * Expression , 'InitialValues' ) - + K ( P ":" , '' ) * SkipSpace * K ( letter^1 , 'Name.Type' )) - + SkipSpace * K ( alphanum ^ 1 , '' ) * SkipSpace - -local Params = Param * ( K ( P "," , '' ) * Param ) ^ 0 - -local StringDoc = K ( P "\"\"\"" , 'String.Doc' ) - * ( K ( (1 - P "\"\"\"" - P "\r" ) ^ 0 , 'String.Doc' ) * EOL ) ^ 0 - * K ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 * P "\"\"\"" , 'String.Doc' ) - + K ( P "'''" , 'String.Doc' ) - * ( K ( (1 - P "'''" - P "\r")^0 , 'String.Doc' ) * EOL ) ^ 0 - * K ( ( 1 - P "'''" - P "\r")^0 * P "'''" , 'String.Doc' ) - -local CommentMath = P "$" * K ( ( 1 - S "$\r" ) ^ 1 , 'Comment.Math' ) * P "$" - - -local Comment = L ( "{\\PitonStyle{Comment}{" ) - * C ( P "#" ) * ( CommentMath + C ( ( 1 - S "$\r" ) ^ 1 ) ) ^ 0 - * L ( "}}" ) - * ( EOL + -1 ) - +% \bigskip +% The following \textsc{lpeg} |CommentLaTeX| is for what is called in that +% document the ``LaTeX comments''. Since the elements that will be catched must +% be sent to LaTeX with standard LaTeX catcodes, we put the capture (done by +% the function~|C|) in a table (by using~|Ct|, which is an alias for |lpeg.Ct|). +% \begin{macrocode} local CommentLaTeX = - P "##" - * L "{\\PitonStyle{Comment.LaTeX}{\\ignorespaces" - * Ct ( C ( ( 1 - P "\r" ) ^ 0 ) ) - * L "}}" - * ( EOL + -1 ) - + P "##" + * Lc "{\\PitonStyle{Comment.LaTeX}{\\ignorespaces" + * L ( ( 1 - P "\r" ) ^ 0 ) + * Lc "}}" + * ( EOL + -1 ) +% \end{macrocode} +% +% \bigskip +% \paragraph{DefFunction} +% +% The following \textsc{lpeg} |Expression| will be used for the parameters in +% the \emph{argspec} of a Python function. It's necessary to use a \emph{grammar} +% because that pattern mainly checks the correct nesting of the delimiters +% (and it's known in the theory of formal languages that this can't be done with +% regular expressions \emph{stricto sensu} only). +% \begin{macrocode} +local Expression = + P { "E" , + E = ( 1 - S "{}()[]\r," ) ^ 0 + * ( + ( P "{" * V "F" * P "}" + + P "(" * V "F" * P ")" + + P "[" * V "F" * P "]" ) * ( 1 - S "{}()[]\r," ) ^ 0 + ) ^ 0 , + F = ( 1 - S "{}()[]\r\"'" ) ^ 0 + * ( ( + P "'" * (P "\\'" + 1 - S"'\r" )^0 * P "'" + + P "\"" * (P "\\\"" + 1 - S"\"\r" )^0 * P "\"" + + P "{" * V "F" * P "}" + + P "(" * V "F" * P ")" + + P "[" * V "F" * P "]" + ) * ( 1 - S "{}()[]\r\"'" ) ^ 0 ) ^ 0 , + } +% \end{macrocode} +% +% \bigskip +% We will now define a \textsc{lpeg} |Params| that will catch the list of +% parameters (that is to say the \emph{argspec}) in the definition of a Python +% function. For example, in the line of code +% \begin{center} +% \piton|def MyFunction(a,b,x=10,n:int): return n| +% \end{center} +% the \textsc{lpeg} |Params| will be used to catch the chunk\enskip |a,b,x=10,n:int|. +% +% Or course, a |Params| is simply a comma-separated list of |Param|, and that's +% why we define first the \textsc{lpeg} |Param|. +% +% \medskip +% \begin{macrocode} +local Param = + SkipSpace * Identifier * SkipSpace + * ( + K ( P "=" * Expression , 'InitialValues' ) + + Q ( P ":" ) * SkipSpace * K ( letter^1 , 'Name.Type' ) + ) ^ -1 +% \end{macrocode} +% +% \medskip +% \begin{macrocode} +local Params = ( Param * ( Q "," * Param ) ^ 0 ) ^ -1 +% \end{macrocode} +% +% \bigskip +% The following \textsc{lpeg} |DefFunction| catches a keyword |def| and the +% following name of function \emph{but also everything else until a potential +% docstring}. That's why this definition of \textsc{lpeg} must occur (in the file +% |piton.sty|) after the definition of several other \textsc{lpeg} such as +% |Comment|, |CommentLaTeX|, |Params|, |StringDoc|... +% \begin{macrocode} local DefFunction = - K ( P "def" , 'Keyword' ) - * ( Space - * K ( identifier , 'Name.Function' ) - * ( SkipSpace * K ( P "(" , '' ) * Params * K ( P ")" , '' ) ) ^ -1 - * ( SkipSpace - * K ( ( 1 - S ":\r" )^0 , 'Post.Function' ) - * K ( P ":" , 'Keyword' ) - * SkipSpace - * ( EOL + CommentLaTeX + Comment ) - * SkipSpace - * StringDoc ) ^ -1 - ) ^ -1 - -local ItemDict = ShortString * SkipSpace * C ( P ":" ) * K ( Expression , 'Dict.Value' ) + K ( P "def" , 'Keyword' ) + * Space + * K ( identifier , 'Name.Function' ) + * SkipSpace + * Q ( P "(" ) * Params * Q ( P ")" ) + * SkipSpace +% \end{macrocode} +% Here, we need a \pkg{piton} style |Post.Function| which will be linked to +% |\@@_piton:n| (that means that the capture will be parsed once again by +% \pkg{piton}). We could avoid that kind of trick by using a non-terminal of a +% grammar but we have probably here a better legibility. +% \begin{macrocode} + * K ( ( 1 - S ":\r" )^0 , 'Post.Function' ) + * Q ( P ":" ) + * ( SkipSpace + * ( EOL + CommentLaTeX + Comment ) -- in all cases, that contains an EOL + * Tab ^ 0 + * SkipSpace + * StringDoc ^ 0 -- there may be additionnal docstrings + ) ^ -1 +% \end{macrocode} +% Remark that, in the previous code, |CommentLaTeX| \emph{must} appear +% before |Comment|: there is no commutativity of the addition for the +% \emph{parsing expression grammars} (\textsc{peg}). +% +% \smallskip +% If the word |def| is not followed by a identifier and parenthesis, it will be +% catched as keyword by the \textsc{lpeg} |Keyword| (useful if, for example, the +% final user wants to speak of the keyword \piton{def}). +% +% \bigskip +% \paragraph{The dictionaries of Python} +% +% We have \textsc{lpeg} dealings with dictionaries of Python because, in +% typesettings of explicit Python dictionnaries, one may prefer to have all the +% values formatted in black (in order to see more clearly the keys which are +% usually Python strings). That's why we have a \pkg{piton} style |Dict.Value|. +% +% The initial value of that \pkg{piton} style is |\@@_piton:n|, which means that +% the value of the entry of the dictionary is parsed once again by \pkg{piton} +% (and nothing special is done for the dictionary). In the following example, we +% have set the \pkg{piton} style |Dict.Value| to |\color{black}|: +% +% \medskip +% \begingroup +% \SetPitonStyle{Dict.Value = \color{black}} +% \piton|mydict = { 'name' : 'Paul', 'sex' : 'male', 'age' : 31 } | +% \endgroup +% +% \medskip +% At this time, this mechanism works only for explicit dictionaries on a single line! +% +% \begin{macrocode} +local ItemDict = + ShortString * SkipSpace * Q ( P ":" ) * K ( Expression , 'Dict.Value' ) local ItemOfSet = SkipSpace * ( ItemDict + ShortString ) * SkipSpace -local Set = C ( P "{" ) - * ItemOfSet * ( C ( P "," ) * ItemOfSet ) ^ 0 - * C ( P "}" ) - -local Operator = K ( P "!=" + P "==" + P "<<" + P ">>" + S "-~+/*%=<>&.@|" , 'Operator') +local Set = + Q ( P "{" ) + * ItemOfSet * ( Q ( P "," ) * ItemOfSet ) ^ 0 + * Q ( P "}" ) -local OperatorWord = K ( P "in" + P "is" + P "and" + P "or" + P "not" , 'Operator.Word') - -local SyntaxPython = +% \end{macrocode} +% +% +% \bigskip +% \paragraph{The main LPEG} +% +% |SyntaxPython| is the main \textsc{lpeg} of the package \pkg{piton}. We have +% written an auxiliary \textsc{lpeg} |SyntaxPythonAux| only for legibility. +% \begin{macrocode} +local SyntaxPythonAux = +% \end{macrocode} +% We recall that each line in the Python code to parse will be sent back to +% LaTeX between a pair |\@@_begin_line:| -- |\@@_end_line:|\footnote{Remember +% that the \texttt{\textbackslash @@\_end\_line:} must be explicit because it +% will be used as marker in order to delimit the argument of the command +% \texttt{\textbackslash @@\_begin\_line:}}. +% \begin{macrocode} + Lc ( '\\@@_begin_line:' ) * ( ( space - P "\r" ) ^0 * P "\r" ) ^ -1 * ( ( space^1 * -1 ) + EOL + + Tab + Space + Escape + CommentLaTeX @@ -1976,117 +2566,183 @@ local SyntaxPython = + ExceptionInConsole + Set + Delim - + Class * ( Space + Punct + EOL ) - + Namespace * ( Space + Punct + EOL ) +% \end{macrocode} +% |Operator| must be before |Punct|. +% \begin{macrocode} + + Operator + + ShortString + + Punct + + FromImport + ImportAs + RaiseException - + Keyword * ( Space + Punct + EOL ) + DefFunction - + ShortString - + Decorator * ( Space + Punct + EOL ) - + Operator - + OperatorWord * ( Space + Punct + EOL ) - + Builtin * ( Space + Punct + EOL ) + + DefClass + + Keyword * ( Space + Punct + Delim + EOL + -1) + + Decorator + + OperatorWord + + Builtin * ( Space + Punct + Delim + EOL + -1) + Identifier + Number + Word - ) ^0 * -1 - - -local MinimalSyntax = - P { "S" ; - S = K ( (1 - P "\r" ) ^ 0 , '') + EOL * S - } - - + ) ^0 * -1 * Lc ( '\\@@_end_line:' ) +% \end{macrocode} +% +% We have written a auxiliary \textsc{lpeg} |SyntaxPythonAux| for legibility only. +% \begin{macrocode} +local SyntaxPython = Ct ( SyntaxPythonAux ) +% \end{macrocode} +% +% \bigskip +% \subsubsection{The function Parse} +% +% The function |Parse| is the main function of the package \pkg{piton}. +% It parses its argument and sends back to LaTeX the code with interlaced +% formatting LaTeX instructions. In fact, everything is done by the +% \textsc{lpeg} |SyntaxPython| which returns as capture a Lua table containing +% data to send to LaTeX. +% +% \bigskip +% \begin{macrocode} function Parse(code) - local t = Ct(SyntaxPython) : match(code) - tex.sprint( luatexbase.catcodetables.expl , '\\__piton_begin_line:' ) - if t then else t = Ct(MinimalSyntax) : match(code) end - for i = 1 , #t do - if type(t[i]) == 'string' - then - tex.sprint(luatexbase.catcodetables.CatcodeTableOther, t[i]) - else - tex.tprint(t[i]) - end - end - tex.sprint( luatexbase.catcodetables.expl , '\\__piton_end_line:' ) + local t = SyntaxPython : match ( code ) + for _ , s in ipairs(t) do tex.tprint(s) end end - +% \end{macrocode} +% +% \bigskip +% The function |ParseFile| will be used by the LaTeX command |\PitonInputFile|. +% That function merely reads the whole file (that is to say all its lines) and +% then apply the function~|Parse| to the resulting Lua string. +% \begin{macrocode} function ParseFile(name) s = '' for line in io.lines(name) do s = s .. '\r' .. line end Parse(s) end - - -function define_gobble_syntax(n) - GobbleSyntax = ( 1 - P "\r" ) ^ (-n) * C ( ( 1 - P "\r" ) ^0 ) - * ( C ( P "\r" ) - * ( 1 - P "\r" ) ^ (-n) - * C ( ( 1 - P "\r" ) ^0 ) - ) ^ 0 -end - -function GobbleParse(code) - local t = Ct(GobbleSyntax):match(code) - local new_code = "" - for i = 1 , #t do - new_code = new_code .. t[i] +% \end{macrocode} +% +% \bigskip +% \subsubsection{The preprocessors of the function Parse} +% +% We deal now with preprocessors of the function |Parse| which are needed when +% the ``gobble mechanism'' is used. +% +% +% \bigskip +% The function |gobble| gobbles $n$ characters on the left of the code. It uses +% a \textsc{lpeg} that we have to compute dynamically because if depends on the +% value of~$n$. +% \begin{macrocode} +function gobble(n,code) + function concat(acc,new_value) + return acc .. new_value + end + if n==0 + then return code + else + return Cf ( + Cc ( "" ) * + ( 1 - P "\r" ) ^ (-n) * C ( ( 1 - P "\r" ) ^ 0 ) + * ( C ( P "\r" ) + * ( 1 - P "\r" ) ^ (-n) + * C ( ( 1 - P "\r" ) ^ 0 ) + ) ^ 0 , + concat + ) : match ( code ) end - Parse(new_code) end - +% \end{macrocode} +% +% \bigskip +% \begin{macrocode} +function GobbleParse(n,code) + if n==-1 + then n = AutoGobbleLPEG : match(code) + else if n==-2 + then n = EnvGobbleLPEG : match(code) + end + end + Parse(gobble(n,code)) +end +% \end{macrocode} +% +% +% \bigskip +% The following function |add| will be used in the following \textsc{lpeg} +% |AutoGobbleLPEG| and |EnvGobbleLPEG|. +% \begin{macrocode} function add(acc,new_value) return acc + new_value end - - - - ---[[ The following LPEG returns as capture the minimal number of spaces at -the beginning of the lines of code]] -AutoGobbleSyntax = +% \end{macrocode} +% +% \bigskip +% The following \textsc{lpeg} returns as capture the minimal number of spaces at +% the beginning of the lines of code. The main work is done by two \emph{fold +% captures} (|lpeg.Cf|), one using |add| and the other (encompassing the +% previous one) using |math.min| as folding operator. +% \begin{macrocode} +AutoGobbleLPEG = ( space ^ 0 * P "\r" ) ^ -1 * Cf ( ( +% \end{macrocode} +% We don't take into account the empty lines (with only spaces). +% \begin{macrocode} ( P " " ) ^ 0 * P "\r" + Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * ( 1 - P " " ) * ( 1 - P "\r" ) ^ 0 * P "\r" ) ^ 0 +% \end{macrocode} +% Now for the last line of the Python code... +% \begin{macrocode} * ( Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * ( 1 - P " " ) * ( 1 - P "\r" ) ^ 0 ) ^ -1 , math.min ) - -function AutoGobbleParse(code) - local n = AutoGobbleSyntax:match(code) - if n==0 - then Parse(code) - else define_gobble_syntax(n) - GobbleParse(code) - end -end - - ---[[ The following LPEG returns as capture the number of spaces at the last line, -that is to say begin the \end{Piton} ]] - -EnvGobbleSyntax = - ( ( 1 - P "\r" ) ^ 0 * P "\r" ) ^ 0 - * Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * -1 - -function EnvGobbleParse(code) - local n = EnvGobbleSyntax:match(code) - if n==0 - then Parse(code) - else define_gobble_syntax(n) - GobbleParse(code) - end -end +% \end{macrocode} +% +% \bigskip +% The following \textsc{lpeg} returns as capture the number of spaces at the +% last line, that is to say before the |\end{Piton}| (and usually it's also the +% number of spaces before the corresponding |\begin{Piton}| because that's the +% traditionnal way to indent in LaTeX). The main work is done by a \emph{fold +% capture} (|lpeg.Cf|) using the function |add| as folding operator. +% \begin{macrocode} +EnvGobbleLPEG = + ( ( 1 - P "\r" ) ^ 0 * P "\r" ) ^ 0 + * Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * -1 +% \end{macrocode} +% +% \bigskip +% \subsubsection{To count the number of lines} +% +% \medskip +% \begin{macrocode} +function CountLines(code) + local count = 0 + for i in code:gmatch("\r") do count = count + 1 end + tex.sprint( + luatexbase.catcodetables.expl , + '\\int_set:Nn \\l_@@_nb_lines_int {' .. count .. '}' ) +end +% \end{macrocode} +% +% \bigskip +% \begin{macrocode} +function CountLinesFile(name) + local count = 0 + for line in io.lines(name) do count = count + 1 end + tex.sprint( + luatexbase.catcodetables.expl , + '\\int_set:Nn \\l_@@_nb_lines_int {' .. count .. '}' ) +end +% \end{macrocode} +% +% \bigskip +% \begin{macrocode} \end{luacode*} % \end{macrocode} % @@ -2107,6 +2763,12 @@ end % % New key |left-margin|. % +% \subsection*{Changes between versions 0.8 and 0.9} +% +% New key |tab-size|. +% +% Integer value for the key |splittable|. +% % \end{document} % % |