summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2013-09-12 23:36:12 +0000
committerKarl Berry <karl@freefriends.org>2013-09-12 23:36:12 +0000
commit88cc357fc8d3e1b57e21eb762fef36e4ef2dd3f4 (patch)
tree51244daaeef0622ddd48de2d1464daa5b235b1c1 /Master/texmf-dist/scripts
parent3ce8f395f1980ebc8f4e95d66e79ac7f29f99222 (diff)
pythontex (import from $0 symlink works after all)
git-svn-id: svn://tug.org/texlive/trunk@31638 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts')
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/depythontex.py18
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/depythontex2.py1404
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/depythontex3.py1404
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex.py54
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex2.py2360
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex3.py2360
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex_2to3.py81
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex_engines.py922
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex_install_texlive.py343
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex_utils.py372
10 files changed, 9318 insertions, 0 deletions
diff --git a/Master/texmf-dist/scripts/pythontex/depythontex.py b/Master/texmf-dist/scripts/pythontex/depythontex.py
new file mode 100755
index 00000000000..013e268241f
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/depythontex.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+'''
+This is the PythonTeX wrapper script. It automatically detects the version
+of Python, and then imports the correct code from depythontex2.py or
+depythontex3.py.
+
+Copyright (c) 2013, Geoffrey M. Poore
+All rights reserved.
+Licensed under the BSD 3-Clause License:
+ http://www.opensource.org/licenses/BSD-3-Clause
+
+'''
+
+import sys
+if sys.version_info[0] == 2:
+ import depythontex2 as depythontex
+elif sys.version_info[0] == 3:
+ import depythontex3 as depythontex
diff --git a/Master/texmf-dist/scripts/pythontex/depythontex2.py b/Master/texmf-dist/scripts/pythontex/depythontex2.py
new file mode 100755
index 00000000000..028badcb88e
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/depythontex2.py
@@ -0,0 +1,1404 @@
+# -*- coding: utf-8 -*-
+'''
+PythonTeX depythontex script.
+
+This script takes a LaTeX document that uses the PythonTeX package and
+creates a new document that does not depend on PythonTeX. It substitutes all
+externally generated content into a copy of the original LaTeX document.
+This is useful when you need a document that relies on few external packages
+or custom macros (for example, for submission to a journal or conversion to
+another document format).
+
+If you just want to share a document that uses PythonTeX, keep in mind that
+the document can be modified and compiled just like a regular LaTeX document,
+without needing Python or any other external tools, so long as the following
+conditions are met:
+
+ * A copy of pythontex.sty is included with the document.
+ * The pythontex-files-<name> directory is included with the document.
+ * The PythonTeX-specific parts of the document are not modified.
+
+To work, this script requires that the original LaTeX document be compiled
+with the package option `depythontex`. That creates an auxiliary file with
+the extension .depytx that contains information about all content that needs
+to be substituted.
+
+This script is purposely written in a simple, largely linear form to
+facilitate customization. Most of the key substitutions are performed by a
+few functions defined near the beginning of the script, so if you need custom
+substitutions, you should begin there. By default, all typeset code is
+wrapped in `\verb` commands and verbatim environments, since these have the
+greatest generality. However, the command-line option --listing allows code
+to be typeset with the fancyvrb, listings, minted, or PythonTeX packages
+instead.
+
+The script automatically extracts all arguments of all commands and
+environments that it replaces, so that these are available if desired for
+customized substitution. Two additional pieces of information are also
+available for any typeset code: the Pygments lexer (often the same as the
+language) and the starting line number (if line numbering was used).
+
+Keep in mind that some manual adjustments may be required after a document is
+depythontex'ed. While depythontex attempts to create an exact copy of the
+original document, in many cases an identical copy is impossible. For
+example, typeset code may have a different appearance or layout when it is
+typeset with a different package.
+
+
+Copyright (c) 2013, Geoffrey M. Poore
+All rights reserved.
+Licensed under the BSD 3-Clause License:
+ http://www.opensource.org/licenses/BSD-3-Clause
+
+'''
+
+
+# Imports
+#// Python 2
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+#\\ End Python 2
+import sys
+import os
+#// Python 2
+from io import open
+input = raw_input
+#\\ End Python 2
+import argparse
+from collections import defaultdict
+from re import match, sub, search
+import textwrap
+
+
+# Script parameters
+# Version
+version = 'v0.12'
+
+
+
+
+# Functions and parameters for customizing the script output
+
+# Style or package for replacing code listings
+# This is actually set via command-line option --listing
+# It is created here simply for reference
+listing = None #'verbatim', 'fancyvrb', 'listings', 'minted', 'pythontex'
+
+# List of things to add to the preamble
+# It can be appended to via the command-line option --preamble
+# It is also appended to based on the code listing style that is used
+# And it could be manually edited here as well, as long as it remains a list
+preamble_additions = list()
+
+# Lexer dict
+# If you are using Pygments lexers that don't directly correspond to the
+# languages used by the listings package, you can submit replacements via the
+# command line option --lexer-dict, or edit this dict manually here. When
+# listings is used, all lexers are checked against this dict to see if a
+# substitution should be made. This approach could easily be modified to
+# work with another, non-Pygments highlighting package.
+lexer_dict = dict()
+
+
+def replace_code_cmd(name, arglist, linenum, code_replacement,
+ code_replacement_mode, after, lexer, firstnumber):
+ '''
+ Typeset code from a command with a command.
+
+ It is only ever called if there is indeed code to typeset.
+
+ Usually, code from a command is also typeset with a command. This
+ function primarily deals with that case. In cases where code from a
+ command is typeset with an environment (for example, `\inputpygments`),
+ this function performs some preprocessing and then uses
+ replace_code_env() to do the real work. This approach prevents the two
+ functions from unnecessarily duplicating each other, while still giving
+ the desired output.
+
+ Args:
+ name (str): name of the command
+ arglist (list, of str/None): all arguments given to the original
+ command; the last argument is what is typeset, unless a
+ code_replacement is specified or other instructions are given
+ linenum (int): line number in the original TeX document
+ code_replacement (str/None): replacement for the code; usually None
+ for commands, because typically the code to be typeset is the
+ last argument passed to the command, rather than something
+ captured elsewhere (like the body of an environment) or something
+ preprocessed (like a console environment's content)
+ code_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX; generally unused for code),
+ verb (inline), or verbatim (environment)
+ after (str): text immediately following the command; usually
+ shouldn't be needed
+ lexer (str/None): Pygments lexer
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ '''
+ # Get the correct replacement
+ if code_replacement is None:
+ code_replacement = arglist[-1]
+
+ # We only consider two possible modes of typesetting, verbatim and inline
+ # verbatim
+ if code_replacement_mode == 'verbatim':
+ # Sometimes we must replace a command with an environment, for
+ # example, for `\inputpygments`
+
+ # Make sure the introduction of an environment where a command was
+ # previously won't produce errors with following content; make sure
+ # that any following content is on a separate line
+ if bool(match('[ \t]*\S', after)):
+ after = '\n' + after
+ # Rather than duplicating much of replace_code_env(), just use it
+ return replace_code_env(name, arglist, linenum, code_replacement,
+ code_replacement_mode, after, lexer, firstnumber)
+ else:
+ # Usually, we're replacing a command with a command
+
+ # Wrap the replacement in appropriate delimiters
+ if (listing in ('verbatim', 'fancyvrb', 'minted') or
+ (listing in ('listings', 'pythontex') and
+ ('{' in code_replacement or '}' in code_replacement))):
+ for delim in ('|', '/', '`', '!', '&', '#', '@', ':', '%', '~', '$',
+ '=', '+', '-', '^', '_', '?', ';'):
+ if delim not in code_replacement:
+ break
+ code_replacement = delim + code_replacement + delim
+ else:
+ code_replacement = '{' + code_replacement + '}'
+ # Assemble the actual replacement
+ if listing in ('verbatim', 'minted'): # `\mint` isn't for inline use
+ code_replacement = r'\verb' + code_replacement
+ elif listing == 'fancyvrb':
+ code_replacement = r'\Verb' + code_replacement
+ elif listing == 'listings':
+ if lexer is None:
+ code_replacement = r'\lstinline[language={}]' + code_replacement
+ else:
+ if lexer in lexer_dict:
+ lexer = lexer_dict[lexer]
+ code_replacement = r'\lstinline[language=' + lexer + ']' + code_replacement
+ elif listing == 'pythontex':
+ if lexer is None:
+ code_replacement = r'\pygment{text}' + code_replacement
+ else:
+ code_replacement = r'\pygment{' + lexer + '}' + code_replacement
+ return (code_replacement, after)
+
+
+def replace_code_env(name, arglist, linenum, code_replacement,
+ code_replacement_mode, after, lexer, firstnumber):
+ '''
+ Typeset code from an environment with an environment.
+
+ It is only ever called if there is indeed code to typeset.
+
+ Usually it is only used to typeset code from an environment. However,
+ some commands bring in code that must be typeset as an environment. In
+ those cases, replace_code_cmd() is called initially, and after it
+ performs some preprocessing, this function is called. This approach
+ avoids unnecessary duplication between the two functions.
+
+ Args:
+ name (str): name of the environment
+ arglist (list, of str/None): all arguments given to the original
+ environment
+ linenum (int): line number in the original TeX document where
+ the environment began
+ code_replacement (str): replacement for the code; unlike the case of
+ commands, this is always not None if the function is called
+ code_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX; generally unused for code),
+ verb (inline), or verbatim (environment)
+ after (str): text immediately following the environment; usually
+ shouldn't be needed
+ lexer (str/None): Pygments lexer
+ firstnumber (str/None): the first number of the listing, if the listing
+ had numbered lines
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ '''
+ # Currently, there is no need to test for code_replacement_mode, because
+ # this function is only ever called if the mode is 'verbatim'. That may
+ # change in the future, but it seems unlikely that code entered in an
+ # environment would end up typeset with a command.
+ if listing == 'verbatim':
+ pre = '\\begin{verbatim}'
+ post = '\\end{verbatim}'
+ elif listing == 'fancyvrb':
+ if firstnumber is None:
+ pre = '\\begin{Verbatim}'
+ else:
+ pre = '\\begin{{Verbatim}}[numbers=left,firstnumber={0}]'.format(firstnumber)
+ post = '\\end{Verbatim}'
+ elif listing == 'listings':
+ if lexer is None:
+ if firstnumber is None:
+ pre = '\\begin{lstlisting}[language={}]'
+ else:
+ pre = '\\begin{{lstlisting}}[language={{}},numbers=left,firstnumber={0}]'.format(firstnumber)
+ else:
+ if lexer in lexer_dict:
+ lexer = lexer_dict[lexer]
+ if firstnumber is None:
+ pre = '\\begin{{lstlisting}}[language={0}]'.format(lexer)
+ else:
+ pre = '\\begin{{lstlisting}}[language={0},numbers=left,firstnumber={1}]'.format(lexer, firstnumber)
+ post = '\\end{lstlisting}'
+ elif listing == 'minted':
+ if lexer is None:
+ if firstnumber is None:
+ pre = '\\begin{minted}{text}'
+ else:
+ pre = '\\begin{{minted}}[linenos,firstnumber={0}]{{text}}'.format(firstnumber)
+ else:
+ if firstnumber is None:
+ pre = '\\begin{{minted}}{{{0}}}'.format(lexer)
+ else:
+ pre = '\\begin{{minted}}[linenos,firstnumber={0}]{{{1}}}'.format(firstnumber, lexer)
+ post = '\\end{minted}'
+ elif listing == 'pythontex':
+ if lexer is None:
+ if firstnumber is None:
+ pre = '\\begin{pygments}{text}'
+ else:
+ pre = '\\begin{{pygments}}[numbers=left,firstnumber={0}]{{text}}'.format(firstnumber)
+ else:
+ if firstnumber is None:
+ pre = '\\begin{{pygments}}{{{0}}}'.format(lexer)
+ else:
+ pre = '\\begin{{pygments}}[numbers=left,firstnumber={0}]{{{1}}}'.format(firstnumber, lexer)
+ post = '\\end{pygments}'
+ code_replacement = pre + code_replacement + post
+ return (code_replacement, after)
+
+
+# We will need to issue a warning every time that a substitution of printed
+# content results in a forced double space. We could just do this as we go,
+# but it's easier for the user to read if we just collect all the warnings
+# of this type, and print them once.
+forced_double_space_list = list()
+
+
+def replace_print_cmd(name, arglist, linenum,
+ print_replacement, print_replacement_mode, source,
+ after):
+ '''
+ Typeset printed content from a command.
+
+ It is only ever called if there is indeed printed content to typeset.
+
+ Args:
+ name (str): name of the command
+ arglist (list, of str/None): all arguments given to the original
+ command
+ linenum (int): line number in the original TeX document
+ print_replacement (str): printed content, read directly from file
+ into a single string
+ print_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX), inlineverb (or v) (as inline),
+ or verb (as environment)
+ source (str/None): source of the replacement content
+ after (str): text immediately following the command; important in
+ some situations, because spacing can depend on what's next
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ '''
+ if print_replacement_mode == 'verb':
+ if print_replacement.count('\n') > 1:
+ print('* DePythonTeX error:')
+ print(' Attempt to print multiple lines of content near line ' + str(linenum))
+ print(' This is not possible in inline verbatim mode')
+ sys.exit(1)
+ print_replacement = print_replacement.rstrip('\n')
+ for delim in ('|', '/', '`', '!', '&', '#', '@', ':', '%', '~', '$',
+ '=', '+', '-', '^', '_', '?', ';'):
+ if delim not in print_replacement:
+ break
+ print_replacement = r'\verb' + delim + print_replacement + delim
+ elif print_replacement_mode == 'verbatim':
+ if bool(match('\s*?\n', after)):
+ # Usually, we would end the verbatim environment with a newline.
+ # This is fine if there is content in `after` before the next
+ # newline---in fact, it's desirable, because the verbatim package
+ # doesn't allow for content on the same line as the end of the
+ # environment. But if `after` is an empty line, then adding a
+ # newline will throw off spacing and must be avoided
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}'
+ else:
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}\n'
+ else:
+ # When printed content from a file is included as LaTeX code, we have
+ # to be particularly careful to ensure that the content produces the
+ # same output when substituted as when brought in by `\input`. In
+ # particular, `\input` strips newlines from each line of content and
+ # adds a space at the end of each line. This space is inside the
+ # `\input`, so it will not merge with following spaces. So when we
+ # substitute the content, sometimes we need to replace the final
+ # newline with a space that cannot be gobbled.
+ #
+ # It gets more complicated. This final space is often not
+ # desirable. It can be prevented by either printing an `\endinput`
+ # command, to terminate the `\input`, or printing a percent
+ # character % in the last line of the content, which comments out the
+ # final newline. So we must check for `\endinput` anywhere in
+ # printed content, and % in the final line, and remove any content
+ # after them. It's also possible that the print is followed by
+ # an `\unskip` that eats the space, so we need to check for that too.
+ #
+ # It turns out that the same approach is needed when a command like
+ # `\py` brings in content ending in a newline
+ if (print_replacement.endswith('\\endinput\n') and
+ not print_replacement.endswith('\\string\\endinput\n')):
+ # If `\endinput` is present, everything from it on should be
+ # discarded, unless the `\endinput` is not actually a command
+ # but rather a typeset name (for example, `\string\endinput` or
+ # `\verb|\endinput|`). It's impossible to check for all cases in
+ # which `\endinput` is not a command (at least, without actually
+ # using LaTeX), and even checking for most of them would require
+ # a good bit of parsing. We assume that `\endinput`, as a
+ # command, will only ever occur at the immediate end of the
+ # printed content. Later, we issue a warning in case it appears
+ # anywhere else.
+ print_replacement = print_replacement.rsplit(r'\endinput', 1)[0]
+ elif (print_replacement.endswith('%\n') and
+ not print_replacement.endswith('\\%\n') and
+ not print_replacement.endswith('\\string%\n')):
+ # Perform an analogous check for a terminating percent characer %.
+ # This case would be a bit easier to parse fully, since a percent
+ # that comments out the last newline would have to be in the
+ # final line of the replacement. But it would still be
+ # very difficult to perform a complete check. Later, we issue a
+ # warning if there is reason to think that a percent character
+ # was active in the last line.
+ print_replacement = print_replacement.rsplit(r'%', 1)[0]
+ elif print_replacement.endswith('\n'):
+ # We can't just use `else` because that would catch content
+ # from `\py` and similar
+ # By default, LaTeX strips newlines and adds a space at the end
+ # of each line of content that is brought in by `\input`. This
+ # may or may not be desirable, but we replicate the effect here
+ # for consistency with the original document. We use `\space{}`
+ # because plain `\space` would gobble a following space, which
+ # isn't consistent with the `\input` behavior being replicated.
+ if bool(match(r'\\unskip\s+\S', after)):
+ # If there's an `\unskip`, fix the spacing and remove the
+ # `\unskip`. Since this is inline, the `\unskip` must
+ # immediately follow the command to do any good; otherwise,
+ # it eliminates spaces that precede it, but doesn't get into
+ # the `\input` content.
+ print_replacement = print_replacement.rstrip(' \t\n')
+ after = sub(r'^\\unskip\s+', '', after)
+ elif bool(match('\S', after)):
+ # If the next character is not whitespace, we can just leave
+ # the `\n`, and it will yield a space.
+ pass
+ elif bool(match('\s*$', after)):
+ # If the rest of the current line, and the next line, are
+ # whitespace, we will get the correct spacing without needing
+ # `\space{}`. We could leave `\n`, but it would be
+ # extraneous whitespace.
+ print_replacement = print_replacement[:-1]
+ else:
+ # Otherwise, we do need to insert `\space{}`
+ # We keep the newline at the end of printed content, in case
+ # it's at the end of an environment, and thus is needed to
+ # protect the following content
+ print_replacement += '\\space{}'
+ after = sub('^\s+', '', after)
+ forced_double_space_list.append((name, linenum))
+ else:
+ if bool(match('\s+\S', after)):
+ # If the following line starts with whitespace, replace it
+ # with a newline, to protect in the event that the printed
+ # content ended with an end-of-environment command
+ after = sub('^\s+', '\n', after)
+ # Issue warnings, if warranted
+ # Warn about `\endinput`
+ if (r'\endinput' in print_replacement and
+ print_replacement.count(r'\endinput') != print_replacement.count(r'\string\endinput')):
+ print('* DePythonTeX warning:')
+ print(' "\\endinput" was present in printed content near line ' + str(linenum))
+ print(' If this "\\endinput" was verbatim, you have nothing to worry about')
+ print(' If this "\\endinput" is to be active, it should be printed last')
+ print(' If you need "\\endinput" elsewhere, customize depythontex.py')
+ # Warn if it looks like there are active `%` that could comment
+ # out part of the original document. We only need to check the
+ # last line of printed content, because only there could
+ # percent characters escape from their original confines within
+ # `\input`, and comment out part of the document.
+ if print_replacement.endswith('\n'):
+ if print_replacement.count('\n') > 1:
+ last_line = print_replacement.rsplit('\n', 2)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following newline')
+ else:
+ if '\n' in print_replacement:
+ last_line = print_replacement.rsplit('\n', 1)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following text')
+ # Warn if there were `\unskip`'s in the output, in case they weren't
+ # positioned correctly
+ if bool(match(r'\s*\\unskip', after)):
+ print('* DePythonTeX warning:')
+ print(' "\\unskip" survived substitution near line ' + str(linenum))
+ print(' If it should have adjusted the spacing of printed content')
+ print(' you should double-check the spacing')
+ return (print_replacement, after)
+
+
+def replace_print_env(name, arglist, linenum,
+ print_replacement, print_replacement_mode, source,
+ after):
+ '''
+ Typeset printed content from an environment.
+
+ It is only ever called if there is indeed printed content to typeset.
+
+ This should be similar to replace_print_cmd(). The main difference is
+ that the environment context typically ends with a newline, so
+ substitution has to be a little different to ensure that spacing after
+ the environment isn't modified.
+
+ Args:
+ name (str): name of the environment
+ arglist (list, of str/None): all arguments given to the original
+ environment
+ linenum (int): line number in the original TeX document where the
+ environment began
+ print_replacement (str): printed content, read directly from file
+ into a single string
+ print_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX), inlineverb (or v) (as inline),
+ or verb (as environment)
+ source (str/None): source of the replacement content
+ after (str): text immediately following the command; important in
+ some situations, because spacing can depend on what's next
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ #### The inlineverb and verb modes should work, but haven't been tested
+ since there are currently no environments that use them; they are only
+ used by `\printpythontex`, which is a command.
+ '''
+ if print_replacement_mode == 'verb':
+ if print_replacement.count('\n') > 1:
+ print('* DePythonTeX error:')
+ print(' Attempt to print multiple lines of content near line ' + str(linenum))
+ print(' This is not possible in inline verbatim mode')
+ sys.exit(1)
+ print_replacement = print_replacement.rstrip('\n')
+ for delim in ('|', '/', '`', '!', '&', '#', '@', ':', '%', '~', '$',
+ '=', '+', '-', '^', '_', '?', ';'):
+ if delim not in print_replacement:
+ break
+ print_replacement = r'\verb' + delim + print_replacement + delim
+ if not bool(match('[ \t]+\S', after)):
+ # If there is text on the same line as the end of the
+ # environment, we're fine (this is unusual). Otherwise,
+ # we need to toss the newline at the end of the environment
+ # and gobble leading spaces. Leading spaces need to be
+ # gobbled because previously they were at the beginning of a
+ # line, where they would have been discarded.
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*?\n\s*', '', after)
+ elif print_replacement_mode == 'verbatim':
+ if bool(match('\s*?\n', after)):
+ # Usually, we would end the verbatim environment with a newline.
+ # This is fine if there is content in `after` before the next
+ # newline---in fact, it's desirable, because the verbatim package
+ # doesn't allow for content on the same line as the end of the
+ # environment. But if `after` is an empty line, then adding a
+ # newline will throw off spacing and must be avoided
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}'
+ else:
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}\n'
+ else:
+ # When printed content is included as LaTeX code, we have to be
+ # particularly careful to ensure that the content produces the same
+ # output when substituted as when brought in by `\input`. In
+ # particular, `\input` strips newlines from each line of content and
+ # adds a space at the end of each line. This space is inside the
+ # `\input`, so it will not merge with following spaces. So when we
+ # substitute the content, sometimes we need to replace the final
+ # newline with a space that cannot be gobbled.
+ #
+ # It gets more complicated. This final space is often not
+ # desirable. It can be prevented by either printing an `\endinput`
+ # command, to terminate the `\input`, or printing a percent
+ # character % in the last line of the content, which comments out the
+ # final newline. So we must check for `\endinput` anywhere in
+ # printed content, and % in the final line, and remove any content
+ # after them. It's also possible that the print is followed by
+ # an `\unskip` that eats the space, so we need to check for that too.
+ if (print_replacement.endswith('\\endinput\n') and
+ not print_replacement.endswith('\\string\\endinput\n')):
+ # If `\endinput` is present, everything from it on should be
+ # discarded, unless the `\endinput` is not actually a command
+ # but rather a typeset name (for example, `\string\endinput` or
+ # `\verb|\endinput|`). It's impossible to check for all cases in
+ # which `\endinput` is not a command (at least, without actually
+ # using LaTeX), and even checking for most of them would require
+ # a good bit of parsing. We assume that `\endinput`, as a
+ # command, will only ever occur at the immediate end of the
+ # printed content. Later, we issue a warning in case it appears
+ # anywhere else.
+ print_replacement = print_replacement.rsplit(r'\endinput', 1)[0]
+ if not bool(match('[ \t]+\S', after)):
+ # If there is text on the same line as the end of the
+ # environment, we're fine (this is unusual). Otherwise,
+ # we need to toss the newline at the end of the environment
+ # and gobble leading spaces. Leading spaces need to be
+ # gobbled because previously they were at the beginning of a
+ # line, where they would have been discarded.
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*?\n\s*', '', after)
+ elif (print_replacement.endswith('%\n') and
+ not print_replacement.endswith('\\%\n') and
+ not print_replacement.endswith('\\string%\n')):
+ # Perform an analogous check for a terminating percent characer %.
+ # This case would be a bit easier to parse fully, since a percent
+ # that comments out the last newline would have to be in the
+ # final line of the replacement. But it would still be
+ # very difficult to perform a complete check. Later, we issue a
+ # warning if there is reason to think that a percent character
+ # was active in the last line.
+ print_replacement = print_replacement.rsplit(r'%', 1)[0]
+ if not bool(match('[ \t]+\S', after)):
+ # If there is text on the same line as the end of the
+ # environment, we're fine (this is unusual). Otherwise,
+ # we need to toss the newline at the end of the environment
+ # and gobble leading spaces. Leading spaces need to be
+ # gobbled because previously they were at the beginning of a
+ # line, where they would have been discarded.
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*?\n\s*', '', after)
+ else:
+ # By default, LaTeX strips newlines and adds a space at the end
+ # of each line of content that is brought in by `\input`. This
+ # may or may not be desirable, but we replicate the effect here
+ # for consistency with the original document. We use `\space{}`
+ # because plain `\space` would gobble a following space, which
+ # isn't consistent with the `\input` behavior being replicated.
+ if bool(match(r'\s*\\unskip\s+\S', after)):
+ # If there's an `\unskip`, fix the spacing and remove the
+ # `\unskip`
+ print_replacement = print_replacement.rstrip(' \t\n')
+ after = sub(r'^\s*\\unskip\s+', '', after)
+ elif bool(match('[ \t]+\S', after)):
+ # If the next character after the end of the environment is
+ # not whitespace (usually not allowed), we can just leave
+ # the `\n` in printed content, and it will yield a space.
+ # So we need do nothing. But if there is text on that line
+ # we need `\space{}`.
+ after = sub('^\s+', '\\space', after)
+ forced_double_space_list.append((name, linenum))
+ else:
+ # If the line at the end of the environment is blank,
+ # we can just discard it and keep the newline at the end of
+ # the printed content; the newline gives us the needed space
+ after = after.split('\n', 1)[1]
+ # Issue warnings, if warranted
+ # Warn about `\endinput`
+ if (r'\endinput' in print_replacement and
+ print_replacement.count(r'\endinput') != print_replacement.count(r'\string\endinput')):
+ print('* DePythonTeX warning:')
+ print(' "\\endinput" was present in printed content near line ' + str(linenum))
+ print(' If this "\\endinput" was verbatim, you have nothing to worry about')
+ print(' If this "\\endinput" is to be active, it should be printed last')
+ print(' If you need "\\endinput" elsewhere, customize depythontex.py')
+ # Warn if it looks like there are active `%` that could comment
+ # out part of the original document. We only need to check the
+ # last line of printed content, because only there could
+ # percent characters escape from their original confines within
+ # `\input`, and comment out part of the document.
+ if print_replacement.endswith('\n'):
+ if print_replacement.count('\n') > 1:
+ last_line = print_replacement.rsplit('\n', 2)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following newline')
+ else:
+ if '\n' in print_replacement:
+ last_line = print_replacement.rsplit('\n', 1)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following text')
+ # Warn if there were `\unskip`'s in the output, in case they weren't
+ # positioned correctly
+ if bool(match(r'\s*\\unskip', after)):
+ print('* DePythonTeX warning:')
+ print(' "\\unskip" survived substitution near line ' + str(linenum))
+ print(' If it should have adjusted the spacing of printed content')
+ print(' you should double-check the spacing')
+ return (print_replacement, after)
+
+
+
+
+# Deal with argv
+# Parse argv
+parser = argparse.ArgumentParser()
+parser.add_argument('--version', action='version',
+ version='DePythonTeX {0}'.format(version))
+parser.add_argument('--encoding', default='utf-8',
+ help='encoding for all text files (see codecs module for encodings)')
+parser.add_argument('--overwrite', default=False, action='store_true',
+ help='overwrite existing output, if it exists (off by default)')
+parser.add_argument('--listing', default='verbatim',
+ choices=('verbatim', 'fancyvrb', 'listings', 'minted', 'pythontex'),
+ help='style or package used for typesetting code')
+parser.add_argument('--lexer-dict', default=None,
+ help='add mappings from Pygments lexer names to the language names of other highlighting packages; should be a comma-separated list of the form "<Pygments lexer>:<language>, <Pygments lexer>:<language>, ..."')
+parser.add_argument('--preamble', default=None,
+ help='line of commands to add to output preamble')
+parser.add_argument('--graphicspath', default=False, action='store_true',
+ help=r'Add the outputdir to the graphics path, by modifying an existing \graphicspath command or adding one.')
+parser.add_argument('TEXNAME',
+ help='LaTeX file')
+parser.add_argument('OUTFILE', nargs='?', default=None,
+ help='output file; by default, <filename>.<ext> is converted into depythontex_<filename>.<ext>')
+args = parser.parse_args()
+
+# Process argv
+encoding = args.encoding
+listing = args.listing
+if args.preamble is not None:
+ preamble_additions.append(args.preamble)
+if args.lexer_dict is not None:
+ args.lexer_dict = args.lexer_dict.replace(' ', '').replace("'", "").replace('"','').strip('{}')
+ for entry in args.lexer_dict.split(','):
+ k, v = entry.split(':')
+ lexer_dict[k] = v
+if args.listing == 'verbatim':
+ # In some contexts, the verbatim package might be desirable.
+ # But we assume that the user wants minimal packages.
+ # Also, the default verbatim environment doesn't allow text to follow the
+ # end-of-environment command.
+ # If the verbatim package is ever desired, simply uncomment the following:
+ # preamble_additions.append('\\usepackage{verbatim}')
+ pass
+elif args.listing == 'fancyvrb':
+ preamble_additions.append('\\usepackage{fancyvrb}')
+elif args.listing == 'listings':
+ preamble_additions.append('\\usepackage{listings}')
+elif args.listing == 'minted':
+ preamble_additions.append('\\usepackage{minted}')
+elif args.listing == 'pythontex':
+ preamble_additions.append('\\usepackage{pythontex}')
+
+
+
+
+# Let the user know things have started
+print('This is DePythonTeX {0}'.format(version))
+sys.stdout.flush()
+
+
+
+
+# Make sure we have a valid texfile
+texfile_name = os.path.expanduser(os.path.normcase(args.TEXNAME))
+if not os.path.isfile(texfile_name):
+ resolved = False
+ if not texfile_name.endswith('.tex'):
+ for ext in ('.tex', '.ltx', '.dtx'):
+ if os.path.isfile(texfile_name + ext):
+ texfile_name = texfile_name + ext
+ resolved = True
+ break
+ if not resolved:
+ print('* DePythonTeX error:')
+ print(' Could not locate file "' + texfile_name + '"')
+ sys.exit(1)
+# Make sure we have a valid outfile
+if args.OUTFILE is None:
+ p, f_name = os.path.split(texfile_name)
+ outfile_name = os.path.join(p, 'depythontex_' + f_name)
+else:
+ outfile_name = os.path.expanduser(os.path.normcase(args.OUTFILE))
+if not args.overwrite and os.path.isfile(outfile_name):
+ print('* DePythonTeX warning:')
+ print(' Output file "' + outfile_name + '" already exists')
+ ans = input(' Do you want to overwrite this file? [y,n]\n ')
+ if ans != 'y':
+ sys.exit(1)
+# Make sure the .depytx file exists
+depytxfile_name = texfile_name.rsplit('.')[0] + '.depytx'
+if not os.path.isfile(depytxfile_name):
+ print('* DePythonTeX error:')
+ print(' Could not find DePythonTeX auxiliary file "' + depytxfile_name + '"')
+ print(' Use package option depythontex to creat it')
+ sys.exit(1)
+
+
+
+
+# Start opening files and loading data
+# Read in the LaTeX file
+# We read into a list with an empty first entry, so that we don't have to
+# worry about zero indexing when comparing list index to file line number
+f = open(texfile_name, 'r', encoding=encoding)
+tex = ['']
+tex.extend(f.readlines())
+f.close()
+# Load the .depytx
+f = open(depytxfile_name, 'r', encoding=encoding)
+depytx = f.readlines()
+f.close()
+# Process the .depytx by getting the settings contained in the last few lines
+settings = dict()
+n = len(depytx) - 1
+while depytx[n].startswith('=>DEPYTHONTEX:SETTINGS#'):
+ content = depytx[n].split('#', 1)[1].rsplit('#', 1)[0]
+ k, v = content.split('=', 1)
+ if v in ('true', 'True'):
+ v = True
+ elif v in ('false', 'False'):
+ v = False
+ settings[k] = v
+ depytx[n] = ''
+ n -= 1
+# Check .depytx version to make sure it is compatible
+if settings['version'] != version:
+ print('* DePythonTeX warning:')
+ print(' Version mismatch with DePythonTeX auxiliary file')
+ print(' Do a complete compile cycle to update the auxiliary file')
+ print(' Attempting to proceed')
+# Go ahead and open the outfile, even though we don't need it until the end
+# This lets us change working directories for convenience without worrying
+# about having to modify the outfile path
+outfile = open(outfile_name, 'w', encoding=encoding)
+
+
+
+
+# Change working directory to the document directory
+# Technically, we could get by without this, but that would require a lot of
+# path modification. This way, we can just use all paths straight out of the
+# .depytx without any modification, which is much simpler and less error-prone.
+if os.path.split(texfile_name)[0] != '':
+ os.chdir(os.path.split(texfile_name)[0])
+
+
+
+
+# Open and process the file of macros
+# Read in the macros
+if os.path.isfile(settings['macrofile']):
+ f = open(settings['macrofile'], 'r', encoding=encoding)
+ macros = f.readlines()
+ f.close()
+else:
+ print('* DePythonTeX error:')
+ print(' The macro file could not be found:')
+ print(' "' + settings['macrofile'] + '"')
+ print(' Run PythonTeX to create it')
+ sys.exit(1)
+# Create a dict for storing macros
+macrodict = defaultdict(list)
+# Create variables for keeping track of whether we're inside a macro or
+# environment
+# These must exist before we begin processing
+inside_macro = False
+inside_environment = False
+# Loop through the macros, and extract everything
+# We just extract content; we get content wrappers later, when we process all
+# substituted content
+for line in macros:
+ if inside_macro:
+ # If we're in a macro, look for the end-of-macro command
+ if r'\endpytx@SVMCR' in line:
+ # If the current line contains the end-of-macro command, split
+ # off any content that comes before it. Also reset
+ # `inside_macro`.
+ macrodict[current_macro].append(line.rsplit(r'\endpytx@SVMCR', 1)[0])
+ inside_macro = False
+ else:
+ # If the current line doesn't end the macro, we add the whole
+ # line to the macro dict
+ macrodict[current_macro].append(line)
+ elif inside_environment:
+ if line.startswith(end_environment):
+ # If the environment is ending, we reset inside_environment
+ inside_environment = False
+ else:
+ # If we're still in the environment, add the current line to the
+ # macro dict
+ macrodict[current_macro].append(line)
+ else:
+ # If we're not in a macro or environment, we need to figure out which
+ # we are dealing with (if either; there are blank lines in the macro
+ # file to increase readability). Once we've determined which one,
+ # we need to get its name and extract any content.
+ if line.startswith(r'\begin{'):
+ # Any \begin will indicate a use of fancyvrb to save verbatim
+ # content, since that is the only time an environment is used in
+ # the macro file. All other content is saved in a standard macro.
+ # We extract the name of the macro in which the verbatim content
+ # is saved.
+ current_macro = line.rsplit('{', 1)[1].rstrip('}\n')
+ inside_environment = True
+ # We assemble the end-of-environment string we will need to look
+ # for. We don't assume any particular name, for generality.
+ end_environment = r'\end{' + line.split('}', 1)[0].split('{', 1)[1] + '}'
+ # Code typset in an environment needs to have a leading newline,
+ # because the content of a normal verbatim environment keeps its
+ # leading newline.
+ macrodict[current_macro].append('\n')
+ elif line.startswith(r'\pytx@SVMCR{'):
+ # Any regular macro will use `\pytx@SVMCR`
+ current_macro = line.split('{', 1)[1].split('}', 1)[0]
+ inside_macro = True
+ # Any content will always be on the next line, so we don't need
+ # to check for it
+
+
+
+
+# Do the actual processing
+# Create a variable for keeping track of the current line in the LaTeX file
+# Start at 1, since the first entry in the tex list is `''`
+texlinenum = 1
+# Create a variable for storing the current line(s) we are processing.
+# This contains all lines from immediately after the last successfully
+# processed line up to and including texlinenum. We may have to process
+# multiple lines at once if a macro is split over multiple lines, etc.
+texcontent = tex[texlinenum]
+# Create a list for storing processed content.
+texout = list()
+# Loop through the depytx and process
+for n, depytxline in enumerate(depytx):
+ if depytxline.startswith('=>DEPYTHONTEX#'):
+ # Process info
+ depytxcontent = depytxline.split('#', 1)[1].rstrip('#\n')
+ depy_type, depy_name, depy_args, depy_typeset, depy_linenum, depy_lexer = depytxcontent.split(':')
+ if depy_lexer == '':
+ depy_lexer = None
+
+ # Do a quick check on validity of info
+ # #### Eventually add 'cp' and 'pc'
+ if not (depy_type in ('cmd', 'env') and
+ all([letter in ('o', 'm', 'v', 'n', '|') for letter in depy_args]) and
+ ('|' not in depy_args or (depy_args.count('|') == 1 and depy_args.endswith('|'))) and
+ depy_typeset in ('c', 'p', 'n')):
+ print('* PythonTeX error:')
+ print(' Invalid \\Depythontex string for operation on line ' + str(depy_linenum))
+ print(' The offending string was ' + depytxcontent)
+ sys.exit(1)
+ # If depy_args contains a `|` to indicate `\obeylines`, strip it and
+ # store in a variable. Create a bool to keep track of obeylines
+ # status, which governs whether we can look on the next line for
+ # arguments. (If obeylines is active, a newline terminates the
+ # argument search.)
+ if depy_args.endswith('|'):
+ obeylines = True
+ depy_args = depy_args.rstrip('|')
+ else:
+ obeylines = False
+ # Get the line number as an integer
+ # We don't have to adjust for zero indexing in tex
+ depy_linenum = int(depy_linenum)
+
+
+ # Check for information passed from LaTeX
+ # This will be extra listings information, or replacements to plug in
+ code_replacement = None
+ code_replacement_mode = None
+ print_replacement = None
+ print_replacement_mode = None
+ firstnumber = None
+ source = None
+ scan_ahead_line = n + 1
+ nextdepytxline = depytx[scan_ahead_line]
+ while not nextdepytxline.startswith('=>DEPYTHONTEX#'):
+ if nextdepytxline.startswith('LISTING:'):
+ listingcontent = nextdepytxline.split(':', 1)[1].rstrip('\n')
+ if bool(match(r'firstnumber=\d+$', listingcontent)):
+ firstnumber = listingcontent.split('=', 1)[1]
+ else:
+ print('* DePythonTeX error:')
+ print(' Unknown information in listings data on line ' + str(depy_linenum))
+ print(' The listings content was "' + listingcontent + '"')
+ sys.exit(1)
+ elif nextdepytxline.startswith('MACRO:'):
+ source = 'macro'
+ try:
+ typeset, macro = nextdepytxline.rstrip('\n').split(':', 2)[1:]
+ except:
+ print('* DePythonTeX error:')
+ print(' Improperly formatted macro information on line ' + str(depy_linenum))
+ print(' The macro information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ if macro not in macrodict:
+ print('* DePythonTeX error:')
+ print(' Could not find replacement content for macro "' + macro + '"')
+ print(' This is probably because the document needs to be recompiled')
+ sys.exit(1)
+ if typeset == 'c':
+ if depy_type == 'cmd':
+ code_replacement = ''.join(macrodict[macro]).strip('\n')
+ else:
+ code_replacement = ''.join(macrodict[macro])
+ elif typeset == 'p':
+ print_replacement = ''.join(macrodict[macro])
+ else:
+ print('* DePythonTeX error:')
+ print(' Improper typesetting information for macro information on line ' + str(depy_linenum))
+ print(' The macro information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ elif nextdepytxline.startswith('FILE:'):
+ source = 'file'
+ try:
+ typeset, f_name = nextdepytxline.rstrip('\n').split(':', 2)[1:]
+ except:
+ print('* DePythonTeX error:')
+ print(' Improperly formatted file information on line ' + str(depy_linenum))
+ print(' The file information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ # Files that are brought in have an optional mode that
+ # determines if they need special handling (for example, verbatim)
+ if ':mode=' in f_name:
+ f_name, mode = f_name.split(':mode=')
+ else:
+ mode = None
+ f = open(f_name, 'r', encoding=encoding)
+ replacement = f.read()
+ f.close()
+ if typeset == 'c':
+ code_replacement_mode = mode
+ if depy_type == 'cmd' and code_replacement_mode != 'verbatim':
+ # Usually, code from commands is typeset with commands
+ # and code from environments is typeset in
+ # environments. The except is code from commands
+ # that bring in external files, like `\inputpygments`
+ code_replacement = replacement
+ else:
+ # If we're replacing an environment of code with a
+ # file, then we lose the newline at the beginning
+ # of the environment, and need to get it back.
+ code_replacement = '\n' + replacement
+ elif typeset == 'p':
+ print_replacement_mode = mode
+ print_replacement = replacement
+ else:
+ print('* DePythonTeX error:')
+ print(' Improper typesetting information for file information on line ' + str(depy_linenum))
+ print(' The file information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ # Increment the line in depytx to check for more information
+ # from LaTeX
+ scan_ahead_line += 1
+ if scan_ahead_line == len(depytx):
+ break
+ else:
+ nextdepytxline = depytx[scan_ahead_line]
+
+
+ # If the line we're looking for is within the range currently held by
+ # texcontent, do nothing. Otherwise, transfer content from tex
+ # to texout until we get to the line of tex that we're looking for
+ if depy_linenum > texlinenum:
+ texout.append(texcontent)
+ texlinenum += 1
+ while texlinenum < depy_linenum:
+ texout.append(tex[texlinenum])
+ texlinenum += 1
+ texcontent = tex[texlinenum]
+
+
+ # Deal with arguments
+ # All arguments are parsed and stored in a list variables, even if
+ # they are not used, for completeness; this makes it easy to add
+ # functionality
+ # Start by splitting the current line into what comes before the
+ # command or environment, and what is after it
+ if depy_type == 'cmd':
+ try:
+ before, after = texcontent.split('\\' + depy_name, 1)
+ except:
+ print('* DePythonTeX error:')
+ print(' Could not find command "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ else: # depy_type == 'env':
+ try:
+ before, after = texcontent.split(r'\begin{' + depy_name + '}', 1)
+ except:
+ print('* DePythonTeX error:')
+ print(' Could not find environment "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ # We won't need the content from before the command or environment
+ # again, so we go ahead and store it
+ texout.append(before)
+
+ # Parse the arguments
+ # Create a list for storing the recovered arguments
+ arglist = list()
+ for argindex, arg in enumerate(depy_args):
+ if arg == 'n':
+ pass
+ elif arg == 'o':
+ if after[0] == '[':
+ # Account for possible line breaks before end of arg
+ while ']' not in after:
+ texlinenum += 1
+ after += tex[texlinenum]
+ optarg, after = after[1:].split(']', 1)
+ else:
+ if obeylines:
+ # Take into account possible whitespace before arg
+ if bool(match('[ \t]*\[', after)):
+ after = after.split('[', 1)[1]
+ while ']' not in after:
+ texlinenum += 1
+ after += tex[texlinenum]
+ optarg, after = after.split(']', 1)
+ else:
+ optarg = None
+ # If this is the last arg, and it wasn't found,
+ # the macro should eat all whitespace following it
+ if argindex == len(depy_args) - 1:
+ after = sub('^[ \t]*', '', after)
+ else:
+ # Allow peeking ahead a line for the argument
+ if bool(match('\s*$', after)) and after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ # Take into account possible whitespace before arg
+ if bool(match('\s*\[', after)):
+ after = after.split('[', 1)[1]
+ while ']' not in after:
+ texlinenum += 1
+ after += tex[texlinenum]
+ optarg, after = after.split(']', 1)
+ else:
+ optarg = None
+ # Account for eating whitespace afterward, if arg not found
+ if argindex == len(depy_args) - 1:
+ if bool(match('\s*$', after)) and after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*', '', after)
+ arglist.append(optarg)
+ elif arg == 'm':
+ # Account for possible line breaks or spaces before arg
+ if after[0] == '{':
+ after = after[1:]
+ else:
+ if obeylines:
+ # Account for possible leading whitespace
+ if bool(match('[ \t\f\v]*\{', after)):
+ after = after.split('{', 1)[1]
+ else:
+ print('* DePythonTeX error:')
+ print(' Flawed mandatory argument for "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ else:
+ # Peek ahead a line if needed
+ if bool(match('\s*$', after)) and after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ if bool(match('\s*\{', after)):
+ after = after.split('{', 1)[1]
+ else:
+ print('* DePythonTeX error:')
+ print(' Flawed mandatory argument for "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ # Go through the argument character by character to find the
+ # closing brace.
+ # If possible, use a very simple approach
+ if (r'\{' not in after and r'\}' not in after and
+ r'\string' not in after and
+ after.count('{') + 1 == after.count('}')):
+ pos = 0
+ lbraces = 1
+ rbraces = 0
+ while True:
+ if after[pos] == '{':
+ lbraces += 1
+ elif after[pos] == '}':
+ rbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ if pos == len(after):
+ texlinenum += 1
+ after += tex[texlinenum]
+ # If a simple parsing approach won't work, parse in much
+ # greater depth
+ else:
+ pos = 0
+ lbraces = 1
+ rbraces = 0
+ while True:
+ if after[pos] == '{':
+ # If the current character is a brace, we count it
+ lbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ elif after[pos] == '}':
+ # If the current character is a brace, we count it
+ rbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ elif after[pos:].startswith(r'\string'):
+ # If the current position marks the beginning of `\string`, we
+ # resolve the `\string` command
+ # First, jump ahead to after `\string`
+ pos += 7 #+= len(r'\string')
+ # See if `\string` is followed by a regular macro
+ # If so, jump past it; otherwise, figure out if a
+ # single-character macro, or just a single character, is next,
+ # and jump past it
+ standard_macro = match(r'\\[a-zA-Z]+', line[pos:])
+ if bool(standard_macro):
+ pos += standard_macro.end()
+ elif line[pos] == '\\':
+ pos += 2
+ else:
+ pos += 1
+ elif line[pos] == '\\':
+ # If the current position is a backslash, figure out what
+ # macro is used, and jump past it
+ # The macro must either be a standard alphabetic macro,
+ # or a single-character macro
+ standard_macro = match(r'\\[a-zA-Z]+', line[pos:])
+ if bool(standard_macro):
+ pos += standard_macro.end()
+ else:
+ pos += 2
+ else:
+ pos += 1
+ if pos == len(after):
+ texlinenum += 1
+ after += tex[texlinenum]
+ mainarg = after[:pos]
+ after = after[pos+1:]
+ arglist.append(mainarg)
+ elif arg == 'v':
+ if after[0] == '{':
+ # Account for the possibility of matched brace delimiters
+ # Not all verbatim commands allow for these
+ pos = 1
+ lbraces = 1
+ rbraces = 0
+ while True:
+ if after[pos] == '{':
+ lbraces += 1
+ elif after[pos] == '}':
+ rbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ if pos == len(after):
+ texlinenum += 1
+ after += tex[texlinenum]
+ mainarg = after[1:pos]
+ after = after[pos+1:]
+ else:
+ # Deal with matched character delims
+ delim = after[0]
+ while after.count(delim) < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ mainarg, after = after[1:].split(delim, 1)
+ arglist.append(mainarg)
+
+
+ # Do substitution, depending on what is required
+ # Need a variable for processed content to be added to texout
+ processed = None
+ if depy_typeset == 'c':
+ if depy_type == 'cmd':
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_code_cmd(depy_name, arglist,
+ depy_linenum,
+ code_replacement,
+ code_replacement_mode,
+ after, depy_lexer,
+ firstnumber)
+ else: # depy_type == 'env'
+ end_environment = r'\end{' + depy_name + '}'
+ if code_replacement is None:
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ after += tex[texlinenum]
+ if end_environment in tex[texlinenum]:
+ break
+ code_replacement, after = after.split(end_environment, 1)
+ # If there's content on the line with the end-environment
+ # command, it should be discarded, to imitate TeX
+ if not code_replacement.endswith('\n'):
+ code_replacement = code_replacement.rsplit('\n', 1)[0] + '\n'
+ # Take care of `gobble`
+ if settings['gobble'] == 'auto':
+ code_replacement = textwrap.dedent(code_replacement)
+ else:
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ if end_environment in tex[texlinenum]:
+ after = tex[texlinenum]
+ break
+ after = after.split(end_environment, 1)[1]
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_code_env(depy_name, arglist,
+ depy_linenum,
+ code_replacement,
+ code_replacement_mode,
+ after, depy_lexer,
+ firstnumber)
+ elif depy_typeset == 'p' and print_replacement is not None:
+ if depy_type == 'cmd':
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_print_cmd(depy_name, arglist,
+ depy_linenum,
+ print_replacement,
+ print_replacement_mode,
+ source,
+ after)
+ else: # depy_type == 'env'
+ end_environment = r'\end{' + depy_name + '}'
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ if end_environment in tex[texlinenum]:
+ after = tex[texlinenum]
+ break
+ after = after.split(end_environment, 1)[1]
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_print_env(depy_name, arglist,
+ depy_linenum,
+ print_replacement,
+ print_replacement_mode,
+ source,
+ after)
+ else: # depy_typeset == 'n' or (depy_typeset == 'p' and print_replacement is None):
+ if depy_type == 'cmd':
+ texcontent = after
+ else: # depy_type == 'env'
+ end_environment = r'\end{' + depy_name + '}'
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ if end_environment in tex[texlinenum]:
+ after = tex[texlinenum]
+ break
+ after = after.split(end_environment, 1)[1]
+ if bool(match('\s*\n', after)):
+ # If the line following `after` is whitespace, it should
+ # be stripped, since most environments throw away
+ # anything after the end of the environment
+ after = after.split('\n')[1]
+ texcontent = after
+ # #### Once it's supported on the TeX side, need to add support for
+ # pc and cp
+
+
+ # Store any processed content
+ if processed is not None:
+ texout.append(processed)
+
+
+# Transfer anything that's left in tex to texout
+texout.append(texcontent)
+texout.extend(tex[texlinenum+1:])
+
+
+
+
+# Replace the `\usepackage{pythontex}`
+for n, line in enumerate(texout):
+ if '{pythontex}' in line:
+ startline = n
+ while '\\usepackage' not in texout[startline] and startline >= 0:
+ startline -= 1
+ if startline == n:
+ if bool(search(r'\\usepackage(?:\[.*?\]){0,1}\{pythontex\}', line)):
+ texout[n] = sub(r'\\usepackage(?:\[.*?\]){0,1}\{pythontex\}', '', line)
+ break
+ else:
+ content = ''.join(texout[startline:n+1])
+ if bool(search(r'(?s)\\usepackage(?:\[.*?\]\s*){0,1}\{pythontex\}', content)):
+ replacement = sub(r'(?s)\\usepackage(?:\[.*?\]\s*){0,1}\{pythontex\}', '', content)
+ texout[startline] = replacement
+ for l in range(startline+1, n+1):
+ texout[l] = ''
+ break
+ elif line.startswith(r'\begin{document}'):
+ break
+if preamble_additions:
+ texout[n] += '\n'.join(preamble_additions) + '\n'
+# Take care of graphicspath
+if args.graphicspath and settings['graphicx']:
+ for n, line in enumerate(texout):
+ if '\\graphicspath' in line and not bool(match('\s*%', line)):
+ texout[n] = line.replace('\\graphicspath{', '\\graphicspath{{' + settings['outputdir'] +'/}')
+ break
+ elif line.startswith(r'\begin{document}'):
+ texout[n] = '\\graphicspath{{' + settings['outputdir'] + '/}}\n' + line
+ break
+
+
+
+
+# Print any final messages
+if forced_double_space_list:
+ print('* DePythonTeX warning:')
+ print(' A trailing double space was forced with "\\space{}" for the following')
+ print(' This can happen when printed content is included inline')
+ print(' The forced double space is only an issue if it is not intentional')
+ for name, linenum in forced_double_space_list:
+ print(' "' + name + '" near line ' + str(linenum))
+
+
+
+
+# Write output
+for line in texout:
+ outfile.write(line)
+outfile.close()
diff --git a/Master/texmf-dist/scripts/pythontex/depythontex3.py b/Master/texmf-dist/scripts/pythontex/depythontex3.py
new file mode 100755
index 00000000000..996b8447316
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/depythontex3.py
@@ -0,0 +1,1404 @@
+# -*- coding: utf-8 -*-
+'''
+PythonTeX depythontex script.
+
+This script takes a LaTeX document that uses the PythonTeX package and
+creates a new document that does not depend on PythonTeX. It substitutes all
+externally generated content into a copy of the original LaTeX document.
+This is useful when you need a document that relies on few external packages
+or custom macros (for example, for submission to a journal or conversion to
+another document format).
+
+If you just want to share a document that uses PythonTeX, keep in mind that
+the document can be modified and compiled just like a regular LaTeX document,
+without needing Python or any other external tools, so long as the following
+conditions are met:
+
+ * A copy of pythontex.sty is included with the document.
+ * The pythontex-files-<name> directory is included with the document.
+ * The PythonTeX-specific parts of the document are not modified.
+
+To work, this script requires that the original LaTeX document be compiled
+with the package option `depythontex`. That creates an auxiliary file with
+the extension .depytx that contains information about all content that needs
+to be substituted.
+
+This script is purposely written in a simple, largely linear form to
+facilitate customization. Most of the key substitutions are performed by a
+few functions defined near the beginning of the script, so if you need custom
+substitutions, you should begin there. By default, all typeset code is
+wrapped in `\verb` commands and verbatim environments, since these have the
+greatest generality. However, the command-line option --listing allows code
+to be typeset with the fancyvrb, listings, minted, or PythonTeX packages
+instead.
+
+The script automatically extracts all arguments of all commands and
+environments that it replaces, so that these are available if desired for
+customized substitution. Two additional pieces of information are also
+available for any typeset code: the Pygments lexer (often the same as the
+language) and the starting line number (if line numbering was used).
+
+Keep in mind that some manual adjustments may be required after a document is
+depythontex'ed. While depythontex attempts to create an exact copy of the
+original document, in many cases an identical copy is impossible. For
+example, typeset code may have a different appearance or layout when it is
+typeset with a different package.
+
+
+Copyright (c) 2013, Geoffrey M. Poore
+All rights reserved.
+Licensed under the BSD 3-Clause License:
+ http://www.opensource.org/licenses/BSD-3-Clause
+
+'''
+
+
+# Imports
+#// Python 2
+#from __future__ import absolute_import
+#from __future__ import division
+#from __future__ import print_function
+#from __future__ import unicode_literals
+#\\ End Python 2
+import sys
+import os
+#// Python 2
+#from io import open
+#input = raw_input
+#\\ End Python 2
+import argparse
+from collections import defaultdict
+from re import match, sub, search
+import textwrap
+
+
+# Script parameters
+# Version
+version = 'v0.12'
+
+
+
+
+# Functions and parameters for customizing the script output
+
+# Style or package for replacing code listings
+# This is actually set via command-line option --listing
+# It is created here simply for reference
+listing = None #'verbatim', 'fancyvrb', 'listings', 'minted', 'pythontex'
+
+# List of things to add to the preamble
+# It can be appended to via the command-line option --preamble
+# It is also appended to based on the code listing style that is used
+# And it could be manually edited here as well, as long as it remains a list
+preamble_additions = list()
+
+# Lexer dict
+# If you are using Pygments lexers that don't directly correspond to the
+# languages used by the listings package, you can submit replacements via the
+# command line option --lexer-dict, or edit this dict manually here. When
+# listings is used, all lexers are checked against this dict to see if a
+# substitution should be made. This approach could easily be modified to
+# work with another, non-Pygments highlighting package.
+lexer_dict = dict()
+
+
+def replace_code_cmd(name, arglist, linenum, code_replacement,
+ code_replacement_mode, after, lexer, firstnumber):
+ '''
+ Typeset code from a command with a command.
+
+ It is only ever called if there is indeed code to typeset.
+
+ Usually, code from a command is also typeset with a command. This
+ function primarily deals with that case. In cases where code from a
+ command is typeset with an environment (for example, `\inputpygments`),
+ this function performs some preprocessing and then uses
+ replace_code_env() to do the real work. This approach prevents the two
+ functions from unnecessarily duplicating each other, while still giving
+ the desired output.
+
+ Args:
+ name (str): name of the command
+ arglist (list, of str/None): all arguments given to the original
+ command; the last argument is what is typeset, unless a
+ code_replacement is specified or other instructions are given
+ linenum (int): line number in the original TeX document
+ code_replacement (str/None): replacement for the code; usually None
+ for commands, because typically the code to be typeset is the
+ last argument passed to the command, rather than something
+ captured elsewhere (like the body of an environment) or something
+ preprocessed (like a console environment's content)
+ code_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX; generally unused for code),
+ verb (inline), or verbatim (environment)
+ after (str): text immediately following the command; usually
+ shouldn't be needed
+ lexer (str/None): Pygments lexer
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ '''
+ # Get the correct replacement
+ if code_replacement is None:
+ code_replacement = arglist[-1]
+
+ # We only consider two possible modes of typesetting, verbatim and inline
+ # verbatim
+ if code_replacement_mode == 'verbatim':
+ # Sometimes we must replace a command with an environment, for
+ # example, for `\inputpygments`
+
+ # Make sure the introduction of an environment where a command was
+ # previously won't produce errors with following content; make sure
+ # that any following content is on a separate line
+ if bool(match('[ \t]*\S', after)):
+ after = '\n' + after
+ # Rather than duplicating much of replace_code_env(), just use it
+ return replace_code_env(name, arglist, linenum, code_replacement,
+ code_replacement_mode, after, lexer, firstnumber)
+ else:
+ # Usually, we're replacing a command with a command
+
+ # Wrap the replacement in appropriate delimiters
+ if (listing in ('verbatim', 'fancyvrb', 'minted') or
+ (listing in ('listings', 'pythontex') and
+ ('{' in code_replacement or '}' in code_replacement))):
+ for delim in ('|', '/', '`', '!', '&', '#', '@', ':', '%', '~', '$',
+ '=', '+', '-', '^', '_', '?', ';'):
+ if delim not in code_replacement:
+ break
+ code_replacement = delim + code_replacement + delim
+ else:
+ code_replacement = '{' + code_replacement + '}'
+ # Assemble the actual replacement
+ if listing in ('verbatim', 'minted'): # `\mint` isn't for inline use
+ code_replacement = r'\verb' + code_replacement
+ elif listing == 'fancyvrb':
+ code_replacement = r'\Verb' + code_replacement
+ elif listing == 'listings':
+ if lexer is None:
+ code_replacement = r'\lstinline[language={}]' + code_replacement
+ else:
+ if lexer in lexer_dict:
+ lexer = lexer_dict[lexer]
+ code_replacement = r'\lstinline[language=' + lexer + ']' + code_replacement
+ elif listing == 'pythontex':
+ if lexer is None:
+ code_replacement = r'\pygment{text}' + code_replacement
+ else:
+ code_replacement = r'\pygment{' + lexer + '}' + code_replacement
+ return (code_replacement, after)
+
+
+def replace_code_env(name, arglist, linenum, code_replacement,
+ code_replacement_mode, after, lexer, firstnumber):
+ '''
+ Typeset code from an environment with an environment.
+
+ It is only ever called if there is indeed code to typeset.
+
+ Usually it is only used to typeset code from an environment. However,
+ some commands bring in code that must be typeset as an environment. In
+ those cases, replace_code_cmd() is called initially, and after it
+ performs some preprocessing, this function is called. This approach
+ avoids unnecessary duplication between the two functions.
+
+ Args:
+ name (str): name of the environment
+ arglist (list, of str/None): all arguments given to the original
+ environment
+ linenum (int): line number in the original TeX document where
+ the environment began
+ code_replacement (str): replacement for the code; unlike the case of
+ commands, this is always not None if the function is called
+ code_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX; generally unused for code),
+ verb (inline), or verbatim (environment)
+ after (str): text immediately following the environment; usually
+ shouldn't be needed
+ lexer (str/None): Pygments lexer
+ firstnumber (str/None): the first number of the listing, if the listing
+ had numbered lines
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ '''
+ # Currently, there is no need to test for code_replacement_mode, because
+ # this function is only ever called if the mode is 'verbatim'. That may
+ # change in the future, but it seems unlikely that code entered in an
+ # environment would end up typeset with a command.
+ if listing == 'verbatim':
+ pre = '\\begin{verbatim}'
+ post = '\\end{verbatim}'
+ elif listing == 'fancyvrb':
+ if firstnumber is None:
+ pre = '\\begin{Verbatim}'
+ else:
+ pre = '\\begin{{Verbatim}}[numbers=left,firstnumber={0}]'.format(firstnumber)
+ post = '\\end{Verbatim}'
+ elif listing == 'listings':
+ if lexer is None:
+ if firstnumber is None:
+ pre = '\\begin{lstlisting}[language={}]'
+ else:
+ pre = '\\begin{{lstlisting}}[language={{}},numbers=left,firstnumber={0}]'.format(firstnumber)
+ else:
+ if lexer in lexer_dict:
+ lexer = lexer_dict[lexer]
+ if firstnumber is None:
+ pre = '\\begin{{lstlisting}}[language={0}]'.format(lexer)
+ else:
+ pre = '\\begin{{lstlisting}}[language={0},numbers=left,firstnumber={1}]'.format(lexer, firstnumber)
+ post = '\\end{lstlisting}'
+ elif listing == 'minted':
+ if lexer is None:
+ if firstnumber is None:
+ pre = '\\begin{minted}{text}'
+ else:
+ pre = '\\begin{{minted}}[linenos,firstnumber={0}]{{text}}'.format(firstnumber)
+ else:
+ if firstnumber is None:
+ pre = '\\begin{{minted}}{{{0}}}'.format(lexer)
+ else:
+ pre = '\\begin{{minted}}[linenos,firstnumber={0}]{{{1}}}'.format(firstnumber, lexer)
+ post = '\\end{minted}'
+ elif listing == 'pythontex':
+ if lexer is None:
+ if firstnumber is None:
+ pre = '\\begin{pygments}{text}'
+ else:
+ pre = '\\begin{{pygments}}[numbers=left,firstnumber={0}]{{text}}'.format(firstnumber)
+ else:
+ if firstnumber is None:
+ pre = '\\begin{{pygments}}{{{0}}}'.format(lexer)
+ else:
+ pre = '\\begin{{pygments}}[numbers=left,firstnumber={0}]{{{1}}}'.format(firstnumber, lexer)
+ post = '\\end{pygments}'
+ code_replacement = pre + code_replacement + post
+ return (code_replacement, after)
+
+
+# We will need to issue a warning every time that a substitution of printed
+# content results in a forced double space. We could just do this as we go,
+# but it's easier for the user to read if we just collect all the warnings
+# of this type, and print them once.
+forced_double_space_list = list()
+
+
+def replace_print_cmd(name, arglist, linenum,
+ print_replacement, print_replacement_mode, source,
+ after):
+ '''
+ Typeset printed content from a command.
+
+ It is only ever called if there is indeed printed content to typeset.
+
+ Args:
+ name (str): name of the command
+ arglist (list, of str/None): all arguments given to the original
+ command
+ linenum (int): line number in the original TeX document
+ print_replacement (str): printed content, read directly from file
+ into a single string
+ print_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX), inlineverb (or v) (as inline),
+ or verb (as environment)
+ source (str/None): source of the replacement content
+ after (str): text immediately following the command; important in
+ some situations, because spacing can depend on what's next
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ '''
+ if print_replacement_mode == 'verb':
+ if print_replacement.count('\n') > 1:
+ print('* DePythonTeX error:')
+ print(' Attempt to print multiple lines of content near line ' + str(linenum))
+ print(' This is not possible in inline verbatim mode')
+ sys.exit(1)
+ print_replacement = print_replacement.rstrip('\n')
+ for delim in ('|', '/', '`', '!', '&', '#', '@', ':', '%', '~', '$',
+ '=', '+', '-', '^', '_', '?', ';'):
+ if delim not in print_replacement:
+ break
+ print_replacement = r'\verb' + delim + print_replacement + delim
+ elif print_replacement_mode == 'verbatim':
+ if bool(match('\s*?\n', after)):
+ # Usually, we would end the verbatim environment with a newline.
+ # This is fine if there is content in `after` before the next
+ # newline---in fact, it's desirable, because the verbatim package
+ # doesn't allow for content on the same line as the end of the
+ # environment. But if `after` is an empty line, then adding a
+ # newline will throw off spacing and must be avoided
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}'
+ else:
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}\n'
+ else:
+ # When printed content from a file is included as LaTeX code, we have
+ # to be particularly careful to ensure that the content produces the
+ # same output when substituted as when brought in by `\input`. In
+ # particular, `\input` strips newlines from each line of content and
+ # adds a space at the end of each line. This space is inside the
+ # `\input`, so it will not merge with following spaces. So when we
+ # substitute the content, sometimes we need to replace the final
+ # newline with a space that cannot be gobbled.
+ #
+ # It gets more complicated. This final space is often not
+ # desirable. It can be prevented by either printing an `\endinput`
+ # command, to terminate the `\input`, or printing a percent
+ # character % in the last line of the content, which comments out the
+ # final newline. So we must check for `\endinput` anywhere in
+ # printed content, and % in the final line, and remove any content
+ # after them. It's also possible that the print is followed by
+ # an `\unskip` that eats the space, so we need to check for that too.
+ #
+ # It turns out that the same approach is needed when a command like
+ # `\py` brings in content ending in a newline
+ if (print_replacement.endswith('\\endinput\n') and
+ not print_replacement.endswith('\\string\\endinput\n')):
+ # If `\endinput` is present, everything from it on should be
+ # discarded, unless the `\endinput` is not actually a command
+ # but rather a typeset name (for example, `\string\endinput` or
+ # `\verb|\endinput|`). It's impossible to check for all cases in
+ # which `\endinput` is not a command (at least, without actually
+ # using LaTeX), and even checking for most of them would require
+ # a good bit of parsing. We assume that `\endinput`, as a
+ # command, will only ever occur at the immediate end of the
+ # printed content. Later, we issue a warning in case it appears
+ # anywhere else.
+ print_replacement = print_replacement.rsplit(r'\endinput', 1)[0]
+ elif (print_replacement.endswith('%\n') and
+ not print_replacement.endswith('\\%\n') and
+ not print_replacement.endswith('\\string%\n')):
+ # Perform an analogous check for a terminating percent characer %.
+ # This case would be a bit easier to parse fully, since a percent
+ # that comments out the last newline would have to be in the
+ # final line of the replacement. But it would still be
+ # very difficult to perform a complete check. Later, we issue a
+ # warning if there is reason to think that a percent character
+ # was active in the last line.
+ print_replacement = print_replacement.rsplit(r'%', 1)[0]
+ elif print_replacement.endswith('\n'):
+ # We can't just use `else` because that would catch content
+ # from `\py` and similar
+ # By default, LaTeX strips newlines and adds a space at the end
+ # of each line of content that is brought in by `\input`. This
+ # may or may not be desirable, but we replicate the effect here
+ # for consistency with the original document. We use `\space{}`
+ # because plain `\space` would gobble a following space, which
+ # isn't consistent with the `\input` behavior being replicated.
+ if bool(match(r'\\unskip\s+\S', after)):
+ # If there's an `\unskip`, fix the spacing and remove the
+ # `\unskip`. Since this is inline, the `\unskip` must
+ # immediately follow the command to do any good; otherwise,
+ # it eliminates spaces that precede it, but doesn't get into
+ # the `\input` content.
+ print_replacement = print_replacement.rstrip(' \t\n')
+ after = sub(r'^\\unskip\s+', '', after)
+ elif bool(match('\S', after)):
+ # If the next character is not whitespace, we can just leave
+ # the `\n`, and it will yield a space.
+ pass
+ elif bool(match('\s*$', after)):
+ # If the rest of the current line, and the next line, are
+ # whitespace, we will get the correct spacing without needing
+ # `\space{}`. We could leave `\n`, but it would be
+ # extraneous whitespace.
+ print_replacement = print_replacement[:-1]
+ else:
+ # Otherwise, we do need to insert `\space{}`
+ # We keep the newline at the end of printed content, in case
+ # it's at the end of an environment, and thus is needed to
+ # protect the following content
+ print_replacement += '\\space{}'
+ after = sub('^\s+', '', after)
+ forced_double_space_list.append((name, linenum))
+ else:
+ if bool(match('\s+\S', after)):
+ # If the following line starts with whitespace, replace it
+ # with a newline, to protect in the event that the printed
+ # content ended with an end-of-environment command
+ after = sub('^\s+', '\n', after)
+ # Issue warnings, if warranted
+ # Warn about `\endinput`
+ if (r'\endinput' in print_replacement and
+ print_replacement.count(r'\endinput') != print_replacement.count(r'\string\endinput')):
+ print('* DePythonTeX warning:')
+ print(' "\\endinput" was present in printed content near line ' + str(linenum))
+ print(' If this "\\endinput" was verbatim, you have nothing to worry about')
+ print(' If this "\\endinput" is to be active, it should be printed last')
+ print(' If you need "\\endinput" elsewhere, customize depythontex.py')
+ # Warn if it looks like there are active `%` that could comment
+ # out part of the original document. We only need to check the
+ # last line of printed content, because only there could
+ # percent characters escape from their original confines within
+ # `\input`, and comment out part of the document.
+ if print_replacement.endswith('\n'):
+ if print_replacement.count('\n') > 1:
+ last_line = print_replacement.rsplit('\n', 2)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following newline')
+ else:
+ if '\n' in print_replacement:
+ last_line = print_replacement.rsplit('\n', 1)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following text')
+ # Warn if there were `\unskip`'s in the output, in case they weren't
+ # positioned correctly
+ if bool(match(r'\s*\\unskip', after)):
+ print('* DePythonTeX warning:')
+ print(' "\\unskip" survived substitution near line ' + str(linenum))
+ print(' If it should have adjusted the spacing of printed content')
+ print(' you should double-check the spacing')
+ return (print_replacement, after)
+
+
+def replace_print_env(name, arglist, linenum,
+ print_replacement, print_replacement_mode, source,
+ after):
+ '''
+ Typeset printed content from an environment.
+
+ It is only ever called if there is indeed printed content to typeset.
+
+ This should be similar to replace_print_cmd(). The main difference is
+ that the environment context typically ends with a newline, so
+ substitution has to be a little different to ensure that spacing after
+ the environment isn't modified.
+
+ Args:
+ name (str): name of the environment
+ arglist (list, of str/None): all arguments given to the original
+ environment
+ linenum (int): line number in the original TeX document where the
+ environment began
+ print_replacement (str): printed content, read directly from file
+ into a single string
+ print_replacement_mode (str/None): mode in which the replacement is
+ to be typeset; raw/None (as TeX), inlineverb (or v) (as inline),
+ or verb (as environment)
+ source (str/None): source of the replacement content
+ after (str): text immediately following the command; important in
+ some situations, because spacing can depend on what's next
+ Returns:
+ (replacement, after) (tuple, of str)
+
+ #### The inlineverb and verb modes should work, but haven't been tested
+ since there are currently no environments that use them; they are only
+ used by `\printpythontex`, which is a command.
+ '''
+ if print_replacement_mode == 'verb':
+ if print_replacement.count('\n') > 1:
+ print('* DePythonTeX error:')
+ print(' Attempt to print multiple lines of content near line ' + str(linenum))
+ print(' This is not possible in inline verbatim mode')
+ sys.exit(1)
+ print_replacement = print_replacement.rstrip('\n')
+ for delim in ('|', '/', '`', '!', '&', '#', '@', ':', '%', '~', '$',
+ '=', '+', '-', '^', '_', '?', ';'):
+ if delim not in print_replacement:
+ break
+ print_replacement = r'\verb' + delim + print_replacement + delim
+ if not bool(match('[ \t]+\S', after)):
+ # If there is text on the same line as the end of the
+ # environment, we're fine (this is unusual). Otherwise,
+ # we need to toss the newline at the end of the environment
+ # and gobble leading spaces. Leading spaces need to be
+ # gobbled because previously they were at the beginning of a
+ # line, where they would have been discarded.
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*?\n\s*', '', after)
+ elif print_replacement_mode == 'verbatim':
+ if bool(match('\s*?\n', after)):
+ # Usually, we would end the verbatim environment with a newline.
+ # This is fine if there is content in `after` before the next
+ # newline---in fact, it's desirable, because the verbatim package
+ # doesn't allow for content on the same line as the end of the
+ # environment. But if `after` is an empty line, then adding a
+ # newline will throw off spacing and must be avoided
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}'
+ else:
+ print_replacement = '\\begin{verbatim}\n' + print_replacement + '\\end{verbatim}\n'
+ else:
+ # When printed content is included as LaTeX code, we have to be
+ # particularly careful to ensure that the content produces the same
+ # output when substituted as when brought in by `\input`. In
+ # particular, `\input` strips newlines from each line of content and
+ # adds a space at the end of each line. This space is inside the
+ # `\input`, so it will not merge with following spaces. So when we
+ # substitute the content, sometimes we need to replace the final
+ # newline with a space that cannot be gobbled.
+ #
+ # It gets more complicated. This final space is often not
+ # desirable. It can be prevented by either printing an `\endinput`
+ # command, to terminate the `\input`, or printing a percent
+ # character % in the last line of the content, which comments out the
+ # final newline. So we must check for `\endinput` anywhere in
+ # printed content, and % in the final line, and remove any content
+ # after them. It's also possible that the print is followed by
+ # an `\unskip` that eats the space, so we need to check for that too.
+ if (print_replacement.endswith('\\endinput\n') and
+ not print_replacement.endswith('\\string\\endinput\n')):
+ # If `\endinput` is present, everything from it on should be
+ # discarded, unless the `\endinput` is not actually a command
+ # but rather a typeset name (for example, `\string\endinput` or
+ # `\verb|\endinput|`). It's impossible to check for all cases in
+ # which `\endinput` is not a command (at least, without actually
+ # using LaTeX), and even checking for most of them would require
+ # a good bit of parsing. We assume that `\endinput`, as a
+ # command, will only ever occur at the immediate end of the
+ # printed content. Later, we issue a warning in case it appears
+ # anywhere else.
+ print_replacement = print_replacement.rsplit(r'\endinput', 1)[0]
+ if not bool(match('[ \t]+\S', after)):
+ # If there is text on the same line as the end of the
+ # environment, we're fine (this is unusual). Otherwise,
+ # we need to toss the newline at the end of the environment
+ # and gobble leading spaces. Leading spaces need to be
+ # gobbled because previously they were at the beginning of a
+ # line, where they would have been discarded.
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*?\n\s*', '', after)
+ elif (print_replacement.endswith('%\n') and
+ not print_replacement.endswith('\\%\n') and
+ not print_replacement.endswith('\\string%\n')):
+ # Perform an analogous check for a terminating percent characer %.
+ # This case would be a bit easier to parse fully, since a percent
+ # that comments out the last newline would have to be in the
+ # final line of the replacement. But it would still be
+ # very difficult to perform a complete check. Later, we issue a
+ # warning if there is reason to think that a percent character
+ # was active in the last line.
+ print_replacement = print_replacement.rsplit(r'%', 1)[0]
+ if not bool(match('[ \t]+\S', after)):
+ # If there is text on the same line as the end of the
+ # environment, we're fine (this is unusual). Otherwise,
+ # we need to toss the newline at the end of the environment
+ # and gobble leading spaces. Leading spaces need to be
+ # gobbled because previously they were at the beginning of a
+ # line, where they would have been discarded.
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*?\n\s*', '', after)
+ else:
+ # By default, LaTeX strips newlines and adds a space at the end
+ # of each line of content that is brought in by `\input`. This
+ # may or may not be desirable, but we replicate the effect here
+ # for consistency with the original document. We use `\space{}`
+ # because plain `\space` would gobble a following space, which
+ # isn't consistent with the `\input` behavior being replicated.
+ if bool(match(r'\s*\\unskip\s+\S', after)):
+ # If there's an `\unskip`, fix the spacing and remove the
+ # `\unskip`
+ print_replacement = print_replacement.rstrip(' \t\n')
+ after = sub(r'^\s*\\unskip\s+', '', after)
+ elif bool(match('[ \t]+\S', after)):
+ # If the next character after the end of the environment is
+ # not whitespace (usually not allowed), we can just leave
+ # the `\n` in printed content, and it will yield a space.
+ # So we need do nothing. But if there is text on that line
+ # we need `\space{}`.
+ after = sub('^\s+', '\\space', after)
+ forced_double_space_list.append((name, linenum))
+ else:
+ # If the line at the end of the environment is blank,
+ # we can just discard it and keep the newline at the end of
+ # the printed content; the newline gives us the needed space
+ after = after.split('\n', 1)[1]
+ # Issue warnings, if warranted
+ # Warn about `\endinput`
+ if (r'\endinput' in print_replacement and
+ print_replacement.count(r'\endinput') != print_replacement.count(r'\string\endinput')):
+ print('* DePythonTeX warning:')
+ print(' "\\endinput" was present in printed content near line ' + str(linenum))
+ print(' If this "\\endinput" was verbatim, you have nothing to worry about')
+ print(' If this "\\endinput" is to be active, it should be printed last')
+ print(' If you need "\\endinput" elsewhere, customize depythontex.py')
+ # Warn if it looks like there are active `%` that could comment
+ # out part of the original document. We only need to check the
+ # last line of printed content, because only there could
+ # percent characters escape from their original confines within
+ # `\input`, and comment out part of the document.
+ if print_replacement.endswith('\n'):
+ if print_replacement.count('\n') > 1:
+ last_line = print_replacement.rsplit('\n', 2)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following newline')
+ else:
+ if '\n' in print_replacement:
+ last_line = print_replacement.rsplit('\n', 1)[1]
+ else:
+ last_line = print_replacement
+ if last_line.count('%') != last_line.count(r'\%') + last_line.count(r'\string%'):
+ print('* DePythonTeX warning:')
+ print(' One or more percent characters are on the last line of ')
+ print(' printed content near line ' + str(linenum))
+ print(' If they are not verbatim, they could comment out the following text')
+ # Warn if there were `\unskip`'s in the output, in case they weren't
+ # positioned correctly
+ if bool(match(r'\s*\\unskip', after)):
+ print('* DePythonTeX warning:')
+ print(' "\\unskip" survived substitution near line ' + str(linenum))
+ print(' If it should have adjusted the spacing of printed content')
+ print(' you should double-check the spacing')
+ return (print_replacement, after)
+
+
+
+
+# Deal with argv
+# Parse argv
+parser = argparse.ArgumentParser()
+parser.add_argument('--version', action='version',
+ version='DePythonTeX {0}'.format(version))
+parser.add_argument('--encoding', default='utf-8',
+ help='encoding for all text files (see codecs module for encodings)')
+parser.add_argument('--overwrite', default=False, action='store_true',
+ help='overwrite existing output, if it exists (off by default)')
+parser.add_argument('--listing', default='verbatim',
+ choices=('verbatim', 'fancyvrb', 'listings', 'minted', 'pythontex'),
+ help='style or package used for typesetting code')
+parser.add_argument('--lexer-dict', default=None,
+ help='add mappings from Pygments lexer names to the language names of other highlighting packages; should be a comma-separated list of the form "<Pygments lexer>:<language>, <Pygments lexer>:<language>, ..."')
+parser.add_argument('--preamble', default=None,
+ help='line of commands to add to output preamble')
+parser.add_argument('--graphicspath', default=False, action='store_true',
+ help=r'Add the outputdir to the graphics path, by modifying an existing \graphicspath command or adding one.')
+parser.add_argument('TEXNAME',
+ help='LaTeX file')
+parser.add_argument('OUTFILE', nargs='?', default=None,
+ help='output file; by default, <filename>.<ext> is converted into depythontex_<filename>.<ext>')
+args = parser.parse_args()
+
+# Process argv
+encoding = args.encoding
+listing = args.listing
+if args.preamble is not None:
+ preamble_additions.append(args.preamble)
+if args.lexer_dict is not None:
+ args.lexer_dict = args.lexer_dict.replace(' ', '').replace("'", "").replace('"','').strip('{}')
+ for entry in args.lexer_dict.split(','):
+ k, v = entry.split(':')
+ lexer_dict[k] = v
+if args.listing == 'verbatim':
+ # In some contexts, the verbatim package might be desirable.
+ # But we assume that the user wants minimal packages.
+ # Also, the default verbatim environment doesn't allow text to follow the
+ # end-of-environment command.
+ # If the verbatim package is ever desired, simply uncomment the following:
+ # preamble_additions.append('\\usepackage{verbatim}')
+ pass
+elif args.listing == 'fancyvrb':
+ preamble_additions.append('\\usepackage{fancyvrb}')
+elif args.listing == 'listings':
+ preamble_additions.append('\\usepackage{listings}')
+elif args.listing == 'minted':
+ preamble_additions.append('\\usepackage{minted}')
+elif args.listing == 'pythontex':
+ preamble_additions.append('\\usepackage{pythontex}')
+
+
+
+
+# Let the user know things have started
+print('This is DePythonTeX {0}'.format(version))
+sys.stdout.flush()
+
+
+
+
+# Make sure we have a valid texfile
+texfile_name = os.path.expanduser(os.path.normcase(args.TEXNAME))
+if not os.path.isfile(texfile_name):
+ resolved = False
+ if not texfile_name.endswith('.tex'):
+ for ext in ('.tex', '.ltx', '.dtx'):
+ if os.path.isfile(texfile_name + ext):
+ texfile_name = texfile_name + ext
+ resolved = True
+ break
+ if not resolved:
+ print('* DePythonTeX error:')
+ print(' Could not locate file "' + texfile_name + '"')
+ sys.exit(1)
+# Make sure we have a valid outfile
+if args.OUTFILE is None:
+ p, f_name = os.path.split(texfile_name)
+ outfile_name = os.path.join(p, 'depythontex_' + f_name)
+else:
+ outfile_name = os.path.expanduser(os.path.normcase(args.OUTFILE))
+if not args.overwrite and os.path.isfile(outfile_name):
+ print('* DePythonTeX warning:')
+ print(' Output file "' + outfile_name + '" already exists')
+ ans = input(' Do you want to overwrite this file? [y,n]\n ')
+ if ans != 'y':
+ sys.exit(1)
+# Make sure the .depytx file exists
+depytxfile_name = texfile_name.rsplit('.')[0] + '.depytx'
+if not os.path.isfile(depytxfile_name):
+ print('* DePythonTeX error:')
+ print(' Could not find DePythonTeX auxiliary file "' + depytxfile_name + '"')
+ print(' Use package option depythontex to creat it')
+ sys.exit(1)
+
+
+
+
+# Start opening files and loading data
+# Read in the LaTeX file
+# We read into a list with an empty first entry, so that we don't have to
+# worry about zero indexing when comparing list index to file line number
+f = open(texfile_name, 'r', encoding=encoding)
+tex = ['']
+tex.extend(f.readlines())
+f.close()
+# Load the .depytx
+f = open(depytxfile_name, 'r', encoding=encoding)
+depytx = f.readlines()
+f.close()
+# Process the .depytx by getting the settings contained in the last few lines
+settings = dict()
+n = len(depytx) - 1
+while depytx[n].startswith('=>DEPYTHONTEX:SETTINGS#'):
+ content = depytx[n].split('#', 1)[1].rsplit('#', 1)[0]
+ k, v = content.split('=', 1)
+ if v in ('true', 'True'):
+ v = True
+ elif v in ('false', 'False'):
+ v = False
+ settings[k] = v
+ depytx[n] = ''
+ n -= 1
+# Check .depytx version to make sure it is compatible
+if settings['version'] != version:
+ print('* DePythonTeX warning:')
+ print(' Version mismatch with DePythonTeX auxiliary file')
+ print(' Do a complete compile cycle to update the auxiliary file')
+ print(' Attempting to proceed')
+# Go ahead and open the outfile, even though we don't need it until the end
+# This lets us change working directories for convenience without worrying
+# about having to modify the outfile path
+outfile = open(outfile_name, 'w', encoding=encoding)
+
+
+
+
+# Change working directory to the document directory
+# Technically, we could get by without this, but that would require a lot of
+# path modification. This way, we can just use all paths straight out of the
+# .depytx without any modification, which is much simpler and less error-prone.
+if os.path.split(texfile_name)[0] != '':
+ os.chdir(os.path.split(texfile_name)[0])
+
+
+
+
+# Open and process the file of macros
+# Read in the macros
+if os.path.isfile(settings['macrofile']):
+ f = open(settings['macrofile'], 'r', encoding=encoding)
+ macros = f.readlines()
+ f.close()
+else:
+ print('* DePythonTeX error:')
+ print(' The macro file could not be found:')
+ print(' "' + settings['macrofile'] + '"')
+ print(' Run PythonTeX to create it')
+ sys.exit(1)
+# Create a dict for storing macros
+macrodict = defaultdict(list)
+# Create variables for keeping track of whether we're inside a macro or
+# environment
+# These must exist before we begin processing
+inside_macro = False
+inside_environment = False
+# Loop through the macros, and extract everything
+# We just extract content; we get content wrappers later, when we process all
+# substituted content
+for line in macros:
+ if inside_macro:
+ # If we're in a macro, look for the end-of-macro command
+ if r'\endpytx@SVMCR' in line:
+ # If the current line contains the end-of-macro command, split
+ # off any content that comes before it. Also reset
+ # `inside_macro`.
+ macrodict[current_macro].append(line.rsplit(r'\endpytx@SVMCR', 1)[0])
+ inside_macro = False
+ else:
+ # If the current line doesn't end the macro, we add the whole
+ # line to the macro dict
+ macrodict[current_macro].append(line)
+ elif inside_environment:
+ if line.startswith(end_environment):
+ # If the environment is ending, we reset inside_environment
+ inside_environment = False
+ else:
+ # If we're still in the environment, add the current line to the
+ # macro dict
+ macrodict[current_macro].append(line)
+ else:
+ # If we're not in a macro or environment, we need to figure out which
+ # we are dealing with (if either; there are blank lines in the macro
+ # file to increase readability). Once we've determined which one,
+ # we need to get its name and extract any content.
+ if line.startswith(r'\begin{'):
+ # Any \begin will indicate a use of fancyvrb to save verbatim
+ # content, since that is the only time an environment is used in
+ # the macro file. All other content is saved in a standard macro.
+ # We extract the name of the macro in which the verbatim content
+ # is saved.
+ current_macro = line.rsplit('{', 1)[1].rstrip('}\n')
+ inside_environment = True
+ # We assemble the end-of-environment string we will need to look
+ # for. We don't assume any particular name, for generality.
+ end_environment = r'\end{' + line.split('}', 1)[0].split('{', 1)[1] + '}'
+ # Code typset in an environment needs to have a leading newline,
+ # because the content of a normal verbatim environment keeps its
+ # leading newline.
+ macrodict[current_macro].append('\n')
+ elif line.startswith(r'\pytx@SVMCR{'):
+ # Any regular macro will use `\pytx@SVMCR`
+ current_macro = line.split('{', 1)[1].split('}', 1)[0]
+ inside_macro = True
+ # Any content will always be on the next line, so we don't need
+ # to check for it
+
+
+
+
+# Do the actual processing
+# Create a variable for keeping track of the current line in the LaTeX file
+# Start at 1, since the first entry in the tex list is `''`
+texlinenum = 1
+# Create a variable for storing the current line(s) we are processing.
+# This contains all lines from immediately after the last successfully
+# processed line up to and including texlinenum. We may have to process
+# multiple lines at once if a macro is split over multiple lines, etc.
+texcontent = tex[texlinenum]
+# Create a list for storing processed content.
+texout = list()
+# Loop through the depytx and process
+for n, depytxline in enumerate(depytx):
+ if depytxline.startswith('=>DEPYTHONTEX#'):
+ # Process info
+ depytxcontent = depytxline.split('#', 1)[1].rstrip('#\n')
+ depy_type, depy_name, depy_args, depy_typeset, depy_linenum, depy_lexer = depytxcontent.split(':')
+ if depy_lexer == '':
+ depy_lexer = None
+
+ # Do a quick check on validity of info
+ # #### Eventually add 'cp' and 'pc'
+ if not (depy_type in ('cmd', 'env') and
+ all([letter in ('o', 'm', 'v', 'n', '|') for letter in depy_args]) and
+ ('|' not in depy_args or (depy_args.count('|') == 1 and depy_args.endswith('|'))) and
+ depy_typeset in ('c', 'p', 'n')):
+ print('* PythonTeX error:')
+ print(' Invalid \\Depythontex string for operation on line ' + str(depy_linenum))
+ print(' The offending string was ' + depytxcontent)
+ sys.exit(1)
+ # If depy_args contains a `|` to indicate `\obeylines`, strip it and
+ # store in a variable. Create a bool to keep track of obeylines
+ # status, which governs whether we can look on the next line for
+ # arguments. (If obeylines is active, a newline terminates the
+ # argument search.)
+ if depy_args.endswith('|'):
+ obeylines = True
+ depy_args = depy_args.rstrip('|')
+ else:
+ obeylines = False
+ # Get the line number as an integer
+ # We don't have to adjust for zero indexing in tex
+ depy_linenum = int(depy_linenum)
+
+
+ # Check for information passed from LaTeX
+ # This will be extra listings information, or replacements to plug in
+ code_replacement = None
+ code_replacement_mode = None
+ print_replacement = None
+ print_replacement_mode = None
+ firstnumber = None
+ source = None
+ scan_ahead_line = n + 1
+ nextdepytxline = depytx[scan_ahead_line]
+ while not nextdepytxline.startswith('=>DEPYTHONTEX#'):
+ if nextdepytxline.startswith('LISTING:'):
+ listingcontent = nextdepytxline.split(':', 1)[1].rstrip('\n')
+ if bool(match(r'firstnumber=\d+$', listingcontent)):
+ firstnumber = listingcontent.split('=', 1)[1]
+ else:
+ print('* DePythonTeX error:')
+ print(' Unknown information in listings data on line ' + str(depy_linenum))
+ print(' The listings content was "' + listingcontent + '"')
+ sys.exit(1)
+ elif nextdepytxline.startswith('MACRO:'):
+ source = 'macro'
+ try:
+ typeset, macro = nextdepytxline.rstrip('\n').split(':', 2)[1:]
+ except:
+ print('* DePythonTeX error:')
+ print(' Improperly formatted macro information on line ' + str(depy_linenum))
+ print(' The macro information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ if macro not in macrodict:
+ print('* DePythonTeX error:')
+ print(' Could not find replacement content for macro "' + macro + '"')
+ print(' This is probably because the document needs to be recompiled')
+ sys.exit(1)
+ if typeset == 'c':
+ if depy_type == 'cmd':
+ code_replacement = ''.join(macrodict[macro]).strip('\n')
+ else:
+ code_replacement = ''.join(macrodict[macro])
+ elif typeset == 'p':
+ print_replacement = ''.join(macrodict[macro])
+ else:
+ print('* DePythonTeX error:')
+ print(' Improper typesetting information for macro information on line ' + str(depy_linenum))
+ print(' The macro information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ elif nextdepytxline.startswith('FILE:'):
+ source = 'file'
+ try:
+ typeset, f_name = nextdepytxline.rstrip('\n').split(':', 2)[1:]
+ except:
+ print('* DePythonTeX error:')
+ print(' Improperly formatted file information on line ' + str(depy_linenum))
+ print(' The file information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ # Files that are brought in have an optional mode that
+ # determines if they need special handling (for example, verbatim)
+ if ':mode=' in f_name:
+ f_name, mode = f_name.split(':mode=')
+ else:
+ mode = None
+ f = open(f_name, 'r', encoding=encoding)
+ replacement = f.read()
+ f.close()
+ if typeset == 'c':
+ code_replacement_mode = mode
+ if depy_type == 'cmd' and code_replacement_mode != 'verbatim':
+ # Usually, code from commands is typeset with commands
+ # and code from environments is typeset in
+ # environments. The except is code from commands
+ # that bring in external files, like `\inputpygments`
+ code_replacement = replacement
+ else:
+ # If we're replacing an environment of code with a
+ # file, then we lose the newline at the beginning
+ # of the environment, and need to get it back.
+ code_replacement = '\n' + replacement
+ elif typeset == 'p':
+ print_replacement_mode = mode
+ print_replacement = replacement
+ else:
+ print('* DePythonTeX error:')
+ print(' Improper typesetting information for file information on line ' + str(depy_linenum))
+ print(' The file information was "' + nextdepytxline + '"')
+ sys.exit(1)
+ # Increment the line in depytx to check for more information
+ # from LaTeX
+ scan_ahead_line += 1
+ if scan_ahead_line == len(depytx):
+ break
+ else:
+ nextdepytxline = depytx[scan_ahead_line]
+
+
+ # If the line we're looking for is within the range currently held by
+ # texcontent, do nothing. Otherwise, transfer content from tex
+ # to texout until we get to the line of tex that we're looking for
+ if depy_linenum > texlinenum:
+ texout.append(texcontent)
+ texlinenum += 1
+ while texlinenum < depy_linenum:
+ texout.append(tex[texlinenum])
+ texlinenum += 1
+ texcontent = tex[texlinenum]
+
+
+ # Deal with arguments
+ # All arguments are parsed and stored in a list variables, even if
+ # they are not used, for completeness; this makes it easy to add
+ # functionality
+ # Start by splitting the current line into what comes before the
+ # command or environment, and what is after it
+ if depy_type == 'cmd':
+ try:
+ before, after = texcontent.split('\\' + depy_name, 1)
+ except:
+ print('* DePythonTeX error:')
+ print(' Could not find command "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ else: # depy_type == 'env':
+ try:
+ before, after = texcontent.split(r'\begin{' + depy_name + '}', 1)
+ except:
+ print('* DePythonTeX error:')
+ print(' Could not find environment "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ # We won't need the content from before the command or environment
+ # again, so we go ahead and store it
+ texout.append(before)
+
+ # Parse the arguments
+ # Create a list for storing the recovered arguments
+ arglist = list()
+ for argindex, arg in enumerate(depy_args):
+ if arg == 'n':
+ pass
+ elif arg == 'o':
+ if after[0] == '[':
+ # Account for possible line breaks before end of arg
+ while ']' not in after:
+ texlinenum += 1
+ after += tex[texlinenum]
+ optarg, after = after[1:].split(']', 1)
+ else:
+ if obeylines:
+ # Take into account possible whitespace before arg
+ if bool(match('[ \t]*\[', after)):
+ after = after.split('[', 1)[1]
+ while ']' not in after:
+ texlinenum += 1
+ after += tex[texlinenum]
+ optarg, after = after.split(']', 1)
+ else:
+ optarg = None
+ # If this is the last arg, and it wasn't found,
+ # the macro should eat all whitespace following it
+ if argindex == len(depy_args) - 1:
+ after = sub('^[ \t]*', '', after)
+ else:
+ # Allow peeking ahead a line for the argument
+ if bool(match('\s*$', after)) and after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ # Take into account possible whitespace before arg
+ if bool(match('\s*\[', after)):
+ after = after.split('[', 1)[1]
+ while ']' not in after:
+ texlinenum += 1
+ after += tex[texlinenum]
+ optarg, after = after.split(']', 1)
+ else:
+ optarg = None
+ # Account for eating whitespace afterward, if arg not found
+ if argindex == len(depy_args) - 1:
+ if bool(match('\s*$', after)) and after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ if not bool(match('\s*$', after)):
+ after = sub('^\s*', '', after)
+ arglist.append(optarg)
+ elif arg == 'm':
+ # Account for possible line breaks or spaces before arg
+ if after[0] == '{':
+ after = after[1:]
+ else:
+ if obeylines:
+ # Account for possible leading whitespace
+ if bool(match('[ \t\f\v]*\{', after)):
+ after = after.split('{', 1)[1]
+ else:
+ print('* DePythonTeX error:')
+ print(' Flawed mandatory argument for "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ else:
+ # Peek ahead a line if needed
+ if bool(match('\s*$', after)) and after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ if bool(match('\s*\{', after)):
+ after = after.split('{', 1)[1]
+ else:
+ print('* DePythonTeX error:')
+ print(' Flawed mandatory argument for "' + depy_name + '" on line ' + str(depy_linenum))
+ sys.exit(1)
+ # Go through the argument character by character to find the
+ # closing brace.
+ # If possible, use a very simple approach
+ if (r'\{' not in after and r'\}' not in after and
+ r'\string' not in after and
+ after.count('{') + 1 == after.count('}')):
+ pos = 0
+ lbraces = 1
+ rbraces = 0
+ while True:
+ if after[pos] == '{':
+ lbraces += 1
+ elif after[pos] == '}':
+ rbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ if pos == len(after):
+ texlinenum += 1
+ after += tex[texlinenum]
+ # If a simple parsing approach won't work, parse in much
+ # greater depth
+ else:
+ pos = 0
+ lbraces = 1
+ rbraces = 0
+ while True:
+ if after[pos] == '{':
+ # If the current character is a brace, we count it
+ lbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ elif after[pos] == '}':
+ # If the current character is a brace, we count it
+ rbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ elif after[pos:].startswith(r'\string'):
+ # If the current position marks the beginning of `\string`, we
+ # resolve the `\string` command
+ # First, jump ahead to after `\string`
+ pos += 7 #+= len(r'\string')
+ # See if `\string` is followed by a regular macro
+ # If so, jump past it; otherwise, figure out if a
+ # single-character macro, or just a single character, is next,
+ # and jump past it
+ standard_macro = match(r'\\[a-zA-Z]+', line[pos:])
+ if bool(standard_macro):
+ pos += standard_macro.end()
+ elif line[pos] == '\\':
+ pos += 2
+ else:
+ pos += 1
+ elif line[pos] == '\\':
+ # If the current position is a backslash, figure out what
+ # macro is used, and jump past it
+ # The macro must either be a standard alphabetic macro,
+ # or a single-character macro
+ standard_macro = match(r'\\[a-zA-Z]+', line[pos:])
+ if bool(standard_macro):
+ pos += standard_macro.end()
+ else:
+ pos += 2
+ else:
+ pos += 1
+ if pos == len(after):
+ texlinenum += 1
+ after += tex[texlinenum]
+ mainarg = after[:pos]
+ after = after[pos+1:]
+ arglist.append(mainarg)
+ elif arg == 'v':
+ if after[0] == '{':
+ # Account for the possibility of matched brace delimiters
+ # Not all verbatim commands allow for these
+ pos = 1
+ lbraces = 1
+ rbraces = 0
+ while True:
+ if after[pos] == '{':
+ lbraces += 1
+ elif after[pos] == '}':
+ rbraces += 1
+ if lbraces == rbraces:
+ break
+ pos += 1
+ if pos == len(after):
+ texlinenum += 1
+ after += tex[texlinenum]
+ mainarg = after[1:pos]
+ after = after[pos+1:]
+ else:
+ # Deal with matched character delims
+ delim = after[0]
+ while after.count(delim) < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ mainarg, after = after[1:].split(delim, 1)
+ arglist.append(mainarg)
+
+
+ # Do substitution, depending on what is required
+ # Need a variable for processed content to be added to texout
+ processed = None
+ if depy_typeset == 'c':
+ if depy_type == 'cmd':
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_code_cmd(depy_name, arglist,
+ depy_linenum,
+ code_replacement,
+ code_replacement_mode,
+ after, depy_lexer,
+ firstnumber)
+ else: # depy_type == 'env'
+ end_environment = r'\end{' + depy_name + '}'
+ if code_replacement is None:
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ after += tex[texlinenum]
+ if end_environment in tex[texlinenum]:
+ break
+ code_replacement, after = after.split(end_environment, 1)
+ # If there's content on the line with the end-environment
+ # command, it should be discarded, to imitate TeX
+ if not code_replacement.endswith('\n'):
+ code_replacement = code_replacement.rsplit('\n', 1)[0] + '\n'
+ # Take care of `gobble`
+ if settings['gobble'] == 'auto':
+ code_replacement = textwrap.dedent(code_replacement)
+ else:
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ if end_environment in tex[texlinenum]:
+ after = tex[texlinenum]
+ break
+ after = after.split(end_environment, 1)[1]
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_code_env(depy_name, arglist,
+ depy_linenum,
+ code_replacement,
+ code_replacement_mode,
+ after, depy_lexer,
+ firstnumber)
+ elif depy_typeset == 'p' and print_replacement is not None:
+ if depy_type == 'cmd':
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_print_cmd(depy_name, arglist,
+ depy_linenum,
+ print_replacement,
+ print_replacement_mode,
+ source,
+ after)
+ else: # depy_type == 'env'
+ end_environment = r'\end{' + depy_name + '}'
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ if end_environment in tex[texlinenum]:
+ after = tex[texlinenum]
+ break
+ after = after.split(end_environment, 1)[1]
+ # Make sure that `after` contains two lines of content
+ # This is needed for some replacements that need to look ahead
+ if after.count('\n') < 2:
+ texlinenum += 1
+ after += tex[texlinenum]
+ processed, texcontent = replace_print_env(depy_name, arglist,
+ depy_linenum,
+ print_replacement,
+ print_replacement_mode,
+ source,
+ after)
+ else: # depy_typeset == 'n' or (depy_typeset == 'p' and print_replacement is None):
+ if depy_type == 'cmd':
+ texcontent = after
+ else: # depy_type == 'env'
+ end_environment = r'\end{' + depy_name + '}'
+ if end_environment not in after:
+ while True:
+ texlinenum += 1
+ if end_environment in tex[texlinenum]:
+ after = tex[texlinenum]
+ break
+ after = after.split(end_environment, 1)[1]
+ if bool(match('\s*\n', after)):
+ # If the line following `after` is whitespace, it should
+ # be stripped, since most environments throw away
+ # anything after the end of the environment
+ after = after.split('\n')[1]
+ texcontent = after
+ # #### Once it's supported on the TeX side, need to add support for
+ # pc and cp
+
+
+ # Store any processed content
+ if processed is not None:
+ texout.append(processed)
+
+
+# Transfer anything that's left in tex to texout
+texout.append(texcontent)
+texout.extend(tex[texlinenum+1:])
+
+
+
+
+# Replace the `\usepackage{pythontex}`
+for n, line in enumerate(texout):
+ if '{pythontex}' in line:
+ startline = n
+ while '\\usepackage' not in texout[startline] and startline >= 0:
+ startline -= 1
+ if startline == n:
+ if bool(search(r'\\usepackage(?:\[.*?\]){0,1}\{pythontex\}', line)):
+ texout[n] = sub(r'\\usepackage(?:\[.*?\]){0,1}\{pythontex\}', '', line)
+ break
+ else:
+ content = ''.join(texout[startline:n+1])
+ if bool(search(r'(?s)\\usepackage(?:\[.*?\]\s*){0,1}\{pythontex\}', content)):
+ replacement = sub(r'(?s)\\usepackage(?:\[.*?\]\s*){0,1}\{pythontex\}', '', content)
+ texout[startline] = replacement
+ for l in range(startline+1, n+1):
+ texout[l] = ''
+ break
+ elif line.startswith(r'\begin{document}'):
+ break
+if preamble_additions:
+ texout[n] += '\n'.join(preamble_additions) + '\n'
+# Take care of graphicspath
+if args.graphicspath and settings['graphicx']:
+ for n, line in enumerate(texout):
+ if '\\graphicspath' in line and not bool(match('\s*%', line)):
+ texout[n] = line.replace('\\graphicspath{', '\\graphicspath{{' + settings['outputdir'] +'/}')
+ break
+ elif line.startswith(r'\begin{document}'):
+ texout[n] = '\\graphicspath{{' + settings['outputdir'] + '/}}\n' + line
+ break
+
+
+
+
+# Print any final messages
+if forced_double_space_list:
+ print('* DePythonTeX warning:')
+ print(' A trailing double space was forced with "\\space{}" for the following')
+ print(' This can happen when printed content is included inline')
+ print(' The forced double space is only an issue if it is not intentional')
+ for name, linenum in forced_double_space_list:
+ print(' "' + name + '" near line ' + str(linenum))
+
+
+
+
+# Write output
+for line in texout:
+ outfile.write(line)
+outfile.close()
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex.py b/Master/texmf-dist/scripts/pythontex/pythontex.py
new file mode 100755
index 00000000000..1be1f6d70ea
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/pythontex.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+This is the PythonTeX wrapper script. It automatically detects the version
+of Python, and then imports the correct code from pythontex2.py or
+pythontex3.py.
+
+Licensed under the BSD 3-Clause License:
+
+Copyright (c) 2012-2013, Geoffrey M. Poore
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the <organization> nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+
+
+import sys
+if sys.version_info.major == 2:
+ if sys.version_info.minor >= 7:
+ import pythontex2 as pythontex
+ else:
+ sys.exit('PythonTeX require Python 2.7; you are using 2.{0}'.format(sys.version_info.minor))
+elif sys.version_info.major == 3:
+ if sys.version_info.minor >= 2:
+ import pythontex3 as pythontex
+ else:
+ sys.exit('PythonTeX require Python 3.2+; you are using 3.{0}'.format(sys.version_info.minor))
+
+# The "if" statement is needed for multiprocessing under Windows; see the
+# multiprocessing documentation.
+if __name__ == '__main__':
+ pythontex.main()
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex2.py b/Master/texmf-dist/scripts/pythontex/pythontex2.py
new file mode 100755
index 00000000000..c123adee22b
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/pythontex2.py
@@ -0,0 +1,2360 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+This is the main PythonTeX script. It should be launched via pythontex.py.
+
+Two versions of this script are provided. One, with name ending in "2", runs
+under Python 2.7. The other, with name ending in "3", runs under Python 3.2+.
+
+This script needs to be able to import pythontex_engines.py; in general it
+should be in the same directory.
+
+
+Licensed under the BSD 3-Clause License:
+
+Copyright (c) 2012-2013, Geoffrey M. Poore
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the <organization> nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+
+
+# Imports
+#// Python 2
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+#\\ End Python 2
+import sys
+import os
+import argparse
+import codecs
+import time
+from hashlib import sha1
+from collections import defaultdict, OrderedDict, namedtuple
+from re import match, sub, search
+import subprocess
+import multiprocessing
+from pygments.styles import get_all_styles
+from pythontex_engines import *
+import textwrap
+
+if sys.version_info[0] == 2:
+ try:
+ import cPickle as pickle
+ except:
+ import pickle
+ from io import open
+else:
+ import pickle
+
+
+
+
+# Script parameters
+# Version
+version = 'v0.12'
+
+
+
+
+class Pytxcode(object):
+ def __init__(self, data, gobble):
+ self.delims, self.code = data.split('#\n', 1)
+ self.input_family, self.input_session, self.input_restart, self.input_instance, self.input_command, self.input_context, self.input_args_run, self.input_args_prettyprint, self.input_file, self.input_line = self.delims.split('#')
+ self.input_instance_int = int(self.input_instance)
+ self.input_line_int = int(self.input_line)
+ self.key_run = self.input_family + '#' + self.input_session + '#' + self.input_restart
+ self.key_typeset = self.key_run + '#' + self.input_instance
+ self.hashable_delims_run = self.key_typeset + '#' + self.input_command + '#' + self.input_context + '#' + self.input_args_run
+ self.hashable_delims_typeset = self.key_typeset + '#' + self.input_command + '#' + self.input_context + '#' + self.input_args_run
+ if len(self.input_command) > 1:
+ self.is_inline = False
+ # Environments start on the next line
+ self.input_line_int += 1
+ self.input_line = str(self.input_line_int)
+ else:
+ self.is_inline = True
+ self.is_extfile = True if self.input_session.startswith('EXT:') else False
+ if self.is_extfile:
+ self.extfile = os.path.expanduser(os.path.normcase(self.input_session.replace('EXT:', '', 1)))
+ self.is_cc = True if self.input_family.startswith('CC:') else False
+ self.is_pyg = True if self.input_family.startswith('PYG') else False
+ self.is_verb = True if self.input_restart.endswith('verb') else False
+ if self.is_cc:
+ self.input_instance += 'CC'
+ self.cc_type, self.cc_pos = self.input_family.split(':')[1:]
+ if self.is_verb or self.is_pyg or self.is_cc:
+ self.is_cons = False
+ else:
+ self.is_cons = engine_dict[self.input_family].console
+ self.is_code = False if self.is_verb or self.is_pyg or self.is_cc or self.is_cons else True
+ if self.input_command in ('c', 'code') or (self.input_command == 'i' and not self.is_cons):
+ self.is_typeset = False
+ else:
+ self.is_typeset = True
+
+ if gobble == 'auto':
+ self.code = textwrap.dedent(self.code)
+
+
+
+
+def process_argv(data, temp_data):
+ '''
+ Process command line options using the argparse module.
+
+ Most options are passed via the file of code, rather than via the command
+ line.
+ '''
+
+ # Create a command line argument parser
+ parser = argparse.ArgumentParser()
+ parser.add_argument('TEXNAME',
+ help='LaTeX file, with or without .tex extension')
+ parser.add_argument('--version', action='version',
+ version='PythonTeX {0}'.format(data['version']))
+ parser.add_argument('--encoding', default='UTF-8',
+ help='encoding for all text files (see codecs module for encodings)')
+ parser.add_argument('--error-exit-code', default='true',
+ choices=('true', 'false'),
+ help='return exit code of 1 if there are errors (not desirable with some TeX editors and workflows)')
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('--runall', nargs='?', default='false',
+ const='true', choices=('true', 'false'),
+ help='run ALL code; equivalent to package option')
+ group.add_argument('--rerun', default='errors',
+ choices=('never', 'modified', 'errors', 'warnings', 'always'),
+ help='set conditions for rerunning code; equivalent to package option')
+ parser.add_argument('--hashdependencies', nargs='?', default='false',
+ const='true', choices=('true', 'false'),
+ help='hash dependencies (such as external data) to check for modification, rather than using mtime; equivalent to package option')
+ parser.add_argument('-v', '--verbose', default=False, action='store_true',
+ help='verbose output')
+ parser.add_argument('--interpreter', default=None, help='set a custom interpreter; argument should be in the form "<interpreter>:<command>, <interp>:<cmd>, ..." where <interpreter> is "python", "ruby", etc., and <command> is the command for invoking the interpreter; argument may also be in the form of a Python dictionary')
+ args = parser.parse_args()
+
+ # Store the parsed argv in data and temp_data
+ data['encoding'] = args.encoding
+ if args.error_exit_code == 'true':
+ temp_data['error_exit_code'] = True
+ else:
+ temp_data['error_exit_code'] = False
+ # runall can be mapped onto rerun, so both are stored under rerun
+ if args.runall == 'true':
+ temp_data['rerun'] = 'always'
+ else:
+ temp_data['rerun'] = args.rerun
+ # hashdependencies need only be in temp_data, since changing it would
+ # change hashes (hashes of mtime vs. file contents)
+ if args.hashdependencies == 'true':
+ temp_data['hashdependencies'] = True
+ else:
+ temp_data['hashdependencies'] = False
+ temp_data['verbose'] = args.verbose
+ # Update interpreter_dict based on interpreter
+ if args.interpreter is not None:
+ interp_list = args.interpreter.lstrip('{').rstrip('}').split(',')
+ for interp in interp_list:
+ if interp:
+ try:
+ k, v = interp.split(':')
+ k = k.strip(' \'"')
+ v = v.strip(' \'"')
+ interpreter_dict[k] = v
+ except:
+ print('Invalid --interpreter argument')
+ return sys.exit(2)
+
+
+ if args.TEXNAME is not None:
+ # Determine if we a dealing with just a filename, or a name plus
+ # path. If there's a path, we need to make the document directory
+ # the current working directory.
+ dir, raw_jobname = os.path.split(args.TEXNAME)
+ dir = os.path.expanduser(os.path.normcase(dir))
+ if dir:
+ os.chdir(dir)
+ sys.path.append(dir)
+ # If necessary, strip off an extension to find the raw jobname that
+ # corresponds to the .pytxcode.
+ if not os.path.exists(raw_jobname + '.pytxcode'):
+ raw_jobname = raw_jobname.rsplit('.', 1)[0]
+ if not os.path.exists(raw_jobname + '.pytxcode'):
+ print('* PythonTeX error')
+ print(' Code file ' + raw_jobname + '.pytxcode does not exist.')
+ print(' Run LaTeX to create it.')
+ return sys.exit(1)
+
+ # We need a "sanitized" version of the jobname, with spaces and
+ # asterisks replaced with hyphens. This is done to avoid TeX issues
+ # with spaces in file names, paralleling the approach taken in
+ # pythontex.sty. From now on, we will use the sanitized version every
+ # time we create a file that contains the jobname string. The raw
+ # version will only be used in reference to pre-existing files created
+ # on the TeX side, such as the .pytxcode file.
+ jobname = raw_jobname.replace(' ', '-').replace('"', '').replace('*', '-')
+ # Store the results in data
+ data['raw_jobname'] = raw_jobname
+ data['jobname'] = jobname
+
+ # We need to check to make sure that the "sanitized" jobname doesn't
+ # lead to a collision with a file that already has that name, so that
+ # two files attempt to use the same PythonTeX folder.
+ #
+ # If <jobname>.<ext> and <raw_jobname>.<ext> both exist, where <ext>
+ # is a common LaTeX extension, we exit. We operate under the
+ # assumption that there should be only a single file <jobname> in the
+ # document root directory that has a common LaTeX extension. That
+ # could be false, but if so, the user probably has worse things to
+ # worry about than a potential PythonTeX output collision.
+ # If <jobname>* and <raw_jobname>* both exist, we issue a warning but
+ # attempt to proceed.
+ if jobname != raw_jobname:
+ resolved = False
+ for ext in ('.tex', '.ltx', '.dtx'):
+ if os.path.isfile(raw_jobname + ext):
+ if os.path.isfile(jobname + ext):
+ print('* PythonTeX error')
+ print(' Directory naming collision between the following files:')
+ print(' ' + raw_jobname + ext)
+ print(' ' + jobname + ext)
+ return sys.exit(1)
+ else:
+ resolved = True
+ break
+ if not resolved:
+ ls = os.listdir('.')
+ for file in ls:
+ if file.startswith(jobname):
+ print('* PythonTeX warning')
+ print(' Potential directory naming collision between the following names:')
+ print(' ' + raw_jobname)
+ print(' ' + jobname + '*')
+ print(' Attempting to proceed.')
+ temp_data['warnings'] += 1
+ break
+
+
+
+
+def load_code_get_settings(data, temp_data):
+ '''
+ Load the code file, preprocess the code, and extract the settings.
+ '''
+ # Bring in the .pytxcode file as a single string
+ raw_jobname = data['raw_jobname']
+ encoding = data['encoding']
+ # The error checking here is a little redundant
+ if os.path.isfile(raw_jobname + '.pytxcode'):
+ f = open(raw_jobname + '.pytxcode', 'r', encoding=encoding)
+ pytxcode = f.read()
+ f.close()
+ else:
+ print('* PythonTeX error')
+ print(' Code file ' + raw_jobname + '.pytxcode does not exist.')
+ print(' Run LaTeX to create it.')
+ return sys.exit(1)
+
+ # Split code and settings
+ try:
+ pytxcode, pytxsettings = pytxcode.rsplit('=>PYTHONTEX:SETTINGS#', 1)
+ except:
+ print('The .pytxcode file appears to have an outdated format or be invalid')
+ print('Run LaTeX to make sure the file is current')
+ return sys.exit(1)
+
+
+ # Prepare to process settings
+ #
+ # Create a dict for storing settings.
+ settings = {}
+ # Create a dict for storing Pygments settings.
+ # Each dict entry will itself be a dict.
+ pygments_settings = defaultdict(dict)
+
+ # Create a dict of processing functions, and generic processing functions
+ settings_func = dict()
+ def set_kv_data(k, v):
+ if v == 'true':
+ settings[k] = True
+ elif v == 'false':
+ settings[k] = False
+ else:
+ settings[k] = v
+ # Need a function for when assignment is only needed if not default value
+ def set_kv_temp_data_if_not_default(k, v):
+ if v != 'default':
+ if v == 'true':
+ temp_data[k] = True
+ elif v == 'false':
+ temp_data[k] = False
+ else:
+ temp_data[k] = v
+ def set_kv_data_fvextfile(k, v):
+ # Error checking on TeX side should be enough, but be careful anyway
+ try:
+ v = int(v)
+ except ValueError:
+ print('* PythonTeX error')
+ print(' Unable to parse package option fvextfile.')
+ return sys.exit(1)
+ if v < 0:
+ settings[k] = sys.maxsize
+ elif v == 0:
+ settings[k] = 1
+ print('* PythonTeX warning')
+ print(' Invalid value for package option fvextfile.')
+ temp_data['warnings'] += 1
+ else:
+ settings[k] = v
+ def set_kv_pygments(k, v):
+ input_family, lexer_opts, options = v.replace(' ','').split('|')
+ lexer = None
+ lex_dict = {}
+ opt_dict = {}
+ if lexer_opts:
+ for l in lexer_opts.split(','):
+ if '=' in l:
+ k, v = l.split('=', 1)
+ if k == 'lexer':
+ lexer = l
+ else:
+ lex_dict[k] = v
+ else:
+ lexer = l
+ if options:
+ for o in options.split(','):
+ if '=' in o:
+ k, v = o.split('=', 1)
+ if v in ('true', 'True'):
+ v = True
+ elif v in ('false', 'False'):
+ v = False
+ else:
+ k = option
+ v = True
+ opt_dict[k] = v
+ if input_family != ':GLOBAL':
+ if 'lexer' in pygments_settings[':GLOBAL']:
+ lexer = pygments_settings[':GLOBAL']['lexer']
+ lex_dict.update(pygments_settings[':GLOBAL']['lexer_options'])
+ opt_dict.update(pygments_settings[':GLOBAL']['formatter_options'])
+ if 'style' not in opt_dict:
+ opt_dict['style'] = 'default'
+ opt_dict['commandprefix'] = 'PYG' + opt_dict['style']
+ if lexer is not None:
+ pygments_settings[input_family]['lexer'] = lexer
+ pygments_settings[input_family]['lexer_options'] = lex_dict
+ pygments_settings[input_family]['formatter_options'] = opt_dict
+ settings_func['version'] = set_kv_data
+ settings_func['outputdir'] = set_kv_data
+ settings_func['workingdir'] = set_kv_data
+ settings_func['gobble'] = set_kv_data
+ settings_func['rerun'] = set_kv_temp_data_if_not_default
+ settings_func['hashdependencies'] = set_kv_temp_data_if_not_default
+ settings_func['makestderr'] = set_kv_data
+ settings_func['stderrfilename'] = set_kv_data
+ settings_func['keeptemps'] = set_kv_data
+ settings_func['pyfuture'] = set_kv_data
+ settings_func['pyconfuture'] = set_kv_data
+ settings_func['pygments'] = set_kv_data
+ settings_func['fvextfile'] = set_kv_data_fvextfile
+ settings_func['pygglobal'] = set_kv_pygments
+ settings_func['pygfamily'] = set_kv_pygments
+ settings_func['pyconbanner'] = set_kv_data
+ settings_func['pyconfilename'] = set_kv_data
+ settings_func['depythontex'] = set_kv_data
+
+ # Process settings
+ for line in pytxsettings.split('\n'):
+ if line:
+ key, val = line.split('=', 1)
+ try:
+ settings_func[key](key, val)
+ except KeyError:
+ print('* PythonTeX warning')
+ print(' Unknown option "' + key + '"')
+ temp_data['warnings'] += 1
+
+ # Check for compatility between the .pytxcode and the script
+ if 'version' not in settings or settings['version'] != data['version']:
+ print('* PythonTeX warning')
+ print(' The version of the PythonTeX scripts does not match')
+ print(' the last code saved by the document--run LaTeX to create')
+ print(' an updated version. Attempting to proceed.')
+ sys.stdout.flush()
+
+ # Store all results that haven't already been stored.
+ data['settings'] = settings
+ data['pygments_settings'] = pygments_settings
+
+ # Create a tuple of vital quantities that invalidate old saved data
+ # Don't need to include outputdir, because if that changes, no old output
+ # fvextfile could be checked on a case-by-case basis, which would result
+ # in faster output, but that would involve a good bit of additional
+ # logic, which probably isn't worth it for a feature that will rarely be
+ # changed.
+ data['vitals'] = (data['version'], data['encoding'],
+ settings['gobble'], settings['fvextfile'])
+
+ # Create tuples of vital quantities
+ data['code_vitals'] = (settings['workingdir'], settings['keeptemps'],
+ settings['makestderr'], settings['stderrfilename'])
+ data['cons_vitals'] = (settings['workingdir'])
+ data['typeset_vitals'] = ()
+
+ # Pass any customizations to types
+ for k in engine_dict:
+ engine_dict[k].customize(pyfuture=settings['pyfuture'],
+ pyconfuture=settings['pyconfuture'],
+ pyconbanner=settings['pyconbanner'],
+ pyconfilename=settings['pyconfilename'])
+
+ # Store code
+ # Do this last, so that Pygments settings are available
+ if pytxcode.startswith('=>PYTHONTEX#'):
+ gobble = settings['gobble']
+ temp_data['pytxcode'] = [Pytxcode(c, gobble) for c in pytxcode.split('=>PYTHONTEX#')[1:]]
+ else:
+ temp_data['pytxcode'] = []
+
+
+
+
+def get_old_data(data, old_data, temp_data):
+ '''
+ Load data from the last run, if it exists, into the dict old_data.
+ Determine the path to the PythonTeX scripts, either by using a previously
+ found, saved path or via kpsewhich.
+
+ The old data is used for determining when PythonTeX has been upgraded,
+ when any settings have changed, when code has changed (via hashes), and
+ what files may need to be cleaned up. The location of the PythonTeX
+ scripts is needed so that they can be imported by the scripts created by
+ PythonTeX. The location of the scripts is confirmed even if they were
+ previously located, to make sure that the path is still valid. Finding
+ the scripts depends on having a TeX installation that includes the
+ Kpathsea library (TeX Live and MiKTeX, possibly others).
+
+ All code that relies on old_data is written based on the assumption that
+ if old_data exists and has the current PythonTeX version, then it
+ contains all needed information. Thus, all code relying on old_data must
+ check that it was loaded and that it has the current version. If not,
+ code should adapt gracefully.
+ '''
+
+ # Create a string containing the name of the data file
+ pythontex_data_file = os.path.join(data['settings']['outputdir'], 'pythontex_data.pkl')
+
+ # Load the old data if it exists (read as binary pickle)
+ if os.path.isfile(pythontex_data_file):
+ f = open(pythontex_data_file, 'rb')
+ old = pickle.load(f)
+ f.close()
+ # Check for compabilility
+ if 'vitals' in old and data['vitals'] == old['vitals']:
+ temp_data['loaded_old_data'] = True
+ old_data.update(old)
+ else:
+ temp_data['loaded_old_data'] = False
+ # Clean up all old files
+ if 'files' in old:
+ for key in old['files']:
+ for f in old['files'][key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ if 'pygments_files' in old:
+ for key in old['pygments_files']:
+ for f in old['pygments_files'][key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ else:
+ temp_data['loaded_old_data'] = False
+
+ # Set the utilspath
+ if os.path.isfile(os.path.join(sys.path[0], 'pythontex_utils.py')):
+ # Need the path with forward slashes, so escaping isn't necessary
+ data['utilspath'] = sys.path[0].replace('\\', '/')
+ else:
+ print('* PythonTeX error')
+ print(' Could not determine the utils path from sys.path[0]')
+ print(' The file "pythontex_utils.py" may be missing')
+ return sys.exit(1)
+
+
+
+
+def modified_dependencies(key, data, old_data, temp_data):
+ hashdependencies = temp_data['hashdependencies']
+ if key not in old_data['dependencies']:
+ return False
+ else:
+ old_dep_hash_dict = old_data['dependencies'][key]
+ workingdir = data['settings']['workingdir']
+ for dep in old_dep_hash_dict.keys():
+ # We need to know if the path is relative (based off the
+ # working directory) or absolute. We can't use
+ # os.path.isabs() alone for determining the distinction,
+ # because we must take into account the possibility of an
+ # initial ~ (tilde) standing for the home directory.
+ dep_file = os.path.expanduser(os.path.normcase(dep))
+ if not os.path.isabs(dep_file):
+ dep_file = os.path.join(workingdir, dep_file)
+ if not os.path.isfile(dep_file):
+ print('* PythonTeX error')
+ print(' Cannot find dependency "' + dep + '"')
+ print(' It belongs to ' + key.replace('#', ':'))
+ print(' Relative paths to dependencies must be specified from the working directory.')
+ temp_data['errors'] += 1
+ # A removed dependency should trigger an error, but it
+ # shouldn't cause code to execute. Running the code
+ # again would just give more errors when it can't find
+ # the dependency. (There won't be issues when a
+ # dependency is added or removed, because that would
+ # involve modifying code, which would trigger
+ # re-execution.)
+ elif hashdependencies:
+ # Read and hash the file in binary. Opening in text mode
+ # would require an unnecessary decoding and encoding cycle.
+ f = open(dep_file, 'rb')
+ hasher = sha1()
+ hash = hasher(f.read()).hexdigest()
+ f.close()
+ if hash != old_dep_hash_dict[dep][1]:
+ return True
+ else:
+ mtime = os.path.getmtime(dep_file)
+ if mtime != old_dep_hash_dict[dep][0]:
+ return True
+ return False
+
+def should_rerun(hash, old_hash, old_exit_status, key, rerun, data, old_data, temp_data):
+ # #### Need to clean up arg passing here
+ if rerun == 'never':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):
+ print('* PythonTeX warning')
+ print(' Session ' + key.replace('#', ':') + ' has rerun=never')
+ print(' But its code or dependencies have been modified')
+ temp_data['warnings'] += 1
+ return False
+ elif rerun == 'modified':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):
+ return True
+ else:
+ return False
+ elif rerun == 'errors':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or
+ old_exit_status[0] != 0):
+ return True
+ else:
+ return False
+ elif rerun == 'warnings':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or
+ old_exit_status != (0, 0)):
+ return True
+ else:
+ return False
+ elif rerun == 'always':
+ return True
+
+
+
+
+def hash_all(data, temp_data, old_data, engine_dict):
+ '''
+ Hash the code to see what has changed and needs to be updated.
+
+ Save the hashes in hashdict. Create update_code, a list of bools
+ regarding whether code should be executed. Create update_pygments, a
+ list of bools determining what needs updated Pygments highlighting.
+ Update pygments_settings to account for Pygments (as opposed to PythonTeX)
+ commands and environments.
+ '''
+
+ # Note that the PythonTeX information that accompanies code must be
+ # hashed in addition to the code itself; the code could stay the same,
+ # but its context or args could change, which might require that code be
+ # executed. All of the PythonTeX information is hashed except for the
+ # input line number. Context-dependent code is going too far if
+ # it depends on that.
+
+ # Create variables to more easily access parts of data
+ pytxcode = temp_data['pytxcode']
+ encoding = data['encoding']
+ loaded_old_data = temp_data['loaded_old_data']
+ rerun = temp_data['rerun']
+ pygments_settings = data['pygments_settings']
+
+ # Calculate cumulative hashes for all code that is executed
+ # Calculate individual hashes for all code that will be typeset
+ code_hasher = defaultdict(sha1)
+ cons_hasher = defaultdict(sha1)
+ cc_hasher = defaultdict(sha1)
+ typeset_hasher = defaultdict(sha1)
+ for c in pytxcode:
+ if c.is_code:
+ code_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))
+ code_encoded = c.code.encode(encoding)
+ code_hasher[c.key_run].update(code_encoded)
+ if c.is_typeset:
+ typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
+ typeset_hasher[c.key_typeset].update(code_encoded)
+ elif c.is_cons:
+ cons_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))
+ code_encoded = c.code.encode(encoding)
+ cons_hasher[c.key_run].update(code_encoded)
+ if c.is_typeset:
+ typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
+ typeset_hasher[c.key_typeset].update(code_encoded)
+ elif c.is_cc:
+ cc_hasher[c.cc_type].update(c.hashable_delims_run.encode(encoding))
+ cc_hasher[c.cc_type].update(c.code.encode(encoding))
+ elif c.is_typeset:
+ typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
+ typeset_hasher[c.key_typeset].update(c.code.encode(encoding))
+
+
+ # Store hashes
+ code_hash_dict = {}
+ for key in code_hasher:
+ input_family = key.split('#', 1)[0]
+ code_hash_dict[key] = (code_hasher[key].hexdigest(),
+ cc_hasher[input_family].hexdigest(),
+ engine_dict[input_family].get_hash())
+ data['code_hash_dict'] = code_hash_dict
+
+ cons_hash_dict = {}
+ for key in cons_hasher:
+ input_family = key.split('#', 1)[0]
+ cons_hash_dict[key] = (cons_hasher[key].hexdigest(),
+ cc_hasher[input_family].hexdigest(),
+ engine_dict[input_family].get_hash())
+ data['cons_hash_dict'] = cons_hash_dict
+
+ typeset_hash_dict = {}
+ for key in typeset_hasher:
+ typeset_hash_dict[key] = typeset_hasher[key].hexdigest()
+ data['typeset_hash_dict'] = typeset_hash_dict
+
+
+ # See what needs to be updated.
+ # In the process, copy over macros and files that may be reused.
+ code_update = {}
+ cons_update = {}
+ pygments_update = {}
+ macros = defaultdict(list)
+ files = defaultdict(list)
+ pygments_macros = {}
+ pygments_files = {}
+ typeset_cache = {}
+ dependencies = defaultdict(dict)
+ exit_status = {}
+ pygments_settings_changed = {}
+ if loaded_old_data:
+ old_macros = old_data['macros']
+ old_files = old_data['files']
+ old_pygments_macros = old_data['pygments_macros']
+ old_pygments_files = old_data['pygments_files']
+ old_typeset_cache = old_data['typeset_cache']
+ old_dependencies = old_data['dependencies']
+ old_exit_status = old_data['exit_status']
+ old_code_hash_dict = old_data['code_hash_dict']
+ old_cons_hash_dict = old_data['cons_hash_dict']
+ old_typeset_hash_dict = old_data['typeset_hash_dict']
+ old_pygments_settings = old_data['pygments_settings']
+ for s in pygments_settings:
+ if (s in old_pygments_settings and
+ pygments_settings[s] == old_pygments_settings[s]):
+ pygments_settings_changed[s] = False
+ else:
+ pygments_settings_changed[s] = True
+
+ # If old data was loaded (and thus is compatible) determine what has
+ # changed so that only
+ # modified code may be executed. Otherwise, execute everything.
+ # We don't have to worry about checking for changes in pyfuture, because
+ # custom code and default code are hashed. The treatment of keeptemps
+ # could be made more efficient (if changed to 'none', just delete old temp
+ # files rather than running everything again), but given that it is
+ # intended as a debugging aid, that probable isn't worth it.
+ # We don't have to worry about hashdependencies changing, because if it
+ # does the hashes won't match (file contents vs. mtime) and thus code will
+ # be re-executed.
+ if loaded_old_data and data['code_vitals'] == old_data['code_vitals']:
+ # Compare the hash values, and set which code needs to be run
+ for key in code_hash_dict:
+ if (key in old_code_hash_dict and
+ not should_rerun(code_hash_dict[key], old_code_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):
+ code_update[key] = False
+ macros[key] = old_macros[key]
+ files[key] = old_files[key]
+ dependencies[key] = old_dependencies[key]
+ exit_status[key] = old_exit_status[key]
+ else:
+ code_update[key] = True
+ else:
+ for key in code_hash_dict:
+ code_update[key] = True
+
+ if loaded_old_data and data['cons_vitals'] == old_data['cons_vitals']:
+ # Compare the hash values, and set which code needs to be run
+ for key in cons_hash_dict:
+ if (key in old_cons_hash_dict and
+ not should_rerun(cons_hash_dict[key], old_cons_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):
+ cons_update[key] = False
+ macros[key] = old_macros[key]
+ files[key] = old_files[key]
+ typeset_cache[key] = old_typeset_cache[key]
+ dependencies[key] = old_dependencies[key]
+ exit_status[key] = old_exit_status[key]
+ else:
+ cons_update[key] = True
+ else:
+ for key in cons_hash_dict:
+ cons_update[key] = True
+
+ if loaded_old_data and data['typeset_vitals'] == old_data['typeset_vitals']:
+ for key in typeset_hash_dict:
+ input_family = key.split('#', 1)[0]
+ if input_family in pygments_settings:
+ if (not pygments_settings_changed[input_family] and
+ key in old_typeset_hash_dict and
+ typeset_hash_dict[key] == old_typeset_hash_dict[key]):
+ pygments_update[key] = False
+ if key in old_pygments_macros:
+ pygments_macros[key] = old_pygments_macros[key]
+ if key in old_pygments_files:
+ pygments_files[key] = old_pygments_files[key]
+ else:
+ pygments_update[key] = True
+ else:
+ pygments_update[key] = False
+ # Make sure Pygments styles are up-to-date
+ pygments_style_list = list(get_all_styles())
+ if pygments_style_list != old_data['pygments_style_list']:
+ pygments_style_defs = {}
+ # Lazy import
+ from pygments.formatters import LatexFormatter
+ for s in pygments_style_list:
+ formatter = LatexFormatter(style=s, commandprefix='PYG'+s)
+ pygments_style_defs[s] = formatter.get_style_defs()
+ else:
+ pygments_style_defs = old_data['pygments_style_defs']
+ else:
+ for key in typeset_hash_dict:
+ input_family = key.split('#', 1)[0]
+ if input_family in pygments_settings:
+ pygments_update[key] = True
+ else:
+ pygments_update[key] = False
+ # Create Pygments styles
+ pygments_style_list = list(get_all_styles())
+ pygments_style_defs = {}
+ # Lazy import
+ from pygments.formatters import LatexFormatter
+ for s in pygments_style_list:
+ formatter = LatexFormatter(style=s, commandprefix='PYG'+s)
+ pygments_style_defs[s] = formatter.get_style_defs()
+
+ # Save to data
+ temp_data['code_update'] = code_update
+ temp_data['cons_update'] = cons_update
+ temp_data['pygments_update'] = pygments_update
+ data['macros'] = macros
+ data['files'] = files
+ data['pygments_macros'] = pygments_macros
+ data['pygments_style_list'] = pygments_style_list
+ data['pygments_style_defs'] = pygments_style_defs
+ data['pygments_files'] = pygments_files
+ data['typeset_cache'] = typeset_cache
+ data['dependencies'] = dependencies
+ data['exit_status'] = exit_status
+
+
+ # Clean up for code that will be run again, and for code that no longer
+ # exists.
+ if loaded_old_data:
+ # Take care of code files
+ for key in code_hash_dict:
+ if code_update[key] and key in old_files:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ for key in old_code_hash_dict:
+ if key not in code_hash_dict:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ # Take care of old console files
+ for key in cons_hash_dict:
+ if cons_update[key] and key in old_files:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ for key in old_cons_hash_dict:
+ if key not in cons_hash_dict:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ # Take care of old Pygments files
+ # The approach here is a little different since there isn't a
+ # Pygments-specific hash dict, but there is a Pygments-specific
+ # dict of lists of files.
+ for key in pygments_update:
+ if pygments_update[key] and key in old_pygments_files:
+ for f in old_pygments_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ for key in old_pygments_files:
+ if key not in pygments_update:
+ for f in old_pygments_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+
+
+
+
+
+def parse_code_write_scripts(data, temp_data, engine_dict):
+ '''
+ Parse the code file into separate scripts, and write them to file.
+ '''
+ code_dict = defaultdict(list)
+ cc_dict_begin = defaultdict(list)
+ cc_dict_end = defaultdict(list)
+ cons_dict = defaultdict(list)
+ pygments_list = []
+ # Create variables to ease data access
+ encoding = data['encoding']
+ utilspath = data['utilspath']
+ outputdir = data['settings']['outputdir']
+ workingdir = data['settings']['workingdir']
+ pytxcode = temp_data['pytxcode']
+ code_update = temp_data['code_update']
+ cons_update = temp_data['cons_update']
+ pygments_update = temp_data['pygments_update']
+ files = data['files']
+ # We need to keep track of the last instance for each session, so
+ # that duplicates can be eliminated. Some LaTeX environments process
+ # their content multiple times and thus will create duplicates. We
+ # need to initialize everything at -1, since instances begin at zero.
+ def negative_one():
+ return -1
+ last_instance = defaultdict(negative_one)
+ for c in pytxcode:
+ if c.input_instance_int > last_instance[c.key_run]:
+ last_instance[c.key_run] = c.input_instance_int
+ if c.is_code:
+ if code_update[c.key_run]:
+ code_dict[c.key_run].append(c)
+ if c.is_typeset and pygments_update[c.key_typeset]:
+ pygments_list.append(c)
+ elif c.is_cons:
+ # Only append to Pygments if not run, since Pygments is
+ # automatically taken care of during run for console content
+ if cons_update[c.key_run]:
+ cons_dict[c.key_run].append(c)
+ elif c.is_typeset and pygments_update[c.key_typeset]:
+ pygments_list.append(c)
+ elif (c.is_pyg or c.is_verb) and pygments_update[c.key_typeset]:
+ pygments_list.append(c)
+ elif c.is_cc:
+ if c.cc_pos == 'begin':
+ cc_dict_begin[c.cc_type].append(c)
+ else:
+ cc_dict_end[c.cc_type].append(c)
+
+ # Save
+ temp_data['code_dict'] = code_dict
+ temp_data['cc_dict_begin'] = cc_dict_begin
+ temp_data['cc_dict_end'] = cc_dict_end
+ temp_data['cons_dict'] = cons_dict
+ temp_data['pygments_list'] = pygments_list
+
+ # Save the code sessions that need to be updated
+ # Keep track of the files that are created
+ # Also accumulate error indices for handling stderr
+ code_index_dict = {}
+ for key in code_dict:
+ input_family, input_session, input_restart = key.split('#')
+ fname = os.path.join(outputdir, input_family + '_' + input_session + '_' + input_restart + '.' + engine_dict[input_family].extension)
+ files[key].append(fname)
+ sessionfile = open(fname, 'w', encoding=encoding)
+ script, code_index = engine_dict[input_family].get_script(encoding,
+ utilspath,
+ workingdir,
+ cc_dict_begin[input_family],
+ code_dict[key],
+ cc_dict_end[input_family])
+ for lines in script:
+ sessionfile.write(lines)
+ sessionfile.close()
+ code_index_dict[key] = code_index
+ temp_data['code_index_dict'] = code_index_dict
+
+
+
+
+def do_multiprocessing(data, temp_data, old_data, engine_dict):
+ jobname = data['jobname']
+ encoding = data['encoding']
+ outputdir = data['settings']['outputdir']
+ workingdir = data['settings']['workingdir']
+ keeptemps = data['settings']['keeptemps']
+ fvextfile = data['settings']['fvextfile']
+ pygments_settings = data['pygments_settings']
+ verbose = temp_data['verbose']
+
+ code_dict = temp_data['code_dict']
+ cons_dict = temp_data['cons_dict']
+ cc_dict_begin = temp_data['cc_dict_begin']
+ cc_dict_end = temp_data['cc_dict_end']
+ pygments_list = temp_data['pygments_list']
+ pygments_style_defs = data['pygments_style_defs']
+
+ files = data['files']
+ macros = data['macros']
+ pygments_files = data['pygments_files']
+ pygments_macros = data['pygments_macros']
+ typeset_cache = data['typeset_cache']
+
+ errors = temp_data['errors']
+ warnings = temp_data['warnings']
+
+ makestderr = data['settings']['makestderr']
+ stderrfilename = data['settings']['stderrfilename']
+ code_index_dict = temp_data['code_index_dict']
+
+ hashdependencies = temp_data['hashdependencies']
+ dependencies = data['dependencies']
+ exit_status = data['exit_status']
+ start_time = data['start_time']
+
+
+ # Set maximum number of concurrent processes for multiprocessing
+ # Accoding to the docs, cpu_count() may raise an error
+ try:
+ max_processes = multiprocessing.cpu_count()
+ except NotImplementedError:
+ max_processes = 1
+ pool = multiprocessing.Pool(max_processes)
+ tasks = []
+
+ # If verbose, print a list of processes
+ if verbose:
+ print('\n* PythonTeX will run the following processes:')
+
+ # Add code processes. Note that everything placed in the codedict
+ # needs to be executed, based on previous testing, except for custom code.
+ for key in code_dict:
+ input_family = key.split('#')[0]
+ # Uncomment the following for debugging, and comment out what follows
+ '''run_code(encoding, outputdir, workingdir, code_dict[key],
+ engine_dict[input_family].language,
+ engine_dict[input_family].command,
+ engine_dict[input_family].created,
+ engine_dict[input_family].extension,
+ makestderr, stderrfilename,
+ code_index_dict[key],
+ engine_dict[input_family].errors,
+ engine_dict[input_family].warnings,
+ engine_dict[input_family].linenumbers,
+ engine_dict[input_family].lookbehind,
+ keeptemps, hashdependencies)'''
+ tasks.append(pool.apply_async(run_code, [encoding, outputdir,
+ workingdir, code_dict[key],
+ engine_dict[input_family].language,
+ engine_dict[input_family].command,
+ engine_dict[input_family].created,
+ engine_dict[input_family].extension,
+ makestderr, stderrfilename,
+ code_index_dict[key],
+ engine_dict[input_family].errors,
+ engine_dict[input_family].warnings,
+ engine_dict[input_family].linenumbers,
+ engine_dict[input_family].lookbehind,
+ keeptemps, hashdependencies]))
+ if verbose:
+ print(' - Code process ' + key.replace('#', ':'))
+
+ # Add console processes
+ for key in cons_dict:
+ input_family = key.split('#')[0]
+ if engine_dict[input_family].language.startswith('python'):
+ if input_family in pygments_settings:
+ # Uncomment the following for debugging
+ '''python_console(jobname, encoding, outputdir, workingdir,
+ fvextfile, pygments_settings[input_family],
+ cc_dict_begin[input_family], cons_dict[key],
+ cc_dict_end[input_family], engine_dict[input_family].startup,
+ engine_dict[input_family].banner,
+ engine_dict[input_family].filename)'''
+ tasks.append(pool.apply_async(python_console, [jobname, encoding,
+ outputdir, workingdir,
+ fvextfile,
+ pygments_settings[input_family],
+ cc_dict_begin[input_family],
+ cons_dict[key],
+ cc_dict_end[input_family],
+ engine_dict[input_family].startup,
+ engine_dict[input_family].banner,
+ engine_dict[input_family].filename]))
+ else:
+ tasks.append(pool.apply_async(python_console, [jobname, encoding,
+ outputdir, workingdir,
+ fvextfile,
+ None,
+ cc_dict_begin[input_family],
+ cons_dict[key],
+ cc_dict_end[input_family],
+ engine_dict[input_family].startup,
+ engine_dict[input_family].banner,
+ engine_dict[input_family].filename]))
+ else:
+ print('* PythonTeX error')
+ print(' Currently, non-Python consoles are not supported')
+ errors += 1
+ if verbose:
+ print(' - Console process ' + key.replace('#', ':'))
+
+ # Add a Pygments process
+ if pygments_list:
+ tasks.append(pool.apply_async(do_pygments, [encoding, outputdir,
+ fvextfile,
+ pygments_list,
+ pygments_settings,
+ typeset_cache]))
+ if verbose:
+ print(' - Pygments process')
+
+ # Execute the processes
+ pool.close()
+ pool.join()
+
+ # Get the outputs of processes
+ # Get the files and macros created. Get the number of errors and warnings
+ # produced. Get any messages returned. Get the exit_status, which is a
+ # dictionary of code that failed and thus must be run again (its hash is
+ # set to a null string). Keep track of whether there were any new files,
+ # so that the last time of file creation in .pytxmcr can be updated.
+ new_files = False
+ messages = []
+ for task in tasks:
+ result = task.get()
+ if result['process'] == 'code':
+ key = result['key']
+ files[key].extend(result['files'])
+ if result['files']:
+ new_files = True
+ macros[key].extend(result['macros'])
+ dependencies[key] = result['dependencies']
+ errors += result['errors']
+ warnings += result['warnings']
+ exit_status[key] = (result['errors'], result['warnings'])
+ messages.extend(result['messages'])
+ elif result['process'] == 'console':
+ key = result['key']
+ files[key].extend(result['files'])
+ if result['files']:
+ new_files = True
+ macros[key].extend(result['macros'])
+ pygments_files.update(result['pygments_files'])
+ pygments_macros.update(result['pygments_macros'])
+ dependencies[key] = result['dependencies']
+ typeset_cache[key] = result['typeset_cache']
+ errors += result['errors']
+ warnings += result['warnings']
+ exit_status[key] = (result['errors'], result['warnings'])
+ messages.extend(result['messages'])
+ elif result['process'] == 'pygments':
+ pygments_files.update(result['pygments_files'])
+ for k in result['pygments_files']:
+ if result['pygments_files'][k]:
+ new_files = True
+ break
+ pygments_macros.update(result['pygments_macros'])
+ errors += result['errors']
+ warnings += result['warnings']
+ messages.extend(result['messages'])
+
+ # Do a quick check to see if any dependencies were modified since the
+ # beginning of the run. If so, reset them so they will run next time and
+ # issue a warning
+ unresolved_dependencies = False
+ unresolved_sessions= []
+ for key in dependencies:
+ for dep, val in dependencies[key].items():
+ if val[0] > start_time:
+ unresolved_dependencies = True
+ dependencies[key][dep] = (None, None)
+ unresolved_sessions.append(key.replace('#', ':'))
+ if unresolved_dependencies:
+ print('* PythonTeX warning')
+ print(' The following have dependencies that have been modified')
+ print(' Run PythonTeX again to resolve dependencies')
+ for s in set(unresolved_sessions):
+ print(' - ' + s)
+ warnings += 1
+
+
+ # Save all content (only needs to be done if code was indeed run).
+ # Save a commented-out time corresponding to the last time PythonTeX ran
+ # and created files, so that tools like latexmk can easily detect when
+ # another run is needed.
+ if tasks:
+ if new_files or not temp_data['loaded_old_data']:
+ last_new_file_time = start_time
+ else:
+ last_new_file_time = old_data['last_new_file_time']
+ data['last_new_file_time'] = last_new_file_time
+
+ macro_file = open(os.path.join(outputdir, jobname + '.pytxmcr'), 'w', encoding=encoding)
+ macro_file.write('%Last time of file creation: ' + str(last_new_file_time) + '\n\n')
+ for key in macros:
+ macro_file.write(''.join(macros[key]))
+ macro_file.close()
+
+ pygments_macro_file = open(os.path.join(outputdir, jobname + '.pytxpyg'), 'w', encoding=encoding)
+ # Only save Pygments styles that are used
+ style_set = set([pygments_settings[k]['formatter_options']['style'] for k in pygments_settings if k != ':GLOBAL'])
+ for key in pygments_style_defs:
+ if key in style_set:
+ pygments_macro_file.write(''.join(pygments_style_defs[key]))
+ for key in pygments_macros:
+ pygments_macro_file.write(''.join(pygments_macros[key]))
+ pygments_macro_file.close()
+
+ pythontex_data_file = os.path.join(outputdir, 'pythontex_data.pkl')
+ f = open(pythontex_data_file, 'wb')
+ pickle.dump(data, f, -1)
+ f.close()
+
+ # Print any errors and warnings.
+ if messages:
+ print('\n'.join(messages))
+ sys.stdout.flush()
+ # Store errors and warnings back into temp_data
+ # This is needed because they are ints and thus immutable
+ temp_data['errors'] = errors
+ temp_data['warnings'] = warnings
+
+
+
+
+def run_code(encoding, outputdir, workingdir, code_list, language, command,
+ command_created, extension, makestderr, stderrfilename,
+ code_index, errorsig, warningsig, linesig, stderrlookbehind,
+ keeptemps, hashdependencies):
+ '''
+ Function for multiprocessing code files
+ '''
+ import shlex
+
+ # Create what's needed for storing results
+ input_family = code_list[0].input_family
+ input_session = code_list[0].input_session
+ key_run = code_list[0].key_run
+ files = []
+ macros = []
+ dependencies = {}
+ errors = 0
+ warnings = 0
+ unknowns = 0
+ messages = []
+
+ # Create message lists only for stderr, one for undelimited stderr and
+ # one for delimited, so it's easy to keep track of if there is any
+ # stderr. These are added onto messages at the end.
+ err_messages_ud = []
+ err_messages_d = []
+
+ # We need to let the user know we are switching code files
+ # We check at the end to see if there were indeed any errors and warnings
+ # and if not, clear messages.
+ messages.append('\n---- Messages for ' + key_run.replace('#', ':') + ' ----')
+
+ # Open files for stdout and stderr, run the code, then close the files
+ basename = key_run.replace('#', '_')
+ out_file_name = os.path.join(outputdir, basename + '.out')
+ err_file_name = os.path.join(outputdir, basename + '.err')
+ out_file = open(out_file_name, 'w', encoding=encoding)
+ err_file = open(err_file_name, 'w', encoding=encoding)
+ # Note that command is a string, which must be converted to list
+ # Must double-escape any backslashes so that they survive `shlex.split()`
+ script = os.path.join(outputdir, basename)
+ # `shlex.split()` only works with Unicode after 2.7.2
+ if (sys.version_info.major == 2 and sys.version_info.micro < 3):
+ exec_cmd = shlex.split(bytes(command.format(file=script.replace('\\', '\\\\'))))
+ exec_cmd = [unicode(elem) for elem in exec_cmd]
+ else:
+ exec_cmd = shlex.split(command.format(file=script.replace('\\', '\\\\')))
+ # Add any created files due to the command
+ # This needs to be done before attempts to execute, to prevent orphans
+ for f in command_created:
+ files.append(f.format(file=script))
+ try:
+ proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+ except WindowsError as e:
+ if e.errno == 2:
+ # Batch files won't be found when called without extension. They
+ # would be found if `shell=True`, but then getting the right
+ # exit code is tricky. So we perform some `cmd` trickery that
+ # is essentially equivalent to `shell=True`, but gives correct
+ # exit codes. Note that `subprocess.Popen()` works with strings
+ # under Windows; a list is not required.
+ exec_cmd_string = ' '.join(exec_cmd)
+ exec_cmd_string = 'cmd /C "@echo off & call {0} & if errorlevel 1 exit 1"'.format(exec_cmd_string)
+ proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+ else:
+ raise
+
+ proc.wait()
+ out_file.close()
+ err_file.close()
+
+ # Process saved stdout into file(s) that are included in the TeX document.
+ #
+ # Go through the saved output line by line, and save any printed content
+ # to its own file, named based on instance.
+ #
+ # The very end of the stdout lists dependencies, if any, so we start by
+ # removing and processing those.
+ if not os.path.isfile(out_file_name):
+ messages.append('* PythonTeX error')
+ messages.append(' Missing output file for ' + key_run.replace('#', ':'))
+ errors += 1
+ else:
+ f = open(out_file_name, 'r', encoding=encoding)
+ out = f.read()
+ f.close()
+ try:
+ out, created = out.rsplit('=>PYTHONTEX:CREATED#\n', 1)
+ out, deps = out.rsplit('=>PYTHONTEX:DEPENDENCIES#\n', 1)
+ valid_stdout = True
+ except:
+ valid_stdout = False
+ if proc.returncode == 0:
+ raise ValueError('Missing "created" and/or "dependencies" delims in stdout; invalid template?')
+
+ if valid_stdout:
+ # Add created files to created list
+ for c in created.splitlines():
+ files.append(c)
+
+ # Create a set of dependencies, to eliminate duplicates in the event
+ # that there are any. This is mainly useful when dependencies are
+ # automatically determined (for example, through redefining open()),
+ # may be specified multiple times as a result, and are hashed (and
+ # of a large enough size that hashing time is non-negligible).
+ deps = set([dep for dep in deps.splitlines()])
+ # Process dependencies; get mtimes and (if specified) hashes
+ for dep in deps:
+ dep_file = os.path.expanduser(os.path.normcase(dep))
+ if not os.path.isabs(dep_file):
+ dep_file = os.path.join(workingdir, dep_file)
+ if not os.path.isfile(dep_file):
+ # If we can't find the file, we return a null hash and issue
+ # an error. We don't need to change the exit status. If the
+ # code does depend on the file, there will be a separate
+ # error when the code attempts to use the file. If the code
+ # doesn't really depend on the file, then the error will be
+ # raised again anyway the next time PythonTeX runs when the
+ # dependency is listed but not found.
+ dependencies[dep] = (None, None)
+ messages.append('* PythonTeX error')
+ messages.append(' Cannot find dependency "' + dep + '"')
+ messages.append(' It belongs to ' + key_run.replace('#', ':'))
+ messages.append(' Relative paths to dependencies must be specified from the working directory.')
+ errors += 1
+ elif hashdependencies:
+ # Read and hash the file in binary. Opening in text mode
+ # would require an unnecessary decoding and encoding cycle.
+ hasher = sha1()
+ f = open(dep_file, 'rb')
+ hasher.update(f.read())
+ f.close()
+ dependencies[dep] = (os.path.getmtime(dep_file), hasher.hexdigest())
+ else:
+ dependencies[dep] = (os.path.getmtime(dep_file), '')
+
+ for block in out.split('=>PYTHONTEX:STDOUT#')[1:]:
+ if block:
+ delims, content = block.split('#\n', 1)
+ if content:
+ input_instance, input_command = delims.split('#')
+ if input_instance.endswith('CC'):
+ messages.append('* PythonTeX warning')
+ messages.append(' Custom code for "' + input_family + '" attempted to print or write to stdout')
+ messages.append(' This is not supported; use a normal code command or environment')
+ messages.append(' The following content was written:')
+ messages.append('')
+ messages.extend([' ' + l for l in content.splitlines()])
+ warnings += 1
+ elif input_command == 'i':
+ content = r'\pytx@SVMCR{pytx@MCR@' + key_run.replace('#', '@') + '@' + input_instance + '}\n' + content.rstrip('\n') + '\\endpytx@SVMCR\n\n'
+ macros.append(content)
+ else:
+ fname = os.path.join(outputdir, basename + '_' + input_instance + '.stdout')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(content)
+ f.close()
+ files.append(fname)
+
+ # Process stderr
+ if not os.path.isfile(err_file_name):
+ messages.append('* PythonTeX error')
+ messages.append(' Missing stderr file for ' + key_run.replace('#', ':'))
+ errors += 1
+ else:
+ # Open error and code files.
+ f = open(err_file_name, encoding=encoding)
+ err = f.readlines()
+ f.close()
+ # Divide stderr into an undelimited and a delimited portion
+ found = False
+ for n, line in enumerate(err):
+ if line.startswith('=>PYTHONTEX:STDERR#'):
+ found = True
+ err_ud = err[:n]
+ err_d = err[n:]
+ break
+ if not found:
+ err_ud = err
+ err_d = []
+ # Create a dict for storing any stderr content that will be saved
+ err_dict = defaultdict(list)
+ # Create the full basename that will be replaced in stderr
+ # We need two versions, one with the correct slashes for the OS,
+ # and one with the opposite slashes. This is needed when a language
+ # doesn't obey the OS's slash convention in paths given in stderr.
+ # For example, Windows uses backslashes, but Ruby under Windows uses
+ # forward in paths given in stderr.
+ fullbasename_correct = os.path.join(outputdir, basename)
+ if '\\' in fullbasename_correct:
+ fullbasename_reslashed = fullbasename_correct.replace('\\', '/')
+ else:
+ fullbasename_reslashed = fullbasename_correct.replace('/', '\\')
+
+ if err_ud:
+ it = iter(code_index.items())
+ index_now = next(it)
+ index_next = index_now
+ start_errgobble = None
+ for n, line in enumerate(err_ud):
+ if basename in line:
+ # Get the gobbleation. This is used to determine if
+ # other lines containing the basename are a continuation,
+ # or separate messages.
+ errgobble = match('(\s*)', line).groups()[0]
+ if start_errgobble is None:
+ start_errgobble = errgobble
+ # Only issue a message and track down the line numer if
+ # this is indeed the start of a new message, rather than
+ # a continuation of an old message that happens to
+ # contain the basename
+ if errgobble == start_errgobble:
+ # Determine the corresponding line number in the document
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ while index_next[1].lines_total < errlinenum:
+ try:
+ index_now, index_next = index_next, next(it)
+ except:
+ break
+ if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+ doclinenum = str(index_now[1].input_line_int + index_now[1].lines_input)
+ else:
+ doclinenum = str(index_now[1].input_line_int + errlinenum - index_now[1].lines_total - 1)
+ input_file = index_now[1].input_file
+ else:
+ doclinenum = '??'
+ input_file = '??'
+
+ # Try to determine if we are dealing with an error or a
+ # warning.
+ found = False
+ index = n
+ if stderrlookbehind:
+ while index >= 0:
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ past_line = err_ud[index]
+ if (index < n and basename in past_line):
+ break
+ for pattern in warningsig:
+ if pattern in past_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in past_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index -= 1
+ else:
+ while index < len(err_ud):
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ future_line = err_ud[index]
+ if (index > n and basename in future_line and
+ future_line.startswith(start_errgobble)):
+ break
+ for pattern in warningsig:
+ if pattern in future_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in future_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index += 1
+ # If an error or warning wasn't positively identified,
+ # increment unknowns.
+ if not found:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ err_messages_ud.append('* PythonTeX stderr - {0} on line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ err_messages_ud.append('* PythonTeX stderr - {0} on line {1}:'.format(alert_type, doclinenum))
+ err_messages_ud.append(' ' + line.replace(outputdir, '<outputdir>').rstrip('\n'))
+ else:
+ err_messages_ud.append(' ' + line.rstrip('\n'))
+
+ # Create .stderr
+ if makestderr and err_messages_ud:
+ process = False
+ it = iter(code_index.items())
+ index_now = next(it)
+ index_next = index_now
+ it_last = it
+ index_now_last = index_now
+ index_next_last = index_next
+ err_key_last_int = -1
+ for n, line in enumerate(err_ud):
+ if basename in line:
+ # Determine the corresponding line number in the document
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ if index_next[1].lines_total >= errlinenum:
+ it = it_last
+ index_now = index_now_last
+ index_next = index_next_last
+ else:
+ it_last = it
+ index_now_last = index_now
+ index_next_last = index_next
+ while index_next[1].lines_total < errlinenum:
+ try:
+ index_now, index_next = index_next, next(it)
+ except:
+ index_now = index_next
+ break
+ if index_now[0].endswith('CC'):
+ process = False
+ else:
+ process = True
+ if len(index_now[1].input_command) > 1:
+ if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+ codelinenum = str(index_now[1].lines_user + index_now[1].lines_input + 1)
+ else:
+ codelinenum = str(index_now[1].lines_user + errlinenum - index_now[1].lines_total - index_now[1].inline_count)
+ else:
+ codelinenum = '1'
+ else:
+ codelinenum = '??'
+ messages.append('* PythonTeX error')
+ messages.append(' Line number ' + str(errlinenum) + ' could not be synced with the document')
+ messages.append(' Content from stderr is not delimited, and cannot be resolved')
+ errors += 1
+ process = False
+
+ if process:
+ if int(index_now[0]) > err_key_last_int:
+ err_key = basename + '_' + index_now[0]
+ err_key_last_int = int(index_now[0])
+ line = line.replace(str(errlinenum), str(codelinenum), 1)
+ if fullbasename_correct in line:
+ fullbasename = fullbasename_correct
+ else:
+ fullbasename = fullbasename_reslashed
+ if stderrfilename == 'full':
+ line = line.replace(fullbasename, basename)
+ elif stderrfilename == 'session':
+ line = line.replace(fullbasename, input_session)
+ elif stderrfilename == 'genericfile':
+ line = line.replace(fullbasename + '.' + extension, '<file>')
+ elif stderrfilename == 'genericscript':
+ line = line.replace(fullbasename + '.' + extension, '<script>')
+ err_dict[err_key].append(line)
+ elif process:
+ err_dict[err_key].append(line)
+
+ if err_d:
+ start_errgobble = None
+ msg = []
+ found_basename = False
+ for n, line in enumerate(err_d):
+ if line.startswith('=>PYTHONTEX:STDERR#'):
+ # Store the last group of messages. Messages
+ # can't be directly appended to the main list, because
+ # a PythonTeX message must be inserted at the beginning
+ # of each chunk of stderr that never references
+ # the script that was executed. If the script is never
+ # referenced, then line numbers aren't automatically
+ # synced. These types of situations are created by
+ # warnings.warn() etc.
+ if msg:
+ if not found_basename:
+ # Get line number for command or beginning of
+ # environment
+ input_instance = last_delim.split('#')[1]
+ doclinenum = str(code_index[input_instance].input_line_int)
+ input_file = code_index[input_instance].input_file
+ # Try to identify alert. We have to parse all
+ # lines for signs of errors and warnings. This
+ # may result in overcounting, but it's the best
+ # we can do--otherwise, we could easily
+ # undercount, or, finding a warning, miss a
+ # subsequent error. When this code is actually
+ # used, it's already a sign that normal parsing
+ # has failed.
+ found_error = False
+ found_warning = False
+ for l in msg:
+ for pattern in warningsig:
+ if pattern in l:
+ warnings += 1
+ found_warning = True
+ for pattern in errorsig:
+ if pattern in l:
+ errors += 1
+ found_warning = True
+ if found_error:
+ alert_type = 'error'
+ elif found_warning:
+ alert_type = 'warning'
+ else:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1}:'.format(alert_type, doclinenum))
+ err_messages_d.extend(msg)
+ msg = []
+ found_basename = False
+ # Never process delimiting info until it is used
+ # Rather, store the index of the last delimiter
+ last_delim = line
+ elif basename in line:
+ found_basename = True
+ # Get the gobbleation. This is used to determine if
+ # other lines containing the basename are a continuation,
+ # or separate messages.
+ errgobble = match('(\s*)', line).groups()[0]
+ if start_errgobble is None:
+ start_errgobble = errgobble
+ # Only issue a message and track down the line numer if
+ # this is indeed the start of a new message, rather than
+ # a continuation of an old message that happens to
+ # contain the basename
+ if errgobble == start_errgobble:
+ # Determine the corresponding line number in the document
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ # Get info from last delim
+ input_instance, input_command = last_delim.split('#')[1:-1]
+ # Calculate the line number in the document
+ ei = code_index[input_instance]
+ if errlinenum > ei.lines_total + ei.lines_input:
+ doclinenum = str(ei.input_line_int + ei.lines_input)
+ else:
+ doclinenum = str(ei.input_line_int + errlinenum - ei.lines_total - 1)
+ input_file = ei.input_file
+ else:
+ doclinenum = '??'
+ input_file = '??'
+
+ # Try to determine if we are dealing with an error or a
+ # warning.
+ found = False
+ index = n
+ if stderrlookbehind:
+ while index >= 0:
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ past_line = err_d[index]
+ if (past_line.startswith('=>PYTHONTEX:STDERR#') or
+ (index < n and basename in past_line)):
+ break
+ for pattern in warningsig:
+ if pattern in past_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in past_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index -= 1
+ else:
+ while index < len(err_d):
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ future_line = err_d[index]
+ if (future_line.startswith('=>PYTHONTEX:STDERR#') or
+ (index > n and basename in future_line and future_line.startswith(start_errgobble))):
+ break
+ for pattern in warningsig:
+ if pattern in future_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in future_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index += 1
+ # If an error or warning wasn't positively identified,
+ # assume error for safety but indicate uncertainty.
+ if not found:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ msg.append('* PythonTeX stderr - {0} on line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ msg.append('* PythonTeX stderr - {0} on line {1}:'.format(alert_type, doclinenum))
+ # Clean up the stderr format a little, to keep it compact
+ line = line.replace(outputdir, '<outputdir>').rstrip('\n')
+ if '/<outputdir>' in line or '\\<outputdir>' in line:
+ line = sub(r'(?:(?:[A-Z]:\\)|(?:~?/)).*<outputdir>', '<outputdir>', line)
+ msg.append(' ' + line)
+ else:
+ msg.append(' ' + line.rstrip('\n'))
+ # Deal with any leftover messages
+ if msg:
+ if not found_basename:
+ # Get line number for command or beginning of
+ # environment
+ input_instance = last_delim.split('#')[1]
+ doclinenum = str(code_index[input_instance].input_line_int)
+ input_file = code_index[input_instance].input_file
+ # Try to identify alert. We have to parse all
+ # lines for signs of errors and warnings. This
+ # may result in overcounting, but it's the best
+ # we can do--otherwise, we could easily
+ # undercount, or, finding a warning, miss a
+ # subsequent error. When this code is actually
+ # used, it's already a sign that normal parsing
+ # has failed.
+ found_error = False
+ found_warning = False
+ for l in msg:
+ for pattern in warningsig:
+ if pattern in l:
+ warnings += 1
+ found_warning = True
+ for pattern in errorsig:
+ if pattern in l:
+ errors += 1
+ found_warning = True
+ if found_error:
+ alert_type = 'error'
+ elif found_warning:
+ alert_type = 'warning'
+ else:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1}:'.format(alert_type, doclinenum))
+ err_messages_d.extend(msg)
+
+ # Create .stderr
+ if makestderr and err_messages_d:
+ process = False
+ for n, line in enumerate(err_d):
+ if line.startswith('=>PYTHONTEX:STDERR#'):
+ input_instance, input_command = line.split('#')[1:-1]
+ if input_instance.endswith('CC'):
+ process = False
+ else:
+ process = True
+ err_key = basename + '_' + input_instance
+ elif process and basename in line:
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ # Calculate the line number in the document
+ # Account for inline
+ ei = code_index[input_instance]
+ # Store the `input_instance` in case it's
+ # incremented later
+ last_input_instance = input_instance
+ # If the error or warning was actually triggered
+ # later on (for example, multiline string with
+ # missing final delimiter), look ahead and
+ # determine the correct input_instance, so that
+ # we get the correct line number. We don't
+ # associate the created stderr with this later
+ # instance, however, but rather with the instance
+ # in which the error began. Doing that might
+ # possibly be preferable in some cases, but would
+ # also require that the current stderr be split
+ # between multiple instances, requiring extra
+ # parsing.
+ while errlinenum > ei.lines_total + ei.lines_input:
+ next_input_instance = str(int(input_instance) + 1)
+ if next_input_instance in code_index:
+ next_ei = code_index[next_input_instance]
+ if errlinenum > next_ei.lines_total:
+ input_instance = next_input_instance
+ ei = next_ei
+ else:
+ break
+ else:
+ break
+ if len(input_command) > 1:
+ if errlinenum > ei.lines_total + ei.lines_input:
+ codelinenum = str(ei.lines_user + ei.lines_input + 1)
+ else:
+ codelinenum = str(ei.lines_user + errlinenum - ei.lines_total - ei.inline_count)
+ else:
+ codelinenum = '1'
+ # Reset `input_instance`, in case incremented
+ input_instance = last_input_instance
+ else:
+ codelinenum = '??'
+ messages.append('* PythonTeX notice')
+ messages.append(' Line number ' + str(errlinenum) + ' could not be synced with the document')
+
+ line = line.replace(str(errlinenum), str(codelinenum), 1)
+ if fullbasename_correct in line:
+ fullbasename = fullbasename_correct
+ else:
+ fullbasename = fullbasename_reslashed
+ if stderrfilename == 'full':
+ line = line.replace(fullbasename, basename)
+ elif stderrfilename == 'session':
+ line = line.replace(fullbasename, input_session)
+ elif stderrfilename == 'genericfile':
+ line = line.replace(fullbasename + '.' + extension, '<file>')
+ elif stderrfilename == 'genericscript':
+ line = line.replace(fullbasename + '.' + extension, '<script>')
+ err_dict[err_key].append(line)
+ elif process:
+ err_dict[err_key].append(line)
+ if err_dict:
+ for err_key in err_dict:
+ stderr_file_name = os.path.join(outputdir, err_key + '.stderr')
+ f = open(stderr_file_name, 'w', encoding=encoding)
+ f.write(''.join(err_dict[err_key]))
+ f.close()
+ files.append(stderr_file_name)
+
+ # Clean up temp files, and update the list of existing files
+ if keeptemps == 'none':
+ for ext in [extension, 'pytxmcr', 'out', 'err']:
+ fname = os.path.join(outputdir, basename + '.' + ext)
+ if os.path.isfile(fname):
+ os.remove(fname)
+ elif keeptemps == 'code':
+ for ext in ['pytxmcr', 'out', 'err']:
+ fname = os.path.join(outputdir, basename + '.' + ext)
+ if os.path.isfile(fname):
+ os.remove(fname)
+ files.append(os.path.join(outputdir, basename + '.' + extension))
+ elif keeptemps == 'all':
+ for ext in [extension, 'pytxmcr', 'out', 'err']:
+ files.append(os.path.join(outputdir, basename + '.' + ext))
+
+ # Take care of any unknowns, based on exit code
+ # Interpret the exit code as an indicator of whether there were errors,
+ # and treat unknowns accordingly. This will cause all warnings to be
+ # misinterpreted as errors if warnings trigger a nonzero exit code.
+ # It will also cause all warnings to be misinterpreted as errors if there
+ # is a single error that causes a nonzero exit code. That isn't ideal,
+ # but shouldn't be a problem, because as soon as the error(s) are fixed,
+ # the exit code will be zero, and then all unknowns will be interpreted
+ # as warnings.
+ if unknowns:
+ if proc.returncode == 0:
+ unknowns_type = 'warnings'
+ warnings += unknowns
+ else:
+ unknowns_type = 'errors'
+ errors += unknowns
+ unknowns_message = '''
+ * PythonTeX notice
+ {0} message(s) could not be classified
+ Based on the return code, they were interpreted as {1}'''
+ messages[0] += textwrap.dedent(unknowns_message.format(unknowns, unknowns_type))
+
+ # Take care of anything that has escaped detection thus far.
+ if proc.returncode == 1 and not errors:
+ errors += 1
+ command_message = '''
+ * PythonTeX error
+ An error occurred but no error messages were identified.
+ This may indicate a bad command or missing program.
+ The following command was executed:
+ "{0}"'''
+ messages[0] += textwrap.dedent(command_message.format(' '.join(exec_cmd)))
+
+ # Add any stderr messages; otherwise, clear the default message header
+ if err_messages_ud:
+ messages.extend(err_messages_ud)
+ if err_messages_d:
+ messages.extend(err_messages_d)
+ if len(messages) == 1:
+ messages = []
+
+ # Return a dict of dicts of results
+ return {'process': 'code',
+ 'key': key_run,
+ 'files': files,
+ 'macros': macros,
+ 'dependencies': dependencies,
+ 'errors': errors,
+ 'warnings': warnings,
+ 'messages': messages}
+
+
+
+
+def do_pygments(encoding, outputdir, fvextfile, pygments_list,
+ pygments_settings, typeset_cache):
+ '''
+ Create Pygments content.
+
+ To be run during multiprocessing.
+ '''
+ # Lazy import
+ from pygments import highlight
+ from pygments.lexers import get_lexer_by_name
+ from pygments.formatters import LatexFormatter
+
+ # Create what's needed for storing results
+ pygments_files = defaultdict(list)
+ pygments_macros = defaultdict(list)
+ errors = 0
+ warnings = 0
+ messages = []
+ messages.append('\n---- Messages for Pygments ----')
+
+ # Create dicts of formatters and lexers.
+ formatter = dict()
+ lexer = dict()
+ for codetype in pygments_settings:
+ if codetype != ':GLOBAL':
+ formatter[codetype] = LatexFormatter(**pygments_settings[codetype]['formatter_options'])
+ lexer[codetype] = get_lexer_by_name(pygments_settings[codetype]['lexer'],
+ **pygments_settings[codetype]['lexer_options'])
+
+ # Actually parse and highlight the code.
+ for c in pygments_list:
+ if c.is_cons:
+ content = typeset_cache[c.key_run][c.input_instance]
+ elif c.is_extfile:
+ if os.path.isfile(c.extfile):
+ f = open(c.extfile, encoding=encoding)
+ content = f.read()
+ f.close()
+ else:
+ content = None
+ messages.append('* PythonTeX error')
+ messages.append(' Could not find external file ' + c.extfile)
+ messages.append(' The file was not pygmentized')
+ else:
+ content = c.code
+ processed = highlight(content, lexer[c.input_family], formatter[c.input_family])
+ if c.is_inline or content.count('\n') < fvextfile:
+ # Highlighted code brought in via macros needs SaveVerbatim
+ processed = sub(r'\\begin{Verbatim}\[(.+)\]',
+ r'\\begin{{SaveVerbatim}}[\1]{{pytx@{0}@{1}@{2}@{3}}}'.format(c.input_family, c.input_session, c.input_restart, c.input_instance), processed, count=1)
+ processed = processed.rsplit('\\', 1)[0] + '\\end{SaveVerbatim}\n\n'
+ pygments_macros[c.key_typeset].append(processed)
+ else:
+ fname = os.path.join(outputdir, c.key_typeset.replace('#', '_') + '.pygtex')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(processed)
+ f.close()
+ pygments_files[c.key_typeset].append(fname)
+
+ if len(messages) == 1:
+ messages = []
+ # Return a dict of dicts of results
+ return {'process': 'pygments',
+ 'pygments_files': pygments_files,
+ 'pygments_macros': pygments_macros,
+ 'errors': errors,
+ 'warnings': warnings,
+ 'messages': messages}
+
+
+
+
+def python_console(jobname, encoding, outputdir, workingdir, fvextfile,
+ pygments_settings, cc_begin_list, cons_list, cc_end_list,
+ startup, banner, filename):
+ '''
+ Use Python's ``code`` module to typeset emulated Python interactive
+ sessions, optionally highlighting with Pygments.
+ '''
+ # Create what's needed for storing results
+ key_run = cons_list[0].key_run
+ files = []
+ macros = []
+ pygments_files = defaultdict(list)
+ pygments_macros = defaultdict(list)
+ typeset_cache = {}
+ dependencies = {}
+ errors = 0
+ warnings = 0
+ messages = []
+ messages.append('\n---- Messages for ' + key_run.replace('#', ':') + ' ----')
+
+ # Lazy import what's needed
+ import code
+ from collections import deque
+ if sys.version_info[0] == 2:
+ # Need a Python 2 interface to io.StringIO that can accept bytes
+ import io
+ class StringIO(io.StringIO):
+ _orig_write = io.StringIO.write
+ def write(self, s):
+ self._orig_write(unicode(s))
+ else:
+ from io import StringIO
+
+ # Create a custom console class
+ class Console(code.InteractiveConsole):
+ '''
+ A subclass of code.InteractiveConsole that takes a list and treats it
+ as a series of console input.
+ '''
+
+ def __init__(self, banner, filename):
+ if banner == 'none':
+ self.banner = 'NULL BANNER'
+ elif banner == 'standard':
+ cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
+ self.banner = 'Python {0} on {1}\n{2}'.format(sys.version, sys.platform, cprt)
+ elif banner == 'pyversion':
+ self.banner = 'Python ' + '.'.join(str(sys.version_info[n]) for n in (0, 1, 2))
+ else:
+ self.banner = None
+ if filename == 'console':
+ self.filename = '<console>'
+ elif filename == 'stdin':
+ self.filename = '<stdin>'
+ else:
+ self.filename = None
+ code.InteractiveConsole.__init__(self, filename=self.filename)
+ self.iostdout = StringIO()
+
+ def consolize(self, startup, cons_list):
+ self.console_code = deque()
+ # Delimiters are passed straight through and need newlines
+ self.console_code.append('=>PYTHONTEX#STARTUP##\n')
+ cons_config = '''
+ import os
+ import sys
+ docdir = os.getcwd()
+ if os.path.isdir('{workingdir}'):
+ os.chdir('{workingdir}')
+ if os.getcwd() not in sys.path:
+ sys.path.append(os.getcwd())
+ else:
+ sys.exit('Cannot find directory {workingdir}')
+
+ if docdir not in sys.path:
+ sys.path.append(docdir)
+
+ del docdir
+ '''
+ cons_config = cons_config.format(workingdir=workingdir)[1:]
+ self.console_code.extend(textwrap.dedent(cons_config).splitlines())
+ # Code is processed and doesn't need newlines
+ self.console_code.extend(startup.splitlines())
+ for c in cons_list:
+ self.console_code.append('=>PYTHONTEX#{0}#{1}#\n'.format(c.input_instance, c.input_command))
+ self.console_code.extend(c.code.splitlines())
+ old_stdout = sys.stdout
+ sys.stdout = self.iostdout
+ self.interact(self.banner)
+ sys.stdout = old_stdout
+ self.session_log = self.iostdout.getvalue()
+
+ def raw_input(self, prompt):
+ # Have to do a lot of looping and trying to make sure we get
+ # something valid to execute
+ try:
+ line = self.console_code.popleft()
+ except IndexError:
+ raise EOFError
+ while line.startswith('=>PYTHONTEX#'):
+ # Get new lines until we get one that doesn't begin with a
+ # delimiter. Then write the last delimited line.
+ old_line = line
+ try:
+ line = self.console_code.popleft()
+ self.write(old_line)
+ except IndexError:
+ raise EOFError
+ if line or prompt == sys.ps2:
+ self.write('{0}{1}\n'.format(prompt, line))
+ else:
+ self.write('\n')
+ return line
+
+ def write(self, data):
+ self.iostdout.write(data)
+
+ # Need to combine all custom code and user code to pass to consolize
+ cons_list = cc_begin_list + cons_list + cc_end_list
+ # Create a dict for looking up exceptions. This is needed for startup
+ # commands and for code commands and environments, since their output
+ # isn't typeset
+ cons_index = {}
+ for c in cons_list:
+ cons_index[c.input_instance] = c.input_line
+
+ # Consolize the code
+ con = Console(banner, filename)
+ con.consolize(startup, cons_list)
+
+ # Set up Pygments, if applicable
+ if pygments_settings is not None:
+ pygmentize = True
+ # Lazy import
+ from pygments import highlight
+ from pygments.lexers import get_lexer_by_name
+ from pygments.formatters import LatexFormatter
+ formatter = LatexFormatter(**pygments_settings['formatter_options'])
+ lexer = get_lexer_by_name(pygments_settings['lexer'],
+ **pygments_settings['lexer_options'])
+ else:
+ pygmentize = False
+
+ # Process the console output
+ output = con.session_log.split('=>PYTHONTEX#')
+ # Extract banner
+ if banner == 'none':
+ banner_text = ''
+ else:
+ banner_text = output[0]
+ # Ignore the beginning, because it's the banner
+ for block in output[1:]:
+ delims, console_content = block.split('#\n', 1)
+ if console_content:
+ input_instance, input_command = delims.split('#')
+ if input_instance == 'STARTUP':
+ exception = False
+ console_content_lines = console_content.splitlines()
+ for line in console_content_lines:
+ if (not line.startswith(sys.ps1) and
+ not line.startswith(sys.ps2) and
+ line and not line.isspace()):
+ exception = True
+ break
+ if exception:
+ if 'Error:' in console_content:
+ errors += 1
+ alert_type = 'error'
+ elif 'Warning:' in console_content:
+ warnings += 1
+ alert_type = 'warning'
+ else:
+ errors += 1
+ alert_type = 'error (?)'
+ messages.append('* PythonTeX stderr - {0} in console startup code:'.format(alert_type))
+ for line in console_content_lines:
+ messages.append(' ' + line)
+ elif input_command in ('c', 'code'):
+ exception = False
+ console_content_lines = console_content.splitlines()
+ for line in console_content_lines:
+ if (line and not line.startswith(sys.ps1) and
+ not line.startswith(sys.ps2) and
+ not line.isspace()):
+ print('X' + line + 'X')
+ exception = True
+ break
+ if exception:
+ if 'Error:' in console_content:
+ errors += 1
+ alert_type = 'error'
+ elif 'Warning:' in console_content:
+ warnings += 1
+ alert_type = 'warning'
+ else:
+ errors += 1
+ alert_type = 'error (?)'
+ if input_instance.endswith('CC'):
+ messages.append('* PythonTeX stderr - {0} near line {1} in custom code for console:'.format(alert_type, cons_index[input_instance]))
+ else:
+ messages.append('* PythonTeX stderr - {0} near line {1} in console code:'.format(alert_type, cons_index[input_instance]))
+ messages.append(' Console code is not typeset, and should have no output')
+ for line in console_content_lines:
+ messages.append(' ' + line)
+ else:
+ if input_command == 'i':
+ # Currently, there isn't any error checking for invalid
+ # content; it is assumed that a single line of commands
+ # was entered, producing one or more lines of output.
+ # Given that the current ``\pycon`` command doesn't
+ # allow line breaks to be written to the .pytxcode, that
+ # should be a reasonable assumption.
+ console_content = console_content.split('\n', 1)[1]
+ if banner_text is not None and input_command == 'console':
+ # Append banner to first appropriate environment
+ console_content = banner_text + console_content
+ banner_text = None
+ # Cache
+ key_typeset = key_run + '#' + input_instance
+ typeset_cache[input_instance] = console_content
+ # Process for LaTeX
+ if pygmentize:
+ processed = highlight(console_content, lexer, formatter)
+ if console_content.count('\n') < fvextfile:
+ processed = sub(r'\\begin{Verbatim}\[(.+)\]',
+ r'\\begin{{SaveVerbatim}}[\1]{{pytx@{0}}}'.format(key_typeset.replace('#', '@')),
+ processed, count=1)
+ processed = processed.rsplit('\\', 1)[0] + '\\end{SaveVerbatim}\n\n'
+ pygments_macros[key_typeset].append(processed)
+ else:
+ fname = os.path.join(outputdir, key_typeset.replace('#', '_') + '.pygtex')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(processed)
+ f.close()
+ pygments_files[key_typeset].append(fname)
+ else:
+ if console_content.count('\n') < fvextfile:
+ processed = ('\\begin{{SaveVerbatim}}{{pytx@{0}}}\n'.format(key_typeset.replace('#', '@')) +
+ console_content + '\\end{SaveVerbatim}\n\n')
+ macros.append(processed)
+ else:
+ processed = ('\\begin{Verbatim}\n' + console_content +
+ '\\end{Verbatim}\n\n')
+ fname = os.path.join(outputdir, key_typeset.replace('#', '_') + '.tex')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(processed)
+ f.close()
+ files.append(fname)
+
+ if len(messages) == 1:
+ messages = []
+
+ # Return a dict of dicts of results
+ return {'process': 'console',
+ 'key': key_run,
+ 'files': files,
+ 'macros': macros,
+ 'pygments_files': pygments_files,
+ 'pygments_macros': pygments_macros,
+ 'typeset_cache': typeset_cache,
+ 'dependencies': dependencies,
+ 'errors': errors,
+ 'warnings': warnings,
+ 'messages': messages}
+
+
+
+
+def main():
+ # Create dictionaries for storing data.
+ #
+ # All data that must be saved for subsequent runs is stored in "data".
+ # (We start off by saving the script version, a global var, in this dict.)
+ # All data that is only created for this run is stored in "temp_data".
+ # (We start off by creating keys for keeping track of errors and warnings.)
+ # All old data will eventually be loaded into "old_data".
+ # Since dicts are mutable data types, these dicts can be modified
+ # from within functions, as long as the dicts are passed to the functions.
+ # For simplicity, variables will often be created within functions to
+ # refer to dictionary values.
+ data = {'version': version, 'start_time': time.time()}
+ temp_data = {'errors': 0, 'warnings': 0}
+ old_data = dict()
+
+
+ # Process command-line options.
+ #
+ # This gets the raw_jobname (actual job name), jobname (a sanitized job
+ # name, used for creating files named after the jobname), and any options.
+ process_argv(data, temp_data)
+ # If there aren't errors in argv, and the program is going to run
+ # (rather than just exit due to --version or --help command-line options),
+ # print PythonTeX version. Flush to make the message go out immediately,
+ # so that the user knows PythonTeX has started.
+ print('This is PythonTeX ' + version)
+ sys.stdout.flush()
+ # Once we have the encoding (from argv), we set stdout and stderr to use
+ # this encoding. Later, we will parse the saved stderr of scripts
+ # executed via multiprocessing subprocesses, and print the parsed results
+ # to stdout. The saved stderr uses the same encoding that was used
+ # for the files that created it (this is important for code containing
+ # unicode characters), so we also need stdout for the main PythonTeX
+ # script to support this encoding. Setting stderr encoding is primarily
+ # a matter of symmetry. Ideally, pythontex*.py will be bug-free,
+ # and stderr won't be needed!
+ if sys.version_info[0] == 2:
+ sys.stdout = codecs.getwriter(data['encoding'])(sys.stdout, 'strict')
+ sys.stderr = codecs.getwriter(data['encoding'])(sys.stderr, 'strict')
+ else:
+ sys.stdout = codecs.getwriter(data['encoding'])(sys.stdout.buffer, 'strict')
+ sys.stderr = codecs.getwriter(data['encoding'])(sys.stderr.buffer, 'strict')
+
+
+ # Load the code and process the settings it passes from the TeX side.
+ #
+ # This gets a list containing the code (the part of the code file
+ # containing the settings is removed) and the processed settings.
+ load_code_get_settings(data, temp_data)
+ # Now that the settings are loaded, check if outputdir exits.
+ # If not, create it.
+ if not os.path.isdir(data['settings']['outputdir']):
+ os.mkdir(data['settings']['outputdir'])
+
+
+ # Load/create old_data
+ get_old_data(data, old_data, temp_data)
+
+
+ # Hash the code. Determine what needs to be executed. Determine whether
+ # Pygments should be used. Update pygments_settings to account for
+ # Pygments commands and environments (as opposed to PythonTeX commands
+ # and environments).
+ hash_all(data, temp_data, old_data, engine_dict)
+
+
+ # Parse the code and write scripts for execution.
+ parse_code_write_scripts(data, temp_data, engine_dict)
+
+
+ # Execute the code and perform Pygments highlighting via multiprocessing.
+ do_multiprocessing(data, temp_data, old_data, engine_dict)
+
+
+ # Print exit message
+ print('\n--------------------------------------------------')
+ # If some rerun settings are used, there may be unresolved errors or
+ # warnings; if so, print a summary of those along with the current
+ # error and warning summary
+ unresolved_errors = 0
+ unresolved_warnings = 0
+ if temp_data['rerun'] in ('errors', 'modified', 'never'):
+ global_update = {}
+ global_update.update(temp_data['code_update'])
+ global_update.update(temp_data['cons_update'])
+ for key in data['exit_status']:
+ if not global_update[key]:
+ unresolved_errors += data['exit_status'][key][0]
+ unresolved_warnings += data['exit_status'][key][1]
+ if unresolved_warnings != 0 or unresolved_errors != 0:
+ print('PythonTeX: {0}'.format(data['raw_jobname']))
+ print(' - Old: {0} error(s), {1} warnings(s)'.format(unresolved_errors, unresolved_warnings))
+ print(' - Current: {0} error(s), {1} warnings(s)'.format(temp_data['errors'], temp_data['warnings']))
+ else:
+ print('PythonTeX: {0} - {1} error(s), {2} warning(s)\n'.format(data['raw_jobname'], temp_data['errors'], temp_data['warnings']))
+
+ # Exit with appropriate exit code based on user settings.
+ if temp_data['error_exit_code'] and temp_data['errors'] > 0:
+ sys.exit(1)
+ else:
+ sys.exit()
+
+
+
+# The "if" statement is needed for multiprocessing under Windows; see the
+# multiprocessing documentation. It is also needed in this case when the
+# script is invoked via the wrapper.
+if __name__ == '__main__':
+ main() \ No newline at end of file
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex3.py b/Master/texmf-dist/scripts/pythontex/pythontex3.py
new file mode 100755
index 00000000000..1712e75e096
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/pythontex3.py
@@ -0,0 +1,2360 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+This is the main PythonTeX script. It should be launched via pythontex.py.
+
+Two versions of this script are provided. One, with name ending in "2", runs
+under Python 2.7. The other, with name ending in "3", runs under Python 3.2+.
+
+This script needs to be able to import pythontex_engines.py; in general it
+should be in the same directory.
+
+
+Licensed under the BSD 3-Clause License:
+
+Copyright (c) 2012-2013, Geoffrey M. Poore
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the <organization> nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+
+
+# Imports
+#// Python 2
+#from __future__ import absolute_import
+#from __future__ import division
+#from __future__ import print_function
+#from __future__ import unicode_literals
+#\\ End Python 2
+import sys
+import os
+import argparse
+import codecs
+import time
+from hashlib import sha1
+from collections import defaultdict, OrderedDict, namedtuple
+from re import match, sub, search
+import subprocess
+import multiprocessing
+from pygments.styles import get_all_styles
+from pythontex_engines import *
+import textwrap
+
+if sys.version_info[0] == 2:
+ try:
+ import cPickle as pickle
+ except:
+ import pickle
+ from io import open
+else:
+ import pickle
+
+
+
+
+# Script parameters
+# Version
+version = 'v0.12'
+
+
+
+
+class Pytxcode(object):
+ def __init__(self, data, gobble):
+ self.delims, self.code = data.split('#\n', 1)
+ self.input_family, self.input_session, self.input_restart, self.input_instance, self.input_command, self.input_context, self.input_args_run, self.input_args_prettyprint, self.input_file, self.input_line = self.delims.split('#')
+ self.input_instance_int = int(self.input_instance)
+ self.input_line_int = int(self.input_line)
+ self.key_run = self.input_family + '#' + self.input_session + '#' + self.input_restart
+ self.key_typeset = self.key_run + '#' + self.input_instance
+ self.hashable_delims_run = self.key_typeset + '#' + self.input_command + '#' + self.input_context + '#' + self.input_args_run
+ self.hashable_delims_typeset = self.key_typeset + '#' + self.input_command + '#' + self.input_context + '#' + self.input_args_run
+ if len(self.input_command) > 1:
+ self.is_inline = False
+ # Environments start on the next line
+ self.input_line_int += 1
+ self.input_line = str(self.input_line_int)
+ else:
+ self.is_inline = True
+ self.is_extfile = True if self.input_session.startswith('EXT:') else False
+ if self.is_extfile:
+ self.extfile = os.path.expanduser(os.path.normcase(self.input_session.replace('EXT:', '', 1)))
+ self.is_cc = True if self.input_family.startswith('CC:') else False
+ self.is_pyg = True if self.input_family.startswith('PYG') else False
+ self.is_verb = True if self.input_restart.endswith('verb') else False
+ if self.is_cc:
+ self.input_instance += 'CC'
+ self.cc_type, self.cc_pos = self.input_family.split(':')[1:]
+ if self.is_verb or self.is_pyg or self.is_cc:
+ self.is_cons = False
+ else:
+ self.is_cons = engine_dict[self.input_family].console
+ self.is_code = False if self.is_verb or self.is_pyg or self.is_cc or self.is_cons else True
+ if self.input_command in ('c', 'code') or (self.input_command == 'i' and not self.is_cons):
+ self.is_typeset = False
+ else:
+ self.is_typeset = True
+
+ if gobble == 'auto':
+ self.code = textwrap.dedent(self.code)
+
+
+
+
+def process_argv(data, temp_data):
+ '''
+ Process command line options using the argparse module.
+
+ Most options are passed via the file of code, rather than via the command
+ line.
+ '''
+
+ # Create a command line argument parser
+ parser = argparse.ArgumentParser()
+ parser.add_argument('TEXNAME',
+ help='LaTeX file, with or without .tex extension')
+ parser.add_argument('--version', action='version',
+ version='PythonTeX {0}'.format(data['version']))
+ parser.add_argument('--encoding', default='UTF-8',
+ help='encoding for all text files (see codecs module for encodings)')
+ parser.add_argument('--error-exit-code', default='true',
+ choices=('true', 'false'),
+ help='return exit code of 1 if there are errors (not desirable with some TeX editors and workflows)')
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('--runall', nargs='?', default='false',
+ const='true', choices=('true', 'false'),
+ help='run ALL code; equivalent to package option')
+ group.add_argument('--rerun', default='errors',
+ choices=('never', 'modified', 'errors', 'warnings', 'always'),
+ help='set conditions for rerunning code; equivalent to package option')
+ parser.add_argument('--hashdependencies', nargs='?', default='false',
+ const='true', choices=('true', 'false'),
+ help='hash dependencies (such as external data) to check for modification, rather than using mtime; equivalent to package option')
+ parser.add_argument('-v', '--verbose', default=False, action='store_true',
+ help='verbose output')
+ parser.add_argument('--interpreter', default=None, help='set a custom interpreter; argument should be in the form "<interpreter>:<command>, <interp>:<cmd>, ..." where <interpreter> is "python", "ruby", etc., and <command> is the command for invoking the interpreter; argument may also be in the form of a Python dictionary')
+ args = parser.parse_args()
+
+ # Store the parsed argv in data and temp_data
+ data['encoding'] = args.encoding
+ if args.error_exit_code == 'true':
+ temp_data['error_exit_code'] = True
+ else:
+ temp_data['error_exit_code'] = False
+ # runall can be mapped onto rerun, so both are stored under rerun
+ if args.runall == 'true':
+ temp_data['rerun'] = 'always'
+ else:
+ temp_data['rerun'] = args.rerun
+ # hashdependencies need only be in temp_data, since changing it would
+ # change hashes (hashes of mtime vs. file contents)
+ if args.hashdependencies == 'true':
+ temp_data['hashdependencies'] = True
+ else:
+ temp_data['hashdependencies'] = False
+ temp_data['verbose'] = args.verbose
+ # Update interpreter_dict based on interpreter
+ if args.interpreter is not None:
+ interp_list = args.interpreter.lstrip('{').rstrip('}').split(',')
+ for interp in interp_list:
+ if interp:
+ try:
+ k, v = interp.split(':')
+ k = k.strip(' \'"')
+ v = v.strip(' \'"')
+ interpreter_dict[k] = v
+ except:
+ print('Invalid --interpreter argument')
+ return sys.exit(2)
+
+
+ if args.TEXNAME is not None:
+ # Determine if we a dealing with just a filename, or a name plus
+ # path. If there's a path, we need to make the document directory
+ # the current working directory.
+ dir, raw_jobname = os.path.split(args.TEXNAME)
+ dir = os.path.expanduser(os.path.normcase(dir))
+ if dir:
+ os.chdir(dir)
+ sys.path.append(dir)
+ # If necessary, strip off an extension to find the raw jobname that
+ # corresponds to the .pytxcode.
+ if not os.path.exists(raw_jobname + '.pytxcode'):
+ raw_jobname = raw_jobname.rsplit('.', 1)[0]
+ if not os.path.exists(raw_jobname + '.pytxcode'):
+ print('* PythonTeX error')
+ print(' Code file ' + raw_jobname + '.pytxcode does not exist.')
+ print(' Run LaTeX to create it.')
+ return sys.exit(1)
+
+ # We need a "sanitized" version of the jobname, with spaces and
+ # asterisks replaced with hyphens. This is done to avoid TeX issues
+ # with spaces in file names, paralleling the approach taken in
+ # pythontex.sty. From now on, we will use the sanitized version every
+ # time we create a file that contains the jobname string. The raw
+ # version will only be used in reference to pre-existing files created
+ # on the TeX side, such as the .pytxcode file.
+ jobname = raw_jobname.replace(' ', '-').replace('"', '').replace('*', '-')
+ # Store the results in data
+ data['raw_jobname'] = raw_jobname
+ data['jobname'] = jobname
+
+ # We need to check to make sure that the "sanitized" jobname doesn't
+ # lead to a collision with a file that already has that name, so that
+ # two files attempt to use the same PythonTeX folder.
+ #
+ # If <jobname>.<ext> and <raw_jobname>.<ext> both exist, where <ext>
+ # is a common LaTeX extension, we exit. We operate under the
+ # assumption that there should be only a single file <jobname> in the
+ # document root directory that has a common LaTeX extension. That
+ # could be false, but if so, the user probably has worse things to
+ # worry about than a potential PythonTeX output collision.
+ # If <jobname>* and <raw_jobname>* both exist, we issue a warning but
+ # attempt to proceed.
+ if jobname != raw_jobname:
+ resolved = False
+ for ext in ('.tex', '.ltx', '.dtx'):
+ if os.path.isfile(raw_jobname + ext):
+ if os.path.isfile(jobname + ext):
+ print('* PythonTeX error')
+ print(' Directory naming collision between the following files:')
+ print(' ' + raw_jobname + ext)
+ print(' ' + jobname + ext)
+ return sys.exit(1)
+ else:
+ resolved = True
+ break
+ if not resolved:
+ ls = os.listdir('.')
+ for file in ls:
+ if file.startswith(jobname):
+ print('* PythonTeX warning')
+ print(' Potential directory naming collision between the following names:')
+ print(' ' + raw_jobname)
+ print(' ' + jobname + '*')
+ print(' Attempting to proceed.')
+ temp_data['warnings'] += 1
+ break
+
+
+
+
+def load_code_get_settings(data, temp_data):
+ '''
+ Load the code file, preprocess the code, and extract the settings.
+ '''
+ # Bring in the .pytxcode file as a single string
+ raw_jobname = data['raw_jobname']
+ encoding = data['encoding']
+ # The error checking here is a little redundant
+ if os.path.isfile(raw_jobname + '.pytxcode'):
+ f = open(raw_jobname + '.pytxcode', 'r', encoding=encoding)
+ pytxcode = f.read()
+ f.close()
+ else:
+ print('* PythonTeX error')
+ print(' Code file ' + raw_jobname + '.pytxcode does not exist.')
+ print(' Run LaTeX to create it.')
+ return sys.exit(1)
+
+ # Split code and settings
+ try:
+ pytxcode, pytxsettings = pytxcode.rsplit('=>PYTHONTEX:SETTINGS#', 1)
+ except:
+ print('The .pytxcode file appears to have an outdated format or be invalid')
+ print('Run LaTeX to make sure the file is current')
+ return sys.exit(1)
+
+
+ # Prepare to process settings
+ #
+ # Create a dict for storing settings.
+ settings = {}
+ # Create a dict for storing Pygments settings.
+ # Each dict entry will itself be a dict.
+ pygments_settings = defaultdict(dict)
+
+ # Create a dict of processing functions, and generic processing functions
+ settings_func = dict()
+ def set_kv_data(k, v):
+ if v == 'true':
+ settings[k] = True
+ elif v == 'false':
+ settings[k] = False
+ else:
+ settings[k] = v
+ # Need a function for when assignment is only needed if not default value
+ def set_kv_temp_data_if_not_default(k, v):
+ if v != 'default':
+ if v == 'true':
+ temp_data[k] = True
+ elif v == 'false':
+ temp_data[k] = False
+ else:
+ temp_data[k] = v
+ def set_kv_data_fvextfile(k, v):
+ # Error checking on TeX side should be enough, but be careful anyway
+ try:
+ v = int(v)
+ except ValueError:
+ print('* PythonTeX error')
+ print(' Unable to parse package option fvextfile.')
+ return sys.exit(1)
+ if v < 0:
+ settings[k] = sys.maxsize
+ elif v == 0:
+ settings[k] = 1
+ print('* PythonTeX warning')
+ print(' Invalid value for package option fvextfile.')
+ temp_data['warnings'] += 1
+ else:
+ settings[k] = v
+ def set_kv_pygments(k, v):
+ input_family, lexer_opts, options = v.replace(' ','').split('|')
+ lexer = None
+ lex_dict = {}
+ opt_dict = {}
+ if lexer_opts:
+ for l in lexer_opts.split(','):
+ if '=' in l:
+ k, v = l.split('=', 1)
+ if k == 'lexer':
+ lexer = l
+ else:
+ lex_dict[k] = v
+ else:
+ lexer = l
+ if options:
+ for o in options.split(','):
+ if '=' in o:
+ k, v = o.split('=', 1)
+ if v in ('true', 'True'):
+ v = True
+ elif v in ('false', 'False'):
+ v = False
+ else:
+ k = option
+ v = True
+ opt_dict[k] = v
+ if input_family != ':GLOBAL':
+ if 'lexer' in pygments_settings[':GLOBAL']:
+ lexer = pygments_settings[':GLOBAL']['lexer']
+ lex_dict.update(pygments_settings[':GLOBAL']['lexer_options'])
+ opt_dict.update(pygments_settings[':GLOBAL']['formatter_options'])
+ if 'style' not in opt_dict:
+ opt_dict['style'] = 'default'
+ opt_dict['commandprefix'] = 'PYG' + opt_dict['style']
+ if lexer is not None:
+ pygments_settings[input_family]['lexer'] = lexer
+ pygments_settings[input_family]['lexer_options'] = lex_dict
+ pygments_settings[input_family]['formatter_options'] = opt_dict
+ settings_func['version'] = set_kv_data
+ settings_func['outputdir'] = set_kv_data
+ settings_func['workingdir'] = set_kv_data
+ settings_func['gobble'] = set_kv_data
+ settings_func['rerun'] = set_kv_temp_data_if_not_default
+ settings_func['hashdependencies'] = set_kv_temp_data_if_not_default
+ settings_func['makestderr'] = set_kv_data
+ settings_func['stderrfilename'] = set_kv_data
+ settings_func['keeptemps'] = set_kv_data
+ settings_func['pyfuture'] = set_kv_data
+ settings_func['pyconfuture'] = set_kv_data
+ settings_func['pygments'] = set_kv_data
+ settings_func['fvextfile'] = set_kv_data_fvextfile
+ settings_func['pygglobal'] = set_kv_pygments
+ settings_func['pygfamily'] = set_kv_pygments
+ settings_func['pyconbanner'] = set_kv_data
+ settings_func['pyconfilename'] = set_kv_data
+ settings_func['depythontex'] = set_kv_data
+
+ # Process settings
+ for line in pytxsettings.split('\n'):
+ if line:
+ key, val = line.split('=', 1)
+ try:
+ settings_func[key](key, val)
+ except KeyError:
+ print('* PythonTeX warning')
+ print(' Unknown option "' + key + '"')
+ temp_data['warnings'] += 1
+
+ # Check for compatility between the .pytxcode and the script
+ if 'version' not in settings or settings['version'] != data['version']:
+ print('* PythonTeX warning')
+ print(' The version of the PythonTeX scripts does not match')
+ print(' the last code saved by the document--run LaTeX to create')
+ print(' an updated version. Attempting to proceed.')
+ sys.stdout.flush()
+
+ # Store all results that haven't already been stored.
+ data['settings'] = settings
+ data['pygments_settings'] = pygments_settings
+
+ # Create a tuple of vital quantities that invalidate old saved data
+ # Don't need to include outputdir, because if that changes, no old output
+ # fvextfile could be checked on a case-by-case basis, which would result
+ # in faster output, but that would involve a good bit of additional
+ # logic, which probably isn't worth it for a feature that will rarely be
+ # changed.
+ data['vitals'] = (data['version'], data['encoding'],
+ settings['gobble'], settings['fvextfile'])
+
+ # Create tuples of vital quantities
+ data['code_vitals'] = (settings['workingdir'], settings['keeptemps'],
+ settings['makestderr'], settings['stderrfilename'])
+ data['cons_vitals'] = (settings['workingdir'])
+ data['typeset_vitals'] = ()
+
+ # Pass any customizations to types
+ for k in engine_dict:
+ engine_dict[k].customize(pyfuture=settings['pyfuture'],
+ pyconfuture=settings['pyconfuture'],
+ pyconbanner=settings['pyconbanner'],
+ pyconfilename=settings['pyconfilename'])
+
+ # Store code
+ # Do this last, so that Pygments settings are available
+ if pytxcode.startswith('=>PYTHONTEX#'):
+ gobble = settings['gobble']
+ temp_data['pytxcode'] = [Pytxcode(c, gobble) for c in pytxcode.split('=>PYTHONTEX#')[1:]]
+ else:
+ temp_data['pytxcode'] = []
+
+
+
+
+def get_old_data(data, old_data, temp_data):
+ '''
+ Load data from the last run, if it exists, into the dict old_data.
+ Determine the path to the PythonTeX scripts, either by using a previously
+ found, saved path or via kpsewhich.
+
+ The old data is used for determining when PythonTeX has been upgraded,
+ when any settings have changed, when code has changed (via hashes), and
+ what files may need to be cleaned up. The location of the PythonTeX
+ scripts is needed so that they can be imported by the scripts created by
+ PythonTeX. The location of the scripts is confirmed even if they were
+ previously located, to make sure that the path is still valid. Finding
+ the scripts depends on having a TeX installation that includes the
+ Kpathsea library (TeX Live and MiKTeX, possibly others).
+
+ All code that relies on old_data is written based on the assumption that
+ if old_data exists and has the current PythonTeX version, then it
+ contains all needed information. Thus, all code relying on old_data must
+ check that it was loaded and that it has the current version. If not,
+ code should adapt gracefully.
+ '''
+
+ # Create a string containing the name of the data file
+ pythontex_data_file = os.path.join(data['settings']['outputdir'], 'pythontex_data.pkl')
+
+ # Load the old data if it exists (read as binary pickle)
+ if os.path.isfile(pythontex_data_file):
+ f = open(pythontex_data_file, 'rb')
+ old = pickle.load(f)
+ f.close()
+ # Check for compabilility
+ if 'vitals' in old and data['vitals'] == old['vitals']:
+ temp_data['loaded_old_data'] = True
+ old_data.update(old)
+ else:
+ temp_data['loaded_old_data'] = False
+ # Clean up all old files
+ if 'files' in old:
+ for key in old['files']:
+ for f in old['files'][key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ if 'pygments_files' in old:
+ for key in old['pygments_files']:
+ for f in old['pygments_files'][key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ else:
+ temp_data['loaded_old_data'] = False
+
+ # Set the utilspath
+ if os.path.isfile(os.path.join(sys.path[0], 'pythontex_utils.py')):
+ # Need the path with forward slashes, so escaping isn't necessary
+ data['utilspath'] = sys.path[0].replace('\\', '/')
+ else:
+ print('* PythonTeX error')
+ print(' Could not determine the utils path from sys.path[0]')
+ print(' The file "pythontex_utils.py" may be missing')
+ return sys.exit(1)
+
+
+
+
+def modified_dependencies(key, data, old_data, temp_data):
+ hashdependencies = temp_data['hashdependencies']
+ if key not in old_data['dependencies']:
+ return False
+ else:
+ old_dep_hash_dict = old_data['dependencies'][key]
+ workingdir = data['settings']['workingdir']
+ for dep in old_dep_hash_dict.keys():
+ # We need to know if the path is relative (based off the
+ # working directory) or absolute. We can't use
+ # os.path.isabs() alone for determining the distinction,
+ # because we must take into account the possibility of an
+ # initial ~ (tilde) standing for the home directory.
+ dep_file = os.path.expanduser(os.path.normcase(dep))
+ if not os.path.isabs(dep_file):
+ dep_file = os.path.join(workingdir, dep_file)
+ if not os.path.isfile(dep_file):
+ print('* PythonTeX error')
+ print(' Cannot find dependency "' + dep + '"')
+ print(' It belongs to ' + key.replace('#', ':'))
+ print(' Relative paths to dependencies must be specified from the working directory.')
+ temp_data['errors'] += 1
+ # A removed dependency should trigger an error, but it
+ # shouldn't cause code to execute. Running the code
+ # again would just give more errors when it can't find
+ # the dependency. (There won't be issues when a
+ # dependency is added or removed, because that would
+ # involve modifying code, which would trigger
+ # re-execution.)
+ elif hashdependencies:
+ # Read and hash the file in binary. Opening in text mode
+ # would require an unnecessary decoding and encoding cycle.
+ f = open(dep_file, 'rb')
+ hasher = sha1()
+ hash = hasher(f.read()).hexdigest()
+ f.close()
+ if hash != old_dep_hash_dict[dep][1]:
+ return True
+ else:
+ mtime = os.path.getmtime(dep_file)
+ if mtime != old_dep_hash_dict[dep][0]:
+ return True
+ return False
+
+def should_rerun(hash, old_hash, old_exit_status, key, rerun, data, old_data, temp_data):
+ # #### Need to clean up arg passing here
+ if rerun == 'never':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):
+ print('* PythonTeX warning')
+ print(' Session ' + key.replace('#', ':') + ' has rerun=never')
+ print(' But its code or dependencies have been modified')
+ temp_data['warnings'] += 1
+ return False
+ elif rerun == 'modified':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):
+ return True
+ else:
+ return False
+ elif rerun == 'errors':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or
+ old_exit_status[0] != 0):
+ return True
+ else:
+ return False
+ elif rerun == 'warnings':
+ if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or
+ old_exit_status != (0, 0)):
+ return True
+ else:
+ return False
+ elif rerun == 'always':
+ return True
+
+
+
+
+def hash_all(data, temp_data, old_data, engine_dict):
+ '''
+ Hash the code to see what has changed and needs to be updated.
+
+ Save the hashes in hashdict. Create update_code, a list of bools
+ regarding whether code should be executed. Create update_pygments, a
+ list of bools determining what needs updated Pygments highlighting.
+ Update pygments_settings to account for Pygments (as opposed to PythonTeX)
+ commands and environments.
+ '''
+
+ # Note that the PythonTeX information that accompanies code must be
+ # hashed in addition to the code itself; the code could stay the same,
+ # but its context or args could change, which might require that code be
+ # executed. All of the PythonTeX information is hashed except for the
+ # input line number. Context-dependent code is going too far if
+ # it depends on that.
+
+ # Create variables to more easily access parts of data
+ pytxcode = temp_data['pytxcode']
+ encoding = data['encoding']
+ loaded_old_data = temp_data['loaded_old_data']
+ rerun = temp_data['rerun']
+ pygments_settings = data['pygments_settings']
+
+ # Calculate cumulative hashes for all code that is executed
+ # Calculate individual hashes for all code that will be typeset
+ code_hasher = defaultdict(sha1)
+ cons_hasher = defaultdict(sha1)
+ cc_hasher = defaultdict(sha1)
+ typeset_hasher = defaultdict(sha1)
+ for c in pytxcode:
+ if c.is_code:
+ code_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))
+ code_encoded = c.code.encode(encoding)
+ code_hasher[c.key_run].update(code_encoded)
+ if c.is_typeset:
+ typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
+ typeset_hasher[c.key_typeset].update(code_encoded)
+ elif c.is_cons:
+ cons_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))
+ code_encoded = c.code.encode(encoding)
+ cons_hasher[c.key_run].update(code_encoded)
+ if c.is_typeset:
+ typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
+ typeset_hasher[c.key_typeset].update(code_encoded)
+ elif c.is_cc:
+ cc_hasher[c.cc_type].update(c.hashable_delims_run.encode(encoding))
+ cc_hasher[c.cc_type].update(c.code.encode(encoding))
+ elif c.is_typeset:
+ typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))
+ typeset_hasher[c.key_typeset].update(c.code.encode(encoding))
+
+
+ # Store hashes
+ code_hash_dict = {}
+ for key in code_hasher:
+ input_family = key.split('#', 1)[0]
+ code_hash_dict[key] = (code_hasher[key].hexdigest(),
+ cc_hasher[input_family].hexdigest(),
+ engine_dict[input_family].get_hash())
+ data['code_hash_dict'] = code_hash_dict
+
+ cons_hash_dict = {}
+ for key in cons_hasher:
+ input_family = key.split('#', 1)[0]
+ cons_hash_dict[key] = (cons_hasher[key].hexdigest(),
+ cc_hasher[input_family].hexdigest(),
+ engine_dict[input_family].get_hash())
+ data['cons_hash_dict'] = cons_hash_dict
+
+ typeset_hash_dict = {}
+ for key in typeset_hasher:
+ typeset_hash_dict[key] = typeset_hasher[key].hexdigest()
+ data['typeset_hash_dict'] = typeset_hash_dict
+
+
+ # See what needs to be updated.
+ # In the process, copy over macros and files that may be reused.
+ code_update = {}
+ cons_update = {}
+ pygments_update = {}
+ macros = defaultdict(list)
+ files = defaultdict(list)
+ pygments_macros = {}
+ pygments_files = {}
+ typeset_cache = {}
+ dependencies = defaultdict(dict)
+ exit_status = {}
+ pygments_settings_changed = {}
+ if loaded_old_data:
+ old_macros = old_data['macros']
+ old_files = old_data['files']
+ old_pygments_macros = old_data['pygments_macros']
+ old_pygments_files = old_data['pygments_files']
+ old_typeset_cache = old_data['typeset_cache']
+ old_dependencies = old_data['dependencies']
+ old_exit_status = old_data['exit_status']
+ old_code_hash_dict = old_data['code_hash_dict']
+ old_cons_hash_dict = old_data['cons_hash_dict']
+ old_typeset_hash_dict = old_data['typeset_hash_dict']
+ old_pygments_settings = old_data['pygments_settings']
+ for s in pygments_settings:
+ if (s in old_pygments_settings and
+ pygments_settings[s] == old_pygments_settings[s]):
+ pygments_settings_changed[s] = False
+ else:
+ pygments_settings_changed[s] = True
+
+ # If old data was loaded (and thus is compatible) determine what has
+ # changed so that only
+ # modified code may be executed. Otherwise, execute everything.
+ # We don't have to worry about checking for changes in pyfuture, because
+ # custom code and default code are hashed. The treatment of keeptemps
+ # could be made more efficient (if changed to 'none', just delete old temp
+ # files rather than running everything again), but given that it is
+ # intended as a debugging aid, that probable isn't worth it.
+ # We don't have to worry about hashdependencies changing, because if it
+ # does the hashes won't match (file contents vs. mtime) and thus code will
+ # be re-executed.
+ if loaded_old_data and data['code_vitals'] == old_data['code_vitals']:
+ # Compare the hash values, and set which code needs to be run
+ for key in code_hash_dict:
+ if (key in old_code_hash_dict and
+ not should_rerun(code_hash_dict[key], old_code_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):
+ code_update[key] = False
+ macros[key] = old_macros[key]
+ files[key] = old_files[key]
+ dependencies[key] = old_dependencies[key]
+ exit_status[key] = old_exit_status[key]
+ else:
+ code_update[key] = True
+ else:
+ for key in code_hash_dict:
+ code_update[key] = True
+
+ if loaded_old_data and data['cons_vitals'] == old_data['cons_vitals']:
+ # Compare the hash values, and set which code needs to be run
+ for key in cons_hash_dict:
+ if (key in old_cons_hash_dict and
+ not should_rerun(cons_hash_dict[key], old_cons_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):
+ cons_update[key] = False
+ macros[key] = old_macros[key]
+ files[key] = old_files[key]
+ typeset_cache[key] = old_typeset_cache[key]
+ dependencies[key] = old_dependencies[key]
+ exit_status[key] = old_exit_status[key]
+ else:
+ cons_update[key] = True
+ else:
+ for key in cons_hash_dict:
+ cons_update[key] = True
+
+ if loaded_old_data and data['typeset_vitals'] == old_data['typeset_vitals']:
+ for key in typeset_hash_dict:
+ input_family = key.split('#', 1)[0]
+ if input_family in pygments_settings:
+ if (not pygments_settings_changed[input_family] and
+ key in old_typeset_hash_dict and
+ typeset_hash_dict[key] == old_typeset_hash_dict[key]):
+ pygments_update[key] = False
+ if key in old_pygments_macros:
+ pygments_macros[key] = old_pygments_macros[key]
+ if key in old_pygments_files:
+ pygments_files[key] = old_pygments_files[key]
+ else:
+ pygments_update[key] = True
+ else:
+ pygments_update[key] = False
+ # Make sure Pygments styles are up-to-date
+ pygments_style_list = list(get_all_styles())
+ if pygments_style_list != old_data['pygments_style_list']:
+ pygments_style_defs = {}
+ # Lazy import
+ from pygments.formatters import LatexFormatter
+ for s in pygments_style_list:
+ formatter = LatexFormatter(style=s, commandprefix='PYG'+s)
+ pygments_style_defs[s] = formatter.get_style_defs()
+ else:
+ pygments_style_defs = old_data['pygments_style_defs']
+ else:
+ for key in typeset_hash_dict:
+ input_family = key.split('#', 1)[0]
+ if input_family in pygments_settings:
+ pygments_update[key] = True
+ else:
+ pygments_update[key] = False
+ # Create Pygments styles
+ pygments_style_list = list(get_all_styles())
+ pygments_style_defs = {}
+ # Lazy import
+ from pygments.formatters import LatexFormatter
+ for s in pygments_style_list:
+ formatter = LatexFormatter(style=s, commandprefix='PYG'+s)
+ pygments_style_defs[s] = formatter.get_style_defs()
+
+ # Save to data
+ temp_data['code_update'] = code_update
+ temp_data['cons_update'] = cons_update
+ temp_data['pygments_update'] = pygments_update
+ data['macros'] = macros
+ data['files'] = files
+ data['pygments_macros'] = pygments_macros
+ data['pygments_style_list'] = pygments_style_list
+ data['pygments_style_defs'] = pygments_style_defs
+ data['pygments_files'] = pygments_files
+ data['typeset_cache'] = typeset_cache
+ data['dependencies'] = dependencies
+ data['exit_status'] = exit_status
+
+
+ # Clean up for code that will be run again, and for code that no longer
+ # exists.
+ if loaded_old_data:
+ # Take care of code files
+ for key in code_hash_dict:
+ if code_update[key] and key in old_files:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ for key in old_code_hash_dict:
+ if key not in code_hash_dict:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ # Take care of old console files
+ for key in cons_hash_dict:
+ if cons_update[key] and key in old_files:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ for key in old_cons_hash_dict:
+ if key not in cons_hash_dict:
+ for f in old_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ # Take care of old Pygments files
+ # The approach here is a little different since there isn't a
+ # Pygments-specific hash dict, but there is a Pygments-specific
+ # dict of lists of files.
+ for key in pygments_update:
+ if pygments_update[key] and key in old_pygments_files:
+ for f in old_pygments_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+ for key in old_pygments_files:
+ if key not in pygments_update:
+ for f in old_pygments_files[key]:
+ f = os.path.expanduser(os.path.normcase(f))
+ if os.path.isfile(f):
+ os.remove(f)
+
+
+
+
+
+def parse_code_write_scripts(data, temp_data, engine_dict):
+ '''
+ Parse the code file into separate scripts, and write them to file.
+ '''
+ code_dict = defaultdict(list)
+ cc_dict_begin = defaultdict(list)
+ cc_dict_end = defaultdict(list)
+ cons_dict = defaultdict(list)
+ pygments_list = []
+ # Create variables to ease data access
+ encoding = data['encoding']
+ utilspath = data['utilspath']
+ outputdir = data['settings']['outputdir']
+ workingdir = data['settings']['workingdir']
+ pytxcode = temp_data['pytxcode']
+ code_update = temp_data['code_update']
+ cons_update = temp_data['cons_update']
+ pygments_update = temp_data['pygments_update']
+ files = data['files']
+ # We need to keep track of the last instance for each session, so
+ # that duplicates can be eliminated. Some LaTeX environments process
+ # their content multiple times and thus will create duplicates. We
+ # need to initialize everything at -1, since instances begin at zero.
+ def negative_one():
+ return -1
+ last_instance = defaultdict(negative_one)
+ for c in pytxcode:
+ if c.input_instance_int > last_instance[c.key_run]:
+ last_instance[c.key_run] = c.input_instance_int
+ if c.is_code:
+ if code_update[c.key_run]:
+ code_dict[c.key_run].append(c)
+ if c.is_typeset and pygments_update[c.key_typeset]:
+ pygments_list.append(c)
+ elif c.is_cons:
+ # Only append to Pygments if not run, since Pygments is
+ # automatically taken care of during run for console content
+ if cons_update[c.key_run]:
+ cons_dict[c.key_run].append(c)
+ elif c.is_typeset and pygments_update[c.key_typeset]:
+ pygments_list.append(c)
+ elif (c.is_pyg or c.is_verb) and pygments_update[c.key_typeset]:
+ pygments_list.append(c)
+ elif c.is_cc:
+ if c.cc_pos == 'begin':
+ cc_dict_begin[c.cc_type].append(c)
+ else:
+ cc_dict_end[c.cc_type].append(c)
+
+ # Save
+ temp_data['code_dict'] = code_dict
+ temp_data['cc_dict_begin'] = cc_dict_begin
+ temp_data['cc_dict_end'] = cc_dict_end
+ temp_data['cons_dict'] = cons_dict
+ temp_data['pygments_list'] = pygments_list
+
+ # Save the code sessions that need to be updated
+ # Keep track of the files that are created
+ # Also accumulate error indices for handling stderr
+ code_index_dict = {}
+ for key in code_dict:
+ input_family, input_session, input_restart = key.split('#')
+ fname = os.path.join(outputdir, input_family + '_' + input_session + '_' + input_restart + '.' + engine_dict[input_family].extension)
+ files[key].append(fname)
+ sessionfile = open(fname, 'w', encoding=encoding)
+ script, code_index = engine_dict[input_family].get_script(encoding,
+ utilspath,
+ workingdir,
+ cc_dict_begin[input_family],
+ code_dict[key],
+ cc_dict_end[input_family])
+ for lines in script:
+ sessionfile.write(lines)
+ sessionfile.close()
+ code_index_dict[key] = code_index
+ temp_data['code_index_dict'] = code_index_dict
+
+
+
+
+def do_multiprocessing(data, temp_data, old_data, engine_dict):
+ jobname = data['jobname']
+ encoding = data['encoding']
+ outputdir = data['settings']['outputdir']
+ workingdir = data['settings']['workingdir']
+ keeptemps = data['settings']['keeptemps']
+ fvextfile = data['settings']['fvextfile']
+ pygments_settings = data['pygments_settings']
+ verbose = temp_data['verbose']
+
+ code_dict = temp_data['code_dict']
+ cons_dict = temp_data['cons_dict']
+ cc_dict_begin = temp_data['cc_dict_begin']
+ cc_dict_end = temp_data['cc_dict_end']
+ pygments_list = temp_data['pygments_list']
+ pygments_style_defs = data['pygments_style_defs']
+
+ files = data['files']
+ macros = data['macros']
+ pygments_files = data['pygments_files']
+ pygments_macros = data['pygments_macros']
+ typeset_cache = data['typeset_cache']
+
+ errors = temp_data['errors']
+ warnings = temp_data['warnings']
+
+ makestderr = data['settings']['makestderr']
+ stderrfilename = data['settings']['stderrfilename']
+ code_index_dict = temp_data['code_index_dict']
+
+ hashdependencies = temp_data['hashdependencies']
+ dependencies = data['dependencies']
+ exit_status = data['exit_status']
+ start_time = data['start_time']
+
+
+ # Set maximum number of concurrent processes for multiprocessing
+ # Accoding to the docs, cpu_count() may raise an error
+ try:
+ max_processes = multiprocessing.cpu_count()
+ except NotImplementedError:
+ max_processes = 1
+ pool = multiprocessing.Pool(max_processes)
+ tasks = []
+
+ # If verbose, print a list of processes
+ if verbose:
+ print('\n* PythonTeX will run the following processes:')
+
+ # Add code processes. Note that everything placed in the codedict
+ # needs to be executed, based on previous testing, except for custom code.
+ for key in code_dict:
+ input_family = key.split('#')[0]
+ # Uncomment the following for debugging, and comment out what follows
+ '''run_code(encoding, outputdir, workingdir, code_dict[key],
+ engine_dict[input_family].language,
+ engine_dict[input_family].command,
+ engine_dict[input_family].created,
+ engine_dict[input_family].extension,
+ makestderr, stderrfilename,
+ code_index_dict[key],
+ engine_dict[input_family].errors,
+ engine_dict[input_family].warnings,
+ engine_dict[input_family].linenumbers,
+ engine_dict[input_family].lookbehind,
+ keeptemps, hashdependencies)'''
+ tasks.append(pool.apply_async(run_code, [encoding, outputdir,
+ workingdir, code_dict[key],
+ engine_dict[input_family].language,
+ engine_dict[input_family].command,
+ engine_dict[input_family].created,
+ engine_dict[input_family].extension,
+ makestderr, stderrfilename,
+ code_index_dict[key],
+ engine_dict[input_family].errors,
+ engine_dict[input_family].warnings,
+ engine_dict[input_family].linenumbers,
+ engine_dict[input_family].lookbehind,
+ keeptemps, hashdependencies]))
+ if verbose:
+ print(' - Code process ' + key.replace('#', ':'))
+
+ # Add console processes
+ for key in cons_dict:
+ input_family = key.split('#')[0]
+ if engine_dict[input_family].language.startswith('python'):
+ if input_family in pygments_settings:
+ # Uncomment the following for debugging
+ '''python_console(jobname, encoding, outputdir, workingdir,
+ fvextfile, pygments_settings[input_family],
+ cc_dict_begin[input_family], cons_dict[key],
+ cc_dict_end[input_family], engine_dict[input_family].startup,
+ engine_dict[input_family].banner,
+ engine_dict[input_family].filename)'''
+ tasks.append(pool.apply_async(python_console, [jobname, encoding,
+ outputdir, workingdir,
+ fvextfile,
+ pygments_settings[input_family],
+ cc_dict_begin[input_family],
+ cons_dict[key],
+ cc_dict_end[input_family],
+ engine_dict[input_family].startup,
+ engine_dict[input_family].banner,
+ engine_dict[input_family].filename]))
+ else:
+ tasks.append(pool.apply_async(python_console, [jobname, encoding,
+ outputdir, workingdir,
+ fvextfile,
+ None,
+ cc_dict_begin[input_family],
+ cons_dict[key],
+ cc_dict_end[input_family],
+ engine_dict[input_family].startup,
+ engine_dict[input_family].banner,
+ engine_dict[input_family].filename]))
+ else:
+ print('* PythonTeX error')
+ print(' Currently, non-Python consoles are not supported')
+ errors += 1
+ if verbose:
+ print(' - Console process ' + key.replace('#', ':'))
+
+ # Add a Pygments process
+ if pygments_list:
+ tasks.append(pool.apply_async(do_pygments, [encoding, outputdir,
+ fvextfile,
+ pygments_list,
+ pygments_settings,
+ typeset_cache]))
+ if verbose:
+ print(' - Pygments process')
+
+ # Execute the processes
+ pool.close()
+ pool.join()
+
+ # Get the outputs of processes
+ # Get the files and macros created. Get the number of errors and warnings
+ # produced. Get any messages returned. Get the exit_status, which is a
+ # dictionary of code that failed and thus must be run again (its hash is
+ # set to a null string). Keep track of whether there were any new files,
+ # so that the last time of file creation in .pytxmcr can be updated.
+ new_files = False
+ messages = []
+ for task in tasks:
+ result = task.get()
+ if result['process'] == 'code':
+ key = result['key']
+ files[key].extend(result['files'])
+ if result['files']:
+ new_files = True
+ macros[key].extend(result['macros'])
+ dependencies[key] = result['dependencies']
+ errors += result['errors']
+ warnings += result['warnings']
+ exit_status[key] = (result['errors'], result['warnings'])
+ messages.extend(result['messages'])
+ elif result['process'] == 'console':
+ key = result['key']
+ files[key].extend(result['files'])
+ if result['files']:
+ new_files = True
+ macros[key].extend(result['macros'])
+ pygments_files.update(result['pygments_files'])
+ pygments_macros.update(result['pygments_macros'])
+ dependencies[key] = result['dependencies']
+ typeset_cache[key] = result['typeset_cache']
+ errors += result['errors']
+ warnings += result['warnings']
+ exit_status[key] = (result['errors'], result['warnings'])
+ messages.extend(result['messages'])
+ elif result['process'] == 'pygments':
+ pygments_files.update(result['pygments_files'])
+ for k in result['pygments_files']:
+ if result['pygments_files'][k]:
+ new_files = True
+ break
+ pygments_macros.update(result['pygments_macros'])
+ errors += result['errors']
+ warnings += result['warnings']
+ messages.extend(result['messages'])
+
+ # Do a quick check to see if any dependencies were modified since the
+ # beginning of the run. If so, reset them so they will run next time and
+ # issue a warning
+ unresolved_dependencies = False
+ unresolved_sessions= []
+ for key in dependencies:
+ for dep, val in dependencies[key].items():
+ if val[0] > start_time:
+ unresolved_dependencies = True
+ dependencies[key][dep] = (None, None)
+ unresolved_sessions.append(key.replace('#', ':'))
+ if unresolved_dependencies:
+ print('* PythonTeX warning')
+ print(' The following have dependencies that have been modified')
+ print(' Run PythonTeX again to resolve dependencies')
+ for s in set(unresolved_sessions):
+ print(' - ' + s)
+ warnings += 1
+
+
+ # Save all content (only needs to be done if code was indeed run).
+ # Save a commented-out time corresponding to the last time PythonTeX ran
+ # and created files, so that tools like latexmk can easily detect when
+ # another run is needed.
+ if tasks:
+ if new_files or not temp_data['loaded_old_data']:
+ last_new_file_time = start_time
+ else:
+ last_new_file_time = old_data['last_new_file_time']
+ data['last_new_file_time'] = last_new_file_time
+
+ macro_file = open(os.path.join(outputdir, jobname + '.pytxmcr'), 'w', encoding=encoding)
+ macro_file.write('%Last time of file creation: ' + str(last_new_file_time) + '\n\n')
+ for key in macros:
+ macro_file.write(''.join(macros[key]))
+ macro_file.close()
+
+ pygments_macro_file = open(os.path.join(outputdir, jobname + '.pytxpyg'), 'w', encoding=encoding)
+ # Only save Pygments styles that are used
+ style_set = set([pygments_settings[k]['formatter_options']['style'] for k in pygments_settings if k != ':GLOBAL'])
+ for key in pygments_style_defs:
+ if key in style_set:
+ pygments_macro_file.write(''.join(pygments_style_defs[key]))
+ for key in pygments_macros:
+ pygments_macro_file.write(''.join(pygments_macros[key]))
+ pygments_macro_file.close()
+
+ pythontex_data_file = os.path.join(outputdir, 'pythontex_data.pkl')
+ f = open(pythontex_data_file, 'wb')
+ pickle.dump(data, f, -1)
+ f.close()
+
+ # Print any errors and warnings.
+ if messages:
+ print('\n'.join(messages))
+ sys.stdout.flush()
+ # Store errors and warnings back into temp_data
+ # This is needed because they are ints and thus immutable
+ temp_data['errors'] = errors
+ temp_data['warnings'] = warnings
+
+
+
+
+def run_code(encoding, outputdir, workingdir, code_list, language, command,
+ command_created, extension, makestderr, stderrfilename,
+ code_index, errorsig, warningsig, linesig, stderrlookbehind,
+ keeptemps, hashdependencies):
+ '''
+ Function for multiprocessing code files
+ '''
+ import shlex
+
+ # Create what's needed for storing results
+ input_family = code_list[0].input_family
+ input_session = code_list[0].input_session
+ key_run = code_list[0].key_run
+ files = []
+ macros = []
+ dependencies = {}
+ errors = 0
+ warnings = 0
+ unknowns = 0
+ messages = []
+
+ # Create message lists only for stderr, one for undelimited stderr and
+ # one for delimited, so it's easy to keep track of if there is any
+ # stderr. These are added onto messages at the end.
+ err_messages_ud = []
+ err_messages_d = []
+
+ # We need to let the user know we are switching code files
+ # We check at the end to see if there were indeed any errors and warnings
+ # and if not, clear messages.
+ messages.append('\n---- Messages for ' + key_run.replace('#', ':') + ' ----')
+
+ # Open files for stdout and stderr, run the code, then close the files
+ basename = key_run.replace('#', '_')
+ out_file_name = os.path.join(outputdir, basename + '.out')
+ err_file_name = os.path.join(outputdir, basename + '.err')
+ out_file = open(out_file_name, 'w', encoding=encoding)
+ err_file = open(err_file_name, 'w', encoding=encoding)
+ # Note that command is a string, which must be converted to list
+ # Must double-escape any backslashes so that they survive `shlex.split()`
+ script = os.path.join(outputdir, basename)
+ # `shlex.split()` only works with Unicode after 2.7.2
+ if (sys.version_info.major == 2 and sys.version_info.micro < 3):
+ exec_cmd = shlex.split(bytes(command.format(file=script.replace('\\', '\\\\'))))
+ exec_cmd = [unicode(elem) for elem in exec_cmd]
+ else:
+ exec_cmd = shlex.split(command.format(file=script.replace('\\', '\\\\')))
+ # Add any created files due to the command
+ # This needs to be done before attempts to execute, to prevent orphans
+ for f in command_created:
+ files.append(f.format(file=script))
+ try:
+ proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+ except WindowsError as e:
+ if e.errno == 2:
+ # Batch files won't be found when called without extension. They
+ # would be found if `shell=True`, but then getting the right
+ # exit code is tricky. So we perform some `cmd` trickery that
+ # is essentially equivalent to `shell=True`, but gives correct
+ # exit codes. Note that `subprocess.Popen()` works with strings
+ # under Windows; a list is not required.
+ exec_cmd_string = ' '.join(exec_cmd)
+ exec_cmd_string = 'cmd /C "@echo off & call {0} & if errorlevel 1 exit 1"'.format(exec_cmd_string)
+ proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+ else:
+ raise
+
+ proc.wait()
+ out_file.close()
+ err_file.close()
+
+ # Process saved stdout into file(s) that are included in the TeX document.
+ #
+ # Go through the saved output line by line, and save any printed content
+ # to its own file, named based on instance.
+ #
+ # The very end of the stdout lists dependencies, if any, so we start by
+ # removing and processing those.
+ if not os.path.isfile(out_file_name):
+ messages.append('* PythonTeX error')
+ messages.append(' Missing output file for ' + key_run.replace('#', ':'))
+ errors += 1
+ else:
+ f = open(out_file_name, 'r', encoding=encoding)
+ out = f.read()
+ f.close()
+ try:
+ out, created = out.rsplit('=>PYTHONTEX:CREATED#\n', 1)
+ out, deps = out.rsplit('=>PYTHONTEX:DEPENDENCIES#\n', 1)
+ valid_stdout = True
+ except:
+ valid_stdout = False
+ if proc.returncode == 0:
+ raise ValueError('Missing "created" and/or "dependencies" delims in stdout; invalid template?')
+
+ if valid_stdout:
+ # Add created files to created list
+ for c in created.splitlines():
+ files.append(c)
+
+ # Create a set of dependencies, to eliminate duplicates in the event
+ # that there are any. This is mainly useful when dependencies are
+ # automatically determined (for example, through redefining open()),
+ # may be specified multiple times as a result, and are hashed (and
+ # of a large enough size that hashing time is non-negligible).
+ deps = set([dep for dep in deps.splitlines()])
+ # Process dependencies; get mtimes and (if specified) hashes
+ for dep in deps:
+ dep_file = os.path.expanduser(os.path.normcase(dep))
+ if not os.path.isabs(dep_file):
+ dep_file = os.path.join(workingdir, dep_file)
+ if not os.path.isfile(dep_file):
+ # If we can't find the file, we return a null hash and issue
+ # an error. We don't need to change the exit status. If the
+ # code does depend on the file, there will be a separate
+ # error when the code attempts to use the file. If the code
+ # doesn't really depend on the file, then the error will be
+ # raised again anyway the next time PythonTeX runs when the
+ # dependency is listed but not found.
+ dependencies[dep] = (None, None)
+ messages.append('* PythonTeX error')
+ messages.append(' Cannot find dependency "' + dep + '"')
+ messages.append(' It belongs to ' + key_run.replace('#', ':'))
+ messages.append(' Relative paths to dependencies must be specified from the working directory.')
+ errors += 1
+ elif hashdependencies:
+ # Read and hash the file in binary. Opening in text mode
+ # would require an unnecessary decoding and encoding cycle.
+ hasher = sha1()
+ f = open(dep_file, 'rb')
+ hasher.update(f.read())
+ f.close()
+ dependencies[dep] = (os.path.getmtime(dep_file), hasher.hexdigest())
+ else:
+ dependencies[dep] = (os.path.getmtime(dep_file), '')
+
+ for block in out.split('=>PYTHONTEX:STDOUT#')[1:]:
+ if block:
+ delims, content = block.split('#\n', 1)
+ if content:
+ input_instance, input_command = delims.split('#')
+ if input_instance.endswith('CC'):
+ messages.append('* PythonTeX warning')
+ messages.append(' Custom code for "' + input_family + '" attempted to print or write to stdout')
+ messages.append(' This is not supported; use a normal code command or environment')
+ messages.append(' The following content was written:')
+ messages.append('')
+ messages.extend([' ' + l for l in content.splitlines()])
+ warnings += 1
+ elif input_command == 'i':
+ content = r'\pytx@SVMCR{pytx@MCR@' + key_run.replace('#', '@') + '@' + input_instance + '}\n' + content.rstrip('\n') + '\\endpytx@SVMCR\n\n'
+ macros.append(content)
+ else:
+ fname = os.path.join(outputdir, basename + '_' + input_instance + '.stdout')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(content)
+ f.close()
+ files.append(fname)
+
+ # Process stderr
+ if not os.path.isfile(err_file_name):
+ messages.append('* PythonTeX error')
+ messages.append(' Missing stderr file for ' + key_run.replace('#', ':'))
+ errors += 1
+ else:
+ # Open error and code files.
+ f = open(err_file_name, encoding=encoding)
+ err = f.readlines()
+ f.close()
+ # Divide stderr into an undelimited and a delimited portion
+ found = False
+ for n, line in enumerate(err):
+ if line.startswith('=>PYTHONTEX:STDERR#'):
+ found = True
+ err_ud = err[:n]
+ err_d = err[n:]
+ break
+ if not found:
+ err_ud = err
+ err_d = []
+ # Create a dict for storing any stderr content that will be saved
+ err_dict = defaultdict(list)
+ # Create the full basename that will be replaced in stderr
+ # We need two versions, one with the correct slashes for the OS,
+ # and one with the opposite slashes. This is needed when a language
+ # doesn't obey the OS's slash convention in paths given in stderr.
+ # For example, Windows uses backslashes, but Ruby under Windows uses
+ # forward in paths given in stderr.
+ fullbasename_correct = os.path.join(outputdir, basename)
+ if '\\' in fullbasename_correct:
+ fullbasename_reslashed = fullbasename_correct.replace('\\', '/')
+ else:
+ fullbasename_reslashed = fullbasename_correct.replace('/', '\\')
+
+ if err_ud:
+ it = iter(code_index.items())
+ index_now = next(it)
+ index_next = index_now
+ start_errgobble = None
+ for n, line in enumerate(err_ud):
+ if basename in line:
+ # Get the gobbleation. This is used to determine if
+ # other lines containing the basename are a continuation,
+ # or separate messages.
+ errgobble = match('(\s*)', line).groups()[0]
+ if start_errgobble is None:
+ start_errgobble = errgobble
+ # Only issue a message and track down the line numer if
+ # this is indeed the start of a new message, rather than
+ # a continuation of an old message that happens to
+ # contain the basename
+ if errgobble == start_errgobble:
+ # Determine the corresponding line number in the document
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ while index_next[1].lines_total < errlinenum:
+ try:
+ index_now, index_next = index_next, next(it)
+ except:
+ break
+ if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+ doclinenum = str(index_now[1].input_line_int + index_now[1].lines_input)
+ else:
+ doclinenum = str(index_now[1].input_line_int + errlinenum - index_now[1].lines_total - 1)
+ input_file = index_now[1].input_file
+ else:
+ doclinenum = '??'
+ input_file = '??'
+
+ # Try to determine if we are dealing with an error or a
+ # warning.
+ found = False
+ index = n
+ if stderrlookbehind:
+ while index >= 0:
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ past_line = err_ud[index]
+ if (index < n and basename in past_line):
+ break
+ for pattern in warningsig:
+ if pattern in past_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in past_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index -= 1
+ else:
+ while index < len(err_ud):
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ future_line = err_ud[index]
+ if (index > n and basename in future_line and
+ future_line.startswith(start_errgobble)):
+ break
+ for pattern in warningsig:
+ if pattern in future_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in future_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index += 1
+ # If an error or warning wasn't positively identified,
+ # increment unknowns.
+ if not found:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ err_messages_ud.append('* PythonTeX stderr - {0} on line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ err_messages_ud.append('* PythonTeX stderr - {0} on line {1}:'.format(alert_type, doclinenum))
+ err_messages_ud.append(' ' + line.replace(outputdir, '<outputdir>').rstrip('\n'))
+ else:
+ err_messages_ud.append(' ' + line.rstrip('\n'))
+
+ # Create .stderr
+ if makestderr and err_messages_ud:
+ process = False
+ it = iter(code_index.items())
+ index_now = next(it)
+ index_next = index_now
+ it_last = it
+ index_now_last = index_now
+ index_next_last = index_next
+ err_key_last_int = -1
+ for n, line in enumerate(err_ud):
+ if basename in line:
+ # Determine the corresponding line number in the document
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ if index_next[1].lines_total >= errlinenum:
+ it = it_last
+ index_now = index_now_last
+ index_next = index_next_last
+ else:
+ it_last = it
+ index_now_last = index_now
+ index_next_last = index_next
+ while index_next[1].lines_total < errlinenum:
+ try:
+ index_now, index_next = index_next, next(it)
+ except:
+ index_now = index_next
+ break
+ if index_now[0].endswith('CC'):
+ process = False
+ else:
+ process = True
+ if len(index_now[1].input_command) > 1:
+ if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+ codelinenum = str(index_now[1].lines_user + index_now[1].lines_input + 1)
+ else:
+ codelinenum = str(index_now[1].lines_user + errlinenum - index_now[1].lines_total - index_now[1].inline_count)
+ else:
+ codelinenum = '1'
+ else:
+ codelinenum = '??'
+ messages.append('* PythonTeX error')
+ messages.append(' Line number ' + str(errlinenum) + ' could not be synced with the document')
+ messages.append(' Content from stderr is not delimited, and cannot be resolved')
+ errors += 1
+ process = False
+
+ if process:
+ if int(index_now[0]) > err_key_last_int:
+ err_key = basename + '_' + index_now[0]
+ err_key_last_int = int(index_now[0])
+ line = line.replace(str(errlinenum), str(codelinenum), 1)
+ if fullbasename_correct in line:
+ fullbasename = fullbasename_correct
+ else:
+ fullbasename = fullbasename_reslashed
+ if stderrfilename == 'full':
+ line = line.replace(fullbasename, basename)
+ elif stderrfilename == 'session':
+ line = line.replace(fullbasename, input_session)
+ elif stderrfilename == 'genericfile':
+ line = line.replace(fullbasename + '.' + extension, '<file>')
+ elif stderrfilename == 'genericscript':
+ line = line.replace(fullbasename + '.' + extension, '<script>')
+ err_dict[err_key].append(line)
+ elif process:
+ err_dict[err_key].append(line)
+
+ if err_d:
+ start_errgobble = None
+ msg = []
+ found_basename = False
+ for n, line in enumerate(err_d):
+ if line.startswith('=>PYTHONTEX:STDERR#'):
+ # Store the last group of messages. Messages
+ # can't be directly appended to the main list, because
+ # a PythonTeX message must be inserted at the beginning
+ # of each chunk of stderr that never references
+ # the script that was executed. If the script is never
+ # referenced, then line numbers aren't automatically
+ # synced. These types of situations are created by
+ # warnings.warn() etc.
+ if msg:
+ if not found_basename:
+ # Get line number for command or beginning of
+ # environment
+ input_instance = last_delim.split('#')[1]
+ doclinenum = str(code_index[input_instance].input_line_int)
+ input_file = code_index[input_instance].input_file
+ # Try to identify alert. We have to parse all
+ # lines for signs of errors and warnings. This
+ # may result in overcounting, but it's the best
+ # we can do--otherwise, we could easily
+ # undercount, or, finding a warning, miss a
+ # subsequent error. When this code is actually
+ # used, it's already a sign that normal parsing
+ # has failed.
+ found_error = False
+ found_warning = False
+ for l in msg:
+ for pattern in warningsig:
+ if pattern in l:
+ warnings += 1
+ found_warning = True
+ for pattern in errorsig:
+ if pattern in l:
+ errors += 1
+ found_warning = True
+ if found_error:
+ alert_type = 'error'
+ elif found_warning:
+ alert_type = 'warning'
+ else:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1}:'.format(alert_type, doclinenum))
+ err_messages_d.extend(msg)
+ msg = []
+ found_basename = False
+ # Never process delimiting info until it is used
+ # Rather, store the index of the last delimiter
+ last_delim = line
+ elif basename in line:
+ found_basename = True
+ # Get the gobbleation. This is used to determine if
+ # other lines containing the basename are a continuation,
+ # or separate messages.
+ errgobble = match('(\s*)', line).groups()[0]
+ if start_errgobble is None:
+ start_errgobble = errgobble
+ # Only issue a message and track down the line numer if
+ # this is indeed the start of a new message, rather than
+ # a continuation of an old message that happens to
+ # contain the basename
+ if errgobble == start_errgobble:
+ # Determine the corresponding line number in the document
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ # Get info from last delim
+ input_instance, input_command = last_delim.split('#')[1:-1]
+ # Calculate the line number in the document
+ ei = code_index[input_instance]
+ if errlinenum > ei.lines_total + ei.lines_input:
+ doclinenum = str(ei.input_line_int + ei.lines_input)
+ else:
+ doclinenum = str(ei.input_line_int + errlinenum - ei.lines_total - 1)
+ input_file = ei.input_file
+ else:
+ doclinenum = '??'
+ input_file = '??'
+
+ # Try to determine if we are dealing with an error or a
+ # warning.
+ found = False
+ index = n
+ if stderrlookbehind:
+ while index >= 0:
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ past_line = err_d[index]
+ if (past_line.startswith('=>PYTHONTEX:STDERR#') or
+ (index < n and basename in past_line)):
+ break
+ for pattern in warningsig:
+ if pattern in past_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in past_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index -= 1
+ else:
+ while index < len(err_d):
+ # The order here is important. If a line matches
+ # both the error and warning patterns, default to
+ # error.
+ future_line = err_d[index]
+ if (future_line.startswith('=>PYTHONTEX:STDERR#') or
+ (index > n and basename in future_line and future_line.startswith(start_errgobble))):
+ break
+ for pattern in warningsig:
+ if pattern in future_line:
+ warnings += 1
+ alert_type = 'warning'
+ found = True
+ break
+ for pattern in errorsig:
+ if pattern in future_line:
+ errors += 1
+ alert_type = 'error'
+ found = True
+ break
+ if found:
+ break
+ index += 1
+ # If an error or warning wasn't positively identified,
+ # assume error for safety but indicate uncertainty.
+ if not found:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ msg.append('* PythonTeX stderr - {0} on line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ msg.append('* PythonTeX stderr - {0} on line {1}:'.format(alert_type, doclinenum))
+ # Clean up the stderr format a little, to keep it compact
+ line = line.replace(outputdir, '<outputdir>').rstrip('\n')
+ if '/<outputdir>' in line or '\\<outputdir>' in line:
+ line = sub(r'(?:(?:[A-Z]:\\)|(?:~?/)).*<outputdir>', '<outputdir>', line)
+ msg.append(' ' + line)
+ else:
+ msg.append(' ' + line.rstrip('\n'))
+ # Deal with any leftover messages
+ if msg:
+ if not found_basename:
+ # Get line number for command or beginning of
+ # environment
+ input_instance = last_delim.split('#')[1]
+ doclinenum = str(code_index[input_instance].input_line_int)
+ input_file = code_index[input_instance].input_file
+ # Try to identify alert. We have to parse all
+ # lines for signs of errors and warnings. This
+ # may result in overcounting, but it's the best
+ # we can do--otherwise, we could easily
+ # undercount, or, finding a warning, miss a
+ # subsequent error. When this code is actually
+ # used, it's already a sign that normal parsing
+ # has failed.
+ found_error = False
+ found_warning = False
+ for l in msg:
+ for pattern in warningsig:
+ if pattern in l:
+ warnings += 1
+ found_warning = True
+ for pattern in errorsig:
+ if pattern in l:
+ errors += 1
+ found_warning = True
+ if found_error:
+ alert_type = 'error'
+ elif found_warning:
+ alert_type = 'warning'
+ else:
+ unknowns += 1
+ alert_type = 'unknown'
+ if input_file:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1} in "{2}":'.format(alert_type, doclinenum, input_file))
+ else:
+ err_messages_d.append('* PythonTeX stderr - {0} near line {1}:'.format(alert_type, doclinenum))
+ err_messages_d.extend(msg)
+
+ # Create .stderr
+ if makestderr and err_messages_d:
+ process = False
+ for n, line in enumerate(err_d):
+ if line.startswith('=>PYTHONTEX:STDERR#'):
+ input_instance, input_command = line.split('#')[1:-1]
+ if input_instance.endswith('CC'):
+ process = False
+ else:
+ process = True
+ err_key = basename + '_' + input_instance
+ elif process and basename in line:
+ found = False
+ for pattern in linesig:
+ try:
+ errlinenum = int(search(pattern, line).groups()[0])
+ found = True
+ break
+ except:
+ pass
+ if found:
+ # Calculate the line number in the document
+ # Account for inline
+ ei = code_index[input_instance]
+ # Store the `input_instance` in case it's
+ # incremented later
+ last_input_instance = input_instance
+ # If the error or warning was actually triggered
+ # later on (for example, multiline string with
+ # missing final delimiter), look ahead and
+ # determine the correct input_instance, so that
+ # we get the correct line number. We don't
+ # associate the created stderr with this later
+ # instance, however, but rather with the instance
+ # in which the error began. Doing that might
+ # possibly be preferable in some cases, but would
+ # also require that the current stderr be split
+ # between multiple instances, requiring extra
+ # parsing.
+ while errlinenum > ei.lines_total + ei.lines_input:
+ next_input_instance = str(int(input_instance) + 1)
+ if next_input_instance in code_index:
+ next_ei = code_index[next_input_instance]
+ if errlinenum > next_ei.lines_total:
+ input_instance = next_input_instance
+ ei = next_ei
+ else:
+ break
+ else:
+ break
+ if len(input_command) > 1:
+ if errlinenum > ei.lines_total + ei.lines_input:
+ codelinenum = str(ei.lines_user + ei.lines_input + 1)
+ else:
+ codelinenum = str(ei.lines_user + errlinenum - ei.lines_total - ei.inline_count)
+ else:
+ codelinenum = '1'
+ # Reset `input_instance`, in case incremented
+ input_instance = last_input_instance
+ else:
+ codelinenum = '??'
+ messages.append('* PythonTeX notice')
+ messages.append(' Line number ' + str(errlinenum) + ' could not be synced with the document')
+
+ line = line.replace(str(errlinenum), str(codelinenum), 1)
+ if fullbasename_correct in line:
+ fullbasename = fullbasename_correct
+ else:
+ fullbasename = fullbasename_reslashed
+ if stderrfilename == 'full':
+ line = line.replace(fullbasename, basename)
+ elif stderrfilename == 'session':
+ line = line.replace(fullbasename, input_session)
+ elif stderrfilename == 'genericfile':
+ line = line.replace(fullbasename + '.' + extension, '<file>')
+ elif stderrfilename == 'genericscript':
+ line = line.replace(fullbasename + '.' + extension, '<script>')
+ err_dict[err_key].append(line)
+ elif process:
+ err_dict[err_key].append(line)
+ if err_dict:
+ for err_key in err_dict:
+ stderr_file_name = os.path.join(outputdir, err_key + '.stderr')
+ f = open(stderr_file_name, 'w', encoding=encoding)
+ f.write(''.join(err_dict[err_key]))
+ f.close()
+ files.append(stderr_file_name)
+
+ # Clean up temp files, and update the list of existing files
+ if keeptemps == 'none':
+ for ext in [extension, 'pytxmcr', 'out', 'err']:
+ fname = os.path.join(outputdir, basename + '.' + ext)
+ if os.path.isfile(fname):
+ os.remove(fname)
+ elif keeptemps == 'code':
+ for ext in ['pytxmcr', 'out', 'err']:
+ fname = os.path.join(outputdir, basename + '.' + ext)
+ if os.path.isfile(fname):
+ os.remove(fname)
+ files.append(os.path.join(outputdir, basename + '.' + extension))
+ elif keeptemps == 'all':
+ for ext in [extension, 'pytxmcr', 'out', 'err']:
+ files.append(os.path.join(outputdir, basename + '.' + ext))
+
+ # Take care of any unknowns, based on exit code
+ # Interpret the exit code as an indicator of whether there were errors,
+ # and treat unknowns accordingly. This will cause all warnings to be
+ # misinterpreted as errors if warnings trigger a nonzero exit code.
+ # It will also cause all warnings to be misinterpreted as errors if there
+ # is a single error that causes a nonzero exit code. That isn't ideal,
+ # but shouldn't be a problem, because as soon as the error(s) are fixed,
+ # the exit code will be zero, and then all unknowns will be interpreted
+ # as warnings.
+ if unknowns:
+ if proc.returncode == 0:
+ unknowns_type = 'warnings'
+ warnings += unknowns
+ else:
+ unknowns_type = 'errors'
+ errors += unknowns
+ unknowns_message = '''
+ * PythonTeX notice
+ {0} message(s) could not be classified
+ Based on the return code, they were interpreted as {1}'''
+ messages[0] += textwrap.dedent(unknowns_message.format(unknowns, unknowns_type))
+
+ # Take care of anything that has escaped detection thus far.
+ if proc.returncode == 1 and not errors:
+ errors += 1
+ command_message = '''
+ * PythonTeX error
+ An error occurred but no error messages were identified.
+ This may indicate a bad command or missing program.
+ The following command was executed:
+ "{0}"'''
+ messages[0] += textwrap.dedent(command_message.format(' '.join(exec_cmd)))
+
+ # Add any stderr messages; otherwise, clear the default message header
+ if err_messages_ud:
+ messages.extend(err_messages_ud)
+ if err_messages_d:
+ messages.extend(err_messages_d)
+ if len(messages) == 1:
+ messages = []
+
+ # Return a dict of dicts of results
+ return {'process': 'code',
+ 'key': key_run,
+ 'files': files,
+ 'macros': macros,
+ 'dependencies': dependencies,
+ 'errors': errors,
+ 'warnings': warnings,
+ 'messages': messages}
+
+
+
+
+def do_pygments(encoding, outputdir, fvextfile, pygments_list,
+ pygments_settings, typeset_cache):
+ '''
+ Create Pygments content.
+
+ To be run during multiprocessing.
+ '''
+ # Lazy import
+ from pygments import highlight
+ from pygments.lexers import get_lexer_by_name
+ from pygments.formatters import LatexFormatter
+
+ # Create what's needed for storing results
+ pygments_files = defaultdict(list)
+ pygments_macros = defaultdict(list)
+ errors = 0
+ warnings = 0
+ messages = []
+ messages.append('\n---- Messages for Pygments ----')
+
+ # Create dicts of formatters and lexers.
+ formatter = dict()
+ lexer = dict()
+ for codetype in pygments_settings:
+ if codetype != ':GLOBAL':
+ formatter[codetype] = LatexFormatter(**pygments_settings[codetype]['formatter_options'])
+ lexer[codetype] = get_lexer_by_name(pygments_settings[codetype]['lexer'],
+ **pygments_settings[codetype]['lexer_options'])
+
+ # Actually parse and highlight the code.
+ for c in pygments_list:
+ if c.is_cons:
+ content = typeset_cache[c.key_run][c.input_instance]
+ elif c.is_extfile:
+ if os.path.isfile(c.extfile):
+ f = open(c.extfile, encoding=encoding)
+ content = f.read()
+ f.close()
+ else:
+ content = None
+ messages.append('* PythonTeX error')
+ messages.append(' Could not find external file ' + c.extfile)
+ messages.append(' The file was not pygmentized')
+ else:
+ content = c.code
+ processed = highlight(content, lexer[c.input_family], formatter[c.input_family])
+ if c.is_inline or content.count('\n') < fvextfile:
+ # Highlighted code brought in via macros needs SaveVerbatim
+ processed = sub(r'\\begin{Verbatim}\[(.+)\]',
+ r'\\begin{{SaveVerbatim}}[\1]{{pytx@{0}@{1}@{2}@{3}}}'.format(c.input_family, c.input_session, c.input_restart, c.input_instance), processed, count=1)
+ processed = processed.rsplit('\\', 1)[0] + '\\end{SaveVerbatim}\n\n'
+ pygments_macros[c.key_typeset].append(processed)
+ else:
+ fname = os.path.join(outputdir, c.key_typeset.replace('#', '_') + '.pygtex')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(processed)
+ f.close()
+ pygments_files[c.key_typeset].append(fname)
+
+ if len(messages) == 1:
+ messages = []
+ # Return a dict of dicts of results
+ return {'process': 'pygments',
+ 'pygments_files': pygments_files,
+ 'pygments_macros': pygments_macros,
+ 'errors': errors,
+ 'warnings': warnings,
+ 'messages': messages}
+
+
+
+
+def python_console(jobname, encoding, outputdir, workingdir, fvextfile,
+ pygments_settings, cc_begin_list, cons_list, cc_end_list,
+ startup, banner, filename):
+ '''
+ Use Python's ``code`` module to typeset emulated Python interactive
+ sessions, optionally highlighting with Pygments.
+ '''
+ # Create what's needed for storing results
+ key_run = cons_list[0].key_run
+ files = []
+ macros = []
+ pygments_files = defaultdict(list)
+ pygments_macros = defaultdict(list)
+ typeset_cache = {}
+ dependencies = {}
+ errors = 0
+ warnings = 0
+ messages = []
+ messages.append('\n---- Messages for ' + key_run.replace('#', ':') + ' ----')
+
+ # Lazy import what's needed
+ import code
+ from collections import deque
+ if sys.version_info[0] == 2:
+ # Need a Python 2 interface to io.StringIO that can accept bytes
+ import io
+ class StringIO(io.StringIO):
+ _orig_write = io.StringIO.write
+ def write(self, s):
+ self._orig_write(unicode(s))
+ else:
+ from io import StringIO
+
+ # Create a custom console class
+ class Console(code.InteractiveConsole):
+ '''
+ A subclass of code.InteractiveConsole that takes a list and treats it
+ as a series of console input.
+ '''
+
+ def __init__(self, banner, filename):
+ if banner == 'none':
+ self.banner = 'NULL BANNER'
+ elif banner == 'standard':
+ cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
+ self.banner = 'Python {0} on {1}\n{2}'.format(sys.version, sys.platform, cprt)
+ elif banner == 'pyversion':
+ self.banner = 'Python ' + '.'.join(str(sys.version_info[n]) for n in (0, 1, 2))
+ else:
+ self.banner = None
+ if filename == 'console':
+ self.filename = '<console>'
+ elif filename == 'stdin':
+ self.filename = '<stdin>'
+ else:
+ self.filename = None
+ code.InteractiveConsole.__init__(self, filename=self.filename)
+ self.iostdout = StringIO()
+
+ def consolize(self, startup, cons_list):
+ self.console_code = deque()
+ # Delimiters are passed straight through and need newlines
+ self.console_code.append('=>PYTHONTEX#STARTUP##\n')
+ cons_config = '''
+ import os
+ import sys
+ docdir = os.getcwd()
+ if os.path.isdir('{workingdir}'):
+ os.chdir('{workingdir}')
+ if os.getcwd() not in sys.path:
+ sys.path.append(os.getcwd())
+ else:
+ sys.exit('Cannot find directory {workingdir}')
+
+ if docdir not in sys.path:
+ sys.path.append(docdir)
+
+ del docdir
+ '''
+ cons_config = cons_config.format(workingdir=workingdir)[1:]
+ self.console_code.extend(textwrap.dedent(cons_config).splitlines())
+ # Code is processed and doesn't need newlines
+ self.console_code.extend(startup.splitlines())
+ for c in cons_list:
+ self.console_code.append('=>PYTHONTEX#{0}#{1}#\n'.format(c.input_instance, c.input_command))
+ self.console_code.extend(c.code.splitlines())
+ old_stdout = sys.stdout
+ sys.stdout = self.iostdout
+ self.interact(self.banner)
+ sys.stdout = old_stdout
+ self.session_log = self.iostdout.getvalue()
+
+ def raw_input(self, prompt):
+ # Have to do a lot of looping and trying to make sure we get
+ # something valid to execute
+ try:
+ line = self.console_code.popleft()
+ except IndexError:
+ raise EOFError
+ while line.startswith('=>PYTHONTEX#'):
+ # Get new lines until we get one that doesn't begin with a
+ # delimiter. Then write the last delimited line.
+ old_line = line
+ try:
+ line = self.console_code.popleft()
+ self.write(old_line)
+ except IndexError:
+ raise EOFError
+ if line or prompt == sys.ps2:
+ self.write('{0}{1}\n'.format(prompt, line))
+ else:
+ self.write('\n')
+ return line
+
+ def write(self, data):
+ self.iostdout.write(data)
+
+ # Need to combine all custom code and user code to pass to consolize
+ cons_list = cc_begin_list + cons_list + cc_end_list
+ # Create a dict for looking up exceptions. This is needed for startup
+ # commands and for code commands and environments, since their output
+ # isn't typeset
+ cons_index = {}
+ for c in cons_list:
+ cons_index[c.input_instance] = c.input_line
+
+ # Consolize the code
+ con = Console(banner, filename)
+ con.consolize(startup, cons_list)
+
+ # Set up Pygments, if applicable
+ if pygments_settings is not None:
+ pygmentize = True
+ # Lazy import
+ from pygments import highlight
+ from pygments.lexers import get_lexer_by_name
+ from pygments.formatters import LatexFormatter
+ formatter = LatexFormatter(**pygments_settings['formatter_options'])
+ lexer = get_lexer_by_name(pygments_settings['lexer'],
+ **pygments_settings['lexer_options'])
+ else:
+ pygmentize = False
+
+ # Process the console output
+ output = con.session_log.split('=>PYTHONTEX#')
+ # Extract banner
+ if banner == 'none':
+ banner_text = ''
+ else:
+ banner_text = output[0]
+ # Ignore the beginning, because it's the banner
+ for block in output[1:]:
+ delims, console_content = block.split('#\n', 1)
+ if console_content:
+ input_instance, input_command = delims.split('#')
+ if input_instance == 'STARTUP':
+ exception = False
+ console_content_lines = console_content.splitlines()
+ for line in console_content_lines:
+ if (not line.startswith(sys.ps1) and
+ not line.startswith(sys.ps2) and
+ line and not line.isspace()):
+ exception = True
+ break
+ if exception:
+ if 'Error:' in console_content:
+ errors += 1
+ alert_type = 'error'
+ elif 'Warning:' in console_content:
+ warnings += 1
+ alert_type = 'warning'
+ else:
+ errors += 1
+ alert_type = 'error (?)'
+ messages.append('* PythonTeX stderr - {0} in console startup code:'.format(alert_type))
+ for line in console_content_lines:
+ messages.append(' ' + line)
+ elif input_command in ('c', 'code'):
+ exception = False
+ console_content_lines = console_content.splitlines()
+ for line in console_content_lines:
+ if (line and not line.startswith(sys.ps1) and
+ not line.startswith(sys.ps2) and
+ not line.isspace()):
+ print('X' + line + 'X')
+ exception = True
+ break
+ if exception:
+ if 'Error:' in console_content:
+ errors += 1
+ alert_type = 'error'
+ elif 'Warning:' in console_content:
+ warnings += 1
+ alert_type = 'warning'
+ else:
+ errors += 1
+ alert_type = 'error (?)'
+ if input_instance.endswith('CC'):
+ messages.append('* PythonTeX stderr - {0} near line {1} in custom code for console:'.format(alert_type, cons_index[input_instance]))
+ else:
+ messages.append('* PythonTeX stderr - {0} near line {1} in console code:'.format(alert_type, cons_index[input_instance]))
+ messages.append(' Console code is not typeset, and should have no output')
+ for line in console_content_lines:
+ messages.append(' ' + line)
+ else:
+ if input_command == 'i':
+ # Currently, there isn't any error checking for invalid
+ # content; it is assumed that a single line of commands
+ # was entered, producing one or more lines of output.
+ # Given that the current ``\pycon`` command doesn't
+ # allow line breaks to be written to the .pytxcode, that
+ # should be a reasonable assumption.
+ console_content = console_content.split('\n', 1)[1]
+ if banner_text is not None and input_command == 'console':
+ # Append banner to first appropriate environment
+ console_content = banner_text + console_content
+ banner_text = None
+ # Cache
+ key_typeset = key_run + '#' + input_instance
+ typeset_cache[input_instance] = console_content
+ # Process for LaTeX
+ if pygmentize:
+ processed = highlight(console_content, lexer, formatter)
+ if console_content.count('\n') < fvextfile:
+ processed = sub(r'\\begin{Verbatim}\[(.+)\]',
+ r'\\begin{{SaveVerbatim}}[\1]{{pytx@{0}}}'.format(key_typeset.replace('#', '@')),
+ processed, count=1)
+ processed = processed.rsplit('\\', 1)[0] + '\\end{SaveVerbatim}\n\n'
+ pygments_macros[key_typeset].append(processed)
+ else:
+ fname = os.path.join(outputdir, key_typeset.replace('#', '_') + '.pygtex')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(processed)
+ f.close()
+ pygments_files[key_typeset].append(fname)
+ else:
+ if console_content.count('\n') < fvextfile:
+ processed = ('\\begin{{SaveVerbatim}}{{pytx@{0}}}\n'.format(key_typeset.replace('#', '@')) +
+ console_content + '\\end{SaveVerbatim}\n\n')
+ macros.append(processed)
+ else:
+ processed = ('\\begin{Verbatim}\n' + console_content +
+ '\\end{Verbatim}\n\n')
+ fname = os.path.join(outputdir, key_typeset.replace('#', '_') + '.tex')
+ f = open(fname, 'w', encoding=encoding)
+ f.write(processed)
+ f.close()
+ files.append(fname)
+
+ if len(messages) == 1:
+ messages = []
+
+ # Return a dict of dicts of results
+ return {'process': 'console',
+ 'key': key_run,
+ 'files': files,
+ 'macros': macros,
+ 'pygments_files': pygments_files,
+ 'pygments_macros': pygments_macros,
+ 'typeset_cache': typeset_cache,
+ 'dependencies': dependencies,
+ 'errors': errors,
+ 'warnings': warnings,
+ 'messages': messages}
+
+
+
+
+def main():
+ # Create dictionaries for storing data.
+ #
+ # All data that must be saved for subsequent runs is stored in "data".
+ # (We start off by saving the script version, a global var, in this dict.)
+ # All data that is only created for this run is stored in "temp_data".
+ # (We start off by creating keys for keeping track of errors and warnings.)
+ # All old data will eventually be loaded into "old_data".
+ # Since dicts are mutable data types, these dicts can be modified
+ # from within functions, as long as the dicts are passed to the functions.
+ # For simplicity, variables will often be created within functions to
+ # refer to dictionary values.
+ data = {'version': version, 'start_time': time.time()}
+ temp_data = {'errors': 0, 'warnings': 0}
+ old_data = dict()
+
+
+ # Process command-line options.
+ #
+ # This gets the raw_jobname (actual job name), jobname (a sanitized job
+ # name, used for creating files named after the jobname), and any options.
+ process_argv(data, temp_data)
+ # If there aren't errors in argv, and the program is going to run
+ # (rather than just exit due to --version or --help command-line options),
+ # print PythonTeX version. Flush to make the message go out immediately,
+ # so that the user knows PythonTeX has started.
+ print('This is PythonTeX ' + version)
+ sys.stdout.flush()
+ # Once we have the encoding (from argv), we set stdout and stderr to use
+ # this encoding. Later, we will parse the saved stderr of scripts
+ # executed via multiprocessing subprocesses, and print the parsed results
+ # to stdout. The saved stderr uses the same encoding that was used
+ # for the files that created it (this is important for code containing
+ # unicode characters), so we also need stdout for the main PythonTeX
+ # script to support this encoding. Setting stderr encoding is primarily
+ # a matter of symmetry. Ideally, pythontex*.py will be bug-free,
+ # and stderr won't be needed!
+ if sys.version_info[0] == 2:
+ sys.stdout = codecs.getwriter(data['encoding'])(sys.stdout, 'strict')
+ sys.stderr = codecs.getwriter(data['encoding'])(sys.stderr, 'strict')
+ else:
+ sys.stdout = codecs.getwriter(data['encoding'])(sys.stdout.buffer, 'strict')
+ sys.stderr = codecs.getwriter(data['encoding'])(sys.stderr.buffer, 'strict')
+
+
+ # Load the code and process the settings it passes from the TeX side.
+ #
+ # This gets a list containing the code (the part of the code file
+ # containing the settings is removed) and the processed settings.
+ load_code_get_settings(data, temp_data)
+ # Now that the settings are loaded, check if outputdir exits.
+ # If not, create it.
+ if not os.path.isdir(data['settings']['outputdir']):
+ os.mkdir(data['settings']['outputdir'])
+
+
+ # Load/create old_data
+ get_old_data(data, old_data, temp_data)
+
+
+ # Hash the code. Determine what needs to be executed. Determine whether
+ # Pygments should be used. Update pygments_settings to account for
+ # Pygments commands and environments (as opposed to PythonTeX commands
+ # and environments).
+ hash_all(data, temp_data, old_data, engine_dict)
+
+
+ # Parse the code and write scripts for execution.
+ parse_code_write_scripts(data, temp_data, engine_dict)
+
+
+ # Execute the code and perform Pygments highlighting via multiprocessing.
+ do_multiprocessing(data, temp_data, old_data, engine_dict)
+
+
+ # Print exit message
+ print('\n--------------------------------------------------')
+ # If some rerun settings are used, there may be unresolved errors or
+ # warnings; if so, print a summary of those along with the current
+ # error and warning summary
+ unresolved_errors = 0
+ unresolved_warnings = 0
+ if temp_data['rerun'] in ('errors', 'modified', 'never'):
+ global_update = {}
+ global_update.update(temp_data['code_update'])
+ global_update.update(temp_data['cons_update'])
+ for key in data['exit_status']:
+ if not global_update[key]:
+ unresolved_errors += data['exit_status'][key][0]
+ unresolved_warnings += data['exit_status'][key][1]
+ if unresolved_warnings != 0 or unresolved_errors != 0:
+ print('PythonTeX: {0}'.format(data['raw_jobname']))
+ print(' - Old: {0} error(s), {1} warnings(s)'.format(unresolved_errors, unresolved_warnings))
+ print(' - Current: {0} error(s), {1} warnings(s)'.format(temp_data['errors'], temp_data['warnings']))
+ else:
+ print('PythonTeX: {0} - {1} error(s), {2} warning(s)\n'.format(data['raw_jobname'], temp_data['errors'], temp_data['warnings']))
+
+ # Exit with appropriate exit code based on user settings.
+ if temp_data['error_exit_code'] and temp_data['errors'] > 0:
+ sys.exit(1)
+ else:
+ sys.exit()
+
+
+
+# The "if" statement is needed for multiprocessing under Windows; see the
+# multiprocessing documentation. It is also needed in this case when the
+# script is invoked via the wrapper.
+if __name__ == '__main__':
+ main() \ No newline at end of file
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex_2to3.py b/Master/texmf-dist/scripts/pythontex/pythontex_2to3.py
new file mode 100755
index 00000000000..baff37b411d
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/pythontex_2to3.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Convert PythonTeX scripts from Python 2 to Python 3
+
+It isn't possible to have a single PythonTeX code base, since unicode text
+needs to be supported. Under Python 2, this means importing unicode_literals
+from __future__, or using the unicode function or "u" prefix. Under Python 3,
+all strings are automatically unicode.
+
+At the same time, the differences between the Python 2 and 3 versions are
+usually very small, involving only a few lines of code. To keep the code base
+unified, while simultaneously fully supporting both Python 2 and 3, the
+following scheme was devised. The code is written for Python 2. Whenever
+code is not compatible with Python 3, it is enclosed with the tags
+"#// Python 2" and "#\\ End Python 2" (each on its own line, by itself). If
+a Python 3 version of the code is needed, it is included between analogous
+tags "#// Python 3" and "#\\ End Python 2". The Python 3 code is commented
+out with "#", at the same indentation level as the Python 3 tags.
+
+This script creates Python 3 scripts from the original Python 2 scripts
+by commenting out everything between the Python 2 tags, and uncommenting
+everything between the Python 3 tags. In this way, full compatibility is
+maintained with both Python 2 and 3 while keeping the code base essentially
+unified. This approach also allows greater customization of version-specific
+code than would be possible if automatic translation with a tool like 2to3
+was required.
+
+Copyright (c) 2012-2013, Geoffrey M. Poore
+All rights reserved.
+Licensed under the BSD 3-Clause License:
+ http://www.opensource.org/licenses/BSD-3-Clause
+
+'''
+
+
+# Imports
+from __future__ import unicode_literals
+from io import open
+import re
+
+
+files_to_process = ('pythontex2.py', 'depythontex2.py')
+encoding = 'utf-8'
+
+
+def from2to3(list_of_code):
+ fixed = []
+ in_2 = False
+ in_3 = False
+ indent = ''
+
+ for line in list_of_code:
+ if r'#// Python 2' in line:
+ in_2 = True
+ indent = line.split('#', 1)[0]
+ elif r'#\\ End Python 2' in line:
+ in_2 = False
+ elif r'#// Python 3' in line:
+ in_3 = True
+ indent = line.split('#', 1)[0]
+ elif r'#\\ End Python 3' in line:
+ in_3 = False
+ elif in_2:
+ line = re.sub(indent, indent + '#', line, count=1)
+ elif in_3:
+ line = re.sub(indent + '#', indent, line, count=1)
+ fixed.append(line)
+ return fixed
+
+
+for file in files_to_process:
+ f = open(file, 'r', encoding=encoding)
+ converted_code = from2to3(f.readlines())
+ f.close()
+ f = open(re.sub('2', '3', file), 'w', encoding=encoding)
+ f.write(''.join(converted_code))
+ f.close()
+
+
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex_engines.py b/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
new file mode 100755
index 00000000000..518820c8a7b
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
@@ -0,0 +1,922 @@
+# -*- coding: utf-8 -*-
+'''
+PythonTeX code engines.
+
+Provides a class for managing the different languages/types of code
+that may be executed. A class instance is created for each language/type of
+code. The class provides a method for assembling the scripts that are
+executed, combining user code with templates. It also creates the records
+needed to synchronize `stderr` with the document.
+
+Each instance of the class is automatically added to the `engines_dict` upon
+creation. Instances are typically accessed via this dictionary.
+
+The class is called `*CodeEngine` by analogy with a template engine, since it
+combines user text (code) with existing templates to produce an output
+document (script for execution).
+
+
+
+Copyright (c) 2012-2013, Geoffrey M. Poore
+All rights reserved.
+Licensed under the BSD 3-Clause License:
+ http://www.opensource.org/licenses/BSD-3-Clause
+
+'''
+
+# Imports
+import sys
+import textwrap
+from hashlib import sha1
+from collections import OrderedDict, namedtuple
+
+
+interpreter_dict = {k:k for k in ('python', 'ruby', 'julia')}
+# The {file} field needs to be replaced by itself, since the actual
+# substitution of the real file can only be done at runtime, whereas the
+# substitution for the interpreter should be done when the engine is
+# initialized.
+interpreter_dict['file'] = '{file}'
+
+
+engine_dict = {}
+
+
+CodeIndex = namedtuple('CodeIndex', ['input_file', 'input_command',
+ 'input_line_int', 'lines_total',
+ 'lines_user', 'lines_input',
+ 'inline_count'])
+
+
+class CodeEngine(object):
+ '''
+ The base class that is used for defining language engines. Each command
+ and environment family is based on an engine.
+
+ The class assembles the individual scripts that PythonTeX executes, using
+ templates and user code. It also creates the records needed for
+ synchronizing `stderr` with the document.
+ '''
+ def __init__(self, name, language, extension, command, template, wrapper,
+ formatter, errors=None, warnings=None,
+ linenumbers=None, lookbehind=False,
+ console=False, startup=None, created=None):
+
+ # Save raw arguments so that they may be reused by subtypes
+ self._rawargs = (name, language, extension, command, template, wrapper,
+ formatter, errors, warnings,
+ linenumbers, lookbehind, console, startup, created)
+
+ # Type check all strings, and make sure everything is Unicode
+ if sys.version_info[0] == 2:
+ if (not isinstance(name, basestring) or
+ not isinstance(language, basestring) or
+ not isinstance(extension, basestring) or
+ not isinstance(command, basestring) or
+ not isinstance(template, basestring) or
+ not isinstance(wrapper, basestring) or
+ not isinstance(formatter, basestring)):
+ raise TypeError('CodeEngine needs string in initialization')
+ self.name = unicode(name)
+ self.language = unicode(language)
+ self.extension = unicode(extension)
+ self.command = unicode(command)
+ self.template = unicode(template)
+ self.wrapper = unicode(wrapper)
+ self.formatter = unicode(formatter)
+ else:
+ if (not isinstance(name, str) or
+ not isinstance(language, str) or
+ not isinstance(extension, str) or
+ not isinstance(command, str) or
+ not isinstance(template, str) or
+ not isinstance(wrapper, str) or
+ not isinstance(formatter, str)):
+ raise TypeError('CodeEngine needs string in initialization')
+ self.name = name
+ self.language = language
+ self.extension = extension
+ self.command = command
+ self.template = template
+ self.wrapper = wrapper
+ self.formatter = formatter
+ # Perform some additional formatting on some strings. Dedent.
+ # Change from {{ }} tags for replacement fields to { } tags that
+ # are compatible with Python's string format() method, which is much
+ # more efficient than a template engine.
+ self.extension = self.extension.lstrip('.')
+ self.command = self._dejinja(self.command)
+ self.template = self._dedent(self._dejinja(self.template))
+ self.wrapper = self._dedent(self._dejinja(self.wrapper))
+ # Make sure formatter string ends with a newline
+ if self.formatter.endswith('\n'):
+ self.formatter = self._dejinja(self.formatter)
+ else:
+ self.formatter = self._dejinja(self.formatter) + '\n'
+
+ # Type check errors, warnings, and linenumbers
+ if errors is None:
+ errors = []
+ else:
+ if sys.version_info[0] == 2:
+ if isinstance(errors, basestring):
+ errors = [errors]
+ elif not isinstance(errors, list) and not isinstance(errors, tuple):
+ raise TypeError('CodeEngine needs "errors" to be a string, list, or tuple')
+ for e in errors:
+ if not isinstance(e, basestring):
+ raise TypeError('CodeEngine needs "errors" to contain strings')
+ errors = [unicode(e) for e in errors]
+ else:
+ if isinstance(errors, str):
+ errors = [errors]
+ elif not isinstance(errors, list) and not isinstance(errors, tuple):
+ raise TypeError('CodeEngine needs "errors" to be a string, list, or tuple')
+ for e in errors:
+ if not isinstance(e, str):
+ raise TypeError('CodeEngine needs "errors" to contain strings')
+ self.errors = errors
+ if warnings is None:
+ warnings = []
+ else:
+ if sys.version_info[0] == 2:
+ if isinstance(warnings, basestring):
+ warnings = [warnings]
+ elif not isinstance(warnings, list) and not isinstance(warnings, tuple):
+ raise TypeError('CodeEngine needs "warnings" to be a string, list, or tuple')
+ for w in warnings:
+ if not isinstance(w, basestring):
+ raise TypeError('CodeEngine needs "warnings" to contain strings')
+ warnings = [unicode(w) for w in warnings]
+ else:
+ if isinstance(warnings, str):
+ warnings = [warnings]
+ elif not isinstance(warnings, list) and not isinstance(warnings, tuple):
+ raise TypeError('CodeEngine needs "warnings" to be a string, list, or tuple')
+ for w in warnings:
+ if not isinstance(w, str):
+ raise TypeError('CodeEngine needs "warnings" to contain strings')
+ self.warnings = warnings
+ if linenumbers is None:
+ linenumbers = 'line {{number}}'
+ if sys.version_info[0] == 2:
+ if isinstance(linenumbers, basestring):
+ linenumbers = [linenumbers]
+ elif not isinstance(linenumbers, list) and not isinstance(linenumbers, tuple):
+ raise TypeError('CodeEngine needs "linenumbers" to be a string, list, or tuple')
+ for l in linenumbers:
+ if not isinstance(l, basestring):
+ raise TypeError('CodeEngine needs "linenumbers" to contain strings')
+ linenumbers = [unicode(l) for l in linenumbers]
+ else:
+ if isinstance(linenumbers, str):
+ linenumbers = [linenumbers]
+ elif not isinstance(linenumbers, list) and not isinstance(linenumbers, tuple):
+ raise TypeError('CodeEngine needs "linenumbers" to be a string, list, or tuple')
+ for l in linenumbers:
+ if not isinstance(l, str):
+ raise TypeError('CodeEngine needs "linenumbers" to contain strings')
+ # Need to replace tags
+ linenumbers = [l.replace('{{number}}', r'(\d+)') for l in linenumbers]
+ self.linenumbers = linenumbers
+
+ # Type check lookbehind
+ if not isinstance(lookbehind, bool):
+ raise TypeError('CodeEngine needs "lookbehind" to be bool')
+ self.lookbehind = lookbehind
+
+ # Type check console
+ if not isinstance(console, bool):
+ raise TypeError('CodeEngine needs "console" to be bool')
+ self.console = console
+
+ # Type check startup
+ if startup is None:
+ startup = ''
+ if startup and not self.console:
+ raise TypeError('PythonTeX can only use "startup" for console types')
+ else:
+ if sys.version_info[0] == 2:
+ if isinstance(startup, basestring):
+ startup = unicode(startup)
+ else:
+ raise TypeError('CodeEngine needs "startup" to be a string')
+ else:
+ if not isinstance(startup, str):
+ raise TypeError('CodeEngine needs "startup" to be a string')
+ if not startup.endswith('\n'):
+ startup += '\n'
+ self.startup = self._dedent(startup)
+
+ # Type check created; make sure it is an iterable and contains Unicode
+ if created is None:
+ created = []
+ else:
+ if sys.version_info[0] == 2:
+ if isinstance(created, basestring):
+ created = [created]
+ elif not isinstance(created, list) and not isinstance(created, tuple):
+ raise TypeError('CodeEngine needs "created" to be a string, list, or tuple')
+ for f in created:
+ if not isinstance(f, basestring):
+ raise TypeError('CodeEngine "created" to contain strings')
+ created = [unicode(f) for f in created]
+ else:
+ if isinstance(created, str):
+ created = [created]
+ elif not isinstance(created, list) and not isinstance(created, tuple):
+ raise TypeError('CodeEngine needs "created" to be a string, list, or tuple')
+ for f in created:
+ if not isinstance(f, str):
+ raise TypeError('CodeEngine needs "created" to contain strings')
+ self.created = created
+
+ # The base PythonTeX type does not support extend; it is used in
+ # subtyping. But a dummy extend is needed to fill the extend field
+ # in templates, if it is provided.
+ self.extend = ''
+
+ # Create dummy variables for console
+ self.banner = ''
+ self.filename = ''
+
+ # Each type needs to add itself to a dict, for later access by name
+ self._register()
+
+ def _dedent(self, s):
+ '''
+ Dedent and strip leading newlines
+ '''
+ s = textwrap.dedent(s)
+ while s.startswith('\n'):
+ s = s[1:]
+ return s
+
+ def _dejinja(self, s):
+ '''
+ Switch all `{{ }}` tags into `{ }`, and all normal braces `{ }` into
+ `{{ }}`, so that Python's string format() method may be used. Also
+ strip any whitespace surrounding the field name.
+
+ This will fail if literal `{{` and `}}` are needed. If those are
+ ever needed, then options for custom tags will be needed.
+ '''
+ lst = [t.replace('{', '{{') for t in s.split('{{')]
+ for n in range(1, len(lst)):
+ lst[n] = lst[n].lstrip(' ')
+ s = '{'.join(lst)
+ lst = [t.replace('}', '}}') for t in s.split('}}')]
+ for n in range(0, len(lst)-1):
+ lst[n] = lst[n].rstrip(' ')
+ s = '}'.join(lst)
+ return s
+
+ def _register(self):
+ '''
+ Add instance to a dict for later access by name
+ '''
+ engine_dict[self.name] = self
+
+ def customize(self, **kwargs):
+ '''
+ Customize the template on the fly.
+
+ This provides customization based on command line arguments
+ (`--interpreter`) and customization from the TeX side (imports from
+ `__future__`). Ideally, this function should be restricted to this
+ and similar cases. The custom code command and environment are
+ insufficient for such cases, because the command is at a level above
+ that of code and because of the requirement that imports from
+ `__future__` be at the very beginning of a script.
+ '''
+ # Take care of `--interpreter`
+ self.command = self.command.format(**interpreter_dict)
+ # Take care of `__future__`
+ if self.language.startswith('python'):
+ if sys.version_info[0] == 2 and 'pyfuture' in kwargs:
+ pyfuture = kwargs['pyfuture']
+ future_imports = None
+ if pyfuture == 'all':
+ future_imports = '''
+ from __future__ import absolute_import
+ from __future__ import division
+ from __future__ import print_function
+ from __future__ import unicode_literals
+ {future}'''
+ elif pyfuture == 'default':
+ future_imports = '''
+ from __future__ import absolute_import
+ from __future__ import division
+ from __future__ import print_function
+ {future}'''
+ if future_imports is not None:
+ future_imports = self._dedent(future_imports)
+ self.template = self.template.replace('{future}', future_imports)
+ if self.console:
+ if sys.version_info[0] == 2 and 'pyconfuture' in kwargs:
+ pyconfuture = kwargs['pyconfuture']
+ future_imports = None
+ if pyconfuture == 'all':
+ future_imports = '''
+ from __future__ import absolute_import
+ from __future__ import division
+ from __future__ import print_function
+ from __future__ import unicode_literals
+ '''
+ elif pyconfuture == 'default':
+ future_imports = '''
+ from __future__ import absolute_import
+ from __future__ import division
+ from __future__ import print_function
+ '''
+ if future_imports is not None:
+ future_imports = self._dedent(future_imports)
+ self.startup = future_imports + self.startup
+ if 'pyconbanner' in kwargs:
+ self.banner = kwargs['pyconbanner']
+ if 'pyconfilename' in kwargs:
+ self.filename = kwargs['pyconfilename']
+
+ _hash = None
+
+ def get_hash(self):
+ '''
+ Return a hash of all vital type information (template, etc.). Create
+ the hash if it doesn't exist, otherwise return a stored hash.
+ '''
+ # This file is encoded in UTF-8, so everything can be encoded in UTF-8.
+ # It's not important that this encoding be the same as that given by
+ # the user, since a unique hash is all that's needed.
+ if self._hash is None:
+ hasher = sha1()
+ hasher.update(self.command.encode('utf8'))
+ hasher.update(self.template.encode('utf8'))
+ hasher.update(self.wrapper.encode('utf8'))
+ hasher.update(self.formatter.encode('utf8'))
+ if self.console:
+ hasher.update(self.startup.encode('utf8'))
+ hasher.update(self.banner.encode('utf8'))
+ hasher.update(self.filename.encode('utf8'))
+ self._hash = hasher.hexdigest()
+ return self._hash
+
+ def _process_future(self, code_list):
+ '''
+ Go through a given list of code and extract all imports from
+ `__future__`, so that they can be relocated to the beginning of the
+ script.
+
+ The approach isn't foolproof and doesn't support compound statements.
+ '''
+ done = False
+ future_imports = []
+ for n, c in enumerate(code_list):
+ in_triplequote = False
+ changed = False
+ code = c.code.split('\n')
+ for l, line in enumerate(code):
+ # Detect __future__ imports
+ if (line.startswith('from __future__') or
+ line.startswith('import __future__') and
+ not in_triplequote):
+ changed = True
+ if ';' in line:
+ raise ValueError('Imports from __future__ should be simple statements; semicolons are not supported')
+ else:
+ future_imports.append(line)
+ code[l] = ''
+ # Ignore comments, empty lines, and lines with complete docstrings
+ elif (line.startswith('\n') or line.startswith('#') or
+ line.isspace() or
+ ('"""' in line and line.count('"""')%2 == 0) or
+ ("'''" in line and line.count("'''")%2 == 0)):
+ pass
+ # Detect if entering or leaving a docstring
+ elif line.count('"""')%2 == 1 or line.count("'''")%2 == 1:
+ in_triplequote = not in_triplequote
+ # Stop looking for future imports as soon as a non-comment,
+ # non-empty, non-docstring, non-future import line is found
+ elif not in_triplequote:
+ done = True
+ break
+ if changed:
+ code_list[n].code = '\n'.join(code)
+ if done:
+ break
+ if future_imports:
+ return '\n'.join(future_imports)
+ else:
+ return ''
+
+ def _get_future(self, cc_list_begin, code_list):
+ '''
+ Process custom code and user code for imports from `__future__`
+ '''
+ cc_future = self._process_future(cc_list_begin)
+ code_future = self._process_future(code_list)
+ if cc_future and code_future:
+ return cc_future + '\n' + code_future
+ else:
+ return cc_future + code_future
+
+ def get_script(self, encoding, utilspath, workingdir,
+ cc_list_begin, code_list, cc_list_end):
+ '''
+ Assemble the script that will be executed. In the process, assemble
+ an index of line numbers that may be used to correlate script line
+ numbers with document line numbers and user code line numbers in the
+ event of errors or warnings.
+ '''
+ lines_total = 0
+ script = []
+ code_index = OrderedDict()
+
+ # Take care of future
+ if self.language.startswith('python'):
+ future = self._get_future(cc_list_begin, code_list)
+ else:
+ future = ''
+
+ # Split template into beginning and ending segments
+ try:
+ script_begin, script_end = self.template.split('{body}')
+ except:
+ raise ValueError('Template for ' + self.name + ' is missing {{body}}')
+
+ # Add beginning to script
+ script_begin = script_begin.format(encoding=encoding, future=future,
+ utilspath=utilspath, workingdir=workingdir,
+ extend=self.extend,
+ input_family=code_list[0].input_family,
+ input_session=code_list[0].input_session,
+ input_restart=code_list[0].input_restart,
+ dependencies_delim='=>PYTHONTEX:DEPENDENCIES#',
+ created_delim='=>PYTHONTEX:CREATED#')
+ script.append(script_begin)
+ lines_total += script_begin.count('\n')
+
+ # Prep wrapper
+ try:
+ wrapper_begin, wrapper_end = self.wrapper.split('{code}')
+ except:
+ raise ValueError('Wrapper for ' + self.name + ' is missing {{code}}')
+ if not self.language.startswith('python'):
+ # In the event of a syntax error at the end of user code, Ruby
+ # (and perhaps others) will use the line number from the NEXT
+ # line of code that is non-empty, not from the line of code where
+ # the error started. In these cases, it's important
+ # to make sure that the line number is triggered immediately
+ # after user code, so that the line number makes sense. Hence,
+ # we need to strip all whitespace from the part of the wrapper
+ # that follows user code. For symetry, we do the same for both
+ # parts of the wrapper.
+ wrapper_begin = wrapper_begin.rstrip(' \t\n') + '\n'
+ wrapper_end = wrapper_end.lstrip(' \t\n')
+ stdout_delim = '=>PYTHONTEX:STDOUT#{input_instance}#{input_command}#'
+ stderr_delim = '=>PYTHONTEX:STDERR#{input_instance}#{input_command}#'
+ wrapper_begin = wrapper_begin.replace('{stdout_delim}', stdout_delim).replace('{stderr_delim}', stderr_delim)
+ wrapper_begin_offset = wrapper_begin.count('\n')
+ wrapper_end_offset = wrapper_end.count('\n')
+
+ # Take care of custom code
+ # Line counters must be reset for cc begin, code, and cc end, since
+ # all three are separate
+ lines_user = 0
+ inline_count = 0
+ for c in cc_list_begin:
+ # Wrapper before
+ lines_total += wrapper_begin_offset
+ script.append(wrapper_begin.format(input_command=c.input_command,
+ input_context=c.input_context,
+ input_args=c.input_args_run,
+ input_instance=c.input_instance,
+ input_line=c.input_line))
+
+ # Actual code
+ lines_input = c.code.count('\n')
+ code_index[c.input_instance] = CodeIndex(c.input_file, c.input_command, c.input_line_int, lines_total, lines_user, lines_input, inline_count)
+ script.append(c.code)
+ if c.is_inline:
+ inline_count += 1
+ lines_total += lines_input
+ lines_user += lines_input
+
+ # Wrapper after
+ script.append(wrapper_end)
+ lines_total += wrapper_end_offset
+
+ # Take care of user code
+ lines_user = 0
+ inline_count = 0
+ for c in code_list:
+ # Wrapper before
+ lines_total += wrapper_begin_offset
+ script.append(wrapper_begin.format(input_command=c.input_command,
+ input_context=c.input_context,
+ input_args=c.input_args_run,
+ input_instance=c.input_instance,
+ input_line=c.input_line))
+
+ # Actual code
+ lines_input = c.code.count('\n')
+ code_index[c.input_instance] = CodeIndex(c.input_file, c.input_command, c.input_line_int, lines_total, lines_user, lines_input, inline_count)
+ if c.input_command == 'i':
+ script.append(self.formatter.format(code=c.code.rstrip('\n')))
+ inline_count += 1
+ else:
+ script.append(c.code)
+ lines_total += lines_input
+ lines_user += lines_input
+
+ # Wrapper after
+ script.append(wrapper_end)
+ lines_total += wrapper_end_offset
+
+ # Take care of custom code
+ lines_user = 0
+ inline_count = 0
+ for c in cc_list_end:
+ # Wrapper before
+ lines_total += wrapper_begin_offset
+ script.append(wrapper_begin.format(input_command=c.input_command,
+ input_context=c.input_context,
+ input_args=c.input_args_run,
+ input_instance=c.input_instance,
+ input_line=c.input_line))
+
+ # Actual code
+ lines_input = c.code.count('\n')
+ code_index[c.input_instance] = CodeIndex(c.input_file, c.input_command, c.input_line_int, lines_total, lines_user, lines_input, inline_count)
+ script.append(c.code)
+ if c.is_inline:
+ inline_count += 1
+ lines_total += lines_input
+ lines_user += lines_input
+
+ # Wrapper after
+ script.append(wrapper_end)
+ lines_total += wrapper_end_offset
+
+ # Finish script
+ script.append(script_end)
+
+ return script, code_index
+
+
+
+
+class SubCodeEngine(CodeEngine):
+ '''
+ Create Engine instances that inherit from existing instances.
+ '''
+ def __init__(self, base, name, language=None, extension=None, command=None,
+ template=None, wrapper=None, formatter=None, errors=None,
+ warnings=None, linenumbers=None, lookbehind=False,
+ console=None, created=None, startup=None, extend=None):
+
+ self._rawargs = (name, language, extension, command, template, wrapper,
+ formatter, errors, warnings,
+ linenumbers, lookbehind, console, startup, created)
+
+ base_rawargs = engine_dict[base]._rawargs
+ args = []
+ for n, arg in enumerate(self._rawargs):
+ if arg is None:
+ args.append(base_rawargs[n])
+ else:
+ args.append(arg)
+
+ CodeEngine.__init__(self, *args)
+
+ self.extend = engine_dict[base].extend
+
+ if extend is not None:
+ if sys.version_info[0] == 2:
+ if not isinstance(extend, basestring):
+ raise TypeError('PythonTeXSubtype needs a string for "extend"')
+ extend = unicode(extend)
+ else:
+ if not isinstance(extend, str):
+ raise TypeError('PythonTeXSubtype needs a string for "extend"')
+ if not extend.endswith('\n'):
+ extend = extend + '\n'
+ self.extend += self._dedent(extend)
+
+
+
+
+class PythonConsoleEngine(CodeEngine):
+ '''
+ This uses the Engine class to store information needed for emulating
+ Python interactive consoles.
+
+ In the current form, it isn't used as a real engine, but rather as a
+ convenient storage class that keeps the treatment of all languages/code
+ types uniform.
+ '''
+ def __init__(self, name, startup=None):
+ CodeEngine.__init__(self, name=name, language='python',
+ extension='', command='', template='',
+ wrapper='', formatter='', errors=None,
+ warnings=None, linenumbers=None, lookbehind=False,
+ console=True, startup=startup, created=None)
+
+
+
+
+python_template = '''
+ # -*- coding: {{encoding}} -*-
+
+ {{future}}
+
+ import os
+ import sys
+ import codecs
+
+ if sys.version_info[0] == 2:
+ sys.stdout = codecs.getwriter('{{encoding}}')(sys.stdout, 'strict')
+ sys.stderr = codecs.getwriter('{{encoding}}')(sys.stderr, 'strict')
+ else:
+ sys.stdout = codecs.getwriter('{{encoding}}')(sys.stdout.buffer, 'strict')
+ sys.stderr = codecs.getwriter('{{encoding}}')(sys.stderr.buffer, 'strict')
+
+ sys.path.append('{{utilspath}}')
+ from pythontex_utils import PythonTeXUtils
+ pytex = PythonTeXUtils()
+
+ pytex.docdir = os.getcwd()
+ if os.path.isdir('{{workingdir}}'):
+ os.chdir('{{workingdir}}')
+ if os.getcwd() not in sys.path:
+ sys.path.append(os.getcwd())
+ else:
+ if len(sys.argv) < 2 or sys.argv[1] != '--manual':
+ sys.exit('Cannot find directory {{workingdir}}')
+ if pytex.docdir not in sys.path:
+ sys.path.append(pytex.docdir)
+
+ {{extend}}
+
+ pytex.input_family = '{{input_family}}'
+ pytex.input_session = '{{input_session}}'
+ pytex.input_restart = '{{input_restart}}'
+
+ {{body}}
+
+ pytex.cleanup()
+ '''
+
+python_wrapper = '''
+ pytex.input_command = '{{input_command}}'
+ pytex.input_context = '{{input_context}}'
+ pytex.input_args = '{{input_args}}'
+ pytex.input_instance = '{{input_instance}}'
+ pytex.input_line = '{{input_line}}'
+
+ print('{{stdout_delim}}')
+ sys.stderr.write('{{stderr_delim}}\\n')
+ pytex.before()
+
+ {{code}}
+
+ pytex.after()
+ '''
+
+
+CodeEngine('python', 'python', '.py', '{{python}} {{file}}.py',
+ python_template, python_wrapper, 'print(pytex.formatter({{code}}))',
+ 'Error:', 'Warning:', ['line {{number}}', ':{{number}}:'])
+
+SubCodeEngine('python', 'py')
+
+SubCodeEngine('python', 'pylab', extend='from pylab import *')
+
+sympy_extend = '''
+ from sympy import *
+ pytex.set_formatter('sympy_latex')
+ '''
+
+SubCodeEngine('python', 'sympy', extend=sympy_extend)
+
+
+
+
+PythonConsoleEngine('pycon')
+
+PythonConsoleEngine('pylabcon', startup='from pylab import *')
+
+PythonConsoleEngine('sympycon', startup='from sympy import *')
+
+
+
+
+ruby_template = '''
+ # -*- coding: {{encoding}} -*-
+
+ $stdout.set_encoding('{{encoding}}')
+ $stderr.set_encoding('{{encoding}}')
+
+ class RubyTeXUtils
+ attr_accessor :input_family, :input_session, :input_restart,
+ :input_command, :input_context, :input_args,
+ :input_instance, :input_line, :dependencies, :created, :docdir
+ def initialize
+ @dependencies = Array.new
+ @created = Array.new
+ end
+ def formatter(expr)
+ return expr.to_s
+ end
+ def before
+ end
+ def after
+ end
+ def add_dependencies(*expr)
+ self.dependencies.push(*expr)
+ end
+ def add_created(*expr)
+ self.created.push(*expr)
+ end
+ def cleanup
+ puts '{{dependencies_delim}}'
+ if @dependencies
+ @dependencies.each { |x| puts x }
+ end
+ puts '{{created_delim}}'
+ if @created
+ @created.each { |x| puts x }
+ end
+ end
+ end
+
+ rbtex = RubyTeXUtils.new
+
+ rbtex.docdir = Dir.pwd
+ if File.directory?('{{workingdir}}')
+ Dir.chdir('{{workingdir}}')
+ $LOAD_PATH.push(Dir.pwd) unless $LOAD_PATH.include?(Dir.pwd)
+ elsif ARGV[0] != '--manual'
+ abort('Cannot change to directory {{workingdir}}')
+ end
+ $LOAD_PATH.push(rbtex.docdir) unless $LOAD_PATH.include?(rbtex.docdir)
+
+ {{extend}}
+
+ rbtex.input_family = '{{input_family}}'
+ rbtex.input_session = '{{input_session}}'
+ rbtex.input_restart = '{{input_restart}}'
+
+ {{body}}
+
+ rbtex.cleanup
+ '''
+
+ruby_wrapper = '''
+ rbtex.input_command = '{{input_command}}'
+ rbtex.input_context = '{{input_context}}'
+ rbtex.input_args = '{{input_args}}'
+ rbtex.input_instance = '{{input_instance}}'
+ rbtex.input_line = '{{input_line}}'
+
+ puts '{{stdout_delim}}'
+ $stderr.puts '{{stderr_delim}}'
+ rbtex.before
+
+ {{code}}
+
+ rbtex.after
+ '''
+
+CodeEngine('ruby', 'ruby', '.rb', '{{ruby}} {{file}}.rb', ruby_template,
+ ruby_wrapper, 'puts rbtex.formatter({{code}})',
+ ['Error)', '(Errno', 'error'], 'warning:', ':{{number}}:')
+
+SubCodeEngine('ruby', 'rb')
+
+
+
+
+julia_template = '''
+ # -*- coding: UTF-8 -*-
+
+ # Currently, Julia only supports UTF-8
+ # So can't set stdout and stderr encoding
+
+ type JuliaTeXUtils
+ input_family::String
+ input_session::String
+ input_restart::String
+ input_command::String
+ input_context::String
+ input_args::String
+ input_instance::String
+ input_line::String
+
+ _dependencies::Array{String}
+ _created::Array{String}
+ docdir::String
+
+ formatter::Function
+ before::Function
+ after::Function
+ add_dependencies::Function
+ add_created::Function
+ cleanup::Function
+
+ self::JuliaTeXUtils
+
+ function JuliaTeXUtils()
+ self = new()
+ self.self = self
+ self._dependencies = Array(String, 0)
+ self._created = Array(String, 0)
+
+ function formatter(expr)
+ string(expr)
+ end
+ self.formatter = formatter
+
+ function null()
+ end
+ self.before = null
+ self.after = null
+
+ function add_dependencies(files...)
+ for file in files
+ push!(self._dependencies, file)
+ end
+ end
+ self.add_dependencies = add_dependencies
+ function add_created(files...)
+ for file in files
+ push!(self._created, file)
+ end
+ end
+ self.add_created = add_created
+
+ function cleanup()
+ println("{{dependencies_delim}}")
+ for f in self._dependencies
+ println(f)
+ end
+ println("{{created_delim}}")
+ for f in self._created
+ println(f)
+ end
+ end
+ self.cleanup = cleanup
+
+ return self
+ end
+ end
+
+ jltex = JuliaTeXUtils()
+
+ jltex.docdir = pwd()
+ try
+ cd("{{workingdir}}")
+ if !(contains(LOAD_PATH, pwd()))
+ push!(LOAD_PATH, pwd())
+ end
+ catch
+ if !(length(ARGS) > 0 && ARGS[1] == "--manual")
+ error("Could not find directory {{workingdir}}")
+ end
+ end
+ if !(contains(LOAD_PATH, jltex.docdir))
+ push!(LOAD_PATH, jltex.docdir)
+ end
+
+ {{extend}}
+
+ jltex.input_family = "{{input_family}}"
+ jltex.input_session = "{{input_session}}"
+ jltex.input_restart = "{{input_restart}}"
+
+ {{body}}
+
+ jltex.cleanup()
+ '''
+
+julia_wrapper = '''
+ jltex.input_command = "{{input_command}}"
+ jltex.input_context = "{{input_context}}"
+ jltex.input_args = "{{input_args}}"
+ jltex.input_instance = "{{input_instance}}"
+ jltex.input_line = "{{input_line}}"
+
+ println("{{stdout_delim}}")
+ write(STDERR, "{{stderr_delim}}\\n")
+ jltex.before()
+
+ {{code}}
+
+ jltex.after()
+ '''
+
+CodeEngine('julia', 'julia', '.jl', '{{julia}} {{file}}.jl', julia_template,
+ julia_wrapper, 'println(jltex.formatter({{code}}))',
+ 'ERROR:', 'WARNING:', ':{{number}}', True)
+
+SubCodeEngine('julia', 'jl')
+
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex_install_texlive.py b/Master/texmf-dist/scripts/pythontex/pythontex_install_texlive.py
new file mode 100755
index 00000000000..28176e8b704
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/pythontex_install_texlive.py
@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Install PythonTeX
+
+This installation script should work with most TeX distributions. It is
+primarily written for TeX Live. It should work with other TeX distributions
+that use the Kpathsea library (such as MiKTeX), though with reduced
+functionality in some cases. It will require manual input when used with a
+distribution that does not include Kpathsea.
+
+The script will overwrite (and thus update) all previously installed PythonTeX
+files. When Kpathsea is available, files may be installed in TEXMFDIST,
+TEXMFHOME, or a manually specified location. Otherwise, the installation
+location must be specified manually. Installing in TEXMFDIST is useful if
+you want to install PythonTeX and then update it in the future from CTAN.
+The mktexlsr command is executed at the end of the script, to make the system
+aware of any new files.
+
+The script attempts to create a binary wrapper (Windows) or symlink
+(Linux and OS X) for launching the main PythonTeX scripts, pythontex*.py and
+depythontex*.py.
+
+
+Copyright (c) 2012-2013, Geoffrey M. Poore
+All rights reserved.
+Licensed under the BSD 3-Clause License:
+ http://www.opensource.org/licenses/BSD-3-Clause
+
+'''
+
+
+# Imports
+import sys
+import platform
+from os import path, mkdir, makedirs
+if platform.system() != 'Windows':
+ # Only create symlinks if not under Windows
+ # (os.symlink doesn't exist under Windows)
+ from os import symlink, chmod, unlink
+from subprocess import call, check_call, check_output
+from shutil import copy
+import textwrap
+
+
+# We need a version of input that works under both Python 2 and 3
+try:
+ input = raw_input
+except:
+ pass
+
+
+# Print startup messages and warnings
+print('Preparing to install PythonTeX')
+if platform.system() != 'Windows':
+ message = '''
+ You may need to run this script with elevated permissions
+ and/or specify the environment. For example, you may need
+ "sudo env PATH=$PATH". That is typically necessary when your
+ system includes a TeX distribution, and you have manually
+ installed another distribution (common with Ubuntu etc.). If
+ the installation path you want is not automatically detected,
+ it may indicate a permissions issue.
+ '''
+ print(textwrap.dedent(message))
+
+
+# Make sure all necessary files are present
+# The pythontex_gallery and pythontex_quickstart are optional; we check for them when installing doc
+needed_files = ['pythontex.py', 'pythontex2.py', 'pythontex3.py',
+ 'pythontex_engines.py', 'pythontex_utils.py',
+ 'depythontex.py', 'depythontex2.py', 'depythontex3.py',
+ 'pythontex.sty', 'pythontex.ins', 'pythontex.dtx',
+ 'pythontex.pdf', 'README']
+missing_files = False
+# Print a list of all files that are missing, and exit if any are
+for eachfile in needed_files:
+ if not path.exists(eachfile):
+ print('Could not find file ' + eachfile)
+ missing_files = True
+if missing_files:
+ print('Exiting.')
+ sys.exit(1)
+
+
+# Retrieve the location of valid TeX trees
+# Attempt to use kpsewhich; otherwise, resort to manual input
+should_exit = False # Can't use sys.exit() in try; will trigger except
+try:
+ if sys.version_info[0] == 2:
+ texmf_dist = check_output(['kpsewhich', '-var-value', 'TEXMFDIST']).rstrip('\r\n')
+ texmf_local = check_output(['kpsewhich', '-var-value', 'TEXMFLOCAL']).rstrip('\r\n')
+ texmf_home = check_output(['kpsewhich', '-var-value', 'TEXMFHOME']).rstrip('\r\n')
+ else:
+ texmf_dist = check_output(['kpsewhich', '-var-value', 'TEXMFDIST']).decode('utf-8').rstrip('\r\n')
+ texmf_local = check_output(['kpsewhich', '-var-value', 'TEXMFLOCAL']).decode('utf-8').rstrip('\r\n')
+ texmf_home = check_output(['kpsewhich', '-var-value', 'TEXMFHOME']).decode('utf-8').rstrip('\r\n')
+ message = '''
+ Choose an installation location.
+
+ TEXMFDIST is a good choice if you want to update PythonTeX
+ in the future using your TeX distribution's package manager.
+
+ 1. TEXMFDIST
+ {0}
+ 2. TEXMFLOCAL
+ {1}
+ 3. TEXMFHOME
+ {2}
+ 4. Manual location
+ '''.format(texmf_dist, texmf_local, texmf_home)
+ print(textwrap.dedent(message))
+ path_choice = input('Installation location (number): ')
+ if path_choice not in ('1', '2', '3', '4'):
+ should_exit = True
+ else:
+ if path_choice == '1':
+ texmf_path = texmf_dist
+ elif path_choice == '2':
+ texmf_path = texmf_local
+ elif path_choice == '3':
+ texmf_path = texmf_home
+ else:
+ texmf_path = input('Enter a path:\n')
+except:
+ print('Cannot automatically find TEXMF paths.')
+ print('kpsewhich does not exist or could not be used.')
+ texmf_path = input('Please enter a valid installation path:\n')
+if should_exit:
+ sys.exit()
+# Make sure path slashes are compatible with the operating system
+# Kpathsea returns forward slashes, but Windows needs back slashes
+texmf_path = path.expandvars(path.expanduser(path.normcase(texmf_path)))
+
+# Check to make sure the path is valid
+# This is only really needed for manual input
+# The '' check is for empty manual input
+if texmf_path == '' or not path.exists(texmf_path):
+ print('Invalid installation path. Exiting.')
+ sys.exit(1)
+
+
+# Now check that all other needed paths are present
+if path_choice != '2':
+ doc_path = path.join(texmf_path, 'doc', 'latex')
+ package_path = path.join(texmf_path, 'tex', 'latex')
+ scripts_path = path.join(texmf_path, 'scripts')
+ source_path = path.join(texmf_path, 'source', 'latex')
+else:
+ doc_path = path.join(texmf_path, 'doc', 'latex', 'local')
+ package_path = path.join(texmf_path, 'tex', 'latex', 'local')
+ scripts_path = path.join(texmf_path, 'scripts', 'local')
+ source_path = path.join(texmf_path, 'source', 'latex', 'local')
+make_paths = False
+for eachpath in [doc_path, package_path, scripts_path, source_path]:
+ if not path.exists(eachpath):
+ if make_paths:
+ makedirs(eachpath)
+ print(' * Created ' + eachpath)
+ else:
+ choice = input('Some directories do not exist. Create them? [y/n]\n')
+ if choice not in ('y', 'n'):
+ sys.exit('Invalid choice')
+ elif choice == 'y':
+ make_paths = True
+ makedirs(eachpath)
+ print(' * Created ' + eachpath)
+ else:
+ message = '''
+ Paths were not created. The following will be needed.
+ * {0}
+ * {1}
+ * {2}
+ * {3}
+
+ Exiting.
+ '''.format(doc_path, package_path, scripts_path, source_path)
+ print(textwrap.dedent(message))
+ sys.exit()
+# Modify the paths by adding the pythontex directory, which will be created
+doc_path = path.join(doc_path, 'pythontex')
+package_path = path.join(package_path, 'pythontex')
+scripts_path = path.join(scripts_path, 'pythontex')
+source_path = path.join(source_path, 'pythontex')
+
+
+# Install files
+# Use a try/except in case elevated permissions are needed (Linux and OS X)
+print('\nPythonTeX will be installed in \n ' + texmf_path)
+try:
+ # Install docs
+ if not path.exists(doc_path):
+ mkdir(doc_path)
+ copy('pythontex.pdf', doc_path)
+ copy('README', doc_path)
+ for doc in ('pythontex_quickstart.tex', 'pythontex_quickstart.pdf',
+ 'pythontex_gallery.tex', 'pythontex_gallery.pdf'):
+ if path.isfile(doc):
+ copy(doc, doc_path)
+ else:
+ doc = path.join('..', doc.rsplit('.', 1)[0], doc)
+ if path.isfile(doc):
+ copy(doc, doc_path)
+ # Install package
+ if not path.exists(package_path):
+ mkdir(package_path)
+ copy('pythontex.sty', package_path)
+ # Install scripts
+ if not path.exists(scripts_path):
+ mkdir(scripts_path)
+ copy('pythontex.py', scripts_path)
+ copy('depythontex.py', scripts_path)
+ copy('pythontex_utils.py', scripts_path)
+ copy('pythontex_engines.py', scripts_path)
+ for ver in [2, 3]:
+ copy('pythontex{0}.py'.format(ver), scripts_path)
+ copy('depythontex{0}.py'.format(ver), scripts_path)
+ # Install source
+ if not path.exists(source_path):
+ mkdir(source_path)
+ copy('pythontex.ins', source_path)
+ copy('pythontex.dtx', source_path)
+except OSError as e:
+ if e.errno == 13:
+ print('Insufficient permission to install PythonTeX')
+ print('(For example, you may need "sudo", or possibly "sudo env PATH=$PATH")\n')
+ sys.exit(1)
+ else:
+ raise
+
+
+# Install binary wrappers, create symlinks, or suggest the creation of
+# wrappers/batch files/symlinks. This part is operating system dependent.
+if platform.system() == 'Windows':
+ # If under Windows, we create a binary wrapper if under TeX Live and
+ # otherwise alert the user regarding the need for a wrapper or batch file.
+
+ # Assemble the binary path, assuming TeX Live
+ # The directory bin/ should be at the same level as texmf
+ bin_path = path.join(path.split(texmf_path)[0], 'bin', 'win32')
+ if path.exists(path.join(bin_path, 'runscript.exe')):
+ copy(path.join(bin_path, 'runscript.exe'), path.join(bin_path, 'pythontex.exe'))
+ copy(path.join(bin_path, 'runscript.exe'), path.join(bin_path, 'depythontex.exe'))
+ print('\nCreated binary wrapper...')
+ else:
+ message = '''
+ Could not create a wrapper for launching pythontex.py and
+ depythontex.py; did not find runscript.exe. You will need
+ to create a wrapper manually, or use a batch file. Sample
+ batch files are included with the main PythonTeX files.
+ The wrapper or batch file should be in a location on the
+ Windows PATH. The bin/ directory in your TeX distribution
+ may be a good location.
+
+ The scripts pythontex.py and depythontex.py are located in
+ the following directory:
+ {0}
+ '''.format(scripts_path)
+ print(textwrap.dedent(message))
+else:
+ # Optimistically proceed as if every system other than Windows can share
+ # one set of code.
+ root_path = path.split(texmf_path)[0]
+ # Create a list of all possible subdirectories of bin/ for TeX Live
+ # Source: http://www.tug.org/texlive/doc/texlive-en/texlive-en.html#x1-250003.2.1
+ texlive_platforms = ['alpha-linux', 'amd64-freebsd', 'amd64-kfreebsd',
+ 'armel-linux', 'i386-cygwin', 'i386-freebsd',
+ 'i386-kfreebsd', 'i386-linux', 'i386-solaris',
+ 'mips-irix', 'mipsel-linux', 'powerpc-aix',
+ 'powerpc-linux', 'sparc-solaris', 'universal-darwin',
+ 'x86_64-darwin', 'x86_64-linux', 'x86_64-solaris']
+ symlink_created = False
+ # Try to create a symlink in the standard TeX Live locations
+ for pltfrm in texlive_platforms:
+ bin_path = path.join(root_path, 'bin', pltfrm)
+ if path.exists(bin_path):
+ # Create symlink for pythontex*.py
+ link = path.join(bin_path, 'pythontex.py')
+ # Unlink any old symlinks if they exist, and create new ones
+ # Not doing this gave permissions errors under Ubuntu
+ if path.exists(link):
+ unlink(link)
+ symlink(path.join(scripts_path, 'pythontex.py'), link)
+ chmod(link, 0o775)
+ # Now repeat for depythontex*.py
+ link = path.join(bin_path, 'depythontex.py')
+ if path.exists(link):
+ unlink(link)
+ symlink(path.join(scripts_path, 'depythontex.py'), link)
+ chmod(link, 0o775)
+ symlink_created = True
+
+ # If the standard TeX Live bin/ locations didn't work, try the typical
+ # location for MacPorts TeX Live. This should typically be
+ # /opt/local/bin, but instead of assuming that location, we just climb
+ # two levels up from texmf-dist and then look for a bin/ directory that
+ # contains a tex executable. (For MacPorts, texmf-dist should be at
+ # /opt/local/share/texmf-dist.)
+ if not symlink_created and platform.system() == 'Darwin':
+ bin_path = path.join(path.split(root_path)[0], 'bin')
+ if path.exists(bin_path):
+ try:
+ # Make sure this bin/ is the bin/ we're looking for, by
+ # seeing if pdftex exists
+ check_output([path.join(bin_path, 'pdftex'), '--version'])
+ # Create symlinks
+ link = path.join(bin_path, 'pythontex.py')
+ if path.exists(link):
+ unlink(link)
+ symlink(path.join(scripts_path, 'pythontex.py'), link)
+ chmod(link, 0o775)
+ link = path.join(bin_path, 'depythontex.py')
+ if path.exists(link):
+ unlink(link)
+ symlink(path.join(scripts_path, 'depythontex.py'), link)
+ chmod(link, 0o775)
+ symlink_created = True
+ except:
+ pass
+ if symlink_created:
+ print("\nCreated symlink in Tex's bin/ directory...")
+ else:
+ print('\nCould not automatically create a symlink to pythontex*.py and depythontex*.py.')
+ print('You may wish to create one manually, and make it executable via chmod.')
+ print('The scripts pythontex*.py and depythontex*.py are located in the following directory:')
+ print(' ' + scripts_path)
+
+
+# Alert TeX to the existence of the package via mktexlsr
+try:
+ print('\nRunning mktexlsr to make TeX aware of new files...')
+ check_call(['mktexlsr'])
+except:
+ print('Could not run mktexlsr.')
+ print('Your system may not be aware of newly installed files.')
+
+
+if platform.system() == 'Windows':
+ # Pause so that the user can see any errors or other messages
+ # input('\n[Press ENTER to exit]')
+ print('\n')
+ call(['pause'], shell=True)
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex_utils.py b/Master/texmf-dist/scripts/pythontex/pythontex_utils.py
new file mode 100755
index 00000000000..75d3115a395
--- /dev/null
+++ b/Master/texmf-dist/scripts/pythontex/pythontex_utils.py
@@ -0,0 +1,372 @@
+# -*- coding: utf-8 -*-
+'''
+PythonTeX utilities class for Python scripts.
+
+The utilities class provides variables and methods for the individual
+Python scripts created and executed by PythonTeX. An instance of the class
+named "pytex" is automatically created in each individual script.
+
+Copyright (c) 2012-2013, Geoffrey M. Poore
+All rights reserved.
+Licensed under the BSD 3-Clause License:
+ http://www.opensource.org/licenses/BSD-3-Clause
+
+'''
+
+
+# Imports
+import sys
+import warnings
+
+# Most imports are only needed for SymPy; these are brought in via
+# "lazy import." Importing unicode_literals here shouldn't ever be necessary
+# under Python 2. If unicode_literals is imported in the main script, then
+# all strings in this script will be treated as bytes, and the main script
+# will try to decode the strings from this script as necessary. The decoding
+# shouldn't cause any problems, since all strings in this file may be decoded
+# as valid ASCII. (The actual file is encoded in utf-8, but only characters
+# within the ASCII subset are actually used).
+
+
+class PythonTeXUtils(object):
+ '''
+ A class of PythonTeX utilities.
+
+ Provides variables for keeping track of TeX-side information, and methods
+ for formatting and saving data.
+
+ The following variables and methods will be created within instances
+ of the class during execution.
+
+ String variables for keeping track of TeX information. Most are
+ actually needed; the rest are included for completeness.
+ * input_family
+ * input_session
+ * input_restart
+ * input_command
+ * input_context
+ * input_args
+ * input_instance
+ * input_line
+
+ Future file handle for output that is saved via macros
+ * macrofile
+
+ Future formatter function that is used to format output
+ * formatter
+ '''
+
+ def __init__(self, fmtr='str'):
+ '''
+ Initialize
+ '''
+ self.set_formatter(fmtr)
+
+
+ # We need a context-aware interface to SymPy's latex printer. The
+ # appearance of typeset math should depend on where it appears in a
+ # document. (We will refer to the latex printer, rather than the LaTeX
+ # printer, because the two are separate. Compare sympy.printing.latex
+ # and sympy.galgebra.latex_ex.)
+ #
+ # Creating this interface takes some work. We don't want to import
+ # anything from SymPy unless it is actually used, to keep things clean and
+ # fast.
+
+ # First we create a tuple containing all LaTeX math styles. These are
+ # the contexts that SymPy's latex printer must adapt to.
+ # The style order doesn't matter, but it corresponds to that of \mathchoice
+ _sympy_latex_styles = ('display', 'text', 'script', 'scriptscript')
+
+ # Create the public functions for the user, and private functions that
+ # they call. Two layers are necessary, because we need to be able to
+ # redefine the functions that do the actual work, once things are
+ # initialized. But we don't want to redefine the public functions, since
+ # that could cause problems if the user defines a new function to be one
+ # of the public functions--the user's function would not change when
+ # the method was redefined.
+ def _sympy_latex(self, expr, **settings):
+ self._init_sympy_latex()
+ return self._sympy_latex(expr, **settings)
+
+ def sympy_latex(self, expr, **settings):
+ return self._sympy_latex(expr, **settings)
+
+ def _set_sympy_latex(self, style, **kwargs):
+ self._init_sympy_latex()
+ self._set_sympy_latex(style, **kwargs)
+
+ def set_sympy_latex(self, style, **kwargs):
+ self._set_sympy_latex(style, **kwargs)
+ # Temporary compatibility with deprecated methods
+ def init_sympy_latex(self):
+ warnings.warn('Method init_sympy_latex() is deprecated; init is now automatic.')
+ self._init_sympy_latex()
+
+ # Next we create a method that initializes the actual context-aware
+ # interface to SymPy's latex printer.
+ def _init_sympy_latex(self):
+ '''
+ Initialize a context-aware interface to SymPy's latex printer.
+
+ This consists of creating the dictionary of settings and creating the
+ sympy_latex method that serves as an interface to SymPy's
+ LatexPrinter. This last step is actually performed by calling
+ self._make_sympy_latex().
+ '''
+ # Create dictionaries of settings for different contexts.
+ #
+ # Currently, the main goal is to use pmatrix (or an equivalent)
+ # in \displaystyle contexts, and smallmatrix in \textstyle,
+ # \scriptstyle (superscript or subscript), and \scriptscriptstyle
+ # (superscript or subscript of a superscript or subscript)
+ # contexts. Basically, we want matrix size to automatically
+ # scale based on context. It is expected that additional
+ # customization may prove useful as SymPy's LatexPrinter is
+ # further developed.
+ #
+ # The 'fold_frac_powers' option is probably the main other
+ # setting that might sometimes be nice to invoke in a
+ # context-dependent manner.
+ #
+ # In the default settings below, all matrices are set to use
+ # parentheses rather than square brackets. This is largely a
+ # matter of personal preference. The use of parentheses is based
+ # on the rationale that parentheses are less easily confused with
+ # the determinant and are easier to write by hand than are square
+ # brackets. The settings for 'script' and 'scriptscript' are set
+ # to those of 'text', since all of these should in general
+ # require a more compact representation of things.
+ self._sympy_latex_settings = {'display': {'mat_str': 'pmatrix', 'mat_delim': None},
+ 'text': {'mat_str': 'smallmatrix', 'mat_delim': '('},
+ 'script': {'mat_str': 'smallmatrix', 'mat_delim': '('},
+ 'scriptscript': {'mat_str': 'smallmatrix', 'mat_delim': '('} }
+ # Now we create a function for updating the settings.
+ #
+ # Note that EVERY time the settings are changed, we must call
+ # self._make_sympy_latex(). This is because the _sympy_latex()
+ # method is defined based on the settings, and every time the
+ # settings change, it may need to be redefined. It would be
+ # possible to define _sympy_latex() so that its definition remained
+ # constant, simply drawing on the settings. But most common
+ # combinations of settings allow more efficient versions of
+ # _sympy_latex() to be defined.
+ def _set_sympy_latex(style, **kwargs):
+ if style in self._sympy_latex_styles:
+ self._sympy_latex_settings[style].update(kwargs)
+ elif style == 'all':
+ for s in self._sympy_latex_styles:
+ self._sympy_latex_settings[s].update(kwargs)
+ else:
+ warnings.warn('Unknown LaTeX math style ' + str(style))
+ self._make_sympy_latex()
+ self._set_sympy_latex = _set_sympy_latex
+
+ # Now that the dictionaries of settings have been created, and
+ # the function for modifying the settings is in place, we are ready
+ # to create the actual interface.
+ self._make_sympy_latex()
+
+ # Finally, create the actual interface to SymPy's LatexPrinter
+ def _make_sympy_latex(self):
+ '''
+ Create a context-aware interface to SymPy's LatexPrinter class.
+
+ This is an interface to the LatexPrinter class, rather than
+ to the latex function, because the function is simply a
+ wrapper for accessing the class and because settings may be
+ passed to the class more easily.
+
+ Context dependence is accomplished via LaTeX's \mathchoice macro.
+ This macros takes four arguments:
+ \mathchoice{<display>}{<text>}{<script>}{<scriptscript>}
+ All four arguments are typeset by LaTeX, and then the appropriate one
+ is actually typeset in the document based on the current style. This
+ may seem like a very inefficient way of doing things, but this
+ approach is necessary because LaTeX doesn't know the math style at a
+ given point until after ALL mathematics have been typeset. This is
+ because macros such as \over and \atop change the math style of things
+ that PRECEDE them. See the following discussion for more information:
+ http://tex.stackexchange.com/questions/1223/is-there-a-test-for-the-different-styles-inside-maths-mode
+
+ The interface takes optional settings. These optional
+ settings override the default context-dependent settings.
+ Accomplishing this mixture of settings requires (deep)copying
+ the default settings, then updating the copies with the optional
+ settings. This leaves the default settings intact, with their
+ original values, for the next usage.
+
+ The interface is created in various ways depending on the specific
+ combination of context-specific settings. While a general, static
+ interface could be created, that would involve invoking LatexPrinter
+ four times, once for each math style. It would also require that
+ LaTeX process a \mathchoice macro for everything returned by
+ _sympy_latex(), which would add more inefficiency. In practice, there
+ will generally be enough overlap between the different settings, and
+ the settings will be focused enough, that more efficient
+ implementations of _sympy_latex() are possible.
+
+ Note that we perform a "lazy import" here. We don't want to import
+ the LatexPrinter unless we are sure to use it, since the import brings
+ along a number of other dependencies from SymPy. We don't want
+ unnecessary overhead from SymPy imports.
+ '''
+ # sys has already been imported
+ import copy
+ try:
+ from sympy.printing.latex import LatexPrinter
+ except ImportError:
+ sys.exit('Could not import from SymPy')
+
+ # Go through a number of possible scenarios, to create an efficient
+ # implementation of sympy_latex()
+ if all(self._sympy_latex_settings[style] == {} for style in self._sympy_latex_styles):
+ def _sympy_latex(expr, **settings):
+ '''
+ Deal with the case where there are no context-specific
+ settings.
+ '''
+ return LatexPrinter(settings).doprint(expr)
+ elif all(self._sympy_latex_settings[style] == self._sympy_latex_settings['display'] for style in self._sympy_latex_styles):
+ def _sympy_latex(expr, **settings):
+ '''
+ Deal with the case where all settings are identical, and thus
+ the settings are really only being used to set defaults,
+ rather than context-specific behavior.
+
+ Check for empty settings, so as to avoid deepcopy
+ '''
+ if not settings:
+ return LatexPrinter(self._sympy_latex_settings['display']).doprint(expr)
+ else:
+ final_settings = copy.deepcopy(self._sympy_latex_settings['display'])
+ final_settings.update(settings)
+ return LatexPrinter(final_settings).doprint(expr)
+ elif all(self._sympy_latex_settings[style] == self._sympy_latex_settings['text'] for style in ('script', 'scriptscript')):
+ def _sympy_latex(expr, **settings):
+ '''
+ Deal with the case where only 'display' has different settings.
+
+ This should be the most common case.
+ '''
+ if not settings:
+ display = LatexPrinter(self._sympy_latex_settings['display']).doprint(expr)
+ text = LatexPrinter(self._sympy_latex_settings['text']).doprint(expr)
+ else:
+ display_settings = copy.deepcopy(self._sympy_latex_settings['display'])
+ display_settings.update(settings)
+ display = LatexPrinter(display_settings).doprint(expr)
+ text_settings = copy.deepcopy(self._sympy_latex_settings['text'])
+ text_settings.update(settings)
+ text = LatexPrinter(text_settings).doprint(expr)
+ if display == text:
+ return display
+ else:
+ return r'\mathchoice{' + display + '}{' + text + '}{' + text + '}{' + text + '}'
+ else:
+ def _sympy_latex(expr, **settings):
+ '''
+ If all attempts at simplification fail, create the most
+ general interface.
+
+ The main disadvantage here is that LatexPrinter is invoked
+ four times and we must create many temporary variables.
+ '''
+ if not settings:
+ display = LatexPrinter(self._sympy_latex_settings['display']).doprint(expr)
+ text = LatexPrinter(self._sympy_latex_settings['text']).doprint(expr)
+ script = LatexPrinter(self._sympy_latex_settings['script']).doprint(expr)
+ scriptscript = LatexPrinter(self._sympy_latex_settings['scriptscript']).doprint(expr)
+ else:
+ display_settings = copy.deepcopy(self._sympy_latex_settings['display'])
+ display_settings.update(settings)
+ display = LatexPrinter(display_settings).doprint(expr)
+ text_settings = copy.deepcopy(self._sympy_latex_settings['text'])
+ text_settings.update(settings)
+ text = LatexPrinter(text_settings).doprint(expr)
+ script_settings = copy.deepcopy(self._sympy_latex_settings['script'])
+ script_settings.update(settings)
+ script = LatexPrinter(script_settings).doprint(expr)
+ scriptscript_settings = copy.deepcopy(self._sympy_latex_settings['scripscript'])
+ scriptscript_settings.update(settings)
+ scriptscript = LatexPrinter(scriptscript_settings).doprint(expr)
+ if display == text and display == script and display == scriptscript:
+ return display
+ else:
+ return r'\mathchoice{' + display + '}{' + text + '}{' + script + '}{' + scriptscript+ '}'
+ self._sympy_latex = _sympy_latex
+
+ # Now we are ready to create non-SymPy formatters and a method for
+ # setting formatters
+ def identity_formatter(self, expr):
+ '''
+ For generality, we need an identity formatter, a formatter that does
+ nothing to its argument and simply returns it unchanged.
+ '''
+ return expr
+
+ def set_formatter(self, fmtr='str'):
+ '''
+ Set the formatter method.
+
+ This is used to process output that is brought in via macros. It is
+ also available for the user in formatting printed or saved output.
+ '''
+ if fmtr == 'str':
+ if sys.version_info[0] == 2:
+ self.formatter = unicode
+ else:
+ self.formatter = str
+ elif fmtr == 'sympy_latex':
+ self.formatter = self.sympy_latex
+ elif fmtr in ('None', 'none', 'identity') or fmtr is None:
+ self.formatter = self.identity_formatter
+ else:
+ raise ValueError('Unsupported formatter type')
+
+ # We need functions that can be executed immediately before and after
+ # each chunk of code. By default, these should do nothing; they are for
+ # user customization, or customization via packages.
+ def before(self):
+ pass
+ def after(self):
+ pass
+
+
+ # We need a way to keep track of dependencies
+ # We create a list that stores specified dependencies, and a method that
+ # adds dependencies to the list. The contents of this list must be
+ # written to stdout at the end of the file, to be transmitted back to the
+ # main script. So we create a method that prints them to stdout. This is
+ # called via a generic cleanup method that is always invoked at the end of
+ # the script.
+ _dependencies = list()
+ def add_dependencies(self, *args):
+ self._dependencies.extend(list(args))
+ def _save_dependencies(self):
+ print('=>PYTHONTEX:DEPENDENCIES#')
+ if self._dependencies:
+ for dep in self._dependencies:
+ print(dep)
+
+ # We need a way to keep track of created files, so that they can be
+ # automatically cleaned up. By default, all files are created within the
+ # pythontex-files_<jobname> folder, and are thus contained. If a custom
+ # working directory is used, or files are otherwise created in a custom
+ # location, it may be desirable to track them and keep them cleaned up.
+ # Furthermore, even when files are contained in the default directory, it
+ # may be desirable to delete files when they are no longer needed due to
+ # program changes, renaming, etc.
+ _created = list()
+ def add_created(self, *args):
+ self._created.extend(list(args))
+ def _save_created(self):
+ print('=>PYTHONTEX:CREATED#')
+ if self._created:
+ for creation in self._created:
+ print(creation)
+
+ def cleanup(self):
+ self._save_dependencies()
+ self._save_created()
+