From c42603e27f2d6af845d82162be47078a816fc715 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Tue, 24 Sep 2019 20:07:38 +0000 Subject: pythontex (24sep19) git-svn-id: svn://tug.org/texlive/trunk@52174 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/latex/pythontex/NEWS.rst | 51 +++++ Master/texmf-dist/doc/latex/pythontex/README | 9 +- .../texmf-dist/doc/latex/pythontex/pythontex.pdf | Bin 969134 -> 975950 bytes .../texmf-dist/scripts/pythontex/depythontex2.py | 2 +- .../texmf-dist/scripts/pythontex/depythontex3.py | 2 +- Master/texmf-dist/scripts/pythontex/pythontex2.py | 157 +++++++++++--- Master/texmf-dist/scripts/pythontex/pythontex3.py | 157 +++++++++++--- .../scripts/pythontex/pythontex_engines.py | 235 +++++++++++++++++++-- .../source/latex/pythontex/pythontex.dtx | 183 ++++++++++++---- .../source/latex/pythontex/pythontex.ins | 4 +- .../texmf-dist/tex/latex/pythontex/pythontex.sty | 87 +++++--- 11 files changed, 739 insertions(+), 148 deletions(-) diff --git a/Master/texmf-dist/doc/latex/pythontex/NEWS.rst b/Master/texmf-dist/doc/latex/pythontex/NEWS.rst index 016a7407948..b0eccb7df58 100644 --- a/Master/texmf-dist/doc/latex/pythontex/NEWS.rst +++ b/Master/texmf-dist/doc/latex/pythontex/NEWS.rst @@ -7,6 +7,57 @@ Version History =============== +v0.17 (2019/09/22) +------------------ + +* Pygments syntax highlighting for the Python console (``pycon`` lexer) now + uses the ``python3`` option, and the default Python lexer is now + ``python3`` (#156). + +* Added support for JavaScript (#147; thanks to Nathan Carter). + +* Updated Julia support for Julia versions 0.6 (#107), and 0.7 and 1.0 (#126, + #130). + +* There are now meaningful error messages for the Julia console when Weave.jl + is not installed or raises errors (#131). + +* ``pythontexcustomcode`` and ``\pythontexcustomc`` now set + ``pytex.context`` (#65). + +* Added support for R. The ``R`` family of commands and environments + (``\R``, ``\Rc``, ``Rcode``, ...) executes code as a script. + There is currently no utilities class or equivalent. The ``Rcon`` family + (``Rconsole``) executes code to emulate an interactive R session (#121). + +* ``fancyvrb`` settings from ``\setpythontexfv`` and console + environments now work with Julia and R consoles. + +* ``pythontexcustomcode`` now works with ``juliacon``. There are now proper + ``juliaconcode`` and ``Rconcode`` environments that execute code but + typeset nothing, to parallel ``pyconcode`` (#134). + +* Added support for Perl with the ``perl`` and ``pl`` families of commands + and environments. There is currently no utilities class or equivalent. + +* Added support for Perl 6 with the ``perlsix`` and ``psix`` families of + commands and environments (#104). There is currently no utilities class or + equivalent. + +* Updated Rust support by using ``dyn`` with traits in utilities object. + +* Under Windows, capitalization of script paths in ``stderr`` is now + preserved. + +* Fixed a bug that prevented the ``sub`` environment from working with + ``depythontex`` (#155). + +* Fixed a bug in checking mtime of dependencies to see if they have been + modified while ``pythontex`` is running. The check failed for dependencies + that do not exist or were deleted before ``pythontex`` can read them + (#136). + + v0.16 (2017/07/20) ------------------ diff --git a/Master/texmf-dist/doc/latex/pythontex/README b/Master/texmf-dist/doc/latex/pythontex/README index 56259936766..3994eca34c9 100644 --- a/Master/texmf-dist/doc/latex/pythontex/README +++ b/Master/texmf-dist/doc/latex/pythontex/README @@ -3,15 +3,16 @@ =================================================================== :Author: Geoffrey Poore -:Version: 0.16 (2017/07/20) +:Version: 0.17 (2019/09/22) :License: LPPL v1.3 or later (LaTeX code) and BSD 3-Clause (Python code) :Development: https://github.com/gpoore/pythontex :Requirements: Python 2.7 or 3.2+; Pygments -The PythonTeX package allows you to enter Python code within a LaTeX -document, execute the code, and access its output in the original document. -It also supports the Ruby, Julia, Octave, Sage, Bash, and Rust languages. +The PythonTeX package allows you to enter Python code within a LaTeX document, +execute the code, and access its output in the original document. It also +supports the Ruby, Julia, Octave, Sage, Bash, Rust, R, Perl, Perl 6, and +JavaScript languages. PythonTeX provides fast, user-friendly access to Python from within LaTeX. Python code is only executed when it has been modified, or when it meets diff --git a/Master/texmf-dist/doc/latex/pythontex/pythontex.pdf b/Master/texmf-dist/doc/latex/pythontex/pythontex.pdf index bd919b5a556..ad9183d3acd 100644 Binary files a/Master/texmf-dist/doc/latex/pythontex/pythontex.pdf and b/Master/texmf-dist/doc/latex/pythontex/pythontex.pdf differ diff --git a/Master/texmf-dist/scripts/pythontex/depythontex2.py b/Master/texmf-dist/scripts/pythontex/depythontex2.py index 7687e0e7244..f49fd229778 100755 --- a/Master/texmf-dist/scripts/pythontex/depythontex2.py +++ b/Master/texmf-dist/scripts/pythontex/depythontex2.py @@ -86,7 +86,7 @@ import codecs # Script parameters # Version -__version__ = '0.16' +__version__ = '0.17' # Functions and parameters for customizing the script output diff --git a/Master/texmf-dist/scripts/pythontex/depythontex3.py b/Master/texmf-dist/scripts/pythontex/depythontex3.py index bdcea2185e0..2d86edb5914 100755 --- a/Master/texmf-dist/scripts/pythontex/depythontex3.py +++ b/Master/texmf-dist/scripts/pythontex/depythontex3.py @@ -86,7 +86,7 @@ import codecs # Script parameters # Version -__version__ = '0.16' +__version__ = '0.17' # Functions and parameters for customizing the script output diff --git a/Master/texmf-dist/scripts/pythontex/pythontex2.py b/Master/texmf-dist/scripts/pythontex/pythontex2.py index 871d394e375..34e151f4177 100755 --- a/Master/texmf-dist/scripts/pythontex/pythontex2.py +++ b/Master/texmf-dist/scripts/pythontex/pythontex2.py @@ -13,7 +13,7 @@ should be in the same directory. Licensed under the BSD 3-Clause License: -Copyright (c) 2012-2017, Geoffrey M. Poore +Copyright (c) 2012-2019, Geoffrey M. Poore All rights reserved. @@ -62,6 +62,7 @@ from pygments.styles import get_all_styles from pythontex_engines import * import textwrap import platform +import itertools if sys.version_info[0] == 2: try: @@ -77,7 +78,7 @@ else: # Script parameters # Version -__version__ = '0.16' +__version__ = '0.17' @@ -1213,9 +1214,13 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): # Must double-escape any backslashes so that they survive `shlex.split()` script = basename if os.path.isabs(os.path.expanduser(os.path.normcase(outputdir))): - script_full = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename))) + script_full = os.path.expanduser(os.path.join(outputdir, basename)) else: - script_full = os.path.expanduser(os.path.normcase(os.path.join(orig_cwd, outputdir, basename))) + script_full = os.path.expanduser(os.path.join(orig_cwd, outputdir, basename)) + if platform.system() == 'Windows': + script_full = script_full.replace('/', '\\') + else: + script_full = script_full.replace('\\', '/') # `shlex.split()` only works with Unicode after 2.7.2 if (sys.version_info.major == 2 and sys.version_info.micro < 3): exec_cmd = shlex.split(bytes(command.format(file=script.replace('\\', '\\\\'), File=script_full.replace('\\', '\\\\')))) @@ -1275,7 +1280,11 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): for key in code_dict: family = key.split('#')[0] # Uncomment the following for debugging, and comment out what follows - '''run_code(encoding, outputdir, workingdir, code_dict[key], + '''run_code(encoding, outputdir, + workingdir, + cc_dict_begin[family], + code_dict[key], + cc_dict_end[family], engine_dict[family].language, engine_dict[family].commands, engine_dict[family].created, @@ -1287,9 +1296,12 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): engine_dict[family].linenumbers, engine_dict[family].lookbehind, keeptemps, hashdependencies, - pygments_settings)''' + pygments_settings]))''' tasks.append(pool.apply_async(run_code, [encoding, outputdir, - workingdir, code_dict[key], + workingdir, + cc_dict_begin[family], + code_dict[key], + cc_dict_end[family], engine_dict[family].language, engine_dict[family].commands, engine_dict[family].created, @@ -1412,7 +1424,7 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): unresolved_sessions = [] for key in dependencies: for dep, val in dependencies[key].items(): - if val[0] > start_time: + if val[0] is None or val[0] > start_time: unresolved_dependencies = True dependencies[key][dep] = (None, None) unresolved_sessions.append(key.replace('#', ':')) @@ -1474,7 +1486,8 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): -def run_code(encoding, outputdir, workingdir, code_list, language, commands, +def run_code(encoding, outputdir, workingdir, + cc_begin_list, code_list, cc_end_list, language, commands, command_created, extension, makestderr, stderrfilename, code_index, errorsig, warningsig, linesig, stderrlookbehind, keeptemps, hashdependencies, pygments_settings): @@ -1512,11 +1525,19 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, err_file_name = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename + '.err'))) out_file = open(out_file_name, 'w', encoding=encoding) err_file = open(err_file_name, 'w', encoding=encoding) - script = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename))) + script = os.path.expanduser(os.path.join(outputdir, basename)) + if platform.system() == 'Windows': + script = script.replace('/', '\\') + else: + script = script.replace('\\', '/') if os.path.isabs(script): script_full = script else: - script_full = os.path.expanduser(os.path.normcase(os.path.join(os.getcwd(), outputdir, basename))) + script_full = os.path.expanduser(os.path.join(os.getcwd(), outputdir, basename)) + if platform.system() == 'Windows': + script_full = script_full.replace('/', '\\') + else: + script_full = script_full.replace('\\', '/') # #### Need to revise so that intermediate files can be detected and cleaned up for f in command_created: files.append(f.format(file=script, File=script_full)) @@ -1535,7 +1556,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, # Add any created files due to the command # This needs to be done before attempts to execute, to prevent orphans try: - proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file) + if family != 'Rcon': + proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file) + else: + proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=subprocess.STDOUT) except WindowsError as e: if e.errno == 2: # Batch files won't be found when called without extension. They @@ -1546,7 +1570,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, # under Windows; a list is not required. exec_cmd_string = ' '.join(exec_cmd) exec_cmd_string = 'cmd /C "@echo off & call {0} & if errorlevel 1 exit 1"'.format(exec_cmd_string) - proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file) + if family != 'Rcon': + proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file) + else: + proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=subprocess.STDOUT) else: raise @@ -1567,15 +1594,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, messages.append('* PythonTeX error') messages.append(' Missing output file for ' + key_run.replace('#', ':')) errors += 1 + elif family == 'juliacon' and proc.returncode == 1: + messages.append('* PythonTeX error') + messages.append(' Running code for Julia console failed') + with open(err_file_name, encoding='utf8') as f: + messages.append(f.read()) + errors += 1 else: if family == 'juliacon': with open(out_file_name.rsplit('.', 1)[0] + '.tex', 'r', encoding=encoding) as f: tex_data_lines = f.readlines() - inst = 0 + code_iter = itertools.chain(cc_begin_list, code_list, cc_end_list) for n, line in enumerate(tex_data_lines): if line.rstrip() == '\\begin{juliaterm}': - tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(inst) - inst += 1 + c = next(code_iter) + tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(c.instance) if n != 0: tex_data_lines[n-1] = '' if line.rstrip() == '\\end{juliaterm}': @@ -1583,6 +1616,29 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, tex_data_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n') with open(out_file_name, 'w', encoding=encoding) as f: f.write(''.join(tex_data_lines)) + elif family == 'Rcon': + with open(out_file_name, 'r', encoding=encoding) as f: + stdout_lines = f.readlines() + for n, line in enumerate(stdout_lines): + if line.startswith('> =>PYTHONTEX:'): + stdout_lines[n] = line[2:] + elif '> write("=>PYTHONTEX:' in line: + if line.startswith('> write("=>PYTHONTEX:'): + stdout_lines[n] = '' + else: + # cat() and similar functions can result in the + # prompt not being at the start of a new line. In + # that case, preserve the prompt to accurately + # emulate the console. If there is a following + # console environment, this effectively amounts + # to adding an extra empty line (pressing ENTER) + # between the two. + stdout_lines[n] = line.split('write("=>PYTHONTEX:', 1)[0] + while stdout_lines and (stdout_lines[-1].startswith('>') and not stdout_lines[-1][1:].strip(' \n')): + stdout_lines.pop() + stdout_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n') + with open(out_file_name, 'w', encoding=encoding) as f: + f.write(''.join(stdout_lines)) f = open(out_file_name, 'r', encoding=encoding) out = f.read() @@ -1640,7 +1696,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, else: dependencies[dep] = (os.path.getmtime(dep_file), '') - if family == 'juliacon': + if family in ('juliacon', 'Rcon'): from pygments import highlight from pygments.lexers import get_lexer_by_name from pygments.formatters import LatexFormatter @@ -1655,16 +1711,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, for block in out.split('=>PYTHONTEX:STDOUT#')[1:]: if block: delims, content = block.split('#\n', 1) + if content and not content.endswith('\n'): + # Content might not end with a newline. For example, + # Rcon with something like cat() as the last function. + content += '\n' instance, command = delims.split('#') if content or command in ('s', 'sub'): if instance.endswith('CC'): - messages.append('* PythonTeX warning') - messages.append(' Custom code for "' + family + '" attempted to print or write to stdout') - messages.append(' This is not supported; use a normal code command or environment') - messages.append(' The following content was written:') - messages.append('') - messages.extend([' ' + l for l in content.splitlines()]) - warnings += 1 + if family not in ('juliacon', 'Rcon'): + messages.append('* PythonTeX warning') + messages.append(' Custom code for "' + family + '" attempted to print or write to stdout') + messages.append(' This is not supported; use a normal code command or environment') + messages.append(' The following content was written:') + messages.append('') + messages.extend([' ' + l for l in content.splitlines()]) + warnings += 1 elif command == 'i': content = r'\pytx@SVMCR{pytx@MCR@' + key_run.replace('#', '@') + '@' + instance + '}\n' + content.rstrip('\n') + '\\endpytx@SVMCR\n\n' macros.append(content) @@ -1682,7 +1743,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, # Remove newline added by printing, prevent # LaTeX from adding a space after content content = content.rsplit('\n', 1)[0] + '\\endinput\n' - if family == 'juliacon': + if family in ('juliacon', 'Rcon'): content = highlight(content, lexer[family], formatter[family]) f.write(content) f.close() @@ -1693,7 +1754,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, messages.append('* PythonTeX error') messages.append(' Missing stderr file for ' + key_run.replace('#', ':')) errors += 1 - elif family == 'juliacon': + elif family in ('juliacon', 'Rcon'): pass else: # Open error and code files. @@ -1733,7 +1794,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, index_next = index_now start_errgobble = None for n, line in enumerate(err_ud): - if basename in line: + if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line): # Get the gobbleation. This is used to determine if # other lines containing the basename are a continuation, # or separate messages. @@ -1779,7 +1840,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, # both the error and warning patterns, default to # error. past_line = err_ud[index] - if (index < n and basename in past_line): + if (index < n and basename in past_line and (family not in ('perlsix', 'psix') or '.p6:' in past_line or '.p6 line' in past_line)): break for pattern in warningsig: if pattern in past_line: @@ -1843,8 +1904,9 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, index_now_last = index_now index_next_last = index_next err_key_last_int = -1 + p6_sorry_search = False for n, line in enumerate(err_ud): - if basename in line: + if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line): # Determine the corresponding line number in the document found = False for pattern in linesig: @@ -1906,6 +1968,35 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands, line = line.replace(fullbasename + '.' + extension, '') elif stderrfilename == 'genericscript': line = line.replace(fullbasename + '.' + extension, '