summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2019-09-24 20:07:38 +0000
committerKarl Berry <karl@freefriends.org>2019-09-24 20:07:38 +0000
commitc42603e27f2d6af845d82162be47078a816fc715 (patch)
tree0058aa3c7ad67cf40203a1c66282a8d6fa897963 /Master/texmf-dist/scripts
parent30e223be30ac74b2dd030ec888f906263a3f99c1 (diff)
pythontex (24sep19)
git-svn-id: svn://tug.org/texlive/trunk@52174 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts')
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/depythontex2.py2
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/depythontex3.py2
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex2.py157
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex3.py157
-rwxr-xr-xMaster/texmf-dist/scripts/pythontex/pythontex_engines.py235
5 files changed, 475 insertions, 78 deletions
diff --git a/Master/texmf-dist/scripts/pythontex/depythontex2.py b/Master/texmf-dist/scripts/pythontex/depythontex2.py
index 7687e0e7244..f49fd229778 100755
--- a/Master/texmf-dist/scripts/pythontex/depythontex2.py
+++ b/Master/texmf-dist/scripts/pythontex/depythontex2.py
@@ -86,7 +86,7 @@ import codecs
# Script parameters
# Version
-__version__ = '0.16'
+__version__ = '0.17'
# Functions and parameters for customizing the script output
diff --git a/Master/texmf-dist/scripts/pythontex/depythontex3.py b/Master/texmf-dist/scripts/pythontex/depythontex3.py
index bdcea2185e0..2d86edb5914 100755
--- a/Master/texmf-dist/scripts/pythontex/depythontex3.py
+++ b/Master/texmf-dist/scripts/pythontex/depythontex3.py
@@ -86,7 +86,7 @@ import codecs
# Script parameters
# Version
-__version__ = '0.16'
+__version__ = '0.17'
# Functions and parameters for customizing the script output
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex2.py b/Master/texmf-dist/scripts/pythontex/pythontex2.py
index 871d394e375..34e151f4177 100755
--- a/Master/texmf-dist/scripts/pythontex/pythontex2.py
+++ b/Master/texmf-dist/scripts/pythontex/pythontex2.py
@@ -13,7 +13,7 @@ should be in the same directory.
Licensed under the BSD 3-Clause License:
-Copyright (c) 2012-2017, Geoffrey M. Poore
+Copyright (c) 2012-2019, Geoffrey M. Poore
All rights reserved.
@@ -62,6 +62,7 @@ from pygments.styles import get_all_styles
from pythontex_engines import *
import textwrap
import platform
+import itertools
if sys.version_info[0] == 2:
try:
@@ -77,7 +78,7 @@ else:
# Script parameters
# Version
-__version__ = '0.16'
+__version__ = '0.17'
@@ -1213,9 +1214,13 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
# Must double-escape any backslashes so that they survive `shlex.split()`
script = basename
if os.path.isabs(os.path.expanduser(os.path.normcase(outputdir))):
- script_full = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+ script_full = os.path.expanduser(os.path.join(outputdir, basename))
else:
- script_full = os.path.expanduser(os.path.normcase(os.path.join(orig_cwd, outputdir, basename)))
+ script_full = os.path.expanduser(os.path.join(orig_cwd, outputdir, basename))
+ if platform.system() == 'Windows':
+ script_full = script_full.replace('/', '\\')
+ else:
+ script_full = script_full.replace('\\', '/')
# `shlex.split()` only works with Unicode after 2.7.2
if (sys.version_info.major == 2 and sys.version_info.micro < 3):
exec_cmd = shlex.split(bytes(command.format(file=script.replace('\\', '\\\\'), File=script_full.replace('\\', '\\\\'))))
@@ -1275,7 +1280,11 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
for key in code_dict:
family = key.split('#')[0]
# Uncomment the following for debugging, and comment out what follows
- '''run_code(encoding, outputdir, workingdir, code_dict[key],
+ '''run_code(encoding, outputdir,
+ workingdir,
+ cc_dict_begin[family],
+ code_dict[key],
+ cc_dict_end[family],
engine_dict[family].language,
engine_dict[family].commands,
engine_dict[family].created,
@@ -1287,9 +1296,12 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
engine_dict[family].linenumbers,
engine_dict[family].lookbehind,
keeptemps, hashdependencies,
- pygments_settings)'''
+ pygments_settings]))'''
tasks.append(pool.apply_async(run_code, [encoding, outputdir,
- workingdir, code_dict[key],
+ workingdir,
+ cc_dict_begin[family],
+ code_dict[key],
+ cc_dict_end[family],
engine_dict[family].language,
engine_dict[family].commands,
engine_dict[family].created,
@@ -1412,7 +1424,7 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
unresolved_sessions = []
for key in dependencies:
for dep, val in dependencies[key].items():
- if val[0] > start_time:
+ if val[0] is None or val[0] > start_time:
unresolved_dependencies = True
dependencies[key][dep] = (None, None)
unresolved_sessions.append(key.replace('#', ':'))
@@ -1474,7 +1486,8 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
-def run_code(encoding, outputdir, workingdir, code_list, language, commands,
+def run_code(encoding, outputdir, workingdir,
+ cc_begin_list, code_list, cc_end_list, language, commands,
command_created, extension, makestderr, stderrfilename,
code_index, errorsig, warningsig, linesig, stderrlookbehind,
keeptemps, hashdependencies, pygments_settings):
@@ -1512,11 +1525,19 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
err_file_name = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename + '.err')))
out_file = open(out_file_name, 'w', encoding=encoding)
err_file = open(err_file_name, 'w', encoding=encoding)
- script = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+ script = os.path.expanduser(os.path.join(outputdir, basename))
+ if platform.system() == 'Windows':
+ script = script.replace('/', '\\')
+ else:
+ script = script.replace('\\', '/')
if os.path.isabs(script):
script_full = script
else:
- script_full = os.path.expanduser(os.path.normcase(os.path.join(os.getcwd(), outputdir, basename)))
+ script_full = os.path.expanduser(os.path.join(os.getcwd(), outputdir, basename))
+ if platform.system() == 'Windows':
+ script_full = script_full.replace('/', '\\')
+ else:
+ script_full = script_full.replace('\\', '/')
# #### Need to revise so that intermediate files can be detected and cleaned up
for f in command_created:
files.append(f.format(file=script, File=script_full))
@@ -1535,7 +1556,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# Add any created files due to the command
# This needs to be done before attempts to execute, to prevent orphans
try:
- proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+ if family != 'Rcon':
+ proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+ else:
+ proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=subprocess.STDOUT)
except WindowsError as e:
if e.errno == 2:
# Batch files won't be found when called without extension. They
@@ -1546,7 +1570,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# under Windows; a list is not required.
exec_cmd_string = ' '.join(exec_cmd)
exec_cmd_string = 'cmd /C "@echo off & call {0} & if errorlevel 1 exit 1"'.format(exec_cmd_string)
- proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+ if family != 'Rcon':
+ proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+ else:
+ proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=subprocess.STDOUT)
else:
raise
@@ -1567,15 +1594,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
messages.append('* PythonTeX error')
messages.append(' Missing output file for ' + key_run.replace('#', ':'))
errors += 1
+ elif family == 'juliacon' and proc.returncode == 1:
+ messages.append('* PythonTeX error')
+ messages.append(' Running code for Julia console failed')
+ with open(err_file_name, encoding='utf8') as f:
+ messages.append(f.read())
+ errors += 1
else:
if family == 'juliacon':
with open(out_file_name.rsplit('.', 1)[0] + '.tex', 'r', encoding=encoding) as f:
tex_data_lines = f.readlines()
- inst = 0
+ code_iter = itertools.chain(cc_begin_list, code_list, cc_end_list)
for n, line in enumerate(tex_data_lines):
if line.rstrip() == '\\begin{juliaterm}':
- tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(inst)
- inst += 1
+ c = next(code_iter)
+ tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(c.instance)
if n != 0:
tex_data_lines[n-1] = ''
if line.rstrip() == '\\end{juliaterm}':
@@ -1583,6 +1616,29 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
tex_data_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
with open(out_file_name, 'w', encoding=encoding) as f:
f.write(''.join(tex_data_lines))
+ elif family == 'Rcon':
+ with open(out_file_name, 'r', encoding=encoding) as f:
+ stdout_lines = f.readlines()
+ for n, line in enumerate(stdout_lines):
+ if line.startswith('> =>PYTHONTEX:'):
+ stdout_lines[n] = line[2:]
+ elif '> write("=>PYTHONTEX:' in line:
+ if line.startswith('> write("=>PYTHONTEX:'):
+ stdout_lines[n] = ''
+ else:
+ # cat() and similar functions can result in the
+ # prompt not being at the start of a new line. In
+ # that case, preserve the prompt to accurately
+ # emulate the console. If there is a following
+ # console environment, this effectively amounts
+ # to adding an extra empty line (pressing ENTER)
+ # between the two.
+ stdout_lines[n] = line.split('write("=>PYTHONTEX:', 1)[0]
+ while stdout_lines and (stdout_lines[-1].startswith('>') and not stdout_lines[-1][1:].strip(' \n')):
+ stdout_lines.pop()
+ stdout_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
+ with open(out_file_name, 'w', encoding=encoding) as f:
+ f.write(''.join(stdout_lines))
f = open(out_file_name, 'r', encoding=encoding)
out = f.read()
@@ -1640,7 +1696,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
else:
dependencies[dep] = (os.path.getmtime(dep_file), '')
- if family == 'juliacon':
+ if family in ('juliacon', 'Rcon'):
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import LatexFormatter
@@ -1655,16 +1711,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
for block in out.split('=>PYTHONTEX:STDOUT#')[1:]:
if block:
delims, content = block.split('#\n', 1)
+ if content and not content.endswith('\n'):
+ # Content might not end with a newline. For example,
+ # Rcon with something like cat() as the last function.
+ content += '\n'
instance, command = delims.split('#')
if content or command in ('s', 'sub'):
if instance.endswith('CC'):
- messages.append('* PythonTeX warning')
- messages.append(' Custom code for "' + family + '" attempted to print or write to stdout')
- messages.append(' This is not supported; use a normal code command or environment')
- messages.append(' The following content was written:')
- messages.append('')
- messages.extend([' ' + l for l in content.splitlines()])
- warnings += 1
+ if family not in ('juliacon', 'Rcon'):
+ messages.append('* PythonTeX warning')
+ messages.append(' Custom code for "' + family + '" attempted to print or write to stdout')
+ messages.append(' This is not supported; use a normal code command or environment')
+ messages.append(' The following content was written:')
+ messages.append('')
+ messages.extend([' ' + l for l in content.splitlines()])
+ warnings += 1
elif command == 'i':
content = r'\pytx@SVMCR{pytx@MCR@' + key_run.replace('#', '@') + '@' + instance + '}\n' + content.rstrip('\n') + '\\endpytx@SVMCR\n\n'
macros.append(content)
@@ -1682,7 +1743,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# Remove newline added by printing, prevent
# LaTeX from adding a space after content
content = content.rsplit('\n', 1)[0] + '\\endinput\n'
- if family == 'juliacon':
+ if family in ('juliacon', 'Rcon'):
content = highlight(content, lexer[family], formatter[family])
f.write(content)
f.close()
@@ -1693,7 +1754,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
messages.append('* PythonTeX error')
messages.append(' Missing stderr file for ' + key_run.replace('#', ':'))
errors += 1
- elif family == 'juliacon':
+ elif family in ('juliacon', 'Rcon'):
pass
else:
# Open error and code files.
@@ -1733,7 +1794,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
index_next = index_now
start_errgobble = None
for n, line in enumerate(err_ud):
- if basename in line:
+ if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
# Get the gobbleation. This is used to determine if
# other lines containing the basename are a continuation,
# or separate messages.
@@ -1779,7 +1840,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# both the error and warning patterns, default to
# error.
past_line = err_ud[index]
- if (index < n and basename in past_line):
+ if (index < n and basename in past_line and (family not in ('perlsix', 'psix') or '.p6:' in past_line or '.p6 line' in past_line)):
break
for pattern in warningsig:
if pattern in past_line:
@@ -1843,8 +1904,9 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
index_now_last = index_now
index_next_last = index_next
err_key_last_int = -1
+ p6_sorry_search = False
for n, line in enumerate(err_ud):
- if basename in line:
+ if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
# Determine the corresponding line number in the document
found = False
for pattern in linesig:
@@ -1906,6 +1968,35 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
line = line.replace(fullbasename + '.' + extension, '<file>')
elif stderrfilename == 'genericscript':
line = line.replace(fullbasename + '.' + extension, '<script>')
+ if family in ('perlsix', 'psix'):
+ # Perl 6 "SORRY!" errors during compiling
+ # (before execution) need special processing,
+ # since they lack stderr delims and must
+ # include lines before the current one.
+ if p6_sorry_search: # Already handled
+ pass
+ else:
+ p6_sorry_search = True
+ p6_sorry_index = n - 1
+ while p6_sorry_index >= 0:
+ if not err_ud[p6_sorry_index].startswith('===SORRY!==='):
+ p6_sorry_index -= 1
+ continue
+ if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+ p6_linenum_offset = index_now[1].lines_total
+ else:
+ p6_linenum_offset = index_now[1].lines_total - index_now[1].lines_user + index_now[1].inline_count
+ p6_preceding_err_lines = [sub(r'line ([1-9][0-9]*)', lambda m: 'line {0}'.format(int(m.group(1)) - p6_linenum_offset), x) for x in err_ud[p6_sorry_index:n]]
+ if stderrfilename == 'full':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, basename)
+ elif stderrfilename == 'session':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, session)
+ elif stderrfilename == 'genericfile':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<file>')
+ elif stderrfilename == 'genericscript':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<script>')
+ err_dict[err_key].extend(p6_preceding_err_lines)
+ break
err_dict[err_key].append(line)
elif process:
err_dict[err_key].append(line)
@@ -1967,7 +2058,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# Never process delimiting info until it is used
# Rather, store the index of the last delimiter
last_delim = line
- elif basename in line:
+ elif basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
found_basename = True
# Get the gobbleation. This is used to determine if
# other lines containing the basename are a continuation,
@@ -2122,7 +2213,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
else:
process = True
err_key = basename + '_' + instance
- elif process and basename in line:
+ elif process and basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
found = False
for pattern in linesig:
try:
@@ -2297,6 +2388,8 @@ def do_pygments(encoding, outputdir, fvextfile, pygments_list,
if codetype != ':GLOBAL':
p = pygments_settings[codetype]['formatter_options'].copy()
p['commandprefix'] = 'PYG'
+ if pygments_settings[codetype]['lexer'] == 'pycon':
+ p['python3'] = True
formatter[codetype] = LatexFormatter(**p)
lexer[codetype] = get_lexer_by_name(pygments_settings[codetype]['lexer'], **p)
@@ -2503,6 +2596,8 @@ def python_console(jobname, encoding, outputdir, workingdir, fvextfile,
from pygments.formatters import LatexFormatter
p = pygments_settings['formatter_options'].copy()
p['commandprefix'] = 'PYG'
+ if pygments_settings['lexer'] == 'pycon':
+ p['python3'] = True
formatter = LatexFormatter(**p)
lexer = get_lexer_by_name(pygments_settings['lexer'], **p)
else:
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex3.py b/Master/texmf-dist/scripts/pythontex/pythontex3.py
index cb96f97d336..bc61f912ca6 100755
--- a/Master/texmf-dist/scripts/pythontex/pythontex3.py
+++ b/Master/texmf-dist/scripts/pythontex/pythontex3.py
@@ -13,7 +13,7 @@ should be in the same directory.
Licensed under the BSD 3-Clause License:
-Copyright (c) 2012-2017, Geoffrey M. Poore
+Copyright (c) 2012-2019, Geoffrey M. Poore
All rights reserved.
@@ -62,6 +62,7 @@ from pygments.styles import get_all_styles
from pythontex_engines import *
import textwrap
import platform
+import itertools
if sys.version_info[0] == 2:
try:
@@ -77,7 +78,7 @@ else:
# Script parameters
# Version
-__version__ = '0.16'
+__version__ = '0.17'
@@ -1213,9 +1214,13 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
# Must double-escape any backslashes so that they survive `shlex.split()`
script = basename
if os.path.isabs(os.path.expanduser(os.path.normcase(outputdir))):
- script_full = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+ script_full = os.path.expanduser(os.path.join(outputdir, basename))
else:
- script_full = os.path.expanduser(os.path.normcase(os.path.join(orig_cwd, outputdir, basename)))
+ script_full = os.path.expanduser(os.path.join(orig_cwd, outputdir, basename))
+ if platform.system() == 'Windows':
+ script_full = script_full.replace('/', '\\')
+ else:
+ script_full = script_full.replace('\\', '/')
# `shlex.split()` only works with Unicode after 2.7.2
if (sys.version_info.major == 2 and sys.version_info.micro < 3):
exec_cmd = shlex.split(bytes(command.format(file=script.replace('\\', '\\\\'), File=script_full.replace('\\', '\\\\'))))
@@ -1275,7 +1280,11 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
for key in code_dict:
family = key.split('#')[0]
# Uncomment the following for debugging, and comment out what follows
- '''run_code(encoding, outputdir, workingdir, code_dict[key],
+ '''run_code(encoding, outputdir,
+ workingdir,
+ cc_dict_begin[family],
+ code_dict[key],
+ cc_dict_end[family],
engine_dict[family].language,
engine_dict[family].commands,
engine_dict[family].created,
@@ -1287,9 +1296,12 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
engine_dict[family].linenumbers,
engine_dict[family].lookbehind,
keeptemps, hashdependencies,
- pygments_settings)'''
+ pygments_settings]))'''
tasks.append(pool.apply_async(run_code, [encoding, outputdir,
- workingdir, code_dict[key],
+ workingdir,
+ cc_dict_begin[family],
+ code_dict[key],
+ cc_dict_end[family],
engine_dict[family].language,
engine_dict[family].commands,
engine_dict[family].created,
@@ -1412,7 +1424,7 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
unresolved_sessions = []
for key in dependencies:
for dep, val in dependencies[key].items():
- if val[0] > start_time:
+ if val[0] is None or val[0] > start_time:
unresolved_dependencies = True
dependencies[key][dep] = (None, None)
unresolved_sessions.append(key.replace('#', ':'))
@@ -1474,7 +1486,8 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
-def run_code(encoding, outputdir, workingdir, code_list, language, commands,
+def run_code(encoding, outputdir, workingdir,
+ cc_begin_list, code_list, cc_end_list, language, commands,
command_created, extension, makestderr, stderrfilename,
code_index, errorsig, warningsig, linesig, stderrlookbehind,
keeptemps, hashdependencies, pygments_settings):
@@ -1512,11 +1525,19 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
err_file_name = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename + '.err')))
out_file = open(out_file_name, 'w', encoding=encoding)
err_file = open(err_file_name, 'w', encoding=encoding)
- script = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+ script = os.path.expanduser(os.path.join(outputdir, basename))
+ if platform.system() == 'Windows':
+ script = script.replace('/', '\\')
+ else:
+ script = script.replace('\\', '/')
if os.path.isabs(script):
script_full = script
else:
- script_full = os.path.expanduser(os.path.normcase(os.path.join(os.getcwd(), outputdir, basename)))
+ script_full = os.path.expanduser(os.path.join(os.getcwd(), outputdir, basename))
+ if platform.system() == 'Windows':
+ script_full = script_full.replace('/', '\\')
+ else:
+ script_full = script_full.replace('\\', '/')
# #### Need to revise so that intermediate files can be detected and cleaned up
for f in command_created:
files.append(f.format(file=script, File=script_full))
@@ -1535,7 +1556,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# Add any created files due to the command
# This needs to be done before attempts to execute, to prevent orphans
try:
- proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+ if family != 'Rcon':
+ proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+ else:
+ proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=subprocess.STDOUT)
except WindowsError as e:
if e.errno == 2:
# Batch files won't be found when called without extension. They
@@ -1546,7 +1570,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# under Windows; a list is not required.
exec_cmd_string = ' '.join(exec_cmd)
exec_cmd_string = 'cmd /C "@echo off & call {0} & if errorlevel 1 exit 1"'.format(exec_cmd_string)
- proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+ if family != 'Rcon':
+ proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+ else:
+ proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=subprocess.STDOUT)
else:
raise
@@ -1567,15 +1594,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
messages.append('* PythonTeX error')
messages.append(' Missing output file for ' + key_run.replace('#', ':'))
errors += 1
+ elif family == 'juliacon' and proc.returncode == 1:
+ messages.append('* PythonTeX error')
+ messages.append(' Running code for Julia console failed')
+ with open(err_file_name, encoding='utf8') as f:
+ messages.append(f.read())
+ errors += 1
else:
if family == 'juliacon':
with open(out_file_name.rsplit('.', 1)[0] + '.tex', 'r', encoding=encoding) as f:
tex_data_lines = f.readlines()
- inst = 0
+ code_iter = itertools.chain(cc_begin_list, code_list, cc_end_list)
for n, line in enumerate(tex_data_lines):
if line.rstrip() == '\\begin{juliaterm}':
- tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(inst)
- inst += 1
+ c = next(code_iter)
+ tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(c.instance)
if n != 0:
tex_data_lines[n-1] = ''
if line.rstrip() == '\\end{juliaterm}':
@@ -1583,6 +1616,29 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
tex_data_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
with open(out_file_name, 'w', encoding=encoding) as f:
f.write(''.join(tex_data_lines))
+ elif family == 'Rcon':
+ with open(out_file_name, 'r', encoding=encoding) as f:
+ stdout_lines = f.readlines()
+ for n, line in enumerate(stdout_lines):
+ if line.startswith('> =>PYTHONTEX:'):
+ stdout_lines[n] = line[2:]
+ elif '> write("=>PYTHONTEX:' in line:
+ if line.startswith('> write("=>PYTHONTEX:'):
+ stdout_lines[n] = ''
+ else:
+ # cat() and similar functions can result in the
+ # prompt not being at the start of a new line. In
+ # that case, preserve the prompt to accurately
+ # emulate the console. If there is a following
+ # console environment, this effectively amounts
+ # to adding an extra empty line (pressing ENTER)
+ # between the two.
+ stdout_lines[n] = line.split('write("=>PYTHONTEX:', 1)[0]
+ while stdout_lines and (stdout_lines[-1].startswith('>') and not stdout_lines[-1][1:].strip(' \n')):
+ stdout_lines.pop()
+ stdout_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
+ with open(out_file_name, 'w', encoding=encoding) as f:
+ f.write(''.join(stdout_lines))
f = open(out_file_name, 'r', encoding=encoding)
out = f.read()
@@ -1640,7 +1696,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
else:
dependencies[dep] = (os.path.getmtime(dep_file), '')
- if family == 'juliacon':
+ if family in ('juliacon', 'Rcon'):
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import LatexFormatter
@@ -1655,16 +1711,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
for block in out.split('=>PYTHONTEX:STDOUT#')[1:]:
if block:
delims, content = block.split('#\n', 1)
+ if content and not content.endswith('\n'):
+ # Content might not end with a newline. For example,
+ # Rcon with something like cat() as the last function.
+ content += '\n'
instance, command = delims.split('#')
if content or command in ('s', 'sub'):
if instance.endswith('CC'):
- messages.append('* PythonTeX warning')
- messages.append(' Custom code for "' + family + '" attempted to print or write to stdout')
- messages.append(' This is not supported; use a normal code command or environment')
- messages.append(' The following content was written:')
- messages.append('')
- messages.extend([' ' + l for l in content.splitlines()])
- warnings += 1
+ if family not in ('juliacon', 'Rcon'):
+ messages.append('* PythonTeX warning')
+ messages.append(' Custom code for "' + family + '" attempted to print or write to stdout')
+ messages.append(' This is not supported; use a normal code command or environment')
+ messages.append(' The following content was written:')
+ messages.append('')
+ messages.extend([' ' + l for l in content.splitlines()])
+ warnings += 1
elif command == 'i':
content = r'\pytx@SVMCR{pytx@MCR@' + key_run.replace('#', '@') + '@' + instance + '}\n' + content.rstrip('\n') + '\\endpytx@SVMCR\n\n'
macros.append(content)
@@ -1682,7 +1743,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# Remove newline added by printing, prevent
# LaTeX from adding a space after content
content = content.rsplit('\n', 1)[0] + '\\endinput\n'
- if family == 'juliacon':
+ if family in ('juliacon', 'Rcon'):
content = highlight(content, lexer[family], formatter[family])
f.write(content)
f.close()
@@ -1693,7 +1754,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
messages.append('* PythonTeX error')
messages.append(' Missing stderr file for ' + key_run.replace('#', ':'))
errors += 1
- elif family == 'juliacon':
+ elif family in ('juliacon', 'Rcon'):
pass
else:
# Open error and code files.
@@ -1733,7 +1794,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
index_next = index_now
start_errgobble = None
for n, line in enumerate(err_ud):
- if basename in line:
+ if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
# Get the gobbleation. This is used to determine if
# other lines containing the basename are a continuation,
# or separate messages.
@@ -1779,7 +1840,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# both the error and warning patterns, default to
# error.
past_line = err_ud[index]
- if (index < n and basename in past_line):
+ if (index < n and basename in past_line and (family not in ('perlsix', 'psix') or '.p6:' in past_line or '.p6 line' in past_line)):
break
for pattern in warningsig:
if pattern in past_line:
@@ -1843,8 +1904,9 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
index_now_last = index_now
index_next_last = index_next
err_key_last_int = -1
+ p6_sorry_search = False
for n, line in enumerate(err_ud):
- if basename in line:
+ if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
# Determine the corresponding line number in the document
found = False
for pattern in linesig:
@@ -1906,6 +1968,35 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
line = line.replace(fullbasename + '.' + extension, '<file>')
elif stderrfilename == 'genericscript':
line = line.replace(fullbasename + '.' + extension, '<script>')
+ if family in ('perlsix', 'psix'):
+ # Perl 6 "SORRY!" errors during compiling
+ # (before execution) need special processing,
+ # since they lack stderr delims and must
+ # include lines before the current one.
+ if p6_sorry_search: # Already handled
+ pass
+ else:
+ p6_sorry_search = True
+ p6_sorry_index = n - 1
+ while p6_sorry_index >= 0:
+ if not err_ud[p6_sorry_index].startswith('===SORRY!==='):
+ p6_sorry_index -= 1
+ continue
+ if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+ p6_linenum_offset = index_now[1].lines_total
+ else:
+ p6_linenum_offset = index_now[1].lines_total - index_now[1].lines_user + index_now[1].inline_count
+ p6_preceding_err_lines = [sub(r'line ([1-9][0-9]*)', lambda m: 'line {0}'.format(int(m.group(1)) - p6_linenum_offset), x) for x in err_ud[p6_sorry_index:n]]
+ if stderrfilename == 'full':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, basename)
+ elif stderrfilename == 'session':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, session)
+ elif stderrfilename == 'genericfile':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<file>')
+ elif stderrfilename == 'genericscript':
+ p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<script>')
+ err_dict[err_key].extend(p6_preceding_err_lines)
+ break
err_dict[err_key].append(line)
elif process:
err_dict[err_key].append(line)
@@ -1967,7 +2058,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
# Never process delimiting info until it is used
# Rather, store the index of the last delimiter
last_delim = line
- elif basename in line:
+ elif basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
found_basename = True
# Get the gobbleation. This is used to determine if
# other lines containing the basename are a continuation,
@@ -2122,7 +2213,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
else:
process = True
err_key = basename + '_' + instance
- elif process and basename in line:
+ elif process and basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
found = False
for pattern in linesig:
try:
@@ -2297,6 +2388,8 @@ def do_pygments(encoding, outputdir, fvextfile, pygments_list,
if codetype != ':GLOBAL':
p = pygments_settings[codetype]['formatter_options'].copy()
p['commandprefix'] = 'PYG'
+ if pygments_settings[codetype]['lexer'] == 'pycon':
+ p['python3'] = True
formatter[codetype] = LatexFormatter(**p)
lexer[codetype] = get_lexer_by_name(pygments_settings[codetype]['lexer'], **p)
@@ -2503,6 +2596,8 @@ def python_console(jobname, encoding, outputdir, workingdir, fvextfile,
from pygments.formatters import LatexFormatter
p = pygments_settings['formatter_options'].copy()
p['commandprefix'] = 'PYG'
+ if pygments_settings['lexer'] == 'pycon':
+ p['python3'] = True
formatter = LatexFormatter(**p)
lexer = get_lexer_by_name(pygments_settings['lexer'], **p)
else:
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex_engines.py b/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
index 38fea5efc43..4928826bf1c 100755
--- a/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
+++ b/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
@@ -17,7 +17,7 @@ document (script for execution).
-Copyright (c) 2012-2017, Geoffrey M. Poore
+Copyright (c) 2012-2018, Geoffrey M. Poore
All rights reserved.
Licensed under the BSD 3-Clause License:
http://www.opensource.org/licenses/BSD-3-Clause
@@ -33,7 +33,8 @@ from hashlib import sha1
from collections import OrderedDict, namedtuple
-interpreter_dict = {k:k for k in ('python', 'ruby', 'julia', 'octave', 'bash', 'sage', 'rustc')}
+interpreter_dict = {k:k for k in ('python', 'ruby', 'julia', 'octave', 'bash',
+ 'sage', 'rustc', 'Rscript', 'perl', 'perl6')}
# The {file} field needs to be replaced by itself, since the actual
# substitution of the real file can only be done at runtime, whereas the
# substitution for the interpreter should be done when the engine is
@@ -194,7 +195,7 @@ class CodeEngine(object):
if not isinstance(l, str):
raise TypeError('CodeEngine needs "linenumbers" to contain strings')
# Need to replace tags
- linenumbers = [l.replace('{number}', r'(\d+)') for l in linenumbers]
+ linenumbers = [r'(\d+)'.join(re.escape(x) for x in l.split('{number}')) if '{number}' in l else l for l in linenumbers]
self.linenumbers = linenumbers
# Type check lookbehind
@@ -950,7 +951,7 @@ julia_template = '''
# Currently, Julia only supports UTF-8
# So can't set stdout and stderr encoding
- type JuliaTeXUtils
+ mutable struct JuliaTeXUtils
id::AbstractString
family::AbstractString
session::AbstractString
@@ -983,8 +984,8 @@ julia_template = '''
function JuliaTeXUtils()
self = new()
self.self = self
- self._dependencies = Array(AbstractString, 0)
- self._created = Array(AbstractString, 0)
+ self._dependencies = AbstractString[]
+ self._created = AbstractString[]
self._context_raw = ""
function formatter(expr)
@@ -1095,7 +1096,7 @@ julia_wrapper = '''
jltex.line = "{line}"
println("{stdoutdelim}")
- write(STDERR, "{stderrdelim}\\n")
+ write(stderr, "{stderrdelim}\\n")
jltex.before()
{code}
@@ -1267,6 +1268,7 @@ CodeEngine('octave', 'octave', '.m',
octave_template, octave_wrapper, 'disp({code})', octave_sub,
'error', 'warning', 'line {number}')
+
bash_template = '''
cd "{workingdir}"
{body}
@@ -1297,7 +1299,7 @@ rust_template = '''
use std::{{borrow, collections, fmt, fs, io, iter, ops, path}};
use self::OpenMode::{{ReadMode, WriteMode, AppendMode, TruncateMode, CreateMode, CreateNewMode}};
pub struct UserAction<'u> {{
- _act: Box<FnMut() + 'u>
+ _act: Box<dyn FnMut() + 'u>
}}
impl<'u> UserAction<'u> {{
pub fn new() -> Self {{
@@ -1323,8 +1325,8 @@ rust_template = '''
impl<'u, U: Into<UserAction<'u>> + 'u> ops::Add<U> for UserAction<'u> {{
type Output = UserAction<'u>;
fn add(self, f: U) -> Self::Output {{
- let mut self_act: Box<FnMut() + 'u> = self._act;
- let mut other_act: Box<FnMut() + 'u> = f.into()._act;
+ let mut self_act: Box<dyn FnMut() + 'u> = self._act;
+ let mut other_act: Box<dyn FnMut() + 'u> = f.into()._act;
Self::from(move || {{ self_act.as_mut()(); other_act.as_mut()(); }})
}}
}}
@@ -1335,7 +1337,7 @@ rust_template = '''
}}
}}
impl<'u> ops::Deref for UserAction<'u> {{
- type Target = FnMut() + 'u;
+ type Target = dyn FnMut() + 'u;
fn deref(&self) -> &Self::Target {{
&*self._act
}}
@@ -1346,7 +1348,7 @@ rust_template = '''
}}
}}
pub struct RustTeXUtils<'u> {{
- _formatter: Box<FnMut(&fmt::Display) -> String + 'u>,
+ _formatter: Box<dyn FnMut(&dyn fmt::Display) -> String + 'u>,
pub before: UserAction<'u>,
pub after: UserAction<'u>,
pub family: &'u str,
@@ -1410,7 +1412,7 @@ rust_template = '''
impl<'u> RustTeXUtils<'u> {{
pub fn new() -> Self {{
RustTeXUtils {{
- _formatter: Box::new(|x: &fmt::Display| format!("{{}}", x)),
+ _formatter: Box::new(|x: &dyn fmt::Display| format!("{{}}", x)),
before: UserAction::new(),
after: UserAction::new(),
family: "{family}",
@@ -1428,7 +1430,7 @@ rust_template = '''
pub fn formatter<A: fmt::Display>(&mut self, x: A) -> String {{
(self._formatter)(&x)
}}
- pub fn set_formatter<F: FnMut(&fmt::Display) -> String + 'u>(&mut self, f: F) {{
+ pub fn set_formatter<F: FnMut(&dyn fmt::Display) -> String + 'u>(&mut self, f: F) {{
self._formatter = Box::new(f);
}}
pub fn add_dependencies<SS: IntoIterator>(&mut self, deps: SS)
@@ -1541,3 +1543,208 @@ CodeEngine('rust', 'rust', '.rs',
created='{File}.exe')
SubCodeEngine('rust', 'rs')
+
+
+r_template = '''
+ library(methods)
+ setwd("{workingdir}")
+ pdf(file=NULL)
+ {body}
+ write("{dependencies_delim}", stdout())
+ write("{created_delim}", stdout())
+ '''
+
+r_wrapper = '''
+ write("{stdoutdelim}", stdout())
+ write("{stderrdelim}", stderr())
+ {code}
+ '''
+
+r_sub = '''
+ write("{field_delim}", stdout())
+ write(toString({field}), stdout())
+ '''
+
+CodeEngine('R', 'R', '.R',
+ '{Rscript} "{file}.R"',
+ r_template, r_wrapper, 'write(toString({code}), stdout())', r_sub,
+ ['error', 'Error'], ['warning', 'Warning'],
+ 'line {number}')
+
+
+rcon_template = '''
+ options(echo=TRUE, error=function(){{}})
+ library(methods)
+ setwd("{workingdir}")
+ pdf(file=NULL)
+ {body}
+ '''
+
+rcon_wrapper = '''
+ write("{stdoutdelim}", stdout())
+ {code}
+ '''
+
+CodeEngine('Rcon', 'R', '.R',
+ '{Rscript} "{file}.R"',
+ rcon_template, rcon_wrapper, '', '',
+ ['error', 'Error'], ['warning', 'Warning'],
+ '')
+
+
+perl_template = '''
+ use v5.14;
+ use utf8;
+ use strict;
+ use autodie;
+ use warnings;
+ use warnings qw(FATAL utf8);
+ use feature qw(unicode_strings);
+ use open qw(:encoding(UTF-8) :std);
+ chdir("{workingdir}");
+ {body}
+ print STDOUT "{dependencies_delim}\\n";
+ print STDOUT "{created_delim}\\n";
+ '''
+
+perl_wrapper = '''
+ print STDOUT "{stdoutdelim}\\n";
+ print STDERR "{stderrdelim}\\n";
+ {code}
+ '''
+
+perl_sub = '''
+ print STDOUT "{field_delim}\\n";
+ print STDOUT "" . ({field});
+ '''
+
+CodeEngine('perl', 'perl', '.pl',
+ '{perl} "{file}.pl"',
+ perl_template, perl_wrapper, 'print STDOUT "" . ({code});', perl_sub,
+ ['error', 'Error'], ['warning', 'Warning'],
+ 'line {number}')
+
+SubCodeEngine('perl', 'pl')
+
+
+perl6_template = '''
+ use v6;
+ chdir("{workingdir}");
+ {body}
+ put "{dependencies_delim}";
+ put "{created_delim}";
+ '''
+
+perl6_wrapper = '''
+ put "{stdoutdelim}";
+ note "{stderrdelim}";
+ {code}
+ '''
+
+perl6_sub = '''
+ put "{field_delim}";
+ put ({field});
+ '''
+
+CodeEngine('perlsix', 'perl6', '.p6',
+ '{perl6} "{File}.p6"',
+ perl6_template, perl6_wrapper, 'put ({code});', perl6_sub,
+ ['error', 'Error', 'Cannot'], ['warning', 'Warning'],
+ ['.p6:{number}', '.p6 line {number}'], True)
+
+SubCodeEngine('perlsix', 'psix')
+
+javascript_template = '''
+ jstex = {{
+ before : function () {{ }},
+ after : function () {{ }},
+ _dependencies : [ ],
+ _created : [ ],
+ add_dependencies : function () {{
+ jstex._dependencies = jstex._dependencies.concat(
+ Array.prototype.slice.apply( arguments ) );
+ }},
+ add_created : function () {{
+ jstex._created = jstex._created.concat(
+ Array.prototype.slice.apply( arguments ) );
+ }},
+ cleanup : function () {{
+ console.log( "{dependencies_delim}" );
+ jstex._dependencies.map(
+ dep => console.log( dep ) );
+ console.log( "{created_delim}" );
+ jstex._dependencies.map(
+ cre => console.log( cre ) );
+ }},
+ formatter : function ( x ) {{
+ return String( x );
+ }},
+ escape : function ( x ) {{
+ return String( x ).replace( /_/g, '\\\\_' )
+ .replace( /\\$/g, '\\\\$' )
+ .replace( /\\^/g, '\\\\^' );
+ }},
+ docdir : process.cwd(),
+ context : {{ }},
+ _context_raw : '',
+ set_context : function ( expr ) {{
+ if ( expr != '' && expr != jstex._context_raw ) {{
+ jstex.context = {{ }};
+ expr.split( ',' ).map( pair => {{
+ const halves = pair.split( '=' );
+ jstex.context[halves[0].trim()] = halves[1].trim();
+ }} );
+ }}
+ }}
+ }};
+
+ try {{
+ process.chdir( "{workingdir}" );
+ }} catch ( e ) {{
+ if ( process.argv.indexOf( '--manual' ) == -1 )
+ console.error( e );
+ }}
+ if ( module.paths.indexOf( jstex.docdir ) == -1 )
+ module.paths.unshift( jstex.docdir );
+
+ {extend}
+
+ jstex.id = "{family}_{session}_{restart}";
+ jstex.family = "{family}";
+ jstex.session = "{session}";
+ jstex.restart = "{restart}";
+
+ {body}
+
+ jstex.cleanup();
+ '''
+
+javascript_wrapper = '''
+ jstex.command = "{command}";
+ jstex.set_context( "{context}" );
+ jstex.args = "{args}";
+ jstex.instance = "{instance}";
+ jstex.line = "{line}";
+
+ console.log( "{stdoutdelim}" );
+ console.error( "{stderrdelim}" );
+ jstex.before();
+
+ {code}
+
+ jstex.after();
+ '''
+
+javascript_sub = '''
+ console.log( "{field_delim}" );
+ console.log( {field} );
+ '''
+
+CodeEngine('javascript', 'javascript', '.js',
+ 'node "{file}.js"',
+ javascript_template, javascript_wrapper,
+ 'console.log( jstex.formatter( {code} ) )',
+ javascript_sub,
+ ['error', 'Error'], ['warning', 'Warning'],
+ ':{number}')
+