pythontex (24sep19)

git-svn-id: svn://tug.org/texlive/trunk@52174 c570f23f-e606-0410-a88d-b1316a301751

5 files changed, 475 insertions, 78 deletions

diff --git a/Master/texmf-dist/scripts/pythontex/depythontex2.py b/Master/texmf-dist/scripts/pythontex/depythontex2.py
index 7687e0e7244..f49fd229778 100755
--- a/Master/texmf-dist/scripts/pythontex/depythontex2.py
+++ b/Master/texmf-dist/scripts/pythontex/depythontex2.py
@@ -86,7 +86,7 @@ import codecs
 
 # Script parameters
 # Version
-__version__ = '0.16'
+__version__ = '0.17'
 
 
 # Functions and parameters for customizing the script output
diff --git a/Master/texmf-dist/scripts/pythontex/depythontex3.py b/Master/texmf-dist/scripts/pythontex/depythontex3.py
index bdcea2185e0..2d86edb5914 100755
--- a/Master/texmf-dist/scripts/pythontex/depythontex3.py
+++ b/Master/texmf-dist/scripts/pythontex/depythontex3.py
@@ -86,7 +86,7 @@ import codecs
 
 # Script parameters
 # Version
-__version__ = '0.16'
+__version__ = '0.17'
 
 
 # Functions and parameters for customizing the script output
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex2.py b/Master/texmf-dist/scripts/pythontex/pythontex2.py
index 871d394e375..34e151f4177 100755
--- a/Master/texmf-dist/scripts/pythontex/pythontex2.py
+++ b/Master/texmf-dist/scripts/pythontex/pythontex2.py
@@ -13,7 +13,7 @@ should be in the same directory.
 
 Licensed under the BSD 3-Clause License:
 
-Copyright (c) 2012-2017, Geoffrey M. Poore
+Copyright (c) 2012-2019, Geoffrey M. Poore
 
 All rights reserved.
 
@@ -62,6 +62,7 @@ from pygments.styles import get_all_styles
 from pythontex_engines import *
 import textwrap
 import platform
+import itertools
 
 if sys.version_info[0] == 2:
     try:
@@ -77,7 +78,7 @@ else:
 
 # Script parameters
 # Version
-__version__ = '0.16'
+__version__ = '0.17'
 
 
 
@@ -1213,9 +1214,13 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
         # Must double-escape any backslashes so that they survive `shlex.split()`
         script = basename
         if os.path.isabs(os.path.expanduser(os.path.normcase(outputdir))):
-            script_full = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+            script_full = os.path.expanduser(os.path.join(outputdir, basename))
         else:
-            script_full = os.path.expanduser(os.path.normcase(os.path.join(orig_cwd, outputdir, basename)))
+            script_full = os.path.expanduser(os.path.join(orig_cwd, outputdir, basename))
+        if platform.system() == 'Windows':
+            script_full = script_full.replace('/', '\\')
+        else:
+            script_full = script_full.replace('\\', '/')
         # `shlex.split()` only works with Unicode after 2.7.2
         if (sys.version_info.major == 2 and sys.version_info.micro < 3):
             exec_cmd = shlex.split(bytes(command.format(file=script.replace('\\', '\\\\'), File=script_full.replace('\\', '\\\\'))))
@@ -1275,7 +1280,11 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
     for key in code_dict:
         family = key.split('#')[0]
         # Uncomment the following for debugging, and comment out what follows
-        '''run_code(encoding, outputdir, workingdir, code_dict[key],
+        '''run_code(encoding, outputdir,
+                                                 workingdir,
+                                                 cc_dict_begin[family],
+                                                 code_dict[key],
+                                                 cc_dict_end[family],
                                                  engine_dict[family].language,
                                                  engine_dict[family].commands,
                                                  engine_dict[family].created,
@@ -1287,9 +1296,12 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
                                                  engine_dict[family].linenumbers,
                                                  engine_dict[family].lookbehind,
                                                  keeptemps, hashdependencies,
-                                                 pygments_settings)'''
+                                                 pygments_settings]))'''
         tasks.append(pool.apply_async(run_code, [encoding, outputdir,
-                                                 workingdir, code_dict[key],
+                                                 workingdir,
+                                                 cc_dict_begin[family],
+                                                 code_dict[key],
+                                                 cc_dict_end[family],
                                                  engine_dict[family].language,
                                                  engine_dict[family].commands,
                                                  engine_dict[family].created,
@@ -1412,7 +1424,7 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
     unresolved_sessions = []
     for key in dependencies:
         for dep, val in dependencies[key].items():
-            if val[0] > start_time:
+            if val[0] is None or val[0] > start_time:
                 unresolved_dependencies = True
                 dependencies[key][dep] = (None, None)
                 unresolved_sessions.append(key.replace('#', ':'))
@@ -1474,7 +1486,8 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
 
 
 
-def run_code(encoding, outputdir, workingdir, code_list, language, commands,
+def run_code(encoding, outputdir, workingdir,
+             cc_begin_list, code_list, cc_end_list, language, commands,
              command_created, extension, makestderr, stderrfilename,
              code_index, errorsig, warningsig, linesig, stderrlookbehind,
              keeptemps, hashdependencies, pygments_settings):
@@ -1512,11 +1525,19 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
     err_file_name = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename + '.err')))
     out_file = open(out_file_name, 'w', encoding=encoding)
     err_file = open(err_file_name, 'w', encoding=encoding)
-    script = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+    script = os.path.expanduser(os.path.join(outputdir, basename))
+    if platform.system() == 'Windows':
+        script = script.replace('/', '\\')
+    else:
+        script = script.replace('\\', '/')
     if os.path.isabs(script):
         script_full = script
     else:
-        script_full = os.path.expanduser(os.path.normcase(os.path.join(os.getcwd(), outputdir, basename)))
+        script_full = os.path.expanduser(os.path.join(os.getcwd(), outputdir, basename))
+        if platform.system() == 'Windows':
+            script_full = script_full.replace('/', '\\')
+        else:
+            script_full = script_full.replace('\\', '/')
     # #### Need to revise so that intermediate files can be detected and cleaned up
     for f in command_created:
         files.append(f.format(file=script, File=script_full))
@@ -1535,7 +1556,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
         # Add any created files due to the command
         # This needs to be done before attempts to execute, to prevent orphans
         try:
-            proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+            if family != 'Rcon':
+                proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+            else:
+                proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=subprocess.STDOUT)
         except WindowsError as e:
             if e.errno == 2:
                 # Batch files won't be found when called without extension. They
@@ -1546,7 +1570,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                 # under Windows; a list is not required.
                 exec_cmd_string = ' '.join(exec_cmd)
                 exec_cmd_string = 'cmd /C "@echo off & call {0} & if errorlevel 1 exit 1"'.format(exec_cmd_string)
-                proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+                if family != 'Rcon':
+                    proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+                else:
+                    proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=subprocess.STDOUT)
             else:
                 raise
 
@@ -1567,15 +1594,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
         messages.append('* PythonTeX error')
         messages.append('    Missing output file for ' + key_run.replace('#', ':'))
         errors += 1
+    elif family == 'juliacon' and proc.returncode == 1:
+        messages.append('* PythonTeX error')
+        messages.append('    Running code for Julia console failed')
+        with open(err_file_name, encoding='utf8') as f:
+            messages.append(f.read())
+        errors += 1
     else:
         if family == 'juliacon':
             with open(out_file_name.rsplit('.', 1)[0] + '.tex', 'r', encoding=encoding) as f:
                 tex_data_lines = f.readlines()
-            inst = 0
+            code_iter = itertools.chain(cc_begin_list, code_list, cc_end_list)
             for n, line in enumerate(tex_data_lines):
                 if line.rstrip() == '\\begin{juliaterm}':
-                    tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(inst)
-                    inst += 1
+                    c = next(code_iter)
+                    tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(c.instance)
                     if n != 0:
                         tex_data_lines[n-1] = ''
                 if line.rstrip() == '\\end{juliaterm}':
@@ -1583,6 +1616,29 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
             tex_data_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
             with open(out_file_name, 'w', encoding=encoding) as f:
                 f.write(''.join(tex_data_lines))
+        elif family == 'Rcon':
+            with open(out_file_name, 'r', encoding=encoding) as f:
+                stdout_lines = f.readlines()
+            for n, line in enumerate(stdout_lines):
+                if line.startswith('> =>PYTHONTEX:'):
+                    stdout_lines[n] = line[2:]
+                elif '> write("=>PYTHONTEX:' in line:
+                    if line.startswith('> write("=>PYTHONTEX:'):
+                        stdout_lines[n] = ''
+                    else:
+                        # cat() and similar functions can result in the
+                        # prompt not being at the start of a new line.  In
+                        # that case, preserve the prompt to accurately
+                        # emulate the console.  If there is a following
+                        # console environment, this effectively amounts
+                        # to adding an extra empty line (pressing ENTER)
+                        # between the two.
+                        stdout_lines[n] = line.split('write("=>PYTHONTEX:', 1)[0]
+            while stdout_lines and (stdout_lines[-1].startswith('>') and not stdout_lines[-1][1:].strip(' \n')):
+                stdout_lines.pop()
+            stdout_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
+            with open(out_file_name, 'w', encoding=encoding) as f:
+                f.write(''.join(stdout_lines))
 
         f = open(out_file_name, 'r', encoding=encoding)
         out = f.read()
@@ -1640,7 +1696,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                 else:
                     dependencies[dep] = (os.path.getmtime(dep_file), '')
 
-            if family == 'juliacon':
+            if family in ('juliacon', 'Rcon'):
                 from pygments import highlight
                 from pygments.lexers import get_lexer_by_name
                 from pygments.formatters import LatexFormatter
@@ -1655,16 +1711,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
             for block in out.split('=>PYTHONTEX:STDOUT#')[1:]:
                 if block:
                     delims, content = block.split('#\n', 1)
+                    if content and not content.endswith('\n'):
+                        # Content might not end with a newline.  For example,
+                        # Rcon with something like cat() as the last function.
+                        content += '\n'
                     instance, command = delims.split('#')
                     if content or command in ('s', 'sub'):
                         if instance.endswith('CC'):
-                            messages.append('* PythonTeX warning')
-                            messages.append('    Custom code for "' + family + '" attempted to print or write to stdout')
-                            messages.append('    This is not supported; use a normal code command or environment')
-                            messages.append('    The following content was written:')
-                            messages.append('')
-                            messages.extend(['    ' + l for l in content.splitlines()])
-                            warnings += 1
+                            if family not in ('juliacon', 'Rcon'):
+                                messages.append('* PythonTeX warning')
+                                messages.append('    Custom code for "' + family + '" attempted to print or write to stdout')
+                                messages.append('    This is not supported; use a normal code command or environment')
+                                messages.append('    The following content was written:')
+                                messages.append('')
+                                messages.extend(['    ' + l for l in content.splitlines()])
+                                warnings += 1
                         elif command == 'i':
                             content = r'\pytx@SVMCR{pytx@MCR@' + key_run.replace('#', '@') + '@' + instance + '}\n' + content.rstrip('\n') + '\\endpytx@SVMCR\n\n'
                             macros.append(content)
@@ -1682,7 +1743,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                                     # Remove newline added by printing, prevent
                                     # LaTeX from adding a space after content
                                     content = content.rsplit('\n', 1)[0] + '\\endinput\n'
-                            if family == 'juliacon':
+                            if family in ('juliacon', 'Rcon'):
                                 content = highlight(content, lexer[family], formatter[family])
                             f.write(content)
                             f.close()
@@ -1693,7 +1754,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
         messages.append('* PythonTeX error')
         messages.append('    Missing stderr file for ' + key_run.replace('#', ':'))
         errors += 1
-    elif family == 'juliacon':
+    elif family in ('juliacon', 'Rcon'):
         pass
     else:
         # Open error and code files.
@@ -1733,7 +1794,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
             index_next = index_now
             start_errgobble = None
             for n, line in enumerate(err_ud):
-                if basename in line:
+                if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                     # Get the gobbleation.  This is used to determine if
                     # other lines containing the basename are a continuation,
                     # or separate messages.
@@ -1779,7 +1840,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                                 # both the error and warning patterns, default to
                                 # error.
                                 past_line = err_ud[index]
-                                if (index < n and basename in past_line):
+                                if (index < n and basename in past_line and (family not in ('perlsix', 'psix') or '.p6:' in past_line or '.p6 line' in past_line)):
                                     break
                                 for pattern in warningsig:
                                     if pattern in past_line:
@@ -1843,8 +1904,9 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                 index_now_last = index_now
                 index_next_last = index_next
                 err_key_last_int = -1
+                p6_sorry_search = False
                 for n, line in enumerate(err_ud):
-                    if basename in line:
+                    if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                         # Determine the corresponding line number in the document
                         found = False
                         for pattern in linesig:
@@ -1906,6 +1968,35 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                                 line = line.replace(fullbasename + '.' + extension, '<file>')
                             elif stderrfilename == 'genericscript':
                                 line = line.replace(fullbasename + '.' + extension, '<script>')
+                            if family in ('perlsix', 'psix'):
+                                # Perl 6 "SORRY!" errors during compiling
+                                # (before execution) need special processing,
+                                # since they lack stderr delims and must
+                                # include lines before the current one.
+                                if p6_sorry_search:  # Already handled
+                                    pass
+                                else:
+                                    p6_sorry_search = True
+                                    p6_sorry_index = n - 1
+                                    while p6_sorry_index >= 0:
+                                        if not err_ud[p6_sorry_index].startswith('===SORRY!==='):
+                                            p6_sorry_index -= 1
+                                            continue
+                                        if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+                                            p6_linenum_offset = index_now[1].lines_total
+                                        else:
+                                            p6_linenum_offset = index_now[1].lines_total - index_now[1].lines_user + index_now[1].inline_count
+                                        p6_preceding_err_lines = [sub(r'line ([1-9][0-9]*)', lambda m: 'line {0}'.format(int(m.group(1)) - p6_linenum_offset), x) for x in err_ud[p6_sorry_index:n]]
+                                        if stderrfilename == 'full':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, basename)
+                                        elif stderrfilename == 'session':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, session)
+                                        elif stderrfilename == 'genericfile':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<file>')
+                                        elif stderrfilename == 'genericscript':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<script>')
+                                        err_dict[err_key].extend(p6_preceding_err_lines)
+                                        break
                             err_dict[err_key].append(line)
                     elif process:
                         err_dict[err_key].append(line)
@@ -1967,7 +2058,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                     # Never process delimiting info until it is used
                     # Rather, store the index of the last delimiter
                     last_delim = line
-                elif basename in line:
+                elif basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                     found_basename = True
                     # Get the gobbleation.  This is used to determine if
                     # other lines containing the basename are a continuation,
@@ -2122,7 +2213,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                         else:
                             process = True
                             err_key = basename + '_' + instance
-                    elif process and basename in line:
+                    elif process and basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                         found = False
                         for pattern in linesig:
                             try:
@@ -2297,6 +2388,8 @@ def do_pygments(encoding, outputdir, fvextfile, pygments_list,
         if codetype != ':GLOBAL':
             p = pygments_settings[codetype]['formatter_options'].copy()
             p['commandprefix'] = 'PYG'
+            if pygments_settings[codetype]['lexer'] == 'pycon':
+                p['python3'] = True
             formatter[codetype] = LatexFormatter(**p)
             lexer[codetype] = get_lexer_by_name(pygments_settings[codetype]['lexer'], **p)
 
@@ -2503,6 +2596,8 @@ def python_console(jobname, encoding, outputdir, workingdir, fvextfile,
         from pygments.formatters import LatexFormatter
         p = pygments_settings['formatter_options'].copy()
         p['commandprefix'] = 'PYG'
+        if pygments_settings['lexer'] == 'pycon':
+            p['python3'] = True
         formatter = LatexFormatter(**p)
         lexer = get_lexer_by_name(pygments_settings['lexer'], **p)
     else:
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex3.py b/Master/texmf-dist/scripts/pythontex/pythontex3.py
index cb96f97d336..bc61f912ca6 100755
--- a/Master/texmf-dist/scripts/pythontex/pythontex3.py
+++ b/Master/texmf-dist/scripts/pythontex/pythontex3.py
@@ -13,7 +13,7 @@ should be in the same directory.
 
 Licensed under the BSD 3-Clause License:
 
-Copyright (c) 2012-2017, Geoffrey M. Poore
+Copyright (c) 2012-2019, Geoffrey M. Poore
 
 All rights reserved.
 
@@ -62,6 +62,7 @@ from pygments.styles import get_all_styles
 from pythontex_engines import *
 import textwrap
 import platform
+import itertools
 
 if sys.version_info[0] == 2:
     try:
@@ -77,7 +78,7 @@ else:
 
 # Script parameters
 # Version
-__version__ = '0.16'
+__version__ = '0.17'
 
 
 
@@ -1213,9 +1214,13 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
         # Must double-escape any backslashes so that they survive `shlex.split()`
         script = basename
         if os.path.isabs(os.path.expanduser(os.path.normcase(outputdir))):
-            script_full = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+            script_full = os.path.expanduser(os.path.join(outputdir, basename))
         else:
-            script_full = os.path.expanduser(os.path.normcase(os.path.join(orig_cwd, outputdir, basename)))
+            script_full = os.path.expanduser(os.path.join(orig_cwd, outputdir, basename))
+        if platform.system() == 'Windows':
+            script_full = script_full.replace('/', '\\')
+        else:
+            script_full = script_full.replace('\\', '/')
         # `shlex.split()` only works with Unicode after 2.7.2
         if (sys.version_info.major == 2 and sys.version_info.micro < 3):
             exec_cmd = shlex.split(bytes(command.format(file=script.replace('\\', '\\\\'), File=script_full.replace('\\', '\\\\'))))
@@ -1275,7 +1280,11 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
     for key in code_dict:
         family = key.split('#')[0]
         # Uncomment the following for debugging, and comment out what follows
-        '''run_code(encoding, outputdir, workingdir, code_dict[key],
+        '''run_code(encoding, outputdir,
+                                                 workingdir,
+                                                 cc_dict_begin[family],
+                                                 code_dict[key],
+                                                 cc_dict_end[family],
                                                  engine_dict[family].language,
                                                  engine_dict[family].commands,
                                                  engine_dict[family].created,
@@ -1287,9 +1296,12 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
                                                  engine_dict[family].linenumbers,
                                                  engine_dict[family].lookbehind,
                                                  keeptemps, hashdependencies,
-                                                 pygments_settings)'''
+                                                 pygments_settings]))'''
         tasks.append(pool.apply_async(run_code, [encoding, outputdir,
-                                                 workingdir, code_dict[key],
+                                                 workingdir,
+                                                 cc_dict_begin[family],
+                                                 code_dict[key],
+                                                 cc_dict_end[family],
                                                  engine_dict[family].language,
                                                  engine_dict[family].commands,
                                                  engine_dict[family].created,
@@ -1412,7 +1424,7 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
     unresolved_sessions = []
     for key in dependencies:
         for dep, val in dependencies[key].items():
-            if val[0] > start_time:
+            if val[0] is None or val[0] > start_time:
                 unresolved_dependencies = True
                 dependencies[key][dep] = (None, None)
                 unresolved_sessions.append(key.replace('#', ':'))
@@ -1474,7 +1486,8 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
 
 
 
-def run_code(encoding, outputdir, workingdir, code_list, language, commands,
+def run_code(encoding, outputdir, workingdir,
+             cc_begin_list, code_list, cc_end_list, language, commands,
              command_created, extension, makestderr, stderrfilename,
              code_index, errorsig, warningsig, linesig, stderrlookbehind,
              keeptemps, hashdependencies, pygments_settings):
@@ -1512,11 +1525,19 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
     err_file_name = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename + '.err')))
     out_file = open(out_file_name, 'w', encoding=encoding)
     err_file = open(err_file_name, 'w', encoding=encoding)
-    script = os.path.expanduser(os.path.normcase(os.path.join(outputdir, basename)))
+    script = os.path.expanduser(os.path.join(outputdir, basename))
+    if platform.system() == 'Windows':
+        script = script.replace('/', '\\')
+    else:
+        script = script.replace('\\', '/')
     if os.path.isabs(script):
         script_full = script
     else:
-        script_full = os.path.expanduser(os.path.normcase(os.path.join(os.getcwd(), outputdir, basename)))
+        script_full = os.path.expanduser(os.path.join(os.getcwd(), outputdir, basename))
+        if platform.system() == 'Windows':
+            script_full = script_full.replace('/', '\\')
+        else:
+            script_full = script_full.replace('\\', '/')
     # #### Need to revise so that intermediate files can be detected and cleaned up
     for f in command_created:
         files.append(f.format(file=script, File=script_full))
@@ -1535,7 +1556,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
         # Add any created files due to the command
         # This needs to be done before attempts to execute, to prevent orphans
         try:
-            proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+            if family != 'Rcon':
+                proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=err_file)
+            else:
+                proc = subprocess.Popen(exec_cmd, stdout=out_file, stderr=subprocess.STDOUT)
         except WindowsError as e:
             if e.errno == 2:
                 # Batch files won't be found when called without extension. They
@@ -1546,7 +1570,10 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                 # under Windows; a list is not required.
                 exec_cmd_string = ' '.join(exec_cmd)
                 exec_cmd_string = 'cmd /C "@echo off & call {0} & if errorlevel 1 exit 1"'.format(exec_cmd_string)
-                proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+                if family != 'Rcon':
+                    proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=err_file)
+                else:
+                    proc = subprocess.Popen(exec_cmd_string, stdout=out_file, stderr=subprocess.STDOUT)
             else:
                 raise
 
@@ -1567,15 +1594,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
         messages.append('* PythonTeX error')
         messages.append('    Missing output file for ' + key_run.replace('#', ':'))
         errors += 1
+    elif family == 'juliacon' and proc.returncode == 1:
+        messages.append('* PythonTeX error')
+        messages.append('    Running code for Julia console failed')
+        with open(err_file_name, encoding='utf8') as f:
+            messages.append(f.read())
+        errors += 1
     else:
         if family == 'juliacon':
             with open(out_file_name.rsplit('.', 1)[0] + '.tex', 'r', encoding=encoding) as f:
                 tex_data_lines = f.readlines()
-            inst = 0
+            code_iter = itertools.chain(cc_begin_list, code_list, cc_end_list)
             for n, line in enumerate(tex_data_lines):
                 if line.rstrip() == '\\begin{juliaterm}':
-                    tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(inst)
-                    inst += 1
+                    c = next(code_iter)
+                    tex_data_lines[n] = '=>PYTHONTEX:STDOUT#{0}#code#\n'.format(c.instance)
                     if n != 0:
                         tex_data_lines[n-1] = ''
                 if line.rstrip() == '\\end{juliaterm}':
@@ -1583,6 +1616,29 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
             tex_data_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
             with open(out_file_name, 'w', encoding=encoding) as f:
                 f.write(''.join(tex_data_lines))
+        elif family == 'Rcon':
+            with open(out_file_name, 'r', encoding=encoding) as f:
+                stdout_lines = f.readlines()
+            for n, line in enumerate(stdout_lines):
+                if line.startswith('> =>PYTHONTEX:'):
+                    stdout_lines[n] = line[2:]
+                elif '> write("=>PYTHONTEX:' in line:
+                    if line.startswith('> write("=>PYTHONTEX:'):
+                        stdout_lines[n] = ''
+                    else:
+                        # cat() and similar functions can result in the
+                        # prompt not being at the start of a new line.  In
+                        # that case, preserve the prompt to accurately
+                        # emulate the console.  If there is a following
+                        # console environment, this effectively amounts
+                        # to adding an extra empty line (pressing ENTER)
+                        # between the two.
+                        stdout_lines[n] = line.split('write("=>PYTHONTEX:', 1)[0]
+            while stdout_lines and (stdout_lines[-1].startswith('>') and not stdout_lines[-1][1:].strip(' \n')):
+                stdout_lines.pop()
+            stdout_lines.append('=>PYTHONTEX:DEPENDENCIES#\n=>PYTHONTEX:CREATED#\n')
+            with open(out_file_name, 'w', encoding=encoding) as f:
+                f.write(''.join(stdout_lines))
 
         f = open(out_file_name, 'r', encoding=encoding)
         out = f.read()
@@ -1640,7 +1696,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                 else:
                     dependencies[dep] = (os.path.getmtime(dep_file), '')
 
-            if family == 'juliacon':
+            if family in ('juliacon', 'Rcon'):
                 from pygments import highlight
                 from pygments.lexers import get_lexer_by_name
                 from pygments.formatters import LatexFormatter
@@ -1655,16 +1711,21 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
             for block in out.split('=>PYTHONTEX:STDOUT#')[1:]:
                 if block:
                     delims, content = block.split('#\n', 1)
+                    if content and not content.endswith('\n'):
+                        # Content might not end with a newline.  For example,
+                        # Rcon with something like cat() as the last function.
+                        content += '\n'
                     instance, command = delims.split('#')
                     if content or command in ('s', 'sub'):
                         if instance.endswith('CC'):
-                            messages.append('* PythonTeX warning')
-                            messages.append('    Custom code for "' + family + '" attempted to print or write to stdout')
-                            messages.append('    This is not supported; use a normal code command or environment')
-                            messages.append('    The following content was written:')
-                            messages.append('')
-                            messages.extend(['    ' + l for l in content.splitlines()])
-                            warnings += 1
+                            if family not in ('juliacon', 'Rcon'):
+                                messages.append('* PythonTeX warning')
+                                messages.append('    Custom code for "' + family + '" attempted to print or write to stdout')
+                                messages.append('    This is not supported; use a normal code command or environment')
+                                messages.append('    The following content was written:')
+                                messages.append('')
+                                messages.extend(['    ' + l for l in content.splitlines()])
+                                warnings += 1
                         elif command == 'i':
                             content = r'\pytx@SVMCR{pytx@MCR@' + key_run.replace('#', '@') + '@' + instance + '}\n' + content.rstrip('\n') + '\\endpytx@SVMCR\n\n'
                             macros.append(content)
@@ -1682,7 +1743,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                                     # Remove newline added by printing, prevent
                                     # LaTeX from adding a space after content
                                     content = content.rsplit('\n', 1)[0] + '\\endinput\n'
-                            if family == 'juliacon':
+                            if family in ('juliacon', 'Rcon'):
                                 content = highlight(content, lexer[family], formatter[family])
                             f.write(content)
                             f.close()
@@ -1693,7 +1754,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
         messages.append('* PythonTeX error')
         messages.append('    Missing stderr file for ' + key_run.replace('#', ':'))
         errors += 1
-    elif family == 'juliacon':
+    elif family in ('juliacon', 'Rcon'):
         pass
     else:
         # Open error and code files.
@@ -1733,7 +1794,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
             index_next = index_now
             start_errgobble = None
             for n, line in enumerate(err_ud):
-                if basename in line:
+                if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                     # Get the gobbleation.  This is used to determine if
                     # other lines containing the basename are a continuation,
                     # or separate messages.
@@ -1779,7 +1840,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                                 # both the error and warning patterns, default to
                                 # error.
                                 past_line = err_ud[index]
-                                if (index < n and basename in past_line):
+                                if (index < n and basename in past_line and (family not in ('perlsix', 'psix') or '.p6:' in past_line or '.p6 line' in past_line)):
                                     break
                                 for pattern in warningsig:
                                     if pattern in past_line:
@@ -1843,8 +1904,9 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                 index_now_last = index_now
                 index_next_last = index_next
                 err_key_last_int = -1
+                p6_sorry_search = False
                 for n, line in enumerate(err_ud):
-                    if basename in line:
+                    if basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                         # Determine the corresponding line number in the document
                         found = False
                         for pattern in linesig:
@@ -1906,6 +1968,35 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                                 line = line.replace(fullbasename + '.' + extension, '<file>')
                             elif stderrfilename == 'genericscript':
                                 line = line.replace(fullbasename + '.' + extension, '<script>')
+                            if family in ('perlsix', 'psix'):
+                                # Perl 6 "SORRY!" errors during compiling
+                                # (before execution) need special processing,
+                                # since they lack stderr delims and must
+                                # include lines before the current one.
+                                if p6_sorry_search:  # Already handled
+                                    pass
+                                else:
+                                    p6_sorry_search = True
+                                    p6_sorry_index = n - 1
+                                    while p6_sorry_index >= 0:
+                                        if not err_ud[p6_sorry_index].startswith('===SORRY!==='):
+                                            p6_sorry_index -= 1
+                                            continue
+                                        if errlinenum > index_now[1].lines_total + index_now[1].lines_input:
+                                            p6_linenum_offset = index_now[1].lines_total
+                                        else:
+                                            p6_linenum_offset = index_now[1].lines_total - index_now[1].lines_user + index_now[1].inline_count
+                                        p6_preceding_err_lines = [sub(r'line ([1-9][0-9]*)', lambda m: 'line {0}'.format(int(m.group(1)) - p6_linenum_offset), x) for x in err_ud[p6_sorry_index:n]]
+                                        if stderrfilename == 'full':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, basename)
+                                        elif stderrfilename == 'session':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename, session)
+                                        elif stderrfilename == 'genericfile':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<file>')
+                                        elif stderrfilename == 'genericscript':
+                                            p6_preceding_err_lines[0] = p6_preceding_err_lines[0].replace(fullbasename + '.' + extension, '<script>')
+                                        err_dict[err_key].extend(p6_preceding_err_lines)
+                                        break
                             err_dict[err_key].append(line)
                     elif process:
                         err_dict[err_key].append(line)
@@ -1967,7 +2058,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                     # Never process delimiting info until it is used
                     # Rather, store the index of the last delimiter
                     last_delim = line
-                elif basename in line:
+                elif basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                     found_basename = True
                     # Get the gobbleation.  This is used to determine if
                     # other lines containing the basename are a continuation,
@@ -2122,7 +2213,7 @@ def run_code(encoding, outputdir, workingdir, code_list, language, commands,
                         else:
                             process = True
                             err_key = basename + '_' + instance
-                    elif process and basename in line:
+                    elif process and basename in line and (family not in ('perlsix', 'psix') or '.p6:' in line or '.p6 line' in line):
                         found = False
                         for pattern in linesig:
                             try:
@@ -2297,6 +2388,8 @@ def do_pygments(encoding, outputdir, fvextfile, pygments_list,
         if codetype != ':GLOBAL':
             p = pygments_settings[codetype]['formatter_options'].copy()
             p['commandprefix'] = 'PYG'
+            if pygments_settings[codetype]['lexer'] == 'pycon':
+                p['python3'] = True
             formatter[codetype] = LatexFormatter(**p)
             lexer[codetype] = get_lexer_by_name(pygments_settings[codetype]['lexer'], **p)
 
@@ -2503,6 +2596,8 @@ def python_console(jobname, encoding, outputdir, workingdir, fvextfile,
         from pygments.formatters import LatexFormatter
         p = pygments_settings['formatter_options'].copy()
         p['commandprefix'] = 'PYG'
+        if pygments_settings['lexer'] == 'pycon':
+            p['python3'] = True
         formatter = LatexFormatter(**p)
         lexer = get_lexer_by_name(pygments_settings['lexer'], **p)
     else:
diff --git a/Master/texmf-dist/scripts/pythontex/pythontex_engines.py b/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
index 38fea5efc43..4928826bf1c 100755
--- a/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
+++ b/Master/texmf-dist/scripts/pythontex/pythontex_engines.py
@@ -17,7 +17,7 @@ document (script for execution).
 
 
 
-Copyright (c) 2012-2017, Geoffrey M. Poore
+Copyright (c) 2012-2018, Geoffrey M. Poore
 All rights reserved.
 Licensed under the BSD 3-Clause License:
     http://www.opensource.org/licenses/BSD-3-Clause
@@ -33,7 +33,8 @@ from hashlib import sha1
 from collections import OrderedDict, namedtuple
 
 
-interpreter_dict = {k:k for k in ('python', 'ruby', 'julia', 'octave', 'bash', 'sage', 'rustc')}
+interpreter_dict = {k:k for k in ('python', 'ruby', 'julia', 'octave', 'bash',
+                                  'sage', 'rustc', 'Rscript', 'perl', 'perl6')}
 # The {file} field needs to be replaced by itself, since the actual
 # substitution of the real file can only be done at runtime, whereas the
 # substitution for the interpreter should be done when the engine is
@@ -194,7 +195,7 @@ class CodeEngine(object):
                 if not isinstance(l, str):
                     raise TypeError('CodeEngine needs "linenumbers" to contain strings')
         # Need to replace tags
-        linenumbers = [l.replace('{number}', r'(\d+)') for l in linenumbers]
+        linenumbers = [r'(\d+)'.join(re.escape(x) for x in l.split('{number}')) if '{number}' in l else l for l in linenumbers]
         self.linenumbers = linenumbers
 
         # Type check lookbehind
@@ -950,7 +951,7 @@ julia_template = '''
     # Currently, Julia only supports UTF-8
     # So can't set stdout and stderr encoding
 
-    type JuliaTeXUtils
+    mutable struct JuliaTeXUtils
         id::AbstractString
         family::AbstractString
         session::AbstractString
@@ -983,8 +984,8 @@ julia_template = '''
         function JuliaTeXUtils()
             self = new()
             self.self = self
-            self._dependencies = Array(AbstractString, 0)
-            self._created = Array(AbstractString, 0)
+            self._dependencies = AbstractString[]
+            self._created = AbstractString[]
             self._context_raw = ""
 
             function formatter(expr)
@@ -1095,7 +1096,7 @@ julia_wrapper = '''
     jltex.line = "{line}"
 
     println("{stdoutdelim}")
-    write(STDERR, "{stderrdelim}\\n")
+    write(stderr, "{stderrdelim}\\n")
     jltex.before()
 
     {code}
@@ -1267,6 +1268,7 @@ CodeEngine('octave', 'octave', '.m',
            octave_template, octave_wrapper, 'disp({code})', octave_sub,
            'error', 'warning', 'line {number}')
 
+
 bash_template = '''
     cd "{workingdir}"
     {body}
@@ -1297,7 +1299,7 @@ rust_template = '''
         use std::{{borrow, collections, fmt, fs, io, iter, ops, path}};
         use self::OpenMode::{{ReadMode, WriteMode, AppendMode, TruncateMode, CreateMode, CreateNewMode}};
         pub struct UserAction<'u> {{
-            _act: Box<FnMut() + 'u>
+            _act: Box<dyn FnMut() + 'u>
         }}
         impl<'u> UserAction<'u> {{
             pub fn new() -> Self {{
@@ -1323,8 +1325,8 @@ rust_template = '''
         impl<'u, U: Into<UserAction<'u>> + 'u> ops::Add<U> for UserAction<'u> {{
             type Output = UserAction<'u>;
             fn add(self, f: U) -> Self::Output {{
-                let mut self_act: Box<FnMut() + 'u> = self._act;
-                let mut other_act: Box<FnMut() + 'u> = f.into()._act;
+                let mut self_act: Box<dyn FnMut() + 'u> = self._act;
+                let mut other_act: Box<dyn FnMut() + 'u> = f.into()._act;
                 Self::from(move || {{ self_act.as_mut()(); other_act.as_mut()(); }})
             }}
         }}
@@ -1335,7 +1337,7 @@ rust_template = '''
             }}
         }}
         impl<'u> ops::Deref for UserAction<'u> {{
-            type Target = FnMut() + 'u;
+            type Target = dyn FnMut() + 'u;
             fn deref(&self) -> &Self::Target {{
                 &*self._act
             }}
@@ -1346,7 +1348,7 @@ rust_template = '''
             }}
         }}
         pub struct RustTeXUtils<'u> {{
-            _formatter: Box<FnMut(&fmt::Display) -> String + 'u>,
+            _formatter: Box<dyn FnMut(&dyn fmt::Display) -> String + 'u>,
             pub before: UserAction<'u>,
             pub after: UserAction<'u>,
             pub family: &'u str,
@@ -1410,7 +1412,7 @@ rust_template = '''
         impl<'u> RustTeXUtils<'u> {{
             pub fn new() -> Self {{
                 RustTeXUtils {{
-                    _formatter: Box::new(|x: &fmt::Display| format!("{{}}", x)),
+                    _formatter: Box::new(|x: &dyn fmt::Display| format!("{{}}", x)),
                     before: UserAction::new(),
                     after: UserAction::new(),
                     family: "{family}",
@@ -1428,7 +1430,7 @@ rust_template = '''
             pub fn formatter<A: fmt::Display>(&mut self, x: A) -> String {{
                 (self._formatter)(&x)
             }}
-            pub fn set_formatter<F: FnMut(&fmt::Display) -> String + 'u>(&mut self, f: F) {{
+            pub fn set_formatter<F: FnMut(&dyn fmt::Display) -> String + 'u>(&mut self, f: F) {{
                 self._formatter = Box::new(f);
             }}
             pub fn add_dependencies<SS: IntoIterator>(&mut self, deps: SS)
@@ -1541,3 +1543,208 @@ CodeEngine('rust', 'rust', '.rs',
            created='{File}.exe')
 
 SubCodeEngine('rust', 'rs')
+
+
+r_template = '''
+    library(methods)
+    setwd("{workingdir}")
+    pdf(file=NULL)
+    {body}
+    write("{dependencies_delim}", stdout())
+    write("{created_delim}", stdout())
+    '''
+
+r_wrapper = '''
+    write("{stdoutdelim}", stdout())
+    write("{stderrdelim}", stderr())
+    {code}
+    '''
+
+r_sub = '''
+    write("{field_delim}", stdout())
+    write(toString({field}), stdout())
+    '''
+
+CodeEngine('R', 'R', '.R',
+           '{Rscript} "{file}.R"',
+           r_template, r_wrapper, 'write(toString({code}), stdout())', r_sub,
+           ['error', 'Error'], ['warning', 'Warning'],
+           'line {number}')
+
+
+rcon_template = '''
+    options(echo=TRUE, error=function(){{}})
+    library(methods)
+    setwd("{workingdir}")
+    pdf(file=NULL)
+    {body}
+    '''
+
+rcon_wrapper = '''
+    write("{stdoutdelim}", stdout())
+    {code}
+    '''
+
+CodeEngine('Rcon', 'R', '.R',
+           '{Rscript} "{file}.R"',
+           rcon_template, rcon_wrapper, '', '',
+           ['error', 'Error'], ['warning', 'Warning'],
+           '')
+
+
+perl_template = '''
+    use v5.14;
+    use utf8;
+    use strict;
+    use autodie;
+    use warnings;
+    use warnings qw(FATAL utf8);
+    use feature qw(unicode_strings);
+    use open qw(:encoding(UTF-8) :std);
+    chdir("{workingdir}");
+    {body}
+    print STDOUT "{dependencies_delim}\\n";
+    print STDOUT "{created_delim}\\n";
+    '''
+
+perl_wrapper = '''
+    print STDOUT "{stdoutdelim}\\n";
+    print STDERR "{stderrdelim}\\n";
+    {code}
+    '''
+
+perl_sub = '''
+    print STDOUT "{field_delim}\\n";
+    print STDOUT "" . ({field});
+    '''
+
+CodeEngine('perl', 'perl', '.pl',
+           '{perl} "{file}.pl"',
+           perl_template, perl_wrapper, 'print STDOUT "" . ({code});', perl_sub,
+           ['error', 'Error'], ['warning', 'Warning'],
+           'line {number}')
+
+SubCodeEngine('perl', 'pl')
+
+
+perl6_template = '''
+    use v6;
+    chdir("{workingdir}");
+    {body}
+    put "{dependencies_delim}";
+    put "{created_delim}";
+    '''
+
+perl6_wrapper = '''
+    put "{stdoutdelim}";
+    note "{stderrdelim}";
+    {code}
+    '''
+
+perl6_sub = '''
+    put "{field_delim}";
+    put ({field});
+    '''
+
+CodeEngine('perlsix', 'perl6', '.p6',
+           '{perl6} "{File}.p6"',
+           perl6_template, perl6_wrapper, 'put ({code});', perl6_sub,
+           ['error', 'Error', 'Cannot'], ['warning', 'Warning'],
+           ['.p6:{number}', '.p6 line {number}'], True)
+
+SubCodeEngine('perlsix', 'psix')
+
+javascript_template = '''
+    jstex = {{
+        before : function () {{ }},
+        after : function () {{ }},
+        _dependencies : [ ],
+        _created : [ ],
+        add_dependencies : function () {{
+            jstex._dependencies = jstex._dependencies.concat(
+                Array.prototype.slice.apply( arguments ) );
+        }},
+        add_created : function () {{
+            jstex._created = jstex._created.concat(
+                Array.prototype.slice.apply( arguments ) );
+        }},
+        cleanup : function () {{
+            console.log( "{dependencies_delim}" );
+            jstex._dependencies.map(
+                dep => console.log( dep ) );
+            console.log( "{created_delim}" );
+            jstex._dependencies.map(
+                cre => console.log( cre ) );
+        }},
+        formatter : function ( x ) {{
+            return String( x );
+        }},
+        escape : function ( x ) {{
+            return String( x ).replace( /_/g, '\\\\_' )
+                              .replace( /\\$/g, '\\\\$' )
+                              .replace( /\\^/g, '\\\\^' );
+        }},
+        docdir : process.cwd(),
+        context : {{ }},
+        _context_raw : '',
+        set_context : function ( expr ) {{
+            if ( expr != '' && expr != jstex._context_raw ) {{
+                jstex.context = {{ }};
+                expr.split( ',' ).map( pair => {{
+                    const halves = pair.split( '=' );
+                    jstex.context[halves[0].trim()] = halves[1].trim();
+                }} );
+            }}
+        }}
+    }};
+
+    try {{
+        process.chdir( "{workingdir}" );
+    }} catch ( e ) {{
+        if ( process.argv.indexOf( '--manual' ) == -1 )
+            console.error( e );
+    }}
+    if ( module.paths.indexOf( jstex.docdir ) == -1 )
+        module.paths.unshift( jstex.docdir );
+
+    {extend}
+
+    jstex.id = "{family}_{session}_{restart}";
+    jstex.family = "{family}";
+    jstex.session = "{session}";
+    jstex.restart = "{restart}";
+
+    {body}
+
+    jstex.cleanup();
+    '''
+
+javascript_wrapper = '''
+    jstex.command = "{command}";
+    jstex.set_context( "{context}" );
+    jstex.args = "{args}";
+    jstex.instance = "{instance}";
+    jstex.line = "{line}";
+
+    console.log( "{stdoutdelim}" );
+    console.error( "{stderrdelim}" );
+    jstex.before();
+
+    {code}
+
+    jstex.after();
+    '''
+
+javascript_sub = '''
+    console.log( "{field_delim}" );
+    console.log( {field} );
+    '''
+
+CodeEngine('javascript', 'javascript', '.js',
+           'node "{file}.js"',
+           javascript_template, javascript_wrapper,
+           'console.log( jstex.formatter( {code} ) )',
+           javascript_sub,
+           ['error', 'Error'], ['warning', 'Warning'],
+           ':{number}')
+