summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts/pygmentex
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2014-08-20 21:53:18 +0000
committerKarl Berry <karl@freefriends.org>2014-08-20 21:53:18 +0000
commit95d91547803a6ceab929ef723dfdebe0e31e7a7e (patch)
treeb0bab3bb1fed9f52a46cdb1f36a29d516c3ae2fe /Master/texmf-dist/scripts/pygmentex
parent1dd6e5bec91358ff6bcd1964121979187a676cb4 (diff)
pygmentex (12aug14)
git-svn-id: svn://tug.org/texlive/trunk@34996 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts/pygmentex')
-rwxr-xr-xMaster/texmf-dist/scripts/pygmentex/pygmentex.py523
1 files changed, 523 insertions, 0 deletions
diff --git a/Master/texmf-dist/scripts/pygmentex/pygmentex.py b/Master/texmf-dist/scripts/pygmentex/pygmentex.py
new file mode 100755
index 00000000000..42cef96faea
--- /dev/null
+++ b/Master/texmf-dist/scripts/pygmentex/pygmentex.py
@@ -0,0 +1,523 @@
+#! /usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+"""
+ PygmenTeX
+ ~~~~~~~~~
+
+ PygmenTeX is a converter that do syntax highlighting of snippets of
+ source code extracted from a LaTeX file.
+
+ :copyright: Copyright 2014 by José Romildo Malaquias
+ :license: BSD, see LICENSE for details
+"""
+
+__version__ = '0.8'
+__docformat__ = 'restructuredtext'
+
+import sys
+import getopt
+import re
+from os.path import splitext
+
+from pygments import highlight
+from pygments.styles import get_style_by_name
+from pygments.lexers import get_lexer_by_name
+from pygments.formatters.latex import LatexFormatter, escape_tex, _get_ttype_name
+from pygments.util import get_bool_opt, get_int_opt
+from pygments.lexer import Lexer
+from pygments.token import Token
+
+###################################################
+# The following code is in >=pygments-2.0
+###################################################
+class EnhancedLatexFormatter(LatexFormatter):
+ r"""
+ This is an enhanced LaTeX formatter.
+ """
+ name = 'EnhancedLaTeX'
+ aliases = []
+
+ def __init__(self, **options):
+ LatexFormatter.__init__(self, **options)
+ self.escapeinside = options.get('escapeinside', '')
+ if len(self.escapeinside) == 2:
+ self.left = self.escapeinside[0]
+ self.right = self.escapeinside[1]
+ else:
+ self.escapeinside = ''
+
+ def format_unencoded(self, tokensource, outfile):
+ # TODO: add support for background colors
+ t2n = self.ttype2name
+ cp = self.commandprefix
+
+ if self.full:
+ realoutfile = outfile
+ outfile = StringIO()
+
+ outfile.write(u'\\begin{Verbatim}[commandchars=\\\\\\{\\}')
+ if self.linenos:
+ start, step = self.linenostart, self.linenostep
+ outfile.write(u',numbers=left' +
+ (start and u',firstnumber=%d' % start or u'') +
+ (step and u',stepnumber=%d' % step or u''))
+ if self.mathescape or self.texcomments or self.escapeinside:
+ outfile.write(u',codes={\\catcode`\\$=3\\catcode`\\^=7\\catcode`\\_=8}')
+ if self.verboptions:
+ outfile.write(u',' + self.verboptions)
+ outfile.write(u']\n')
+
+ for ttype, value in tokensource:
+ if ttype in Token.Comment:
+ if self.texcomments:
+ # Try to guess comment starting lexeme and escape it ...
+ start = value[0:1]
+ for i in xrange(1, len(value)):
+ if start[0] != value[i]:
+ break
+ start += value[i]
+
+ value = value[len(start):]
+ start = escape_tex(start, self.commandprefix)
+
+ # ... but do not escape inside comment.
+ value = start + value
+ elif self.mathescape:
+ # Only escape parts not inside a math environment.
+ parts = value.split('$')
+ in_math = False
+ for i, part in enumerate(parts):
+ if not in_math:
+ parts[i] = escape_tex(part, self.commandprefix)
+ in_math = not in_math
+ value = '$'.join(parts)
+ elif self.escapeinside:
+ text = value
+ value = ''
+ while len(text) > 0:
+ a,sep1,text = text.partition(self.left)
+ if len(sep1) > 0:
+ b,sep2,text = text.partition(self.right)
+ if len(sep2) > 0:
+ value += escape_tex(a, self.commandprefix) + b
+ else:
+ value += escape_tex(a + sep1 + b, self.commandprefix)
+ else:
+ value = value + escape_tex(a, self.commandprefix)
+ else:
+ value = escape_tex(value, self.commandprefix)
+ elif ttype not in Token.Escape:
+ value = escape_tex(value, self.commandprefix)
+ styles = []
+ while ttype is not Token:
+ try:
+ styles.append(t2n[ttype])
+ except KeyError:
+ # not in current style
+ styles.append(_get_ttype_name(ttype))
+ ttype = ttype.parent
+ styleval = '+'.join(reversed(styles))
+ if styleval:
+ spl = value.split('\n')
+ for line in spl[:-1]:
+ if line:
+ outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
+ outfile.write('\n')
+ if spl[-1]:
+ outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
+ else:
+ outfile.write(value)
+
+ outfile.write(u'\\end{Verbatim}\n')
+
+ if self.full:
+ realoutfile.write(DOC_TEMPLATE %
+ dict(docclass = self.docclass,
+ preamble = self.preamble,
+ title = self.title,
+ encoding = self.encoding or 'latin1',
+ styledefs = self.get_style_defs(),
+ code = outfile.getvalue()))
+
+class LatexEmbeddedLexer(Lexer):
+ r"""
+
+ This lexer takes one lexer as argument, the lexer for the language
+ being formatted, and the left and right delimiters for escaped text.
+
+ First everything is scanned using the language lexer to obtain
+ strings and comments. All other consecutive tokens are merged and
+ the resulting text is scanned for escaped segments, which are given
+ the Token.Escape type. Finally text that is not escaped is scanned
+ again with the language lexer.
+ """
+ def __init__(self, left, right, lang, **options):
+ self.left = left
+ self.right = right
+ self.lang = lang
+ Lexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ buf = ''
+ for i, t, v in self.lang.get_tokens_unprocessed(text):
+ if t in Token.Comment or t in Token.String:
+ if buf:
+ for x in self.get_tokens_aux(idx, buf):
+ yield x
+ buf = ''
+ yield i, t, v
+ else:
+ if not buf:
+ idx = i
+ buf += v
+ if buf:
+ for x in self.get_tokens_aux(idx, buf):
+ yield x
+
+ def get_tokens_aux(self, index, text):
+ while text:
+ a, sep1, text = text.partition(self.left)
+ if a:
+ for i, t, v in self.lang.get_tokens_unprocessed(a):
+ yield index + i, t, v
+ index += len(a)
+ if sep1:
+ b, sep2, text = text.partition(self.right)
+ if sep2:
+ yield index + len(sep1), Token.Escape, b
+ index += len(sep1) + len(b) + len(sep2)
+ else:
+ yield index, Token.Error, sep1
+ index += len(sep1)
+ text = b
+###################################################
+
+GENERIC_DEFINITIONS_1 = r'''% -*- mode: latex -*-
+
+\makeatletter
+
+\newdimen\LineNumberWidth
+'''
+
+GENERIC_DEFINITIONS_2 = r'''
+\makeatother
+'''
+
+
+INLINE_SNIPPET_TEMPLATE = r'''
+\expandafter\def\csname pygmented@snippet@%(number)s\endcsname{%%
+ \pygmented@snippet@inlined{%%
+%(body)s%%
+}}
+'''
+
+DISPLAY_SNIPPET_TEMPLATE = r'''
+\expandafter\def\csname pygmented@snippet@%(number)s\endcsname{%%
+ \begin{pygmented@snippet@framed}%%
+%(body)s%%
+ \end{pygmented@snippet@framed}%%
+}
+'''
+
+DISPLAY_LINENOS_SNIPPET_TEMPLATE = r'''
+\expandafter\def\csname pygmented@snippet@%(number)s\endcsname{%%
+ \begingroup
+ \def\pygmented@alllinenos{(%(linenumbers)s)}%%
+ \begin{pygmented@snippet@framed}%%
+%(body)s%%
+ \end{pygmented@snippet@framed}%%
+ \endgroup
+}
+'''
+
+
+def pyg(outfile, n, opts, extra_opts, text, usedstyles, inline_delim = ''):
+ try:
+ lexer = get_lexer_by_name(opts['lang'])
+ except ClassNotFound as err:
+ sys.stderr.write('Error: ')
+ sys.stderr.write(str(err))
+ return ""
+
+ # global _fmter
+ _fmter = EnhancedLatexFormatter()
+
+ escapeinside = opts.get('escapeinside', '')
+ if len(escapeinside) == 2:
+ left = escapeinside[0]
+ right = escapeinside[1]
+ _fmter.escapeinside = escapeinside
+ _fmter.left = left
+ _fmter.right = right
+ lexer = LatexEmbeddedLexer(left, right, lexer)
+
+ gobble = abs(get_int_opt(opts, 'gobble', 0))
+ if gobble:
+ lexer.add_filter('gobble', n=gobble)
+
+ tabsize = abs(get_int_opt(opts, 'tabsize', 0))
+ if tabsize:
+ lexer.tabsize = tabsize
+
+ encoding = opts['encoding']
+ if encoding == 'guess':
+ try:
+ import chardet
+ except ImportError:
+ try:
+ text = text.decode('utf-8')
+ if text.startswith(u'\ufeff'):
+ text = text[len(u'\ufeff'):]
+ encoding = 'utf-8'
+ except UnicodeDecodeError:
+ text = text.decode('latin1')
+ encoding = 'latin1'
+ else:
+ encoding = chardet.detect(text)['encoding']
+ text = text.decode(encoding)
+ else:
+ text = text.decode(encoding)
+
+ lexer.encoding = ''
+ _fmter.encoding = encoding
+
+ stylename = opts['sty']
+
+ _fmter.style = get_style_by_name(stylename)
+ _fmter._create_stylesheet()
+
+ _fmter.texcomments = get_bool_opt(opts, 'texcomments', False)
+ _fmter.mathescape = get_bool_opt(opts, 'mathescape', False)
+
+ if stylename not in usedstyles:
+ styledefs = _fmter.get_style_defs() \
+ .replace('#', '##') \
+ .replace(r'\##', r'\#') \
+ .replace(r'\makeatletter', '') \
+ .replace(r'\makeatother', '') \
+ .replace('\n', '%\n')
+ outfile.write(
+ '\\def\\PYstyle{0}{{%\n{1}%\n}}%\n'.format(stylename, styledefs))
+ usedstyles.append(stylename)
+
+ x = highlight(text, lexer, _fmter)
+
+ m = re.match(r'\\begin\{Verbatim}(.*)\n([\s\S]*?)\n\\end\{Verbatim}(\s*)\Z',
+ x)
+ if m:
+ linenos = get_bool_opt(opts, 'linenos', False)
+ linenostart = abs(get_int_opt(opts, 'linenostart', 1))
+ linenostep = abs(get_int_opt(opts, 'linenostep', 1))
+ lines0 = m.group(2).split('\n')
+ numbers = []
+ lines = []
+ counter = linenostart
+ for line in lines0:
+ line = re.sub(r'^ ', r'\\makebox[0pt]{\\phantom{Xy}} ', line)
+ line = re.sub(r' ', '~', line)
+ if linenos:
+ if (counter - linenostart) % linenostep == 0:
+ line = r'\pygmented@lineno@do{' + str(counter) + '}' + line
+ numbers.append(str(counter))
+ counter = counter + 1
+ lines.append(line)
+ if inline_delim:
+ outfile.write(INLINE_SNIPPET_TEMPLATE %
+ dict(number = n,
+ style = stylename,
+ options = extra_opts,
+ body = '\\newline\n'.join(lines)))
+ else:
+ if linenos:
+ template = DISPLAY_LINENOS_SNIPPET_TEMPLATE
+ else:
+ template = DISPLAY_SNIPPET_TEMPLATE
+ outfile.write(template %
+ dict(number = n,
+ style = stylename,
+ options = extra_opts,
+ linenosep = opts['linenosep'],
+ linenumbers = ','.join(numbers),
+ body = '\\newline\n'.join(lines)))
+
+
+
+def parse_opts(basedic, opts):
+ dic = basedic.copy()
+ for opt in re.split(r'\s*,\s*', opts):
+ x = re.split(r'\s*=\s*', opt)
+ if len(x) == 2 and x[0] and x[1]:
+ dic[x[0]] = x[1]
+ elif len(x) == 1 and x[0]:
+ dic[x[0]] = True
+ return dic
+
+
+
+_re_display = re.compile(
+ r'^<@@pygmented@display@(\d+)\n(.*)\n([\s\S]*?)\n>@@pygmented@display@\1$',
+ re.MULTILINE)
+
+_re_inline = re.compile(
+ r'^<@@pygmented@inline@(\d+)\n(.*)\n([\s\S]*?)\n>@@pygmented@inline@\1$',
+ re.MULTILINE)
+
+_re_input = re.compile(
+ r'^<@@pygmented@input@(\d+)\n(.*)\n([\s\S]*?)\n>@@pygmented@input@\1$',
+ re.MULTILINE)
+
+def convert(code, outfile):
+ """
+ Convert ``code``
+ """
+ outfile.write(GENERIC_DEFINITIONS_1)
+
+ opts = { 'lang' : 'c',
+ 'sty' : 'default',
+ 'linenosep' : '0pt',
+ 'tabsize' : '8',
+ 'encoding' : 'guess',
+ }
+
+ usedstyles = [ ]
+ styledefs = ''
+
+ pos = 0
+
+ while pos < len(code):
+ if code[pos].isspace():
+ pos = pos + 1
+ continue
+
+ m = _re_inline.match(code, pos)
+ if m:
+ pyg(outfile,
+ m.group(1),
+ parse_opts(opts.copy(), m.group(2)),
+ '',
+ m.group(3),
+ usedstyles,
+ True)
+ pos = m.end()
+ continue
+
+ m = _re_display.match(code, pos)
+ if m:
+ pyg(outfile,
+ m.group(1),
+ parse_opts(opts.copy(), m.group(2)),
+ '',
+ m.group(3),
+ usedstyles)
+ pos = m.end()
+ continue
+
+ m = _re_input.match(code, pos)
+ if m:
+ try:
+ filecontents = open(m.group(3), 'rb').read()
+ except Exception as err:
+ sys.stderr.write('Error: cannot read input file: ')
+ sys.stderr.write(str(err))
+ else:
+ pyg(outfile,
+ m.group(1),
+ parse_opts(opts, m.group(2)),
+ "",
+ filecontents,
+ usedstyles)
+ pos = m.end()
+ continue
+
+ sys.stderr.write('Error: invalid input file contents: ignoring')
+ break
+
+ outfile.write(GENERIC_DEFINITIONS_2)
+
+
+
+USAGE = """\
+Usage: %s [-o <output file name>] <input file name>
+ %s -h | -V
+
+The input file should consist of a sequence of source code snippets, as
+produced by the `pygmentex` LaTeX package. Each code snippet is
+highlighted using Pygments, and a LaTeX command that expands to the
+highlighted code snippet is written to the output file.
+
+It also writes to the output file a set of LaTeX macro definitions the
+Pygments styles that are used in the code snippets.
+
+If no output file name is given, use `<input file name>.pygmented`.
+
+The -e option enables escaping to LaTex. Text delimited by the <left>
+and <right> characters is read as LaTeX code and typeset accordingly. It
+has no effect in string literals. It has no effect in comments if
+`texcomments` or `mathescape` is set.
+
+The -h option prints this help.
+
+The -V option prints the package version.
+"""
+
+
+def main(args = sys.argv):
+ """
+ Main command line entry point.
+ """
+ usage = USAGE % ((args[0],) * 2)
+
+ try:
+ popts, args = getopt.getopt(args[1:], 'e:o:hV')
+ except getopt.GetoptError as err:
+ sys.stderr.write(usage)
+ return 2
+ opts = {}
+ for opt, arg in popts:
+ opts[opt] = arg
+
+ if not opts and not args:
+ print(usage)
+ return 0
+
+ if opts.pop('-h', None) is not None:
+ print(usage)
+ return 0
+
+ if opts.pop('-V', None) is not None:
+ print('PygmenTeX version %s, (c) 2010 by José Romildo.' % __version__)
+ return 0
+
+ if len(args) != 1:
+ sys.stderr.write(usage)
+ return 2
+ infn = args[0]
+ try:
+ code = open(infn, 'rb').read()
+ except Exception as err:
+ sys.stderr.write('Error: cannot read input file: ')
+ sys.stderr.write(str(err))
+ return 1
+
+ outfn = opts.pop('-o', None)
+ if not outfn:
+ root, ext = splitext(infn)
+ outfn = root + '.pygmented'
+ try:
+ outfile = open(outfn, 'w')
+ except Exception as err:
+ sys.stderr.write('Error: cannot open output file: ')
+ sys.stderr.write(str(err))
+ return 1
+
+ convert(code, outfile)
+
+ return 0
+
+
+if __name__ == '__main__':
+ try:
+ sys.exit(main(sys.argv))
+ except KeyboardInterrupt:
+ sys.exit(1)