diff options
Diffstat (limited to 'biblio/bibtex/utils/mab2bib/latex.py')
-rw-r--r-- | biblio/bibtex/utils/mab2bib/latex.py | 531 |
1 files changed, 531 insertions, 0 deletions
diff --git a/biblio/bibtex/utils/mab2bib/latex.py b/biblio/bibtex/utils/mab2bib/latex.py new file mode 100644 index 0000000000..e8a82b1c38 --- /dev/null +++ b/biblio/bibtex/utils/mab2bib/latex.py @@ -0,0 +1,531 @@ +#!/bin/env python +"""latex.py + +Character translation utilities for LaTeX-formatted text. + +Usage: + - unicode(string,'latex') + - ustring.decode('latex') +are both available just by letting "import latex" find this file. + - unicode(string,'latex+latin1') + - ustring.decode('latex+latin1') +where latin1 can be replaced by any other known encoding, also +become available by calling latex.register(). + +We also make public a dictionary latex_equivalents, +mapping ord(unicode char) to LaTeX code. + +D. Eppstein, October 2003. +source: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/252124 +License: Python license (http://python.org/doc/Copyright.html) +modified for mab2bib 2005/2006 by Henning Hraban Ramm +""" + +#from __future__ import generators +import codecs +import re +from sets import Set + +def register(): + """Enable encodings of the form 'latex+x' where x describes another encoding. + Unicode characters are translated to or from x when possible, otherwise + expanded to latex. + """ + codecs.register(_registry) + +def getregentry(): + """Encodings module API.""" + return _registry('latex') + +def _registry(encoding): + if encoding == 'latex': + encoding = None + elif encoding.startswith('latex+'): + encoding = encoding[6:] + else: + return None + + class Codec(codecs.Codec): + def encode(self,input,errors='strict'): + """Convert unicode string to latex.""" + output = [] + for c in input: + if encoding: + try: + output.append(c.encode(encoding)) + continue + except: + pass + if ord(c) in latex_equivalents: + output.append(latex_equivalents[ord(c)]) + else: + output += ['{\\char', str(ord(c)), '}'] + return ''.join(output), len(input) + + def decode(self,input,errors='strict'): + """Convert latex source string to unicode.""" + if encoding: + input = unicode(input,encoding,errors) + + # Note: we may get buffer objects here. + # It is not permussable to call join on buffer objects + # but we can make them joinable by calling unicode. + # This should always be safe since we are supposed + # to be producing unicode output anyway. + x = map(unicode,_unlatex(input)) + return u''.join(x), len(input) + + class StreamWriter(Codec,codecs.StreamWriter): + pass + + class StreamReader(Codec,codecs.StreamReader): + pass + + return (Codec().encode,Codec().decode,StreamReader,StreamWriter) + +def _tokenize(tex): + """Convert latex source into sequence of single-token substrings.""" + start = 0 + try: + # skip quickly across boring stuff + pos = _stoppers.finditer(tex).next().span()[0] + except StopIteration: + yield tex + return + + while 1: + if pos > start: + yield tex[start:pos] + if tex[start] == '\\' and not (tex[pos-1].isdigit() and tex[start+1].isalpha()): + while pos < len(tex) and tex[pos].isspace(): # skip blanks after csname + pos += 1 + + while pos < len(tex) and tex[pos] in _ignore: + pos += 1 # flush control characters + if pos >= len(tex): + return + start = pos + if tex[pos:pos+2] in {'$$':None, '/~':None}: # protect ~ in urls + pos += 2 + elif tex[pos].isdigit(): + while pos < len(tex) and tex[pos].isdigit(): + pos += 1 + elif tex[pos] == '-': + while pos < len(tex) and tex[pos] == '-': + pos += 1 + elif tex[pos] != '\\' or pos == len(tex) - 1: + pos += 1 + elif not tex[pos+1].isalpha(): + pos += 2 + else: + pos += 1 + while pos < len(tex) and tex[pos].isalpha(): + pos += 1 + if tex[start:pos] == '\\char' or tex[start:pos] == '\\accent': + while pos < len(tex) and tex[pos].isdigit(): + pos += 1 + +class _unlatex: + """Convert tokenized tex into sequence of unicode strings. Helper for decode().""" + + def __iter__(self): + """Turn self into an iterator. It already is one, nothing to do.""" + return self + + def __init__(self,tex): + """Create a new token converter from a string.""" + self.tex = tuple(_tokenize(tex)) # turn tokens into indexable list + self.pos = 0 # index of first unprocessed token + self.lastoutput = 'x' # lastoutput must always be nonempty string + + def __getitem__(self,n): + """Return token at offset n from current pos.""" + p = self.pos + n + t = self.tex + return p < len(t) and t[p] or None + + def next(self): + """Find and return another piece of converted output.""" + if self.pos >= len(self.tex): + raise StopIteration + nextoutput = self.chunk() + if self.lastoutput[0] == '\\' and self.lastoutput[-1].isalpha() and nextoutput[0].isalpha(): + nextoutput = ' ' + nextoutput # add extra space to terminate csname + self.lastoutput = nextoutput + return nextoutput + + def chunk(self): + """Grab another set of input tokens and convert them to an output string.""" + for delta,c in self.candidates(0): + if c in _l2u: + self.pos += delta + return unichr(_l2u[c]) + elif len(c) == 2 and c[1] == 'i' and (c[0],'\\i') in _l2u: + self.pos += delta # correct failure to undot i + return unichr(_l2u[(c[0],'\\i')]) + elif len(c) == 1 and c[0].startswith('\\char') and c[0][5:].isdigit(): + self.pos += delta + return unichr(int(c[0][5:])) + + # nothing matches, just pass through token as-is + self.pos += 1 + return self[-1] + + def candidates(self,offset): + """Generate pairs delta,c where c is a token or tuple of tokens from tex + (after deleting extraneous brackets starting at pos) and delta + is the length of the tokens prior to bracket deletion. + """ + t = self[offset] + if t in _blacklist: + return + elif t == '': + for delta,c in self.candidates(offset+1): + if self[offset+delta+1] == '}': + yield delta+2,c + elif t == '\\mbox': + for delta,c in self.candidates(offset+1): + yield delta+1,c + elif t == '$' and self[offset+2] == '$': + yield 3, (t,self[offset+1],t) + else: + q = self[offset+1] + if q == '' and self[offset+3] == '}': + yield 4, (t,self[offset+2]) + elif q: + yield 2, (t,q) + yield 1, t + +latex_equivalents = { + 0x0009: ' ', + 0x000a: '\n', + 0x0023: '\#', + 0x0026: '\&', + 0x00a0: '~', + 0x00a1: '!`', + 0x00a2: '\\not{c}', + 0x00a3: '\\pounds', + 0x00a7: '\\S', + 0x00a8: '\\"{}', + 0x00a9: '\\copyright', + 0x00af: '\\={}', + 0x00ac: '\\neg', + 0x00ad: '\\-', + 0x00b0: '\\mbox{$^\\circ$}', + 0x00b1: '\\mbox{$\\pm$}', + 0x00b2: '\\mbox{$^2$}', + 0x00b3: '\\mbox{$^3$}', + 0x00b4: "\\'{", + 0x00b5: '\\mbox{$\\mu$}', + 0x00b6: '\\P', + 0x00b7: '\\mbox{$\\cdot$}', + 0x00b8: '\\c{}', + 0x00b9: '\\mbox{$^1$}', + 0x00bf: '?`', + 0x00c0: '\\`A', + 0x00c1: "\\'A", + 0x00c2: '\\^A', + 0x00c3: '\\~A', + 0x00c4: '\\"A', + 0x00c5: '\\AA', + 0x00c6: '\\AE', + 0x00c7: '\\c{C}', + 0x00c8: '\\`E', + 0x00c9: "\\'E", + 0x00ca: '\\^E', + 0x00cb: '\\"E', + 0x00cc: '\\`I', + 0x00cd: "\\'I", + 0x00ce: '\\^I', + 0x00cf: '\\"I', + 0x00d1: '\\~N', + 0x00d2: '\\`O', + 0x00d3: "\\'O", + 0x00d4: '\\^O', + 0x00d5: '\\~O', + 0x00d6: '\\"O', + 0x00d7: '\\mbox{$\\times$}', + 0x00d8: '\\O', + 0x00d9: '\\`U', + 0x00da: "\\'U", + 0x00db: '\\^U', + 0x00dc: '\\"U', + 0x00dd: "\\'Y", + 0x00df: '\\ss', + 0x00e0: '\\`a', + 0x00e1: "\\'a", + 0x00e2: '\\^a', + 0x00e3: '\\~a', + 0x00e4: '\\"a', + 0x00e5: '\\aa', + 0x00e6: '\\ae', + 0x00e7: '\\c{c}', + 0x00e8: '\\`e', + 0x00e9: "\\'e", + 0x00ea: '\\^e', + 0x00eb: '\\"e', + 0x00ec: '\\`\\i', + 0x00ed: "\\'\\i", + 0x00ee: '\\^\\i', + 0x00ef: '\\"\\i', + 0x00f1: '\\~n', + 0x00f2: '\\`o', + 0x00f3: "\\'o", + 0x00f4: '\\^o', + 0x00f5: '\\~o', + 0x00f6: '\\"o', + 0x00f7: '\\mbox{$\\div$}', + 0x00f8: '\\o', + 0x00f9: '\\`u', + 0x00fa: "\\'u", + 0x00fb: '\\^u', + 0x00fc: '\\"u', + 0x00fd: "\\'y", + 0x00ff: '\\"y', + + 0x0100: '\\=A', + 0x0101: '\\=a', + 0x0102: '\\u{A}', + 0x0103: '\\u{a}', + 0x0104: '\\c{A}', + 0x0105: '\\c{a}', + 0x0106: "\\'C", + 0x0107: "\\'c", + 0x0108: "\\^C", + 0x0109: "\\^c", + 0x010a: "\\.C", + 0x010b: "\\.c", + 0x010c: "\\v{C", + 0x010d: "\\v{c", + 0x010e: "\\v{D", + 0x010f: "\\v{d", + 0x0112: '\\=E', + 0x0113: '\\=e', + 0x0114: '\\u{E}', + 0x0115: '\\u{e}', + 0x0116: '\\.E', + 0x0117: '\\.e', + 0x0118: '\\c{E}', + 0x0119: '\\c{e}', + 0x011a: "\\v{E", + 0x011b: "\\v{e", + 0x011c: '\\^G', + 0x011d: '\\^g', + 0x011e: '\\u{G}', + 0x011f: '\\u{g}', + 0x0120: '\\.G', + 0x0121: '\\.g', + 0x0122: '\\c{G}', + 0x0123: '\\c{g}', + 0x0124: '\\^H', + 0x0125: '\\^h', + 0x0128: '\\~I', + 0x0129: '\\~\\i', + 0x012a: '\\=I', + 0x012b: '\\=\\i', + 0x012c: '\\u{I}', + 0x012d: '\\u\\i', + 0x012e: '\\c{I}', + 0x012f: '\\c{i}', + 0x0130: '\\.I', + 0x0131: '\\i', + 0x0132: 'IJ', + 0x0133: 'ij', + 0x0134: '\\^J', + 0x0135: '\\^\\j', + 0x0136: '\\c{K}', + 0x0137: '\\c{k}', + 0x0139: "\\'L", + 0x013a: "\\'l", + 0x013b: "\\c{L", + 0x013c: "\\c{l", + 0x013d: "\\v{L", + 0x013e: "\\v{l", + 0x0141: '\\L', + 0x0142: '\\l', + 0x0143: "\\'N", + 0x0144: "\\'n", + 0x0145: "\\c{N", + 0x0146: "\\c{n", + 0x0147: "\\v{N", + 0x0148: "\\v{n", + 0x014c: '\\=O', + 0x014d: '\\=o', + 0x014e: '\\u{O}', + 0x014f: '\\u{o}', + 0x0150: '\\H{O}', + 0x0151: '\\H{o}', + 0x0152: '\\OE', + 0x0153: '\\oe', + 0x0154: "\\'R", + 0x0155: "\\'r", + 0x0156: "\\c{R", + 0x0157: "\\c{r", + 0x0158: "\\v{R", + 0x0159: "\\v{r", + 0x015a: "\\'S", + 0x015b: "\\'s", + 0x015c: "\\^S", + 0x015d: "\\^s", + 0x015e: "\\c{S", + 0x015f: "\\c{s", + 0x0160: "\\v{S", + 0x0161: "\\v{s", + 0x0162: "\\c{T", + 0x0163: "\\c{t", + 0x0164: "\\v{T", + 0x0165: "\\v{t", + 0x0168: "\\~U", + 0x0169: "\\~u", + 0x016a: "\\=U", + 0x016b: "\\=u", + 0x016c: "\\u{U", + 0x016d: "\\u{u", + 0x016e: "\\r{U", + 0x016f: "\\r{u", + 0x0170: "\\H{U", + 0x0171: "\\H{u", + 0x0172: "\\c{U", + 0x0173: "\\c{u", + 0x0174: "\\^W", + 0x0175: "\\^w", + 0x0176: "\\^Y", + 0x0177: "\\^y", + 0x0178: '\\"Y', + 0x0179: "\\'Z", + 0x017a: "\\'Z", + 0x017b: "\\.Z", + 0x017c: "\\.Z", + 0x017d: "\\v{Z", + 0x017e: "\\v{z", + + 0x01c4: "D\\v{Z", + 0x01c5: "D\\v{z", + 0x01c6: "d\\v{z", + 0x01c7: "LJ", + 0x01c8: "Lj", + 0x01c9: "lj", + 0x01ca: "NJ", + 0x01cb: "Nj", + 0x01cc: "nj", + 0x01cd: "\\v{A", + 0x01ce: "\\v{a", + 0x01cf: "\\v{I", + 0x01d0: "\\v\\i", + 0x01d1: "\\v{O", + 0x01d2: "\\v{o", + 0x01d3: "\\v{U", + 0x01d4: "\\v{u", + 0x01e6: "\\v{G", + 0x01e7: "\\v{g", + 0x01e8: "\\v{K", + 0x01e9: "\\v{k", + 0x01ea: "\\c{O", + 0x01eb: "\\c{o", + 0x01f0: "\\v\\j", + 0x01f1: "DZ", + 0x01f2: "Dz", + 0x01f3: "dz", + 0x01f4: "\\'G", + 0x01f5: "\\'g", + 0x01fc: "\\'\\AE", + 0x01fd: "\\'\\ae", + 0x01fe: "\\'\\O", + 0x01ff: "\\'\\o", + + 0x02c6: '\\^{}', + 0x02dc: '\\~{}', + 0x02d8: '\\u{}', + 0x02d9: '\\.{}', + 0x02da: "\\r{", + 0x02dd: '\\H{}', + 0x02db: '\\c{}', + 0x02c7: '\\v{}', + + 0x03c0: '\\mbox{$\\pi$}', + # consider adding more Greek here + + 0xfb01: 'fi', + 0xfb02: 'fl', + + 0x2013: '--', + 0x2014: '---', + 0x2018: "`", + 0x2019: "'", + 0x201c: "``", + 0x201d: "''", + 0x2020: "\\dag", + 0x2021: "\\ddag", + 0x2122: "\\mbox{$^\\mbox{TM}$", + 0x2022: "\\mbox{$\\bullet$", + 0x2026: "\\ldots", + 0x2202: "\\mbox{$\\partial$", + 0x220f: "\\mbox{$\\prod$", + 0x2211: "\\mbox{$\\sum$", + 0x221a: "\\mbox{$\\surd$", + 0x221e: "\\mbox{$\\infty$", + 0x222b: "\\mbox{$\\int$", + 0x2248: "\\mbox{$\\approx$", + 0x2260: "\\mbox{$\\neq$", + 0x2264: "\\mbox{$\\leq$", + 0x2265: "\\mbox{$\\geq$", +} +for _i in range(0x0020): + if _i not in latex_equivalents: + latex_equivalents[_i] = '' +for _i in range(0x0020,0x007f): + if _i not in latex_equivalents: + latex_equivalents[_i] = chr(_i) + +# Characters that should be ignored and not output in tokenization +_ignore = Set([chr(i) for i in range(32)+[127]]) - Set('\t\n\r') + +# Regexp of chars not in blacklist, for quick start of tokenize +_stoppers = re.compile('[\x00-\x1f!$\\-?\\{~\\\\`\']') + +_blacklist = Set(' \n\r') +_blacklist.add(None) # shortcut candidate generation at end of data + +# Construction of inverse translation table +_l2u = { + '\ ':ord(' ') # unexpanding space makes no sense in non-TeX contexts +} + +for _tex in latex_equivalents: + if _tex <= 0x0020 or (_tex <= 0x007f and len(latex_equivalents[_tex]) <= 1): + continue # boring entry + _toks = tuple(_tokenize(latex_equivalents[_tex])) + if _toks[0] == '' and _toks[-1] == '}': + _toks = _toks[1:-1] + if _toks[0].isalpha(): + continue # don't turn ligatures into single chars + if len(_toks) == 1 and (_toks[0] == "'" or _toks[0] == "`"): + continue # don't turn ascii quotes into curly quotes + if _toks[0] == '\\mbox' and _toks[1] == '' and _toks[-1] == '}': + _toks = _toks[2:-1] + if len(_toks) == 4 and _toks[1] == '' and _toks[3] == '}': + _toks = (_toks[0],_toks[2]) + if len(_toks) == 1: + _toks = _toks[0] + _l2u[_toks] = _tex + +# Shortcut candidate generation for certain useless candidates: +# a character is in _blacklist if it can not be at the start +# of any translation in _l2u. We use this to quickly skip through +# such characters before getting to more difficult-translate parts. +# _blacklist is defined several lines up from here because it must +# be defined in order to call _tokenize, however it is safe to +# delay filling it out until now. + +for i in range(0x0020,0x007f): + _blacklist.add(chr(i)) +_blacklist.remove('{') +_blacklist.remove('$') +for candidate in _l2u: + if isinstance(candidate,tuple): + if not candidate or not candidate[0]: + continue + firstchar = candidate[0][0] + else: + firstchar = candidate[0] + _blacklist.discard(firstchar) |