diff options
Diffstat (limited to 'Master/texmf-dist/doc/latex')
| -rw-r--r-- | Master/texmf-dist/doc/latex/ejpecp/README.md | 2 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/latex/ejpecp/ejpecp.pdf | bin | 180717 -> 180735 bytes | |||
| -rw-r--r-- | Master/texmf-dist/doc/latex/ejpecp/getmref.py | 1809 | ||||
| -rwxr-xr-x | Master/texmf-dist/doc/latex/ejpecp/mgetmref.py | 401 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/latex/ejpecp/sample.pdf | bin | 155832 -> 155933 bytes | |||
| -rw-r--r-- | Master/texmf-dist/doc/latex/ejpecp/sample.tex | 17 |
6 files changed, 1827 insertions, 402 deletions
diff --git a/Master/texmf-dist/doc/latex/ejpecp/README.md b/Master/texmf-dist/doc/latex/ejpecp/README.md index 8deecbb8032..d30626bca4f 100644 --- a/Master/texmf-dist/doc/latex/ejpecp/README.md +++ b/Master/texmf-dist/doc/latex/ejpecp/README.md @@ -68,6 +68,8 @@ https://github.com/vtex-soft/texsupport.ims-ejpecp ## CHANGELOG +- 2020/10/21 v1.9.0 + - Supplement environment added - 2020/08/26 v1.8.3 - Updated URLs - 2020/08/05 diff --git a/Master/texmf-dist/doc/latex/ejpecp/ejpecp.pdf b/Master/texmf-dist/doc/latex/ejpecp/ejpecp.pdf Binary files differindex 7c703f05ea8..290f565d110 100644 --- a/Master/texmf-dist/doc/latex/ejpecp/ejpecp.pdf +++ b/Master/texmf-dist/doc/latex/ejpecp/ejpecp.pdf diff --git a/Master/texmf-dist/doc/latex/ejpecp/getmref.py b/Master/texmf-dist/doc/latex/ejpecp/getmref.py new file mode 100644 index 00000000000..3241aac3aa2 --- /dev/null +++ b/Master/texmf-dist/doc/latex/ejpecp/getmref.py @@ -0,0 +1,1809 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +################################################################################## +# +# getmref.py - gets the references links to MathSciNet through the BatchMRef: +# https://mathscinet.ams.org/batchmref?qdata=xmldocument +# +# Copyright (C) 2017 Sigitas Tolusis, VTeX Ltd., Jim Pitman, Dept. Statistics, +# U.C. Berkeley and Lolita Tolene, VTeX Ltd. +# E-mail: latex-support@vtex.lt +# http://www.stat.berkeley.edu/users/pitman +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# Requires: +# - python ver. >=2.2 +# - [ for option --enc=auto ] +# Universal Encoding Detector library, written by Mark Pilgrim. +# +# Usage: +# getmref.py <bbl or tex file> +# +# Program (description): +# - makes inputfile copy to <inputfilename>.getmref.bak; +# - for each successful bibitem reference search adds line \MR{<mrid>}, +# where <mrid> is data from XML tag <mrid> without front symbols "MR"; +# - writes all adds to <inputfilename>; +# - generates log file <inputfilename>.getmref.log; +# - writes to stdout log info +# +# Changes: +# 2004/04/26 - \bibitem line removed from the query +# 2017/01/12 - input file may contain 'amsrefs', 'bibtex' and 'tex' type +# references (all at once); +# input references can be formatted as 'amsrefs', 'bibtex', +# 'tex' or 'html' type references +# +# +################################################################################## + +__version__ = "GetMRef, v2.4" + +import sys +import os +import re +import string +import urllib +import urllib2 +import ssl +import shutil +import logging +from time import time, sleep +from xml.dom.minidom import parseString +from xml.parsers.expat import ExpatError + +BASICFORMATTER = logging.Formatter('%(message)s') +DEBUGFORMATTER = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) +flog = slog = log + + +class RefTypes(object): + """ This class declares recognized bibliography reference formats + + Formats description + ------------------- + Source: only AMS + "tex": LaTeX code without any specific beginning/ending; + MR number is given in plain text + "html": <a href="http://www.ams.org/mathscinet-getitem?mr=7digits"> + 7digits + </a> + + Source: only user + "bibitem": \bibitem[<name-year info>]{<cite_key>} + ... + \MR{<7 digits>} + \endbibitem, + where '[<name-year info>]' and '\endbibitem' are optional + Requires environment + \begin{thebibliography}{<ref no>} + ... + \end{thebibliography} + + Source: AMS and user + "bibtex": @<ref type>{<cite_key>, + <key1>={<value1>}, + <key2>={<value2>}, + MRNUMBER={<7 digits>} + ...} + "amsrefs": \bib{<cite_key>}{<ref type>}{ + <key1>={<value1>}, + <key2>={<value2>}, + review={\MR{<7 digits>}} + ...} + Requires environment + \begin{biblist} + ... + \end{biblist} + """ + + TEX = "tex" + BIBITEM = "bibitem" + IMS = "ims" + BIBTEX = "bibtex" + AMSREFS = "amsrefs" + HTML = "html" + + # Reference input formats + ITYPES = (BIBITEM, BIBTEX, AMSREFS) + + # Reference output formats + OTYPES = (TEX, BIBTEX, IMS, AMSREFS, HTML) + + +class LessThanFilter(logging.Filter): + """ This class allows to add an upper bound to the logged messages + + Example + ------- + One needs to log all non-error messages to stdout, and all errors + (higher level) only to stderr + """ + + def __init__(self, exclusive_maximum, name=""): + super(LessThanFilter, self).__init__(name) + self.max_level = exclusive_maximum + + def filter(self, record): + # A non-zero return means we log this message + return 1 if record.levelno <= self.max_level else 0 + + +class FilesHandler(RefTypes): + """ This class unites methods and attributes related to + files I/O actions """ + + IN = 'in' + BAK = 'bak' + OUT = 'out' + DATA = 'data' + TMP = 'tmp' + AUX = 'aux' + BIB = 'bib' + HTML = 'html' + LOG = 'log' + ERR = 'err' + + # File status map: + # if True file will be open until closed; + # if False it will be opened on demand + FILE_STATUS = {OUT: True, + LOG: False, + ERR: False, + DATA: True, + BIB: True, + AUX: True, + HTML: True, + TMP: False} + + READ = 'r' + WRITE = 'w' + + GMR_SUFFIX = 'getmref' + + def __init__(self, infile, outputtype): + """ Initiate file handling methods and attributes + + Parameters + ---------- + infile : str or None + Path to input file + outputtype : str or None + Required bibliography reference output format type + """ + + self.infile = infile + self._basename = os.path.splitext(infile)[0] + + # Determining needed file types for given reference output type + msg = ("The given references will be formatted in '%s' format. " + % (outputtype if outputtype is not None else "orig")) + + unnecessary = [self.DATA, self.BIB, self.AUX, self.HTML] + if outputtype in [self.BIBTEX, self.IMS]: + unnecessary = [self.DATA, self.HTML] + # Referring to 'BIB' file as 'DATA' + self.DATA = self.BIB + msg += "Additional files will be created: *.%s, *.%s" \ + % (self.BIB, self.AUX) + elif outputtype == self.HTML: + unnecessary = [self.DATA, self.BIB, self.AUX] + # Referring to 'HTML' file as 'DATA' + self.DATA = self.HTML + msg += "Additional file will be created: *.%s" % self.HTML + elif outputtype in [self.TEX, self.AMSREFS]: + unnecessary = [self.HTML, self.BIB, self.AUX] + msg += "Additional file will be created: *.%s" % self.DATA + + self.files = dict() + for suffix, status in self.FILE_STATUS.items(): + # Deleting old files + self._delete(suffix) + if suffix in unnecessary: + continue + if status: + self.open(suffix) + continue + self.files.update({suffix: self.get_fname(suffix)}) + + flog.info("File: %s" % infile) + if not (os.path.isfile(infile) and os.path.exists(infile)): + logging.shutdown() + for suffix in self.FILE_STATUS: + self.close_and_delete(suffix) + raise ValueError("Provided source file does not exist! " + "Please provide the valid one.") + + flog.debug("Workdir: %s" % os.path.abspath(os.path.dirname(infile))) + flog.debug(msg) + + def set_fname(self, suffix): + """ Set a filepath for a file with the provided suffix + + Parameters + ---------- + suffix : str + File suffix without punctuation + + Returns + ------- + str + """ + return ("%s.%s.%s" % (self._basename, self.GMR_SUFFIX, suffix) + if suffix != self.IN else self.infile) + + def get_fname(self, suffix): + """ Get filepath of a file with the required suffix + + Parameters + ---------- + suffix : str + File suffix without punctuation + + Returns + ------- + str + If requested file is open, returning file object name, + or the filepath otherwise + """ + target = self.files.get(suffix, self.set_fname(suffix)) + if isinstance(target, file): + return target.name + return target + + def open(self, suffix, mask=WRITE): + """ Open file for the selected action + + Parameters + ---------- + suffix : str + File suffix without punctuation + mask : str + Possible actions are read or write + + File is opened and file object is added to the dictionary + for later access + """ + self.files.update({suffix: file(self.get_fname(suffix), mask)}) + + def read(self, suffix): + """ Get the content of a file with the required suffix + + Parameters + ---------- + suffix : str + File suffix without punctuation + + Yields + ------ + str + """ + with open(self.get_fname(suffix), self.READ) as ifile: + for iline in ifile: + yield iline + + def write(self, suffix, msg): + """ Write to the file with the required suffix + only if this file is open + + Parameters + ---------- + suffix : str + File suffix without punctuation + msg : str + """ + target = self.files.get(suffix, None) + if isinstance(target, file): + target.write(msg) + + def close(self, suffix): + """ Close the file with the required suffix + + Parameters + ---------- + suffix : str + File suffix without punctuation + """ + fileobj = self.files.get(suffix, "") + if isinstance(fileobj, file): + fileobj.close() + + def _delete(self, suffix): + """ Delete the file with the required suffix + + Parameters + ---------- + suffix : str + File suffix without punctuation + """ + dfile = self.get_fname(suffix) + try: + os.unlink(dfile) + flog.debug("Deleted: %s" % os.path.split(dfile)[1]) + except OSError: + if os.path.isfile(dfile) and os.path.exists(dfile): + flog.exception("Can't remove file: %s" % dfile) + + def close_and_delete(self, suffix): + """ Close and delete the file with the required suffix + + Parameters + ---------- + suffix : str + File suffix without punctuation + """ + self.close(suffix) + self._delete(suffix) + + def close_files(self): + """ Close all open files and logging instances, + create backup of the input file and + overwrite it with the new content, delete auxiliary files + """ + flog.debug("Closing files...") + for suffix in self.files: + self.close(suffix) + + self._delete(self.TMP) + + bfile = self.get_fname(self.BAK) + if os.path.exists(bfile): + shutil.copy2(self.infile, bfile) + else: + os.rename(self.infile, bfile) + flog.debug("Created backup of the input file: %s" + % os.path.split(bfile)[1]) + + ofile = self.get_fname(self.OUT) + if os.path.exists(ofile): + shutil.copy2(ofile, self.infile) + self._delete(self.OUT) + else: + os.rename(ofile, self.infile) + flog.debug("The input file is overwritten with: %s" + % os.path.split(ofile)[1]) + + logging.shutdown() + + +class RefHandler(RefTypes): + """ This class unites methods and attributes related to bibliography + reference format types and their content modifications """ + + # Bibliography environment + BIBL_ENV = "environment" + BIBL_BEGIN = "begin" + BIBL_END = "end" + + # Declaration of typical reference type ending and + # MR id format for this type + FORMAT_PROPERTIES = { + RefTypes.BIBTEX: { + "ref_ending": "}", + "mr_format": ",\nMRNUMBER={%s},\n" + }, + RefTypes.AMSREFS: { + "ref_ending": "}", + "mr_format": ",\nreview={\MR{%s}},\n" + }, + RefTypes.BIBITEM: { + "ref_ending": "\\endbibitem", + "mr_format": "\n\\MR{%s}\n" + } + } + + # Meaningful reference keys for AMS Batch MR Lookup query + KEYS = {"0AUTH": ("author",), + "1TTL": ("title", "maintitle"), + "2JOUR": ("journal", "journaltitle", "fjournal", "booktitle"), + "3VID": ("volume",), + "4IID": ("number", "series"), + "5PID": ("pages",), + "6YNO": ("year", "date"), + "7ISSN": ("issn", "isrn", "isbn")} + + PATTERN_KEY_VALUE = "^\s*([\w-]+)\s*=\s*(.*?)$" + + PATTERN_LINE_END = r'(\r?\n)+' + PATTERN_PAR = r'(\r?\n){2}' + + PATTERN_BIBL_ENV = (r'\s*\\(?P<envstatus>begin|end)\s*' + r'\{(thebibliography|biblist\*?)\}(.*)$') + + PATTERN_BIBRE = r'^\s*\\bibitem.*' + PATTERN_BIBREF = (r'\s*\\bibitem\s*(?P<biblabel>\[.*?\])*?\s?' + r'\{(?P<citekey>.*?)\}(?P<text>.*)$') + PATTERN_BIBTEX = (r'^\s*(@\S+)(?<!@preamble)\s*' + r'{(?P<citekey>\S+)\s*,(?P<text>.*)$') + PATTERN_AMSREFS = r"\\bib\s*{(?P<citekey>.*)}\s*{(.*)}\s*{(?P<text>.*)$" + + def __init__(self, outputtype): + """ Initiate reference handling methods and attributes + + Parameters + ---------- + outputtype : str or None + Required reference output format type + """ + + self.outputtype = outputtype + + self.re_bibl_env = re.compile(self.PATTERN_BIBL_ENV) + self.re_bibre = re.compile(self.PATTERN_BIBRE) + self.re_bibreF = re.compile(self.PATTERN_BIBREF, re.S) + self.re_bibtex = re.compile(self.PATTERN_BIBTEX, re.M) + self.re_amsrefs = re.compile(self.PATTERN_AMSREFS, re.M) + + self.re_lineend = re.compile(self.PATTERN_LINE_END) + self.re_par = re.compile(self.PATTERN_PAR) + self.re_key_value = re.compile(self.PATTERN_KEY_VALUE, re.DOTALL) + + def find_reference(self, line): + """ Identify reference environment or element by using regex patterns + + Parameters + ---------- + line : str + + Returns + ------- + str or None + If match is found, returns the reference type, None otherwise + dict + Dictionary contains regex pattern group names and their matches + + The value of the key 'text' is the line part without user + defined strings, such as citekey and biblabel, because they may + contain some misleading information for BatchMRef query + """ + + elems = {self.BIBL_ENV: self.re_bibl_env, + self.BIBTEX: self.re_bibtex, + self.AMSREFS: self.re_amsrefs} + + # BIBITEM search starts with an additional check + # which other reference types doesn't have + if self.re_bibre.search(line) is not None: + elems = {self.BIBITEM: self.re_bibreF} + + for reftype, pattern in elems.items(): + match = pattern.search(line) + if match is not None: + return reftype, match.groupdict() + elif reftype == self.BIBITEM: + # If final search for BIBITEM fails, it means that the typical + # structure for this reference type is placed on several lines, + # therefore the current line is prepended to the next input line + return reftype, {"line": line} + return None, dict() + + def extract_keys_data(self, lines): + """ Extract values from selected keys in reference + + Parameters + ---------- + lines : list + + Returns + ------- + str + Output contains extracted values separated by commas + """ + flog.debug(">> Extracting key values from reference") + querystring = "" + user_key = None + found = list() + for line in lines: + match = self.re_key_value.search(line) + if match: + user_key, user_value = match.groups() + user_key = user_key.lower() + for key, value in sorted(self.KEYS.items()): + if user_key in value and (user_key in found or key not in found): + found.append(key) + found.append(user_key) + querystring += "%s, " % user_value.strip().rstrip(",")\ + .strip().strip('"')\ + .strip().rstrip("}")\ + .lstrip("{").strip() + break + elif len(found) > 0 and found[-1] == user_key: + querystring = "%s %s, " % (querystring.strip(", "), + line.strip().rstrip(",").strip().strip('"') + .strip().rstrip("}").lstrip("{") + .strip().rstrip(",").strip()) + + return querystring.strip(", ") + + def insert_mrid(self, reftype, refstring, mrid): + """ Format MR number according to the input reference format and + append it to the input reference + + Parameters + ---------- + reftype : str + Determined input bibliography reference item type + refstring : str + Input bibliography reference item content + mrid : str + MR number returned by query to BatchMRef + + Returns + ------- + str + Output contains input bibliography reference element including + according to reftype formatted mrid. + """ + properties = self.FORMAT_PROPERTIES.get(reftype, None) + if properties is None: + outstring = self.re_lineend.sub('\n', refstring) + return '%s\\MR{%s}\n\n' % (outstring, mrid) + + mr_string = properties["mr_format"] % mrid + ending_index = refstring.rfind(properties["ref_ending"]) + if ending_index == -1: + paragraph = self.re_par.search(refstring) + if paragraph is not None: + ending_index = paragraph.start() + mr_string += "\n" + + if ending_index != -1: + return "%s%s%s" % (refstring[:ending_index].strip().strip(","), + mr_string, + refstring[ending_index:].lstrip()) + + return refstring.strip() + mr_string + "\n" + + def insert_citekey(self, outref, citekey, biblabel, querystring): + """ Add a cite key, extracted from an input reference item, + to the reference content, returned by the query to BatchMRef + (XML tag <outref>), in the required reference output format + + Parameters + ---------- + outref : str or None + Reference item content returned by the query to BatchMRef + citekey : str + Input bibliography reference item cite key + biblabel : str or None + Input bibliography reference item label, + provided in optional parameter of reference type of BIBITEM + querystring : str + Input bibliography reference item formatted for query + to BatchMRef + + Returns + ------- + str or None + Returned string is the outref including the citekey and + the biblabel (if provided) if reference has been found in + the AMS MR DB, else string is formatted according to the + requested output type. + + If allowed output type is not provided, None is returned + """ + + if self.outputtype is None: + return None + + if outref is None: + if self.outputtype == self.TEX: + return ("\\bibitem%s{%s}\n Not Found!\n\n" + % (biblabel if biblabel is not None else "", + citekey)) + if self.outputtype == self.BIBTEX: + return '@MISC {%s,\n NOTE = {Not Found!}\n}\n\n' % citekey + if self.outputtype == self.IMS: + return ('@MISC {%s,\n HOWPUBLISHED = {%s},\n}\n\n' + % (citekey, querystring)) + if self.outputtype == self.AMSREFS: + return ('\\bib{%s}{misc}{\n note = {Not Found!}\n}\n\n' + % citekey) + if self.outputtype == self.HTML: + return '<!-- %s -->\nNot Found!\n<br/><br/>\n\n' % citekey + return None + + outref = outref.strip() + '\n\n' + if self.outputtype == self.TEX: + return ('\\bibitem%s{%s}\n%s' + % (biblabel if biblabel is not None else "", + citekey, outref)) + if self.outputtype in [self.BIBTEX, self.IMS]: + return self.re_bibtex.sub(r'\1 {%s,' % citekey, outref) + if self.outputtype == self.AMSREFS: + return self.re_amsrefs.sub(r'\\bib\0{%s}{\2}' % citekey, outref) + if self.outputtype == self.HTML: + return '<!-- %s -->\n%s<br/><br/>\n' % (citekey, outref) + return None + + +class RefElement(object): + """ This is a container for one bibliography reference item, + containing all data related to it """ + + FORMAT_PROPERTIES = RefHandler.FORMAT_PROPERTIES + + def __init__(self, refid=None, reftype=None, citekey=None, biblabel=None): + """ Initiate reference item container + + Parameters + ---------- + refid : int or None + reftype : str or None + Input bibliography reference type (one of RefTypes.ITYPES). + citekey : str or None + Input bibliography reference cite key + biblabel : str or None + Input bibliography reference label, + provided in the optional parameter of RefTypes.BIBITEM type + reference item + """ + + self.reftype = reftype + self.refid = refid + self.citekey = citekey + self.biblabel = biblabel + + self.orig_lines = list() + self.cleaned_lines = list() + self.query_lines = list() + self.comment_lines = list() + + self.errno = 0 + self._init_querystring = None + self._querystring = None + self._mrid = None + self.outref = None + + def normalize(self, lines): + """ Normalize the reference item content + Parameters + ---------- + lines : list + + Returns + ------- + str + Returned string doesn't contain trailing spaces and + typical ending for the reference of reftype (if found) + """ + nstring = re.sub('\s+', ' ', ''.join(lines)).strip() + ending = self.FORMAT_PROPERTIES.get(self.reftype, + dict()).get("ref_ending", "") + ending_index = nstring.rfind(ending) + if ending_index != -1: + nstring = nstring[:ending_index].strip() + return nstring + + @property + def init_querystring(self): + if self._init_querystring is not None: + return self._init_querystring + + flog.debug(">> Normalizing the reference") + self._init_querystring = self.normalize(self.query_lines) + + return self._init_querystring + + @property + def querystring(self): + if self._querystring is not None: + return self._querystring + return self.init_querystring + + @querystring.setter + def querystring(self, istring): + self._querystring = istring + + @property + def mrid(self): + return self._mrid + + @mrid.setter + def mrid(self, mrid): + """ Normalize MR number, returned by the query to BatchMRef + + Parameters + ---------- + mrid : str + + Returns + ------- + str + If original MR number is shorter than 7 symbols, prepending 0, + till it reaches 7 symbol length + """ + if mrid is not None: + self._mrid = mrid.encode('ascii').lstrip("MR").rjust(7, '0') + + def __repr__(self): + result = "<%s:\n" % self.__class__.__name__ + for key, value in sorted(self.__dict__.items()): + if key.startswith("_"): + continue + result += " %s = %s\n" % (key, repr(value)) + result += " >\n" + return result + + def __str__(self): + return self.__repr__() + + +class RefsContainer(object): + """ This is a container holding as many bibliography reference items as + is allowed by the denoted query to BatchMRef limit and data common + to all of them """ + + def __init__(self): + super(RefsContainer, self).__init__() + self.elems = list() + self.qerrno = 0 + + def append_elem(self, ref_element): + """ Add bibliography reference item instance to the container + + Parameters + ---------- + ref_element : RefElement() instance + """ + + self.elems += (ref_element,) + + def get_elem_by_refid(self, refid): + """ Get bibliography reference item instance by its id + + Parameters + ---------- + refid : int + + Returns + ------- + RefElement() instance or None + If element with required id is not found, None is returned + """ + + elem = [e for e in self.elems if e.refid == refid] + if elem: + return elem[0] + + def __str__(self): + result = "<%s:\n" % self.__class__.__name__ + for key, value in sorted(self.__dict__.items()): + if key == "elems": + for elem in value: + result += " %s" % repr(elem) + elif key not in ["elems", "qresult", "xml"]: + result += " GLOBAL: {} = {}\n".format(key, value) + result += " >\n" + return result + + +class QueryHandler(RefTypes): + """ This class unites methods and attributes related to actions necessary + for the AMS BatchMRef query """ + + AUTO_ENC = "auto" + LATIN1 = 'latin1' + ASCII = "ascii" + + AMS_URL = 'https://mathscinet.ams.org/batchmref' + + # AMS BatchMRef limit of items no per query + QUERY_ITEMS_LIMIT = 100 + + QUERY_XML_HEADING_STRING = '<?xml version="1.0" encoding="UTF-8"?>\n' + + QUERY_HEADING_STRING = ( + '<mref_batch>\n' + ' %s' + '</mref_batch>' + ) + + QUERY_ITEM_STRING = ( + '<mref_item outtype="%s">\n' + ' <inref>\n' + ' %s\n' + ' </inref>\n' + ' <myid>%d</myid>\n' + '</mref_item>\n' + ) + + QUERY_FORMATS = { + RefTypes.TEX: RefTypes.TEX, + RefTypes.BIBTEX: RefTypes.BIBTEX, + RefTypes.IMS: RefTypes.BIBTEX, + RefTypes.AMSREFS: RefTypes.AMSREFS, + RefTypes.HTML: RefTypes.HTML, + None: RefTypes.TEX + } + + PATTERN_MREF_ITEM = '(\<mref_item outtype="(?:bibtex|tex|amsrefs|html)"\>.*?\</mref_item\>)' + PATTERN_BATCH_ERROR = '\<batch_error\>(.*?)\</batch_error\>' + + # AMS gives the following message in HTML if requested website is broken + AMS_MSG = "The AMS Website is temporarily unavailable." + + def __init__(self, encoding, outputtype, refscontainer, address=AMS_URL): + """ Initiate query to BatchMRef handling methods and attributes + + Parameters + ---------- + encoding : str + Input file encoding + outputtype : str or None + Reference output format type passed for BatchMRef query + refscontainer : RefsContainer() instance + address : str + BatchMRef query address + """ + + self.encoding = encoding + flog.debug("Provided encoding format: %s" % encoding) + self.address = address + self.query_format = self.QUERY_FORMATS.get(outputtype, self.TEX) + flog.debug("Query settings: URL = %s, output format = %s" + % (address, self.query_format)) + self.outputtype = outputtype + + self.errno = 0 + self.qresult = None + self.qcode = None + self.xml = None + self.re_mref_item = re.compile(self.PATTERN_MREF_ITEM, re.DOTALL) + self.re_batch_error = re.compile(self.PATTERN_BATCH_ERROR, re.DOTALL) + + self._refscontainer = refscontainer + self.query_elems = list() + + @property + def refscontainer(self): + return self._refscontainer + + def _encode_str(self, istring): + """ Change query string encoding into the ASCII + + Parameters + ---------- + istring : str + + Returns + ------- + str + """ + + str_enc = self.encoding + self.errno = 0 + if str_enc == self.AUTO_ENC: + detector = UniversalDetector() + detector.feed(istring) + detector.close() + str_enc = detector.result.get('encoding', self.ASCII) + flog.debug(">> Determined string encoding: %s" % str_enc) + + if str_enc == self.ASCII: + return istring + if str_enc is None: + flog.debug(">> Encoding determination has FAILED! ") + return istring + + try: + return istring.decode(str_enc.lower()).encode(self.ASCII, + errors='replace') + except: + flog.debug(">> encoding given reference element FAILED!") + msg = (">> encoding given reference element FAILED!\n" + "[Input string]:\n%s\n" % istring) + flog.exception(msg) + self.errno = -2 + return istring + + @staticmethod + def _escape_tex(istring): + """ Convert TeX symbols into XML valid symbols + + Parameters + ---------- + istring : str + + Returns + ------- + str + """ + + flog.debug(">> Converting TeX symbols into XML valid symbols") + return reduce(lambda a, b: string.replace(a, b[0], b[1]), + (istring, ("\\&", '&'), ("<", '<'), (">", '>'), + ("&", '&'), (r"\ndash ", "-"))) + + def _parse_str(self, istring, check=False): + """ Parse string into XML object + + Parameters + ---------- + istring : str + check : bool + If True, checking if string parses to valid XML. + If False, saving parsed XML or an error code + if parsing was unsuccessful + """ + + try: + xml = parseString(istring) + if not check: + self.xml = xml + else: + flog.debug("VALIDATING XML string ...") + flog.debug(">> XML contains no errors") + except ExpatError as err: + flog.debug(">> Parsing given XML FAILED!") + msg = (">> Parsing given XML FAILED!\n", + "[Parse query]:\n%s\n" % istring) + flog.exception(msg) + self.errno = err.code + + def prepare_query_str(self, refid, querystring): + """ Format the reference as an XML string and validate it + + Parameters + ---------- + refid : int + RefElement() instance id + querystring : str + + Returns + ------- + int + If query string was encoded and parsed into valid XML + successfully, it is appended to a future query strings list + and error code is set to 0 + + If something went wrong, non-zero value is returned + """ + + self.errno = 0 + flog.debug("PREPARING query reference") + + single_qstring = self._encode_str( + self.QUERY_ITEM_STRING % (self.query_format, + self._escape_tex(querystring), + refid) + ) + flog.debug(">> Formed query XML:\n" + + "~" * 70 + "\n%s\n" % single_qstring + "~" * 70) + + # Checking if formed string is a valid XML + self._parse_str(single_qstring, check=True) + if self.errno != 0: + return self.errno + + self.query_elems.append(single_qstring) + return self.errno + + def _send_query(self, querystring): + """ Send query to BatchMRef + + Parameters + ---------- + querystring : str + Validated XML query string, containing as many reference items + as QueryHandler.QUERY_ITEMS_LIMIT allows + + If request to BatchMRef was successful, saving query result, + otherwise non-zero error code is saved + """ + + queryinfo = {'qdata': querystring} + queryval = urllib.urlencode(queryinfo) + try: + flog.debug("SENDING query ...") + req = urllib2.Request(url=self.address, data=queryval) + flog.debug(">> Query POST data: %s" % req.get_data()) + context = ssl._create_unverified_context() + batchmref = urllib2.urlopen(req, context=context) + self.qcode = batchmref.getcode() + flog.debug(">> Query result code: %s" % self.qcode) + self.qresult = batchmref.read() + + if self.qcode == 200 and \ + self.qresult.startswith(self.QUERY_XML_HEADING_STRING): + flog.debug(">> Query result string:\n" + + "~"*70 + "\n%s\n" % self.qresult.strip() + "~"*70) + else: + msg = "\n%s" % self.AMS_MSG if self.AMS_MSG in self.qresult else "" + flog.debug(">> Query FAILED! %s" % msg) + flog.error("Query returned an error:\n%s\n\n%s" + % (msg, self.qresult)) + self.errno = self.qcode if self.qcode != 200 else -2 + self.qresult = None + + batchmref.close() + except: + msg = ">> Query FAILED!" + flog.debug(msg) + flog.exception(msg) + self.errno = -2 + self.qresult = None + + @staticmethod + def _extract_xml_data(xml_elem, tag): + """ Extract text data from an XML object + + Parameters + ---------- + xml_elem : XML object + tag : str + XML tag of interest + + Returns + ------- + str or None + Content of XML element with the requested tag. + If element with the tag hasn't been found, None is returned + """ + + childelem = xml_elem.getElementsByTagName(tag) + if childelem: + childnodes = childelem[0].childNodes + if childnodes: + return childnodes[0].data + + def _analyze_xml(self, xml): + """ Extract reference data from the BatchMRef returned XML string, + parsed into XML object + + Parameters + ---------- + xml : XML object + + If no matches have been found in the AMS MR DB, + current RefElement() instance gets a non-zero error code. + Otherwise MR number and reference content (if requested output type + is not None) are saved in the current RefElement() instance + """ + + mref_item = xml.getElementsByTagName("mref_item")[0] + refid = int(self._extract_xml_data(mref_item, "myid")) + elem = self.refscontainer.get_elem_by_refid(refid) + + matches = self._extract_xml_data(mref_item, "matches") + if matches == '1': + flog.debug(">> MRef DB: reference `%s' found!" % elem.citekey) + elem.mrid = self._extract_xml_data(mref_item, "mrid") + flog.debug(">> MRef ID: %s" % elem.mrid) + + if self.outputtype is not None: + elem.outref = self._extract_xml_data(mref_item, "outref") + flog.debug(">> MRef output reference:\n" + + "~"*70 + "\n%s\n" % elem.outref.strip() + "~"*70) + else: + elem.errno = -1 + flog.debug(">> MRef DB: reference `%s' not found!" % elem.citekey) + + def query(self): + """ Send a request to AMS BatchMRef and analyze the returned data + + If query result contains 'batch_error' element or returned + XML string can't be parsed into XML object, + RefsContainer() instance gets a non-zero error code. + """ + + self.errno = 0 + self.qresult = None + + querystring = (self.QUERY_XML_HEADING_STRING + + self.QUERY_HEADING_STRING % ("\n".join(self.query_elems))) + if self.errno == 0: + self._send_query(querystring) + if self.qresult is not None: + error_obj = self.re_batch_error.search(self.qresult) + if error_obj: + flog.debug(">> Query XML contains an ERROR!") + flog.error("[batch_error]:\n%s\n\n[querystring]:\n%s" + % (self._encode_str(error_obj.group(1)), + querystring)) + self.errno = -2 + flog.debug("Splitting query result and analyzing parts separately") + for item_qresult in self.re_mref_item.finditer(self.qresult): + self.xml = None + self._parse_str(self._encode_str(item_qresult.group())) + if self.xml is not None: + self._analyze_xml(self.xml) + + self.refscontainer.qerrno = self.errno + self.query_elems = list() + + +class HandleBBL(RefTypes): + """ This is the main class containing and initiating other classes' + methods and attributes for provided input data processing """ + + # MR number pattern matching all recognized reference formats + PATTERN_MR = r'MRNUMBER=\{.*?\}(,|)|review=\{\\MR\{.*?\}\}(,|)|\\MR\{.*?\}' + + PATTERN_BIBRE_LINE = r'^%.*\r?\n$' + PATTERN_BIBRE_PART = r'\s*(.*?)(?<!\\)%.*\r?\n$' + + PATTERN_TEX_ACCENTS = r"""(?:\{|)\\(?:"|'|`|\^|-|H|~|c|k|=|b|\.|d|r|u|v|A)(?:|\{)([a-zA-Z])\}(?:\}|)""" + PATTERN_BRACED_LETTERS = r"""(\s)(?<!\\)([a-zA-Z]*)\{([A-Z]+)\}""" + + # Mark of the input file ending + EOF = "EOF" + + # Default bibstyle format + PLAIN = 'plain' + + def __init__(self, inputfile, encoding, clean_comments, + itemno, wait, outputtype, bibstyle, debug, version=str()): + """ Initiate all methods and attributes required to process input data + + Parameters + ---------- + inputfile : str or None + encoding : str + Input file encoding + clean_comments : bool + If TeX comments cleaning is selected, + full comment lines will be moved to the beginning of each + identified bibliography reference item + itemno : int + Limit of reference items per query to BatchMRef + wait: int + Pause length after each query to BatchMRef + outputtype : str or None + If not None, additional files with the requested references, + extracted from the AMS MR DB in the requested output format, + will be generated + bibstyle : str or None + Used only if the requested output type is BIBTEX or IMS + debug : int + If debug value is greater than 0, debug messages will be + written to the FileHandler.LOG file. Also, depending on the + given debug value, final data written to the input file will + contain TeX comments with query data. + version : str + """ + + self.refscontainer = RefsContainer() + + self.fh = FilesHandler(inputfile, outputtype) + self.rh = RefHandler(outputtype) + self.qh = QueryHandler(encoding, outputtype, self.refscontainer) + + if itemno < self.qh.QUERY_ITEMS_LIMIT: + self.qh.QUERY_ITEMS_LIMIT = itemno + self.wait = wait + + self.outputtype = outputtype + self.bibstyle = bibstyle + flog.debug("Comments will be cleaned from the output: %s" + % clean_comments) + self.clean_comments = clean_comments + self.debug = debug + self.version = version + + self.re_bibre_line = re.compile(self.PATTERN_BIBRE_LINE) + self.re_bibre_part = re.compile(self.PATTERN_BIBRE_PART) + self.re_MR = re.compile(self.PATTERN_MR) + self.re_tex_accents = re.compile(self.PATTERN_TEX_ACCENTS) + self.re_braced_letters = re.compile(self.PATTERN_BRACED_LETTERS) + + self.eof = False + self.ifile_end_lines = list() + + @property + def icontent(self): + """ Input file content """ + return self.fh.read(self.fh.IN) + + @property + def write(self): + return self.fh.write + + @property + def get_fname(self): + return self.fh.get_fname + + def preprocess_ofiles(self): + """ Depending on the requested bibliography output type, + certain files are pre-filled with required data. + Writing action is fulfilled only if requested file was pre-opened. + """ + self.write(self.fh.AUX, '\\bibstyle{%s}\n' % self.bibstyle) + self.write(self.fh.HTML, "<!DOCTYPE html>\n<html>\n<body>\n\n") + + def postprocess_ofiles(self, refcount): + """ Depending on the requested bibliography output type, + certain files are filled up with the required data. + Writing action is fulfilled only if requested file was pre-opened. + + Parameters + ---------- + refcount: int + If refcount is 0, it means no references have been found + in the input file, and pre-opened additional files are deleted + + If requested bibliography output type is TEX, + number of bibliography items found is written to the first line of + FileHandler.DATA file. Therefore this file is written twice into. + """ + + if refcount == 0: + self.fh.close_and_delete(self.fh.DATA) + self.fh.close_and_delete(self.fh.AUX) + return None + + datafilepath = self.get_fname(self.fh.DATA) + self.write(self.fh.AUX, + '\\bibdata{%s}' % os.path.splitext(datafilepath)[0]) + self.write(self.fh.HTML, "\n</body>\n</html>\n") + + # Formatting the DATA file output according to requested output format + obiblenv = { + self.TEX: { + "begin": "\\begin{thebibliography}{%s}\n" + "\\csname bibmessage\\endcsname\n\n", + "end": "\\end{thebibliography}\n" + }, + self.AMSREFS: { + "begin": "\\begin{bibdiv}\n\\begin{biblist}\n\n", + "end": "\\end{biblist}\n\\end{bibdiv}" + } + } + + strings = obiblenv.get(self.outputtype, None) + if strings is None: + return None + + start_string, finish_string = sorted(strings.values()) + self.write(self.fh.DATA, finish_string) + + # Total items count is known only after processing all references and + # writing to the DATA file, therefore 'thebibliography' environment + # starting string is written to this file when all processing is + # finished + self.fh.close(self.fh.DATA) + os.rename(datafilepath, self.get_fname(self.fh.TMP)) + self.fh.open(self.fh.TMP, self.fh.READ) + + if self.outputtype == self.TEX: + start_string = start_string % refcount + self.fh.open(self.fh.DATA, self.fh.WRITE) + self.write(self.fh.DATA, start_string) + + shutil.copyfileobj(self.fh.files[self.fh.TMP], + self.fh.files[self.fh.DATA]) + + def _remove_tex_comments(self, line): + """ Remove TeX comments + + Parameters + ---------- + line : str + + Returns + ------- + str + """ + fmtline = self.re_bibre_line.sub('', line) + if fmtline: + matchobj = self.re_bibre_part.search(fmtline) + if matchobj is not None: + return "%s\n" % matchobj.groups(1)[0] + return fmtline + return fmtline + + def _remove_tex_accents(self, line): + """ Remove TeX accents and braces around upper case letters + + BatchMRef may not found a reference in the AMS MR DB because of + braces and accents present in reference string (tested), therefore + accented letters "{\'a}" and "\'{a}" are changed to plain "a". + Also "{ABC}" is changed to "ABC". + + Parameters + ---------- + line : str + + Returns + ------- + str + """ + mline = self.re_tex_accents.sub(r'\1', line) + if mline: + return self.re_braced_letters.sub(r'\1\2\3', mline) + return mline + + def gather_records(self, require_env): + """ Extract bibliography reference items from the input file + + Parameters + ---------- + require_env : bool + If True, get bibliography reference items only inside + the bibliography environment. If False, gel all bibliography + reference items found in the input file + + Yields + ------- + str + Denotes reference format type (one of ITYPES), + bibliography environment state (RefHandler.BIBL_BEGIN or + RefHandler.BIBL_END), + or input file end mark (EOF) + + RefElement() instance, str, or None + If reference of one of ITYPES type has been found, + a RefElement() instance is returned with the following + attributes filled in: + reftype, citekey, biblabel, + orig_lines, cleaned_lines, query_lines + + If end of input file has been determined, None is returned + + Otherwise current line is returned + """ + + def sort_comments_out(comment_lines): + """ Assign gathered comment lines to the rightful reference item + + Parameters + ---------- + comment_lines : list + + Returns + ------- + list + Comment lines, belonging to current reference item + list + Comment lines, belonging to the next reference item + """ + + next_elem_comments = [] + reversed_comments = comment_lines[::-1] + reversed_comments_backup = comment_lines[::-1] + advanced_by = 0 + for no, cline in enumerate(reversed_comments): + if len(element.orig_lines) < (no + 1 + advanced_by): + break + while not element.orig_lines[-(no + 1 + advanced_by)].strip(): + # skipping empty lines + advanced_by += 1 + if cline == element.orig_lines[-(no + 1 + advanced_by)]: + reversed_comments_backup.pop(0) + next_elem_comments.append(reversed_comments[no]) + current_elem_comments = reversed_comments_backup[::-1] + return current_elem_comments, next_elem_comments + + # Allowing gathering the references according to + # the bibliography environment status + envmap = {self.rh.BIBL_BEGIN: True, + self.rh.BIBL_END: False, + "not found": False if require_env else True} + gather = envmap["not found"] + search = True + + multiline = "" + element = RefElement() + envstatus = None + for line in self.icontent: + line = multiline + line + clean_line = self._remove_tex_comments(line) + + if not clean_line and element.orig_lines: + element.orig_lines.append(line) + element.comment_lines.append(line) + continue + + reftype = None + if search: + reftype, additional_info = self.rh.find_reference(clean_line) + + if require_env and reftype == self.rh.BIBL_ENV: + if element.reftype is not None: + # Full bibliography item + element.comment_lines, next_elem_comments = \ + sort_comments_out(element.comment_lines) + yield element.reftype, element + element = RefElement() + element.comment_lines = next_elem_comments + + # Bibliography environment + envstatus = additional_info.pop("envstatus", None) + if envstatus in envmap: + gather = envmap[envstatus] + search = gather + yield envstatus, line + continue + + elif reftype in self.ITYPES: + multiline = additional_info.get("line", "") + if multiline: + continue + + if element.reftype is not None: + # Full bibliography item + element.comment_lines, next_elem_comments = \ + sort_comments_out(element.comment_lines) + yield element.reftype, element + element = RefElement() + element.comment_lines = next_elem_comments + + if gather: + element.reftype = reftype + element.citekey = additional_info.get("citekey", None) + element.biblabel = additional_info.get("biblabel", None) + element.orig_lines.append(line) + + mrid_free_line = self.re_MR.sub('', clean_line) + element.cleaned_lines.append(mrid_free_line) + + ref_format_free_line = additional_info.get("text", clean_line) + mrid_free_line = self.re_MR.sub('', ref_format_free_line) + accent_free_line = self._remove_tex_accents(mrid_free_line) + element.query_lines.append(accent_free_line) + continue + + if gather and element.reftype is not None: + element.orig_lines.append(line) + mrid_free_line = self.re_MR.sub('', clean_line) + element.cleaned_lines.append(mrid_free_line) + accent_free_line = self._remove_tex_accents(mrid_free_line) + element.query_lines.append(accent_free_line) + else: + # Before and after the bibliography environment + yield envstatus, line + + if element.reftype is not None: + # The last full bibliography item + element.comment_lines, _ = sort_comments_out(element.comment_lines) + yield element.reftype, element + + yield self.EOF, None + + def transfer_to_file(self): + """ After each query to BatchMRef write gathered data into files + + Returns + ------- + int + Number of references, for which data has been successfully + obtained + """ + + successful = 0 + + for elem in self.refscontainer.elems: + if self.refscontainer.qerrno != 0: + elem.errno = self.refscontainer.qerrno + outstring = ''.join(elem.cleaned_lines if self.clean_comments else + elem.orig_lines) + + elem.outref = self.rh.insert_citekey( + elem.outref, elem.citekey, elem.biblabel, + elem.normalize(elem.cleaned_lines[1:])) + if elem.mrid is not None: + outstring = self.rh.insert_mrid(elem.reftype, outstring, elem.mrid) + slog.info(elem.mrid) + elif elem.errno == -1: + slog.warn('NotFound') + else: + slog.error('QueryError') + + if self.clean_comments: + outstring = "".join(elem.comment_lines) + outstring + + if self.debug == 1: + outstring = '%%%% %s\n%s' % (elem.querystring, outstring) + elif self.debug == 2: + outstring = '%%%% %s\n%s' % (elem.errno, outstring) + elif self.debug == 3: + outstring = '%%%% %s\n%%%% %s\n%s' % (elem.querystring, + elem.errno, + outstring) + + flog.debug("\n" + ">" * 70 + + "\nFINAL reference with MR id in original format:\n" + + "\n%s\n" % outstring.strip()) + + if elem.outref is not None: + flog.debug("FINAL reference in '%s' format:\n" % self.outputtype + + "\n%s\n" % elem.outref.strip() + "<" * 70) + self.write(self.fh.OUT, outstring) + self.write(self.fh.DATA, elem.outref if elem.outref else "") + self.write(self.fh.AUX, '\\citation{%s}\n' % elem.citekey) + + if elem.errno == 0 and self.refscontainer.qerrno == 0: + successful += 1 + + if self.eof: + while self.ifile_end_lines: + self.write(self.fh.OUT, self.ifile_end_lines.pop(0)) + + self.refscontainer = RefsContainer() + self.qh._refscontainer = self.refscontainer + + return successful + + def get_mr_codes(self, require_env): + """ Analyze input file content and process found reference items + + Parameters + ---------- + require_env : bool + If True, and if no bibliography reference items have been found + inside the bibliography environment, or an environment hasn't + been found at all, parameter is set to False this method and + reruns itself in order to search reference items in + the whole input file. + + Returns + ------- + int + Total bibliography reference items found + int + Total number of references, for which data has been + successfully obtained + int + Reference items processed with errors count + + If reference item of ITYPES has been found, current + RefElement() instance attribute 'refid' is assigned a value + """ + + msg = ("in the bibliography environment only" + if require_env else "in the whole input file") + flog.debug("SEARCHING for reference items: %s" % msg) + + total = 0 + valid = 0 + successful = 0 + records = self.gather_records(require_env=require_env) + pseudo_citekey = 0 + for reftype, record in records: + if reftype == self.EOF: + self.eof = True + + elif reftype not in self.ITYPES: + if reftype != self.rh.BIBL_END: + self.write(self.fh.OUT, record) + else: + self.ifile_end_lines.append(record) + continue + + elif valid == 0 or valid % self.qh.QUERY_ITEMS_LIMIT != 0: + total += 1 + + record.refid = total + if not record.citekey: + pseudo_citekey += 1 + record.citekey = '%s' % pseudo_citekey + + flog.debug("=" * 70) + flog.debug("FOUND reference %s: type=%s, cite_key=%s, biblabel=%s" + % (total, reftype, record.citekey, record.biblabel)) + + if reftype != self.BIBITEM: + record.querystring = self.rh.extract_keys_data(record.query_lines) + self.refscontainer.append_elem(record) + record.errno = self.qh.prepare_query_str(record.refid, + record.querystring) + if record.errno == 0: + valid += 1 + + if valid != 0 and (valid % self.qh.QUERY_ITEMS_LIMIT == 0 + or self.eof): + self.qh.query() + successful += self.transfer_to_file() + valid = 0 + if not self.eof: + sleep(self.wait) + + if total == 0 and require_env: + # If no bibliography items were found in the bibliography + # environment, then trying to search for them everywhere + # in the input file + flog.debug("FOUND no references! Changing the search mode ... ") + self.eof = False + self.ifile_end_lines = list() + self.fh.close(self.fh.OUT) + self.fh.open(self.fh.OUT) + return self.get_mr_codes(require_env=False) + + if self.ifile_end_lines: + self.transfer_to_file() + + flog.debug("=" * 70) + errors = total - successful + return total, successful, errors + + def run(self, require_env): + """ Main method + + Parameters + ---------- + require_env : bool + + Returns + ------- + get_mr_codes() output + """ + + slog.info("# %s #\nJob started:" % self.version) + starttime = time() + + self.preprocess_ofiles() + total, successful, errors = self.get_mr_codes(require_env=require_env) + self.postprocess_ofiles(refcount=total) + + flog.info(" total: %s, found: %s, not found: %s, time: %ss" + % (total, successful, errors, int(round(time()-starttime)))) + + slog.info("Job ended") + slog.info("Total: %s, found: %s, not found: %s" + % (total, successful, errors)) + slog.info('Job completed in %ss' % int(round(time()-starttime))) + + self.fh.close_files() + return total, successful, errors + + +if __name__ == '__main__': + import argparse + + # Logging to console + osh = logging.StreamHandler(stream=sys.stdout) + osh.setFormatter(BASICFORMATTER) + osh.setLevel(logging.INFO) + osh.addFilter(LessThanFilter(logging.INFO)) + + esh = logging.StreamHandler(stream=sys.stderr) + esh.setFormatter(BASICFORMATTER) + esh.setLevel(logging.WARN) + + slog = logging.getLogger("%s.StreamLogger" % __name__) + slog.setLevel(logging.INFO) + slog.addHandler(osh) + slog.addHandler(esh) + + # Logging to files + flog = logging.getLogger("%s.FileLogger" % __name__) + flog.setLevel(logging.DEBUG) + + def setup_logging_files(debug, basename=""): + """ Set up logging files + + Parameters + ---------- + debug : int + basename: str + Input file name + + Returns + ------- + logging instance + """ + + if debug == 0: + log_min_level = logging.INFO + log_max_level = logging.INFO + formatter = BASICFORMATTER + else: + log_min_level = logging.DEBUG + log_max_level = logging.WARN + formatter = DEBUGFORMATTER + + ofh = logging.FileHandler(filename="{}.{}.{}".format(basename, + FilesHandler.GMR_SUFFIX, + FilesHandler.LOG), + mode='w', delay=True) + ofh.setFormatter(formatter) + ofh.setLevel(log_min_level) + ofh.addFilter(LessThanFilter(log_max_level)) + flog.addHandler(ofh) + + efh = logging.FileHandler(filename="{}.{}.{}".format(basename, + FilesHandler.GMR_SUFFIX, + FilesHandler.ERR), + mode='w', delay=True) + efh.setFormatter(DEBUGFORMATTER) + efh.setLevel(logging.ERROR) + flog.addHandler(efh) + return flog + + VERSION = __version__.split("-")[0] + DESCRIPTION = ( + "Tool %s, is designed for: " % VERSION + + "(1) getting MR numbers for given references from AMS MRef database, " + + "(2) formatting the given references in one of AMS allowed formats. " + + "Maintainer: L.Tolene <lolita.tolene@vtex.lt>." + ) + + def get_cmd_args(): + """ Command line input parser """ + + parser = argparse.ArgumentParser( + description=DESCRIPTION, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument("filepath", help="References containing file") + parser.add_argument( + "--enc", '-e', type=str, default=QueryHandler.LATIN1, + help="Source file encoding or 'auto'" + ) + parser.add_argument( + "--format", '-f', choices=set(RefTypes.OTYPES), + help="Outputs the given references in provided format. " + "For more information about these formats please " + "consult the AMS MRef tool website. The 'ims' format " + "is almost the same as the 'bibtex' format" + ) + parser.add_argument( + "--bibstyle", '-s', default=HandleBBL.PLAIN, + help="BibTeX style. For more information please consult " + "the BibTeX documentation" + ) + parser.add_argument( + "--nobibenv", action='store_true', + help="If activated, references are searched throughout " + "all source file content; otherwise searching only " + "inside the bibliography environment. Currently " + "recognizable are the 'thebibliography' and 'biblist' " + "environments" + ) + parser.add_argument( + "--clean", '-c', action='store_true', + help="If activated, cleans comments appearing in references" + ) + parser.add_argument( + "--itemno", default=100, type=int, + help="Maximum item count for one AMS query. " + "AMS batchmref has a limit of 100 items per query." + ) + parser.add_argument( + "--wait", default=10, type=int, + help="time (in seconds) to wait between queries to AMS batchmref." + ) + parser.add_argument( + "--debug", '-d', choices={0, 1, 2, 3}, default=0, type=int, + help="Outputs additional info for debugging purposes." + ) + parser.add_argument( + "--version", '-v', action='version', version=VERSION, + help="Module version." + ) + args = parser.parse_args() + return (args.filepath, args.enc, args.format, args.bibstyle, + args.nobibenv, args.clean, args.itemno, args.wait, args.debug) + + # Get input parameter values + inputfile, encoding, output_format, bibstyle, nobibenv, clean, itemno, wait, debug \ + = get_cmd_args() + + # Load additional library is needed + if encoding == QueryHandler.AUTO_ENC: + from chardet.universaldetector import UniversalDetector + + # Setup logging files + flog = setup_logging_files(debug=debug, + basename=os.path.splitext(inputfile)[0]) + + # Create HandleBBL() instance + bblobj = HandleBBL(inputfile=inputfile, encoding=encoding, + clean_comments=clean, itemno=itemno, wait=wait, + outputtype=output_format, bibstyle=bibstyle, + debug=debug, version=VERSION) + + # Process input file + bblobj.run(require_env=not nobibenv) diff --git a/Master/texmf-dist/doc/latex/ejpecp/mgetmref.py b/Master/texmf-dist/doc/latex/ejpecp/mgetmref.py deleted file mode 100755 index 8910556c829..00000000000 --- a/Master/texmf-dist/doc/latex/ejpecp/mgetmref.py +++ /dev/null @@ -1,401 +0,0 @@ -#! /usr/bin/env python -################################################################################## -# -# getmref.py - gets the references links to MathSciNet throught the BatchMRef: -# http://www.ams.org/batchref?qdata=xmldocument -# -# Copyright (C) 2004 Sigitas Tolusis, VTeX Ltd. and Jim Pitman, Dept. Statistics, -# U.C. Berkeley -# E-mail: sigitas@vtex.let -# http://www.stat.berkeley.edu/users/pitman -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# Requires python ver. 2.2 -# -# Usage: -# getmref.py <bbl or tex file> -# -# Program (description): -# - makes inputfile copy to <inputfilename>.getmref.bak; -# - for each successful bibitem reference search adds line \MR{<mrid>}, -# where <mrid> is data from XML tag <mrid> without front symbols "MR"; -# - writes all adds to <inputfilename>; -# - generates log file <inputfilename>.getmref.log; -# - writes to stdout log info -# -# Changes: -# 2004/04/26 - \bibitem line removed from the query -# -# -################################################################################### -SVNinfo = "$Id: getmref.py 46 2006-03-30 07:02:14Z sigitas $" - -import sys, urllib, re, os.path, time, string -from xml.dom.minidom import parseString -import xml.parsers.expat as par - -starttime = time.time() -res = re.search(r'\S+:\s\S+\s+(.*?)\s.*\$', SVNinfo) -if res: - ver = res.group(1) -else: - ver = '0.0' -print "# getmref, v. %s #" % ver - -# -# bbl file parsing /begin -# - -def escapetex(instr): - res = reduce(lambda a,b: string.replace(a, b[0], b[1]), (instr, ("\\&", '&'), ("<", '<'), (">", '>'))) - return res - -def query(instring, bibID, address = 'http://www.ams.org/batchmref'): - domas = None; res = None; err = 0 - escapetexstring = escapetex(instring) - querystring = r'''<?xml version = "1.0" encoding = "UTF-8"?> -<mref_batch> -<mref_item outtype="tex"> -<inref> -%s -</inref> -<myid>%s</myid> -</mref_item> -</mref_batch>''' % (escapetexstring, bibID) - try: - indom = parseString(querystring) - except par.ExpatError, err: - print >>sys.stderr,"[parse query]: %s" % querystring - print >>sys.stderr,sys.exc_info() - pass - else: - queryinfo = {} - queryinfo['qdata'] = querystring - queryval = urllib.urlencode(queryinfo) - try: - batchmref = urllib.urlopen(address, queryval) - res = batchmref.read() - domas = parseString(res) - except err: - print >>sys.stderr,"[parse res]: %s" % res - print >>sys.stderr,sys.exc_info() - pass - return domas, res, err - -def remcomm(line): - "Removes TeX comments" - bibre = re.compile(r'\s*(.*?)(?<!\\)%.*\n$') - fmtline = re.sub('^%.*\n$','', line) - if fmtline: - matchobj = bibre.search(fmtline) - if matchobj: - return matchobj.groups(1)[0] - else: - return fmtline - else: - return fmtline - -def formatbibitem(bibID, domas): - errstring = None; outtype = None; mrid = None; myid = bibID; outref = None; err = 0 - try: - mref = domas.getElementsByTagName("mref_batch")[0] - mref_errors = mref.getElementsByTagName("batch_error") - if len(mref_errors): - errlist = [ mref_error.childNodes[0].nodeValue() for mref_error in mref_errors ] - errstring = ''.join(errlist) - err = -2 - else: - mref_items = [item for item in mref.getElementsByTagName("mref_item")] - matches = mref_items[0].getElementsByTagName("matches")[0].childNodes[0]._get_nodeValue() - if matches == '1': - for item in mref_items: - outtype = dict(item.attributes.items())["outtype"] - mrid = item.getElementsByTagName("mrid")[0].childNodes[0]._get_nodeValue() - err = 0 - if mrid[:2] == "MR": - mrid = mrid[2:] - myids = item.getElementsByTagName("myid") - if len(myids): - myid = myids[0].childNodes[0]._get_nodeValue() - else: - myid = bibID - outref = string.strip(item.getElementsByTagName("outref")[0].childNodes[0]._get_nodeValue()) - else: - err = -1 - except: - err = -3 - print >>sys.stderr,"[formatbibitem]: %s" % bibID - print >>sys.stderr,sys.exc_info() - pass - return mrid, outref, err - - -def handlebibitem(lines, bibID, biblabel=None): - res = 0; err = None; outref = None - outstring = string.strip(''.join(lines)) - lines[:] = [re.sub(r'\\MR\{.*?\}', '', a) for a in lines] - biblines = [x for x in [remcomm(a) for a in lines] if x] - bibstring = re.sub(r'\n', ' ', ''.join(biblines)) - match = re.search(r'\\bibitem\s*?(?:\[.*?\])?\s?\{(?:.*?)\}(.*)(\\endbibitem)?$',bibstring.strip()) - if match: - querystring = match.group(1).strip() - else: - querystring = bibstring - domas = None - try: - domas, xmlres, err = query(querystring, bibID) - except: - res = -2 - print >>sys.stderr,"[parse query]: %s" % querystring - print >>sys.stderr,sys.exc_info() - print 'Error', - else: - mrid, outref, err = formatbibitem(bibID, domas) - if not mrid: - print 'Not Found', - res = -1 - else: - print mrid, - if mrid[:2] == "MR": - outstring = bibstring + '\\MR{%s}' % mrid[2:].rjust(7,'0') - else: - outstring = bibstring + '\\MR{%s}' % mrid.rjust(7,'0') - outstrip, nsub = re.subn(r'\\endbibitem',r'',outstring) - if nsub: - outstrip += '\n\\endbibitem' - outstring = re.sub(r' ', r' ', outstrip) - if not outref: - outref = "Not found!" - else: - outref = re.sub(r'(?<!\\)#',r'\#', outref) - if biblabel: - print >>datafile, '\\bibitem%s{%s}\n%s\n' % (biblabel, bibID, outref) - else: - print >>datafile, '\\bibitem{%s}\n%s\n' % (bibID, outref) - return '%s\n' % outstring, res - -def handleextra(extralines): - if len(extralines): - print >>outputfile, ''.join(extralines), - -def handlebbl(inputfile, out=sys.stdout, data=sys.stdout): - print "Job started:", - total = 0; successful = 0; errors = 0; state = 0; pseudobibID = 0; readbib = '' - bibl_begin = re.compile(r'\s*\\begin\s*\{thebibliography\}.*$') - bibre = re.compile(r'^\s*\\bibitem.*') - bibreF = re.compile(r'\s*\\bibitem\s*(\[.*?\])*?\s?\{(.*?)\}.*$',re.S) - comments = re.compile(r'\s*%.*$') - bibl_end = re.compile(r'\s*\\end\s*\{thebibliography\}.*$') - for line in inputfile: - if len(readbib): - readbib += line - matchobj = bibreF.search(readbib) - if matchobj: - line = "%s" % readbib - readbib = '' - else: - continue - if line: - if state == 0: - matchobj = bibl_begin.search(line) - if matchobj: - print >>data,matchobj.group(0) - print >>data,"\\csname bibmessage\\endcsname\n" - state = 1 - print >>out, line, - continue - elif state == 1: - matchobj = bibre.search(line) - if matchobj: - matchobj = bibreF.search(line) - if matchobj: - biblabel, bibID = matchobj.groups() - if not len(bibID): - pseudobibID += 1 - bibID = '%s' % pseudobibID - state = 2 - lines = [line] - extralines = [] - continue - else: - readbib = line - continue - else: - print >>out, line, - continue - elif state == 2: - matchobj = bibre.search(line) - if matchobj: - matchobj = bibreF.search(line) - if matchobj: - total += 1 - print >>data,line - outstring, sres = handlebibitem(lines, bibID, biblabel) - if not sres: - successful += 1 - else: - errors += 1 - print >>out, outstring, - handleextra(extralines) - lines = [line] - extralines = [] - biblabel, bibID = matchobj.groups() - if not len(bibID): - pseudobibID += 1 - bibID = '%s' % pseudobibID - continue - else: - readbib = line - continue - else: - matchobj = bibl_end.search(line) - if matchobj: - state = 0 - total += 1 - outstring, sres = handlebibitem(lines, bibID, biblabel) - if not sres: - successful += 1 - else: - errors += 1 - print >>out, outstring, - handleextra(extralines) - print >>out, line, - print >>data,matchobj.group(0) - continue - else: - if line[:-1] == '': - state = 3 - extralines = [line] - continue - matchobj = comments.search(line) - if matchobj: - state = 3 - extralines = [line] - continue - lines.append(line) - continue - elif state == 3: - matchobj = bibre.search(line) - if matchobj: - matchobj = bibreF.search(line) - if matchobj: - state = 2 - total += 1 - outstring, sres = handlebibitem(lines, bibID, biblabel) - if not sres: - successful += 1 - else: - errors += 1 - print >>out, outstring, - handleextra(extralines) - lines = [line] - extralines = [] - biblabel, bibID = matchobj.groups() - if not len(bibID): - pseudobibID += 1 - bibID = '%s' % pseudobibID - continue - else: - readbib = line - continue - else: - matchobj = bibl_end.search(line) - if matchobj: - state = 0 - total += 1 - outstring, sres = handlebibitem(lines, bibID, biblabel) - if not sres: - successful += 1 - else: - errors += 1 - print >>out, outstring, - handleextra(extralines) - print >>out, line, - print >>data,matchobj.group(0) - continue - else: - if line[:-1] == '': - extralines.append(line) - continue - matchobj = comments.search(line) - if matchobj: - extralines.append(line) - continue - state = 2 - lines.extend(extralines) - lines.append(line) - extralines = [] - continue - else: - break - print "Job ended" - print "Total: %s, found: %s, errors: %s" % (total, successful, errors) - return (total, successful, errors) - -# -# bbl file parsing /end -# - -if len(sys.argv) < 2: - progname = os.path.basename(sys.argv[0]) - print "Usage:\n %s <bbl or tex file>" % progname - sys.exit(1) -infilename = sys.argv[1] -filebase = os.path.splitext(infilename)[0] -outfilename = "%s.getmref.tmp" % filebase -datafilename = "%s.getmref.data" % filebase -logfilename = "%s.getmref.log" % filebase - -inputfile = file(infilename, 'r') -outputfile = file(outfilename, 'w') -datafile = file(datafilename, 'w') -logfile = file(logfilename, 'w') -if os.path.isfile("%s.getmref.bak" % filebase): - os.unlink("%s.getmref.bak" % filebase) - -sys.stderr = file("%s.getmref.err" % filebase, 'w') -total = 0; successful = 0; errors = 0 -print >>logfile, "File: %s" % infilename -try: - total, successful, errors = handlebbl(inputfile, outputfile, datafile) -except: - print >>sys.stderr,"[handlebbl]" - print >>sys.stderr,sys.exc_info() -print >>logfile, " total: %s, found: %s, errors: %s, time: %ss" % (total, successful, - errors, int(round(time.time()-starttime))) - -inputfile.close() -outputfile.close() -datafile.close() -logfile.close() -sys.stderr.close() -sys.stderr = sys.__stderr__ -if os.path.isfile("%s.getmref.err" % filebase): - if not os.stat("%s.getmref.err" % filebase)[6]: - os.unlink("%s.getmref.err" % filebase) -if os.path.isfile("%s.getmref.bak" % filebase): - os.unlink("%s.getmref.bak" % filebase) -os.rename(infilename, "%s.getmref.bak" % filebase) - -#mes modif -#os.rename(outfilename, infilename) -f=open(outfilename,"r") -g=open(infilename,"w") -x=f.read() -g.write(re.sub(r"\r"," ",x)) - -#fin de la modif - -print 'Job completed in %ss' % int(round(time.time()-starttime)) - - - diff --git a/Master/texmf-dist/doc/latex/ejpecp/sample.pdf b/Master/texmf-dist/doc/latex/ejpecp/sample.pdf Binary files differindex 79f20ac7fa4..aa471752787 100644 --- a/Master/texmf-dist/doc/latex/ejpecp/sample.pdf +++ b/Master/texmf-dist/doc/latex/ejpecp/sample.pdf diff --git a/Master/texmf-dist/doc/latex/ejpecp/sample.tex b/Master/texmf-dist/doc/latex/ejpecp/sample.tex index 2930a08567c..fed48e097e2 100644 --- a/Master/texmf-dist/doc/latex/ejpecp/sample.tex +++ b/Master/texmf-dist/doc/latex/ejpecp/sample.tex @@ -114,7 +114,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \VOLUME{0} -\YEAR{2016} +\YEAR{2020} \PAPERNUM{0} \DOI{10.1214/YY-TN} @@ -459,6 +459,21 @@ and in that case, you should provide the MR number instead of the arXiv identifier. It is acceptable to leave arXiv links in the bibliography (alongside MR links) even if the article has been published. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% %% +%% Supplementary Material, if any, should be provided in %% +%% {supplement} environment with title and short description. %% +%% %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\begin{supplement} +\stitle{Title of Supplement A.} +\sdescription{Short description of Supplement A.} +\end{supplement} +\begin{supplement} +\stitle{Title of Supplement B.} +\sdescription{Short description of Supplement B.} +\end{supplement} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% |