From e0c6872cf40896c7be36b11dcc744620f10adf1d Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Mon, 2 Sep 2019 13:46:59 +0900 Subject: Initial commit --- biblio/bibtex/utils/pybib/BibEntry.py | 662 ++++++++++++++++++++++++++++++ biblio/bibtex/utils/pybib/BibTeX.py | 456 ++++++++++++++++++++ biblio/bibtex/utils/pybib/Bibliography.py | 167 ++++++++ biblio/bibtex/utils/pybib/README | 19 + biblio/bibtex/utils/pybib/bib2html | 146 +++++++ biblio/bibtex/utils/pybib/bibcat | 86 ++++ biblio/bibtex/utils/pybib/bibdvi | 110 +++++ biblio/bibtex/utils/pybib/bibfilter | 110 +++++ biblio/bibtex/utils/pybib/bibgoogle | 264 ++++++++++++ biblio/bibtex/utils/pybib/bibkey | 112 +++++ biblio/bibtex/utils/pybib/biblint | 79 ++++ biblio/bibtex/utils/pybib/biblist | 78 ++++ biblio/bibtex/utils/pybib/bibmerge | 116 ++++++ biblio/bibtex/utils/pybib/bibnames | 89 ++++ biblio/bibtex/utils/pybib/bibsort | 100 +++++ biblio/bibtex/utils/pybib/bibsummary | 81 ++++ 16 files changed, 2675 insertions(+) create mode 100644 biblio/bibtex/utils/pybib/BibEntry.py create mode 100644 biblio/bibtex/utils/pybib/BibTeX.py create mode 100644 biblio/bibtex/utils/pybib/Bibliography.py create mode 100644 biblio/bibtex/utils/pybib/README create mode 100644 biblio/bibtex/utils/pybib/bib2html create mode 100644 biblio/bibtex/utils/pybib/bibcat create mode 100644 biblio/bibtex/utils/pybib/bibdvi create mode 100644 biblio/bibtex/utils/pybib/bibfilter create mode 100644 biblio/bibtex/utils/pybib/bibgoogle create mode 100644 biblio/bibtex/utils/pybib/bibkey create mode 100644 biblio/bibtex/utils/pybib/biblint create mode 100644 biblio/bibtex/utils/pybib/biblist create mode 100644 biblio/bibtex/utils/pybib/bibmerge create mode 100644 biblio/bibtex/utils/pybib/bibnames create mode 100644 biblio/bibtex/utils/pybib/bibsort create mode 100644 biblio/bibtex/utils/pybib/bibsummary (limited to 'biblio/bibtex/utils/pybib') diff --git a/biblio/bibtex/utils/pybib/BibEntry.py b/biblio/bibtex/utils/pybib/BibEntry.py new file mode 100644 index 0000000000..b040d5b096 --- /dev/null +++ b/biblio/bibtex/utils/pybib/BibEntry.py @@ -0,0 +1,662 @@ +# Bibliography entry class +# - holds all information about one bibliographic item +# - provides methods for manipulating/setting/representing that information +# +# TODO: +# __repr__ method needs to do a better job depending on the reference type, similar +# logic is required in bib2html (but it's not their either...) +# + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import sys; +import string; +import re; + +#BadValue = "Bad value"; +#BadField = "Bad field"; +#BadRefType = "Bad reference type"; + +class BibEntry: + fieldDict = {}; + verbose = 0; + bibliography = {}; + + def __init__(self, key, bib): + self.key = key; + self.fieldDict = {}; + self.bibliography = bib; + if BibEntry.verbose: + print >> sys.stderr, "New entry ", key; + + def __repr__(self): + str = '"' + self.getTitle() + '"; '; + try: + str = str + self.getAuthorsNames(); + except: + try: + str = str + "eds. " + self.getEditorsNames(); + except: + pass; + month = self.getMonthName(); + year = self.getYear(); + book = self.getBooktitle(); + if book: + str += ", " + book; + if month: + str += ", " + month; + if year > 0: + str += " " + `year`; + else: + if year > 0: + str += ", " + `year`; + str += '.'; + return str; + + def brief(self, fp=sys.stdout): + print >> fp, self; + + def display(self, fp=sys.stdout): + print >> fp, "%12s: %s" % ("CiteKey", self.key) + for k in self.fieldDict: + if k[0] == '_': + continue; + if k == 'Author': + print >> fp, "%12s: %s" % (k, self.getAuthors()) + else: + print >> fp, "%12s: %s" % (k, self.fieldDict[k]) + + def __getitem__(self, i): + if type(i) is str: + return self.fieldDict[i]; + elif type(i) is int: + return self.fieldDict.keys()[i]; + else: + raise; + + + def check(self): + keys = self.fieldDict.keys(); + missing = []; + reftype = self.getRefType(); + if not (reftype in alltypes): + raise AttributeError, "bad reference type [%s]" % self.getKey(); + for k in required_fields[self.getRefType()]: + if not (string.capitalize(k) in keys): + missing.append(k); + return missing; + + #############################################################3 + # get methods + #############################################################3 + + def getKey(self): + return self.key; + + def getField(self, field): + #print >> sys.stderr, field + #print >> sys.stderr, self.fieldDict[field] + field = field.capitalize(); + if field in self.fieldDict: + return self.fieldDict[field] + else: + return None; + + def getRefType(self): + return self.reftype; + + def isRefType(self, rt): + return self.getRefType().lower() == rt.lower(); + + def getTitle(self): + if 'Title' in self.fieldDict: + title = self.fieldDict['Title']; + title = re.sub(r"""[{}]""", "", title); + title = title.strip('.,\'"'); + return title; + else: + return ""; + + def getURL(self): + if 'Url' in self.fieldDict: + url = self.fieldDict['Url']; + return url; + else: + return ""; + + def getAuthorList(self): + if 'Author' in self.fieldDict: + return self.fieldDict['Author']; + else: + return []; + + def getAuthors(self): + if 'Author' in self.fieldDict: + l = self.fieldDict['Author']; + if len(l) == 1: + return l[0]; + elif len(l) == 2: + return l[0] + " and " + l[1]; + elif len(l) > 2: + return string.join(l[:-1], ", ") + " and " + l[-1]; + else: + return ""; + + + def surname(self, author): + # remove LaTeX accents + def chg(mo): return mo.group(mo.lastindex); + re_accent = re.compile(r'''\\[.'`^"~=uvHcdb]\{(.)\}|\t\{(..)\}'''); + author = re_accent.sub(chg, author) + + # "surname, first names" + m = re.search(r"""^([^,]*),(.*)""", author); + if m: + #print >> sys.stderr, m.group(1), m.group(2) + #return m.group(1) + "," + m.group(2).lstrip()[0]; + return [m.group(1), m.group(2).lstrip()[0]]; + #return m.group(1); + + # "first names surname" + + # take the last component after dot or space + #m = re.search(r"""([a-zA-Z][a-zA-Z-]*)$""", author); + m = re.search(r"""(.*?)([^\. \t]*)$""", author); + if m: + #print >> sys.stderr, author, ":", m.group(2), "|", m.group(1) + return [m.group(2), m.group(1)[0]]; + #return m.group(2) + "," + m.group(1)[0]; + + return ""; + + def getAuthorsSurnameList(self): + if 'Author' in self.fieldDict: + l = self.fieldDict['Author']; + return map(self.surname, l); + + def getAuthorsSurname(self): + l = self.getAuthorsSurnameList(); + try: + l = map(lambda x: x[0], l); + if len(l) == 1: + return l[0]; + elif len(l) == 2: + return l[0] + " and " + l[1]; + elif len(l) > 2: + return string.join(l[:-1], ", ") + " and " + l[-1]; + else: + return ""; + except: + return ""; + + # return initial dot sunrname + def getAuthorsNames(self): + l = self.getAuthorsSurnameList(); + l = map(lambda x: x[1] + ". " + x[0], l); + if len(l) == 1: + return l[0]; + elif len(l) == 2: + return l[0] + " and " + l[1]; + elif len(l) > 2: + return string.join(l[:-1], ", ") + " and " + l[-1]; + else: + return ""; + + # return initial dot sunrname + + def getEditorsSurnameList(self): + if 'Editor' in self.fieldDict: + l = self.fieldDict['Editor']; + return map(self.surname, l); + + def getEditorsNames(self): + l = self.getEditorsSurnameList(); + if not l: + return None; + l = map(lambda x: x[1] + ". " + x[0], l); + if len(l) == 1: + return l[0]; + elif len(l) == 2: + return l[0] + " and " + l[1]; + elif len(l) > 2: + return string.join(l[:-1], ", ") + " and " + l[-1]; + else: + return ""; + + def getBooktitle(self): + if 'Booktitle' in self.fieldDict: + return self.fieldDict['Booktitle']; + else: + return ""; + + def getVolume(self): + if 'Volume' in self.fieldDict: + return self.fieldDict['Volume']; + else: + return -1; + + def getNumber(self): + if 'Number' in self.fieldDict: + return self.fieldDict['Number']; + else: + return -1; + + def getPage(self): + if 'Pages' in self.fieldDict: + return self.fieldDict['Pages']; + else: + return ""; + + def afterDate(self, date): + '''True if the entry occurs after the specified date''' + + if not date: + return True; + elif len(date) == 1: + # simple case, year only + return self.getYear() >= date[0]; + elif len(date) == 2: + # complex case, [month year] + if self.getYear() > date[1]: + return True; + elif (date[1] == self.getYear()) and (self.getMonth() >= date[0]): + return True; + else: + return False; + def beforeDate(self, date): + '''True if the entry occurs before the specified date''' + + if not date: + return True; + elif len(date) == 1: + # simple case, year only + return self.getYear() < date[0]; + elif len(date) == 2: + # complex case, [month year] + if self.getYear() < date[1]: + return True; + elif (date[1] == self.getYear()) and (self.getMonth() < date[0]): + return True; + else: + return False; + + def getYear(self): + if '_year' in self.fieldDict: + return self.fieldDict['_year']; + else: + return -1; + + # return month ordinal in range 1 to 12 + def getMonth(self): + if '_month' in self.fieldDict: + return self.fieldDict['_month']; + else: + return -1; + + monthdict = { + 'january' : 1, + 'february' : 2, + 'march' : 3, + 'april' : 4, + 'may' : 5, + 'june' : 6, + 'july' : 7, + 'august' : 8, + 'september' : 9, + 'october' : 10, + 'november' : 11, + 'december' : 12 }; + + def getMonthName(self): + monthNames = ( + 'january', + 'february', + 'march', + 'april', + 'may', + 'june', + 'july', + 'august', + 'september', + 'october', + 'november', + 'december' ); + m = self.getMonth(); + if m > 0: + return string.capitalize(monthNames[m-1]); + else: + return ""; + + + + #############################################################3 + # set methods + #############################################################3 + + def setType(self, value): + value = string.lower(value); + if not (value in alltypes): + raise AttributeError, "bad reference type [%s]" % self.getKey(); + self.reftype = value; + self.fieldDict['Type'] = value; + + def setField(self, key, value): + key = key.capitalize(); + if not (key in allfields): + raise AttributeError, "bad field <%s> [%s]" % (key, self.getKey()); + if key == 'Year': + self.fieldDict[key] = value; + + # remove all text like "to appear", just leave the digits + year = filter(lambda c : c.isdigit(), value); + try: + self.fieldDict['_year'] = int(year); + except: + if value.find('appear') > -1: + sys.stderr.write("[%s] no year specified, continuing\n" % self.getKey()); + self.fieldDict['_year'] = 0; + else: + self.fieldDict['_year'] = -1; + raise AttributeError, "[%s] bad year <%s>" % (self.getKey(), value); + elif key == 'Month': + # the Month entry has the original string from the file if it is of + # nonstandard form, else is None. + # the hidden entry _month has the ordinal number + self.fieldDict[key] = value; + #print >> sys.stderr, "Month = <%s>" % value; + month = mogrify(value); + for monthname in self.monthdict: + # handle month abbreviations, eg. nov in november + if monthname.find(month) >= 0: + self.fieldDict['_month'] = self.monthdict[monthname]; + #print >> sys.stderr, "_month 1 %d" % self.monthdict[monthname]; + self.fieldDict[key] = None; + + return; + # handle extraneous like november in 'november 12-13' + if month.find(monthname) >= 0: + self.fieldDict['_month'] = self.monthdict[monthname]; + #print >> sys.stderr, "_month 2 %d" % self.monthdict[monthname]; + return; + raise AttributeError, "bad month [%s]" % self.getKey(); + else: + self.fieldDict[key] = value; + #print >> sys.stderr, "<%s> := <%s>\n" % (key, value) + + + + #############################################################3 + # matching methods + #############################################################3 + + def search(self, field, str, caseSens=0): + field = string.capitalize(field); + + if field.lower() == 'all': + for be in self: + for k in self.fieldDict: + if k[0] == '_': + continue; + s = self.fieldDict[k]; + if isinstance(s, list): + s = ' '.join(s); + if s: + if caseSens == 0: + m = re.search(str, s, re.IGNORECASE); + else: + m = re.search(str, s); + if m: + return True; + + else: + # silently ignore search field if not present + if not(field in self.fieldDict): + return False; + s = self.fieldDict[field]; + if isinstance(s, list): + s = ' '.join(s); + if s: + if caseSens == 0: + m = re.search(str, s, re.IGNORECASE); + else: + m = re.search(str, s); + if m: + return True; + + return 0; + + + def matchAuthorList(self, be): + + def split(a): + return re.findall(r"""([a-zA-Z][a-zA-Z-]*[.]?)""", a); + + def matchfrag(s, f): + sdot = s[-1:] == '.'; + fdot = f[-1:] == '.'; + + if (sdot == 0) and (fdot == 0): + return s == f; + elif (sdot == 0) and (fdot == 1): + matchstr = f + '*'; + m = re.match(matchstr, s); + if m: + return m.group(0) == s; + else: + return 0; + elif (sdot == 1) and (fdot == 0): + matchstr = s + '*'; + m = re.match(matchstr, f); + if m: + return m.group(0) == f; + else: + return 0; + elif (sdot == 1) and (fdot == 1): + return s == f; + + def matchAuthor(a1, a2): + l1 = split(a1); + l2 = split(a2); + count = 0; + + for p1 in l1: + for p2 in l2: + if matchfrag(p1,p2): + count += 1; + return count; + + # check if each article has the same number of authors + l1 = self.getAuthorList(); + l2 = be.getAuthorList(); + if len(l1) != len(l2): + return 0; + + # now check the authors match, in order + for i in range( len(l1) ): + if matchAuthor(l1[i], l2[i]) < 2: + return 0; + return 1; + + def matchTitle(self, be, dthresh): + # Levenstein distance between two strings + def distance(a,b): + c = {} + n = len(a); m = len(b) + + for i in range(0,n+1): + c[i,0] = i + for j in range(0,m+1): + c[0,j] = j + + for i in range(1,n+1): + for j in range(1,m+1): + x = c[i-1,j]+1 + y = c[i,j-1]+1 + if a[i-1] == b[j-1]: + z = c[i-1,j-1] + else: + z = c[i-1,j-1]+1 + c[i,j] = min(x,y,z) + return c[n,m] + + d = distance( mogrify(self.getTitle()), mogrify(be.getTitle()) ); + + return d <= dthresh; + + def matchType(self, be): + return self.getRefType() == be.getRefType(); + + def matchYear(self, be): + return fmatch(self.getYear(), be.getYear()); + + def matchMonth(self, be): + return fmatch(self.getMonth(), be.getMonth()); + + def matchVolumeNumber(self, be): + if not fmatch(self.getVolume(), be.getVolume()): + return 0; + if not fmatch(self.getNumber(), be.getNumber()): + return 0; + return 1; + + def matchPage(self, be): + + p1 = self.getPage(); + p2 = be.getPage(); + if p1 and p2: + # both not null + p1 = re.findall("([0-9.]+)", p1); + p2 = re.findall("([0-9.]+)", p2); + if (len(p1) > 0) and (len(p2) > 0): + # optionally compare starting page numbers + if p1[0] != p2[0]: + return 0; + if (len(p1) > 1) and (len(p2) > 1): + # optionally compare ending page numbers + if p1[1] != p2[1]: + return 0; + return 1; + else: + return 1; + + + # see if two bibentries match + def match(self, be, dthresh=2): + # we do the cheapest comparisons first... + if not self.matchType(be): + return 0; + if not self.matchYear(be): + return 0; + if not self.matchMonth(be): + return 0; + if self.isRefType("Article"): + if not self.matchVolumeNumber(be): + return 0; + if not self.matchPage(be): + return 0; + if not self.matchAuthorList(be): + return 0; + if not self.matchTitle(be, dthresh): + return 0; + return 1; + +# we adopt the convention that a numeric value of -1 means not provided, +# so here we match two quantites where either or both is not provided. Only +# return false if both numbers are provided, and they are not equal, otherwise +# give the benefit of the doubt and return true. +def fmatch(n1, n2): + if (n1 > 0) and (n2 > 0): + return n1 == n2; + else: + return 1; + +# remove all punctuation marks and white space that people +# might get wrong +def mogrify(s): + s = string.lower(s); + s = re.sub(r"""[#{}:;,&$ -]""", "", s); + return s; + + +allfields = ('_Reftype', 'Address', 'Author', 'Booktitle', 'Chapter', 'Edition', + 'Editor', 'Howpublished', 'Institution', 'Journal', 'Month', + 'Number', 'Organization', 'Pages', 'Publisher', 'School', + 'Series', 'Title', 'Type', 'Volume', + 'Year', 'Note', 'Code', 'Url', 'Crossref', 'Annote', 'Abstract', 'Date-added', 'Date-modified', 'Read'); + +# list of all reference types +alltypes = ('article', 'book', 'booklet', 'inbook', 'incollection', + 'inproceedings', 'manual', 'mastersthesis', 'misc', 'phdthesis', + 'proceedings', 'techreport', 'unpublished'); + +# list of additional fields, ignored by the standard BibTeX styles +ign = ('crossref', 'code', 'url', 'annote', 'abstract'); + +# lists of required and optional fields for each reference type + +required_fields = { + 'article' : ['Author', 'Title', 'Journal', 'Year'], + 'book' : ['Author', 'Title', 'Publisher', 'Year'], + 'booklet' : ['Title'], + 'inbook' : ['Author', 'Title', 'Chapter', 'Pages', + 'Publisher', 'Year'], + 'incollection' : ['Author', 'Title', 'Booktitle', 'Publisher', 'Year'], + 'inproceedings' : ['Author', 'Title', 'Booktitle', 'Year'], + 'manual' : ['Title'], + 'misc' : [], + 'mastersthesis' : ['Author', 'Title', 'School', 'Year'], + 'phdthesis' : ['Author', 'Title', 'School', 'Year'], + 'proceedings' : ['Title', 'Year'], + 'techreport' : ['Author', 'Title', 'Institution', 'Year'], + 'unpublished' : ['Author', 'Title', 'Note'] +}; + +opt_fields = { + 'article' : ['Volume', 'Number', 'Pages', 'Month', 'Note'], + 'book' : ['Editor', 'Volume', 'Number', 'Series', 'Address', + 'Edition', 'Month', 'Note'], + 'booklet' : ['Author', 'Howpublished', 'Address', 'Month', 'Year', + 'Note'], + 'inbook' : ['Editor', 'Volume', 'Series', 'Address', 'Edition', + 'Month', 'Note'], + 'incollection' : ['Editor', 'Volume', 'Number', 'Series', 'Type', + 'Chapter' 'Pages', 'Address', 'Edition', + 'Month', 'Note'], + 'inproceedings' : ['Editor', 'Pages', 'Organization', 'Publisher', + 'Address', 'Month', 'Note'], + 'manual' : ['Author', 'Organization', 'Address', 'Edition', + 'Month', 'Year', 'Note'], + 'misc' : ['Title', 'Author', 'Howpublished', 'Month', 'Year', + 'Note'], + 'mastersthesis' : ['Address', 'Month', 'Note'], + 'phdthesis' : ['Address', 'Month', 'Note'], + 'proceedings' : ['Editor', 'Publisher', 'Organization', 'Address', + 'Month', 'Note'], + 'techreport' : ['Type', 'Number', 'Address', 'Month', 'Note'], + 'unpublished' : ['Month', 'Year'] +}; diff --git a/biblio/bibtex/utils/pybib/BibTeX.py b/biblio/bibtex/utils/pybib/BibTeX.py new file mode 100644 index 0000000000..5166f3d937 --- /dev/null +++ b/biblio/bibtex/utils/pybib/BibTeX.py @@ -0,0 +1,456 @@ +# Defines two classes: +# +# BibTexEntry, subclass of BibEntry, and provides all BibTeX specific methods such as +# writing an entry to file +# +# BibTex, a subclass of Bibliography, and provides all BibTeX specific methods, in +# particular a parser. + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import string; +import re; +import sys; +import urllib; + +class BibTeXEntry(BibEntry.BibEntry): + + # write a BibTex format entry + def write(self, file=sys.stdout, stringdict=None): + file.write( "@%s{%s,\n" % (self.getRefType(), self.getKey()) ); + count = 0 + for rk in self.fieldDict: + count += 1; + # skip internally used fields + if rk[0] == '_': + continue; + if rk == 'Type': + continue; + + # generate the entry + value = self.fieldDict[rk]; + file.write(" %s = " % rk ); + + if rk in ['Author', 'Editor']: + file.write("{%s}" % " and ".join(value) ); + elif rk == 'Month': + if value: + file.write("{%s}" % value ); + else: + value = self.getMonthName(); + file.write("%s" % value[0:3].lower() ); + else: + # is it an abbrev? + if value in self.bibliography.abbrevDict: + file.write("%s" % value ); + else: + file.write("{%s}" % value ); + + # add comma to all but last fields + if count < len(self.fieldDict): + file.write(",\n"); + else: + file.write("\n"); + file.write("}\n\n"); + + + def setField(self, field, value): + def strStrip(s): + s = string.strip(s, ' '); + if (s[0] == '"') and (s[-1] == '"'): + return s[1:-1]; + if (s[0] == '{') and (s[-1] == '}'): + return s[1:-1]; + return s; + + + # deal specially with author list, convert from bibtex X and Y to + # a list for bibentry class + if field.lower() in ["author", "editor"]: + value = string.split(value, " and "); + value = map(strStrip, value); + try: + # invoke the superclass + BibEntry.BibEntry.setField(self, field, value); + except AttributeError, err: + sys.stderr.write( "%15s: bad value <%s=%s>" % (self.getKey(), field, value)); + +class BibTeX(Bibliography.Bibliography): + + stringDict = {}; + + def parseFile(self, fileName=None, verbose=0, ignore=False): + if fileName == None: + fp = sys.stdin; + else: + fp = self.open(fileName); + + # get the file into one huge string + nbib = 0; + s = fp.read(); + try: + nbib = self.parseString(s, ignore=ignore, verbose=verbose); + except AttributeError, err: + print >> sys.stderr, "Error %s" % err; + + self.close(fp); + return nbib; + + def display(self): + for be in self: + be.display() + + def write(self, file=sys.stdout, resolve=0): + if resolve: + dict = self.stringDict; + else: + dict = None; + + for be in self: + be.write(file, dict) + + def writeStrings(self, file=sys.stdout): + for abbrev, value in self.abbrevDict.items(): + file.write("@string{ %s = {%s} }\n" % (abbrev, value) ); + + # resolve BibTeX's cross reference capability + def resolveCrossRef(self): + for be in self: + try: + xfref = self.getField('crossref'); + except: + return; + + for f in xref: + if not (f in be): + be.setField(f, xref.getField(f)); + + def parseString(self, s, verbose=0, ignore=False): + + # lexical analyzer for bibtex format files + class BibLexer: + + inString = ""; # the string to parse + lineNum = 1; + pos = 0; + + def __init__(self, s): + self.inString = s; + + # an iterator for the class, return next character + def next(self): + if self.pos >= len(self.inString): + raise StopIteration; + c = self.inString[self.pos]; + if c == '\n': + self.lineNum += 1; + self.pos += 1; + return c; + + def __iter__(self): + return self; + + # peek at the next character + def peek(self): + return self.inString[self.pos]; + + # push a character back onto the input + def pushback(self, c): + self.pos -= 1; + if c == '\n': + self.lineNum -= 1; + + # eat whitepsace characters and comments + def skipwhite(self): + + for c in self: + if c == '%': + for c in self: + if c == '\n': + break; + elif (not c.isspace()): + self.pushback(c); + break; + + # print >> sys.stderr, the input buffer + def show(self): + print >> sys.stderr, "[%c]%s" % (self.inString[0], self.inString[1:10]); + + # get the next word from the input stream, this can be + # [alpha][alnum$_-] + # "...." + # {....} + def nextword(self): + + str = ""; + c = self.peek(); + + if c == '"': + # quote delimited string + str = self.next(); + cp = None; # prev char + for c in self: + str += c; + if (c == '"') and (cp != '\\'): + break; + cp = c; + elif c == '{': + # brace delimited string + count = 0; + for c in self: + if c == '{': + count += 1; + if c == '}': + count -= 1; + + str += c; + if count == 0: + break; + else: + # undelimited string + #if (not c.isalpha()): + # print >> sys.stderr, "BAD STRING" + for c in self: + if c.isalnum(): + str += c; + elif c in ".+-_$:'": + str += c; + else: + self.pushback(c); + break; + return str; + + + class Token: + t_ENTRY = 1; + t_DELIM_L = 2; + t_DELIM_R = 3; + t_STRING = 5; + t_EQUAL = 6; + t_COMMA = 7; + + val = None; + type = None; + + def __repr__(self): + if self.type == self.t_ENTRY: + str = "@ %s" % self.val; + elif self.type == self.t_DELIM_R: + str = " }"; + elif self.type == self.t_STRING: + str = "<%s>" % self.val; + elif self.type == self.t_EQUAL: + str = " EQUAL"; + elif self.type == self.t_COMMA: + str = " COMMA"; + else: + str = "BAD TOKEN (%d) <%s>" % (self.type, self.val); + return str; + + def isstring(self): + return self.type == self.t_STRING; + + def isabbrev(self): + return (self.type == self.t_STRING) and self.val.isalnum(); + + def iscomma(self): + return self.type == self.t_COMMA; + + def isequal(self): + return self.type == self.t_EQUAL; + + def isentry(self): + return self.type == self.t_ENTRY; + + def isdelimR(self): + return self.type == self.t_DELIM_R; + + def isdelimL(self): + return self.type == self.t_DELIM_L; + + # + # tokenizer for bibtex format files + # + class BibTokenizer: + + lex = None; + + def __init__(self, s): + self.lex = BibLexer(s); + + # setup an iterator for the next token + def __iter__(self): + return self; + + # return next token + def next(self): + #self.lex.show(); + self.lex.skipwhite(); + c = self.lex.next(); + + t = Token(); + if c == '@': + t.type = t.t_ENTRY; + self.lex.skipwhite(); + t.val = self.lex.nextword(); + self.lex.skipwhite(); + c = self.lex.next(); + if not ((c == '{') or (c == '(')): + print >> sys.stderr, "BAD START OF ENTRY" + + elif c == ',': + t.type = t.t_COMMA; + elif c == '=': + t.type = t.t_EQUAL; + elif (c == '}') or (c == ')'): + t.type = t.t_DELIM_R; + else: + self.lex.pushback(c); + t.type = t.t_STRING; + t.val = self.lex.nextword(); + + return t; + + + class BibParser: + + tok = None; + bibtex = None; + + def __init__(self, s, bt): + self.tok = BibTokenizer(s); + self.bibtex = bt; + + # setup an iterator for the next entry + def __iter__(self): + return self; + + # return next entry + def next(self): + + def strstrip(s): + if s[0] in '"{': + return s[1:-1]; + else: + return s; + + t = self.tok.next(); + if not t.isentry(): + raise SyntaxError, self.tok.lex.lineNum; + if t.val.lower() == 'string': + tn = self.tok.next(); + if not tn.isstring(): + raise SyntaxError, self.tok.lex.lineNum; + t = self.tok.next(); + if not t.isequal(): + raise SyntaxError, self.tok.lex.lineNum; + tv = self.tok.next(); + if not tv.isstring(): + raise SyntaxError, self.tok.lex.lineNum; + # insert string into the string table + self.bibtex.insertAbbrev(tn.val, strstrip(tv.val)); + #print >> sys.stderr, "string", tn.val, tv.val + t = self.tok.next(); + if not t.isdelimR(): + raise SyntaxError, self.tok.lex.lineNum; + elif t.val.lower() == 'comment': + depth = 0; + while True: + tn = self.tok.next(); + if t.isdelimL(): + depth += 1; + if t.isdelimR(): + depth -= 1; + if depth == 0: + break; + else: + # NOT A STRING or COMMENT ENTRY + # assume a normal reference type + + # get the cite key + ck = self.tok.next(); + if not ck.isstring(): + raise SyntaxError, self.tok.lex.lineNum; + + #print >> sys.stderr, t.val, ck.val + be = BibTeXEntry(ck.val, self.bibtex); + be.setType(t.val); + + # get the comma + ck = self.tok.next(); + if not ck.iscomma(): + raise SyntaxError, self.tok.lex.lineNum; + + # get the field value pairs + for tf in self.tok: + # allow for poor syntax with comma before + # end brace + if tf.isdelimR(): + break; + + if not tf.isstring(): + raise SyntaxError, self.tok.lex.lineNum; + t = self.tok.next(); + if not t.isequal(): + raise SyntaxError, self.tok.lex.lineNum; + ts = self.tok.next(); + if not ts.isstring(): + raise SyntaxError, self.tok.lex.lineNum; + #print >> sys.stderr, " ", tf.val, " := ", ts.val; + be.setField(tf.val, strstrip(ts.val)); + + # if it was an abbrev in the file, put it in the + # abbrevDict so it gets written as an abbrev + if ts.isabbrev(): + self.bibtex.insertAbbrev(ts.val, None); + #print >> sys.stderr, "putting unresolved abbrev %s into dict" % ts.val; + + t = self.tok.next(); + if t.iscomma(): + continue; + elif t.isdelimR(): + break; + else: + raise SyntaxError, self.tok.lex.lineNum; + + + self.bibtex.insertEntry(be, ignore); + return; + + bibparser = BibParser(s, self); + bibcount = 0; + try: + for be in bibparser: + bibcount += 1; + pass; + except SyntaxError, err: + print "Syntax error at line " + str(err); + + return bibcount; diff --git a/biblio/bibtex/utils/pybib/Bibliography.py b/biblio/bibtex/utils/pybib/Bibliography.py new file mode 100644 index 0000000000..e61b66c09e --- /dev/null +++ b/biblio/bibtex/utils/pybib/Bibliography.py @@ -0,0 +1,167 @@ +# Bibliography class +# - essentially a container for many BibEntry objects +# - provides methods for reading/writing the bibliography +# - provides iterators, sorting etc + + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +import string; +import BibEntry; +import urllib; +import urlparse; +import os; +import os.path; +import sys; + +NoSuchFile = "No such file"; + +class Bibliography: + + def __init__(self): + self.keyList = []; + self.abbrevDict = {} + + def open(self, filename): + if filename == '-': + self.filename = "stdin"; + return sys.stdin; + urlbits = urlparse.urlparse('~/lib/bib/z.bib'); + if urlbits[0]: + # path is a URL + fp = urllib.urlopen(filename); + self.filename = filename; + else: + # path is a local file + path = os.environ['BIBPATH']; + for p in string.split(path, os.pathsep): + f = os.path.join(p, filename); + if os.path.isfile(f): + break; + else: + raise NoSuchFile; + + fp = open(f, "r"); + home = os.path.expanduser('~'); + f2 = os.path.abspath(f); + common = os.path.commonprefix([home, f2]); + if common: + self.filename = "~" + f2[len(common):] + else: + self.filename = f; + + return fp; + + def close(self, fp): + fp.close(); + + # resolve all abbreviations found in the value fields of all entries + def resolveAbbrev(self): + #print >> sys.stderr, len(self.abbrevDict); + for be in self: + for f in be: + v = be.getField(f); + if isinstance(v,str): + if v in self.abbrevDict: + if self.abbrevDict[v]: + be.setField(f, self.abbrevDict[v]); + + def insertEntry(self, be, ignore=False): + #print >> sys.stderr, "inserting key ", be.getKey() + # should check to see if be is of BibEntry type + key = be.getKey(); + if key in [x.key for x in self.keyList]: + if not ignore: + print >> sys.stderr, "key %s already in dictionary" % (key) + return False; + self.keyList.append(be); + return True; + + def insertAbbrev(self, abbrev, value): + #print >> sys.stderr, "inserting abbrev ", abbrev + if abbrev in self.abbrevDict: + #print >> sys.stderr, "abbrev %s already in list" % (abbrev) + return False; + self.abbrevDict[abbrev] = value; + #be.brief(); + return True; + + def __repr__(self): + print >> sys.stderr + + def brief(self): + for be in self: + be.brief(); + + def getFilename(self): + return self.filename; + + def getAbbrevs(self): + return self.abbrevDict; + + + def display(self): + for be in self: + be.display(); + print >> sys.stderr + + def __contains__(self, key): + return key in [x.key for x in self.keyList]; + + def __getitem__(self, i): + if type(i) is str: + index = [x.key for x in self.keyList].index(i); + return self.keyList[index]; + elif type(i) is int: + return self.keyList[i]; + else: + raise; + + def __len__(self): + return len(self.keyList); + + + def sort(self, sortfunc): + # turn the dictionary of entries into a list so we can sort it + self.keyList.sort(sortfunc); + + + # return list of all bibentry's that match the search spec + def search(self, key, str, type="all", caseSens=0): + if str == '*': + return self.keyList; + + result = []; + if string.lower(type) == "all": + for be in self: + if be.search(key, str, caseSens): + result.append(be); + else: + for be in self: + if be.isRefType(type) and be.search(key, str, caseSens): + result.append(be); + return result; diff --git a/biblio/bibtex/utils/pybib/README b/biblio/bibtex/utils/pybib/README new file mode 100644 index 0000000000..79bd8ee040 --- /dev/null +++ b/biblio/bibtex/utils/pybib/README @@ -0,0 +1,19 @@ +BibEntry.py a general class for a bibliographic entry +Bibliography.py a general container class for bibliographic entries +BibTeX.py a BibTeX specific superclass for BibEntry and Bibliography + +bib2html convert a bibfile to HTML +bibcat concatenate bibfiles, parse and regenerate +bibdvi convert a bibfile to dvi +bibfilter find specific records by field, date, etc +bibgoogle find references on Google scholar +bibkey lookup by citekey +biblint report missing fields in input files +biblist list bibliography in non-BibTeX format +bibmerge merge bibliographies and attempt to remove duplicate entries +bibsort sort by date +bibsummary summary of number of entries by type +bibnames display author names and number of occurences + + +peter corke 2007. diff --git a/biblio/bibtex/utils/pybib/bib2html b/biblio/bibtex/utils/pybib/bib2html new file mode 100644 index 0000000000..36c12c1b4e --- /dev/null +++ b/biblio/bibtex/utils/pybib/bib2html @@ -0,0 +1,146 @@ +#! /usr/bin/env python +# +# Convert bib file to HTML +# +# TODO +# take out tex fragments, backslash, braces etc +# convert tex accents to HTML accents +# do string substitution using templates +# better handling of different reference types +# probably needs more rigour in the HTML code +# use CSS + + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import time; +import optparse; + + +str = ""; + +def append(s): + global str; + + str += s; + +def append_nl(s): + global str; + + str += s + '\n'; + +class HBibEntry(BibEntry.BibEntry): + + def __init__(self, be): + self.fieldDict = be.fieldDict; + + def display(self): + append( "

" ); + # put title + if self.getURL(): + append( '"%s", ' % (self.getURL(), self.getTitle()) ); + else: + append( '"%s", ' % self.getTitle() ); + + # put authors + append( self.getAuthors() + '. ' ); + month = self.getMonthName(); + year = self.getYear(); + + # put more fields + for k in ['Journal', 'Volume', 'Number', 'Booktitle', + 'Address', 'Institution']: + + if k in self.fieldDict: + append( self.fieldDict[k].strip('"') + ', ' ); + eds = self.getEditorsNames(); + if eds: + append("eds. " + eds + ', '); + if month: + append( month ); + if year > 0: + append( " " + `year` ); + else: + if year > 0: + append( ", " + `year` ); + append( " (%s)" % be.getKey() ); + append_nl( "

" ); + + + +## parse switches +p = optparse.OptionParser() +p.add_option('--highlight', dest='highlight', action='store', type='str', + help='highlight the specified word in the output'); +#p.add_option('--resolve', dest='resolve', action='store_true', +# help='resolve cross reference entries'); +p.set_defaults(highlight=None); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib.parseFile(f); +else: + bib.parseFile(); + +bib.resolveAbbrev(); + +## generate HTML +append_nl( "" ); +append_nl( "" ); +append_nl( " Bibliography %s" % ( args[0] if args else '(stdin)',) ); +append_nl( """ """ ); +append_nl( "" ); +append_nl( "" ); + +for be in bib: + hb = HBibEntry(be); + hb.display(); + +if highlight: + str = string.replace(str, highlight, """%s""" % highlight); + +append_nl( "
" ); +append_nl( "

Generated by bib2html at %s. bib2html by Peter Corke

" % time.asctime() ); +append_nl( "" ); +append_nl( "" ); + +print str; + diff --git a/biblio/bibtex/utils/pybib/bibcat b/biblio/bibtex/utils/pybib/bibcat new file mode 100644 index 0000000000..7c261c5c25 --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibcat @@ -0,0 +1,86 @@ +#! /usr/bin/env python +# +# Concatenate bib file(s) to stdout. Each file is parsed then the BibTeX +# records are regenerated. +# + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; + +resolve = False; + + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Concatenate bib file(s) to stdout. +:: Each file is parsed then the BibTeX records are regenerated''' +p = optparse.OptionParser(usage) +p.add_option('--ignore', dest='ignore', action='store_true', + help='ignore duplicate items'); +p.add_option('--nostrings', dest='dumpStrings', action='store_false', + help='dump string definitions'); +p.add_option('-v', '--verbose', dest='verbose', action='store_true', + help='print some extra information'); +p.add_option('--resolve', dest='resolve', action='store_true', + help='resolve cross reference entries'); +p.set_defaults(ignore=False, dumpStrings=True, verbose=False, resolve=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + nbib = bib.parseFile(f, ignore=ignore); + if verbose: + sys.stderr.write( "%d entries read from %s\n" % (len(bib), f) ); +else: + nbib = bib.parseFile(); + if verbose: + sys.stderr.write( "%d entries read from stdin\n" % (len(bib),) ); + +if resolve: + bib.resolveAbbrev(); + +if verbose: + sys.stderr.write( "%d abbreviations to write\n" % len(outbib.getAbbrevs()) ); + sys.stderr.write( "%d entries to write\n" % len(outbib) ); +if dumpStrings: + bib.writeStrings(); +bib.write(resolve=resolve); diff --git a/biblio/bibtex/utils/pybib/bibdvi b/biblio/bibtex/utils/pybib/bibdvi new file mode 100644 index 0000000000..3d49c58f78 --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibdvi @@ -0,0 +1,110 @@ +#! /usr/bin/env python +# +# Convert bib file(s) to dvi +# +# TODO +# handle resolve switch if command line files are given + + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import os; +import optparse; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Convert bib file(s) to dvi''' +p = optparse.OptionParser(usage) +p.add_option('--xdvi', dest='xdvi', action='store_true', + help='launch xdvi when done'); +p.add_option('--bibstyle', dest='bibstyle', action='store', type='str', + help='specify a bibliography style file'); +p.set_defaults(xdvi=False, bibstyle='ieeetr', resolve=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib.parseFile(f); +else: + bib.parseFile(); + +if args: + # a list of files is given, temp file is the same root name as the first argument + texfile = args[0] + texfile = texfile[0:string.rindex(texfile, '.bib')]; + bibfiles = ','.join( [os.path.splitext(x)[0] for x in args] ); +else: + # input from stdin, use stdin as the root filename + texfile = 'stdin'; + fp = open(texfile+'.bib', 'w'); + bib.write(file=fp,resolve=resolve); + fp.close(); + bibfiles = 'stdin'; + +print "Saving to", texfile + + +## create the latex source file +tex = open("%s.tex" % texfile, "w"); + +tex.write( +r"""\documentclass{article} +\begin{document} +"""); + +# write the cite keys +for be in bib: + tex.write("\\nocite{%s}\n" % be.getKey()); + +# add the bibliog commands +tex.write( +r"""\bibliographystyle{%s} +\bibliography{strings,%s} +\end{document} +""" % (bibstyle, bibfiles) ); +tex.close(); + +os.system("pslatex %s" % texfile); +os.system("bibtex %s" % texfile); +os.system("pslatex %s" % texfile); + +if xdvi and os.getenv('DISPLAY'): + os.system("xdvi %s" % texfile); diff --git a/biblio/bibtex/utils/pybib/bibfilter b/biblio/bibtex/utils/pybib/bibfilter new file mode 100644 index 0000000000..6f515047e8 --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibfilter @@ -0,0 +1,110 @@ +#! /usr/bin/env python +# +# Filter bib records that match search criteria +# +# todo: +# handle tex accent characters, utf-16 etc + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; + +startDate = None; +endDate = None; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Filter bib records that match search criteria +:: Multiple rules can be applied at the same time ''' +p = optparse.OptionParser(usage) +p.add_option('--since', dest='since', action='store', type='str', + help='start date for selection, format YYYY or MM/YYYY'); +p.add_option('--before', dest='before', action='store', type='str', + help='end date for selection, format YYYY or MM/YYYY'); +p.add_option('-i', '--case', dest='caseSens', action='store_true', + help='make search case sensitive'); +p.add_option('--type', dest='type', action='store', type='str', + help='reference type to search (default all)'); +p.add_option('--field', dest='field', nargs=2, action='store', type='str', + help='field to search (default all) and the value which matches any substring in the specified field'); +p.add_option('--hasfield', dest='hasfield', action='store', type='str', + help='true if specified field is present'); +p.add_option('--brief', dest='showBrief', action='store_true', + help='show the matching records in brief format (default is BibTeX)'); +p.add_option('--count', dest='showCount', action='store_true', + help='show just the number of matching records'); +p.set_defaults(since=None, before=None, caseSens=False, type='all', hasfield=None, field=['all', '*'], showBrief=False, showCount=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +if since: + startDate = map(int, since.split('/')); +if before: + endDate = map(int, before.split('/')); + + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib.parseFile(f); +else: + bib.parseFile(); + +#print >> sys.stderr, "looking for <%s> in field <%s>, reftype <%s>" % (field[1], field[0], type) + +# search the bibliography for all matches to the field query +l = bib.search(field[0], field[1], type, caseSens); +count = 0; +for be in l: + # check if it has the required field + if hasfield: + if not be.getField(hasfield): + continue; + + # check the date range + if be.afterDate(startDate) and be.beforeDate(endDate): + count += 1; + if not showCount: + if showBrief: + print be; + else: + be.write(); + +if showCount: + print count; diff --git a/biblio/bibtex/utils/pybib/bibgoogle b/biblio/bibtex/utils/pybib/bibgoogle new file mode 100644 index 0000000000..bc829d308c --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibgoogle @@ -0,0 +1,264 @@ +#!/usr/bin/env python +# +# Lookup references on Google Scholar and add a URL which points to a copy of the paper. +# +# Nice to complete the bibliography, and if you want to convert the bibliography to HTML +# (with bib2html). +# + + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import re; +import urllib +import urlparse +import htmllib +import formatter +import time; +import optparse; + +# preferred sources of documents +prefList = ['ieeexplore.ieee.org', 'portal.acm.org', 'doi.ieeecomputersociety.org']; + +class Parser(htmllib.HTMLParser): + # build a list of tuples (anchor text, URL) + + def __init__(self, verbose=0): + self.anchors = []; + f = formatter.NullFormatter() + htmllib.HTMLParser.__init__(self, f, verbose) + + def anchor_bgn(self, href, name, type): + self.save_bgn() + self.href = href + self.name = name + self.type = type + + def anchor_end(self): + text = string.strip(self.save_end()) + if self.href and text: + #self.anchors[text] = self.anchors.get(text, []) + [self.anchor] + #self.anchors[text] = self.anchor + self.anchors.append( (text, self.href) ); + +# trick Google into thinking I'm using Safari +browserName = "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312"; + +class AppURLopener(urllib.FancyURLopener): + version = browserName; + +urllib._urlopener = AppURLopener() + +## lookup the BibEntry on Google scholar +def scholar_lookup(be): + + # Levenstein distance between two strings + def distance(a,b): + c = {} + n = len(a); m = len(b) + + for i in range(0,n+1): + c[i,0] = i + for j in range(0,m+1): + c[0,j] = j + + for i in range(1,n+1): + for j in range(1,m+1): + x = c[i-1,j]+1 + y = c[i,j-1]+1 + if a[i-1] == b[j-1]: + z = c[i-1,j-1] + else: + z = c[i-1,j-1]+1 + c[i,j] = min(x,y,z) + return c[n,m] + + # build the search string from words in the title and authors surnames + # - remove short words and accents, punctuation characters + title = be.getTitle().split(); + newtitle = []; + for word in title: + if len(word) >= 4: + newtitle.append(word); + title = string.join(newtitle, ' '); + title = re.sub(r"""[#{}:;,&$-]""", " ", title); + + search = title.split(); + + # add the year + year = be.getYear(); + if year > 0: + #search.append(repr(year)); + pass + + # add author surnames + search.extend( [x[0] for x in be.getAuthorsSurnameList()]); + + # remove accents and apostrophes, quotes + search2 = []; + for w in search: + w = re.sub(r"""\.|['"]""", "", w); + search2.append(w); + search = search2; + #print string.join(search,' '); + + s = "http://www.scholar.google.com/scholar?q=%s&ie=UTF-8&oe=UTF-8&hl=en&btnG=Search" % ( string.join(search, '+') ); + + # send the query to Scholar + file = urllib.urlopen(s); + html = file.read() + file.close() + + # parse the result + p = Parser() + p.feed(html) + p.close() + + + candidates = []; + + title = be.getTitle().lower(); + # for each returned result, look for the best one + #print p.anchors + for text, url in p.anchors: + #print text, "|", url + + # find the distance between our known title and the title of the article + d = distance(text.lower(), title); + #print d, k + if d < 5: + # consider this a good enough match + i = url.find("http"); + candidates.append( url[i:] ); + + # look for a URL of the form http:....pdf + i = url.find("pdf"); + if i == 0: + i = url.find("http"); + #print " ** PDF ", url[i:] + candidates.append( url[i:] ); + + # now we have a list of candidate URLs + + #print candidates + + # look for a source in our preference list + for url in candidates: + org = urlparse.urlsplit(url)[1]; + if org in prefList: + return url; + + # failing that go for one with a PDF in it + for url in candidates: + if url.find("pdf") > -1: + return url; + + # failing that take the first one + if candidates: + return candidates[0]; + + return None; + + +## main + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Lookup each reference on Google Scholar and add the URL to the bibliography.''' +p = optparse.OptionParser(usage) +p.add_option('-v', '--verbose', dest='verbose', action='store_true', + help='print some extra information'); +p.set_defaults(verbose=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib.parseFile(f); +else: + bib.parseFile(); + +## lookup each reference on Scholar +count = 0; +sourceDict = {}; + +if verbose: + print >> sys.stderr, "Resolving %d references via Google scholar" % len(bib); + +for be in bib: + + rt = be.getRefType(); + if rt in ['article', 'inproceedings']: + # if we already have a URL then skip + if be.getURL(): + continue; + + # do the lookup + url = scholar_lookup(be); + if url: + if verbose: + print >> sys.stderr, be; + print >> sys.stderr, " --> ", url + print >> sys.stderr + be.setField('Url', url); + count = count + 1; + + # build a list of the unique sources of the documents + org = urlparse.urlsplit(url)[1]; + if org in sourceDict: + sourceDict[org] += 1; + else: + sourceDict[org] = 1; + +if verbose: + # print some stats + print >> sys.stderr, "Resolved %d references to URLs (%.1f%%)" % (count, count*100./len(bib)); + + # print the unique source list, sorted in decreasing order of frequency + print >> sys.stderr, "Document sources" + l = sourceDict.items(); + l.sort( lambda x, y: cmp(y[1], x[1]) ); + + for org,n in l: + print >> sys.stderr, " %-30s %d" % (org, n); + +# output the bibligraphy with the URLs set +bib.writeStrings(); +bib.write(); diff --git a/biblio/bibtex/utils/pybib/bibkey b/biblio/bibtex/utils/pybib/bibkey new file mode 100644 index 0000000000..b77029f05c --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibkey @@ -0,0 +1,112 @@ +#! /usr/bin/env python +# +# Display all records with specified citekey(s). +# +# Cite key can come from command line switches or from a .aux file. Useful if you +# want to make a reduced .bib file to match a paper, without all the other junk. +# + + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; +import re; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Display all records with specified citekey +:: citekeys from command line or a .aux file''' +p = optparse.OptionParser(usage) +p.add_option('--key', dest='keys', action='append', type='str', + help='cite key to display (can have multiple of this switch)'); +p.add_option('--aux', dest='aux', action='store', type='str', + help='name of .aux file to parse for keys'); +p.add_option('--strings', dest='dumpStrings', action='store_true', + help='dump the string definitions (abbreviations) as well'); +p.add_option('--brief', dest='showBrief', action='store_true', + help='show the matching records in brief format (default is BibTeX)'); +p.set_defaults(keys=[], aux=None, dumpStrings=False, showBrief=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + + + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +# load extra keys from the specified .aux file +if aux: + f = open(aux, 'r'); + citation = re.compile(r'''^\\citation\{(\w+)\}'''); + for line in f: + m = citation.match(line); + if m: + keys.append( m.group(1) ); +keys2 = keys[:]; + +def action(bib, filename): + found = []; + for k in keys: + try: + be = bib[k]; + found.append(be); + keys2.remove(k); # keep track of keys not found + except: + pass; + if found: + for be in found: + if showBrief: + if f: + print f; + be.brief(); + else: + be.write(); +if args: + for f in args: + bib = BibTeX.BibTeX(); + bib.parseFile(f); + action(bib, f); +else: + bib = BibTeX.BibTeX(); + bib.parseFile(); + action(bib, None); + +if dumpStrings and not showBrief: + bib.writeStrings(); + +if keys2: + print + for k in keys2: + print >> sys.stderr, "%s not found" % k diff --git a/biblio/bibtex/utils/pybib/biblint b/biblio/bibtex/utils/pybib/biblint new file mode 100644 index 0000000000..5b8668cfff --- /dev/null +++ b/biblio/bibtex/utils/pybib/biblint @@ -0,0 +1,79 @@ +#! /usr/bin/env python +# +# Report missing fields and bad values in bibtex records + + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Report missing fields and bad values in bibtex records''' +p = optparse.OptionParser(usage) +#p.add_option('--reverse', dest='reverseSort', action='store_true', +# help='sort into ascending data order (old at top)'); +#p.add_option('--resolve', dest='resolve', action='store_true', +# help='resolve cross reference entries'); +#p.set_defaults(reverseSort=False, resolve=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +## read the input files + +if args: + for f in args: + bib = BibTeX.BibTeX(); + bib.parseFile(f); + print "%d records read from %s" % (len(bib), bib.getFilename()); + + print + for be in bib: + c = be.check(); + if c: + print "%15s: missing " % (be.getKey()), string.join(c, ', ') +else: + bib = BibTeX.BibTeX(); + bib.parseFile(); + print "%d records read from %s" % (len(bib), '(stdin)'); + + print + for be in bib: + c = be.check(); + if c: + print "%15s: missing " % (be.getKey()), string.join(c, ', ') diff --git a/biblio/bibtex/utils/pybib/biblist b/biblio/bibtex/utils/pybib/biblist new file mode 100644 index 0000000000..7fdb95a332 --- /dev/null +++ b/biblio/bibtex/utils/pybib/biblist @@ -0,0 +1,78 @@ +#! /usr/bin/env python +# +# List bibtex file in human readable format +# + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Display bibtex file in human readable format''' +p = optparse.OptionParser(usage) +p.add_option('--brief', dest='showBrief', action='store_true', + help='brief listing (one line per entry)'); +p.add_option('--abbrev', dest='resolveAbbrevs', action='store_true', + help='resolve abbreviations from defined strings'); +#p.add_option('--resolve', dest='resolve', action='store_true', +# help='resolve cross reference entries'); +p.set_defaults(showBrief=False, resolveAbbrevs=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib.parseFile(f); +else: + bib.parseFile(); + +# resolve cross refs and abbreviations +bib.resolveCrossRef(); +if resolveAbbrevs: + bib.resolveAbbrev(); + +# output the readable text +for be in bib: + if showBrief: + be.brief(); + else: + be.display(); + print diff --git a/biblio/bibtex/utils/pybib/bibmerge b/biblio/bibtex/utils/pybib/bibmerge new file mode 100644 index 0000000000..1bd517466b --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibmerge @@ -0,0 +1,116 @@ +#! /usr/bin/env python +# +# Fuzzy merge of bibliographies. +# +# A hiearchy of matching tests is implemented: +# +# Reference type +# Year of publication +# Month of publication (if known) +# Volume number (if article type) +# Page numbers (if known) +# Author surnames +# Fuzzy match on title exclusing white space and punctuation characters, +# using Levenstein distance +# +# Really useful when you are jointly writing a paper and the authors have partially +# overlapping bib files with different cite key conventions. + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Fuzzy merge of bibliographies.''' +p = optparse.OptionParser(usage) +p.add_option('--dthresh', dest='dthresh', action='store', type='int', + help='set the fuzzy match tolerance (Levenstein distance) for title string'); +p.add_option('--showdup', dest='showdup', action='store_true', + help='show information about proposed duplicates'); +p.add_option('-v', '--verbose', dest='verbose', action='store_true', + help='print some extra information'); +p.set_defaults(dthresh=2, showdup=False, verbose=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + + +unique = BibTeX.BibTeX(); +dupcount = 0; + +def action(bib, filename): + global dupcount, unique; + + if verbose: + print >> sys.stderr, "%d records read from %s" % (len(bib), filename) + + # for each new bib entry + for be in bib: + # check against all existing + for ub in unique: + if be.match(ub, dthresh=dthresh): + if verbose: + print >> sys.stderr, " -[%s] %s" % (be.getKey(), be); + dupcount += 1; + if showdup: + print >> sys.stderr, "=============================" + ub.write(sys.stderr); + print >> sys.stderr, "---------- duplicate from %s" % bib.getFilename(); + be.write(sys.stderr); + break; + else: + if verbose: + print >> sys.stderr, " +[%s] %s" % (be.getKey(), be); + unique.insertEntry(be); + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib = BibTeX.BibTeX(); + bib.parseFile(f); + action(bib, f); +else: + bib = BibTeX.BibTeX(); + bib.parseFile(); + action(bib, '(stdin)'); + + +print >> sys.stderr, "New bib has %d records, %d duplicates found" % (len(unique), dupcount); + +unique.write(); diff --git a/biblio/bibtex/utils/pybib/bibnames b/biblio/bibtex/utils/pybib/bibnames new file mode 100644 index 0000000000..d0b07310fa --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibnames @@ -0,0 +1,89 @@ +#! /usr/bin/env python +# +# Display a list of authors and their occurrence in the bibfiles +# +# each output line is of the form: +# +# Surname,I N +# +# where I is their initial and N is the number of occurrences. This can be +# fed throug sort -n -r +1 to get a list of authors in descending order +# of occurrence. Figure out who is your favourite co-author! + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import getopt; +import optparse; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Display a list of authors and their occurrence''' +p = optparse.OptionParser(usage) +#p.add_option('--reverse', dest='reverseSort', action='store_true', +# help='sort into ascending data order (old at top)'); +#p.add_option('--resolve', dest='resolve', action='store_true', +# help='resolve cross reference entries'); +#p.set_defaults(reverseSort=False, resolve=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib.parseFile(f); +else: + bib.parseFile(); + +# Build a list of unique names: Surname,Initial and update occurrence +nameList = {}; +for be in bib: + surnames = be.getAuthorsSurnameList(); + if surnames: + for s in surnames: + s = ','.join(s); + if s in nameList: + nameList[s] += 1 + else: + nameList[s] = 1; + +# display names and occurrence. +for s,v in nameList.iteritems(): + print s, v; diff --git a/biblio/bibtex/utils/pybib/bibsort b/biblio/bibtex/utils/pybib/bibsort new file mode 100644 index 0000000000..15e959832d --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibsort @@ -0,0 +1,100 @@ +#! /usr/bin/env python +# +# Sort bibliographies in chronological order +# + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; + + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Sort bibliographies in chronological order''' +p = optparse.OptionParser(usage) +p.add_option('--reverse', dest='reverseSort', action='store_true', + help='sort into ascending data order (old at top)'); +p.add_option('--resolve', dest='resolve', action='store_true', + help='resolve cross reference entries'); +p.set_defaults(reverseSort=False, resolve=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +count = {}; + +sortReturn = -1 if reverseSort else 1; + +def sortByDate(a, b): + # On input a and b are BibEntry objects + ay = a.getYear(); + by = b.getYear(); + if ay > by: + return -sortReturn; + elif ay < by: + return sortReturn; + else: + am = a.getMonth(); + bm = b.getMonth(); + if am > bm: + return -sortReturn; + elif am < bm: + return sortReturn; + else: + return 0; + +outbib = BibTeX.BibTeX(); + +if args: + for f in args: + bib = BibTeX.BibTeX(); + n = bib.parseFile(f); + + sys.stderr.write( "%d records read from %s\n" % (n, f) ); + +else: + bib = BibTeX.BibTeX(); + bib.parseFile(); + + sys.stderr.write( "%d records read from stdin\n" % len(bib) ); + +# sort it +bib.sort(sortByDate); + +# and output the result +for s in bib: + s.write(); diff --git a/biblio/bibtex/utils/pybib/bibsummary b/biblio/bibtex/utils/pybib/bibsummary new file mode 100644 index 0000000000..b66df9ea7a --- /dev/null +++ b/biblio/bibtex/utils/pybib/bibsummary @@ -0,0 +1,81 @@ +#! /usr/bin/env python +# +# Display a summary of the reference types + +# Copyright (c) 2007, Peter Corke +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * The name of the copyright holder may not be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import Bibliography; +import BibEntry; +import BibTeX; +import string; +import sys; +import optparse; + +## parse switches +usage = '''usage: %prog [options] [bibfiles] + +:: Display a summary of the reference types''' +p = optparse.OptionParser(usage) +#p.add_option('--reverse', dest='reverseSort', action='store_true', +# help='sort into ascending data order (old at top)'); +#p.add_option('--resolve', dest='resolve', action='store_true', +# help='resolve cross reference entries'); +#p.set_defaults(reverseSort=False, resolve=False); +(opts, args) = p.parse_args() +globals().update(opts.__dict__) + +if len(args) == 0 and sys.stdin.isatty(): + p.print_help(); + sys.exit(0); + +## read the input files +bib = BibTeX.BibTeX(); +if args: + for f in args: + bib.parseFile(f); +else: + bib.parseFile(); + +count = {}; +urlCount = 0; + + +for be in bib: + t = be.getRefType(); + if be.getField('Url'): + urlCount += 1; + if t in count: + count[t] += 1; + else: + count[t] = 1; + +for k in count: + print " %15s: %4d" % (k, count[k]); + +if urlCount > 0: + print " %d with URL links" % urlCount; -- cgit v1.2.3