From a4e5c450ebc8a8474ba2b169eefe03e435c6e7da Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Sun, 5 Aug 2007 22:33:16 +0000 Subject: update from jimh git-svn-id: svn://tug.org/texlive/trunk@4700 c570f23f-e606-0410-a88d-b1316a301751 --- Master/tlpkg/lib/TeXLive.py | 1507 ++++++++++++++++++++++++++++--------------- 1 file changed, 990 insertions(+), 517 deletions(-) (limited to 'Master') diff --git a/Master/tlpkg/lib/TeXLive.py b/Master/tlpkg/lib/TeXLive.py index 67b64573c7c..fe302acee9b 100644 --- a/Master/tlpkg/lib/TeXLive.py +++ b/Master/tlpkg/lib/TeXLive.py @@ -2,40 +2,58 @@ # TeXLive.py # -*- encoding: utf-8 -*- # Usage: call with argument of --help -# 2007-Jul-28 Jim Hefferon ftpmaint@tug.ctan.org -""" Read a TeX Live package dB file. +# 2007-Aug-05 Jim Hefferon ftpmaint@tug.ctan.org +"""Understand a TeX Live package and the associated dB file. + +TeX Live has a format for metadata located in trunk/Master/tlpkg (for +the 'src' packages and documentation of the format) and +trunk/Master/texlive.tlpdb (see also -withctan ?). This module reads the +two formats, can transform them by adding package documentation from the +ctanWeb database, and can write them back out. + """ -__version__='0.9.9' +__version__='1.0.0' __author__='Jim Hefferon' -__date__='2007-Jul-15' -__notes__="""This version has been stripped of utility routines for the texlive folks. +__date__='2007-Aug-05' +__notes__=""" +Bugs: +1) Does not expand patterns for the src module, and does not + redo the sizes, etc., for obj files. +2) You can't from the command line change the name of the db files. +3) This version for the TeX Live svn tree has the dB stripped out, so + the option -a doesn't work, and the option -n has no effect. """ import os, os.path, sys import re import optparse import codecs +import tempfile import time import pprint +import glob -# import library -# from util import * -# import dBUtils +## import library +## from util import * +## import dBUtils +## from defaults import DEFAULT_ENCODING, DEFAULT_DATABASE_ENCODING, DEBUG, LOGGING +DEFAULT_ENCODING='UTF-8' TLPDB_FILENAME='/home/svn/texlive/trunk/Master/texlive.tlpdb' THIS_SCRIPT=sys.argv[0] -DEBUG=False -LOGGING=False -LOGFILE_NAME=os.path.splitext(THIS_SCRIPT)[0]+'.log' -VERBOSE=False +DEBUG=True +LOGGING=True +LOGFILE_NAME=os.path.splitext(os.path.basename(THIS_SCRIPT))[0]+'.log' +VERBOSE=True -import locale # What encoding for the terminal? +import locale locale_language, locale_output_encoding = locale.getdefaultlocale() if locale_output_encoding is None: locale_output_encoding='UTF-8' + #---------------------- stuff from util.py import StringIO # for msg import traceback # for msg @@ -432,10 +450,9 @@ def exceptionName(): class tlError(jhError): pass -tlobjKeys=set(['name','category','catalogue','shortdesc','longdesc','depend','execute','revision','srcsize','srcfiles','docsize','docfiles','runsize','runfiles','binsize','binfiles','cataloguedata']) - +tlobjKeys=set(['name','category','catalogue','shortdesc','longdesc','depend','execute','revision','srcsize','srcfiles','docsize','docfiles','runsize','runfiles','binsize','binfiles','cataloguedata','srcpattern','runpattern','docpattern','binpattern']) -class tlpobj(object): +class tlp(object): """Give information based on a TeXLive object file """ def __init__(self,verbose=False,log=None,debug=False): @@ -444,28 +461,13 @@ class tlpobj(object): self.verbose=verbose self.log=log self.debug=debug - self.__initializeUserVars() - self.value={} # map name --> tlpkg - return None - - def __initializeUserVars(self): - self.name=None - self.category=None - self.catalogue=None - self.shortdesc=None - self.longdesc=None - self.depend=None - self.execute=None - self.revision=None - self.srcfiles=None # list of files - self.runfiles=None # list of files - self.docfiles=None # dct fileName--> (dct tagkey --> tagval) - self.binfiles=None # arch --> list of files - self.srcsize=None # int (or None) - self.runsize=None # int (or None) - self.docsize=None # int (or None) - self.binsize=None # arch --> size (or None) - self.cataloguedata=None # cataloguekey --> value + self.name=None # string id of package + self.category=None # string category into which package falls + self.catalogue=None # string id of associated Catalogue package + self.shortdesc=None # string one-liner + self.longdesc=None # string may have newlines + self.depend=None # list of the form Category/Name + self.execute=None # list free-form strings return None def __unicode__(self): @@ -477,16 +479,6 @@ class tlpobj(object): r.append(" longdesc: %s" % (pprint.pformat(self.longdesc),)) r.append(" depend: %s" % (pprint.pformat(self.depend),)) r.append(" execute: %s" % (pprint.pformat(self.execute),)) - r.append(" revision: %s" % (pprint.pformat(self.revision),)) - r.append(" srcsize: %s" % (pprint.pformat(self.srcsize),)) - r.append(" srcfiles: %s" % (pprint.pformat(self.srcfiles),)) - r.append(" docsize: %s" % (pprint.pformat(self.docsize),)) - r.append(" docfiles: %s" % (pprint.pformat(self.docfiles),)) - r.append(" runsize: %s" % (pprint.pformat(self.runsize),)) - r.append(" runfiles: %s" % (pprint.pformat(self.runfiles),)) - r.append(" binsize: %s" % (pprint.pformat(self.binsize),)) - r.append(" binfiles: %s" % (pprint.pformat(self.binfiles),)) - r.append(" cataloguedata: %s" % (pprint.pformat(self.cataloguedata),)) return "\n".join(r) def writeout(self,fn=None): @@ -495,22 +487,25 @@ class tlpobj(object): """ if fn is None: f=sys.stdout + if f.encoding!=DEFAULT_ENCODING: + warn('using STDOUT but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING)) else: + if self.debug: + errors='strict' + else: + errors='replace' try: - f=open(fn,'w') + f=codecs.open(fn,'w',DEFAULT_ENCODING,errors) except Exception, err: - mesg='unable to open file for writing' + mesg='unable to open the file for writing' noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err)) self.writeout_fh(f) f.close() return None - def writeout_fh(self,f): - """Write a string representation of the user variables the filhandle. - Note that files are listed in sorted order. - f filehandle + def _stringToWriteout(self): + """String that will be written out during writeout_fh """ - # Make the string r=[] r.append("name %s" % (self.name,)) if self.category: @@ -518,198 +513,93 @@ class tlpobj(object): if self.shortdesc: r.append("shortdesc %s" % (self.shortdesc,)) if self.longdesc: - for line in self.longdesc: + for line in self.longdesc.split("\n"): r.append("longdesc %s" % (line,)) - if self.revision: - r.append("revision %s" % (self.revision,)) if self.depend: for line in self.depend: r.append("depend %s" % (line,)) if self.execute: for line in self.execute: r.append("execute %s" % (line,)) - if self.docfiles: - if self.docsize is None: - mesg='docsize is not there but there are docfiles' - noteException(mesg,devel=mesg,exception=tlError,log=self.log,debug=self.debug) - r.append("docfiles size=%s" % (self.docsize,)) - dFiles=self.docfiles.keys() - dFiles.sort() - for line in dFiles: - s=[] - for (k,v) in self.docfiles[line].items(): - s.append("%s=%s" % (k,v)) - dLine=" "+line - if s: - dLine+=" "+(" ".join(s)) - r.append(dLine) - if self.srcfiles: - if self.srcsize is None: - mesg='srcsize is not there but there are srcfiles' - noteException(mesg,devel=mesg,exception=tlError,log=self.log,debug=self.debug) - r.append("srcfiles size=%s" % (self.srcsize,)) - self.srcfiles.sort() - for line in self.srcfiles: - r.append(" "+line) - if self.runfiles: - if self.runsize is None: - mesg='runsize is not there but there are runfiles' - noteException(mesg,devel=mesg,exception=tlError,log=self.log,debug=self.debug) - r.append("runfiles size=%s" % (self.runsize,)) - self.runfiles.sort() - for line in self.runfiles: - r.append(" "+line) - if self.binfiles: - arches=self.binfiles.keys() - arches.sort() - for arch in arches: - if self.binsize[arch] is None: - mesg='binsize is not there but there are binfiles' - noteException(mesg,devel=mesg+" for %(arch)s",exception=tlError,log=self.log,debug=self.debug,arch=repr(arch)) - r.append("binfiles arch=%s size=%s" % (arch,self.binsize[arch],)) - sBinfiles=self.binfiles[arch] - sBinfiles.sort() - for line in sBinfiles: - r.append(" "+line) - if self.cataloguedata: - cdKeys=self.cataloguedata.keys() - for k in cdKeys: - r.append("catalogue-%s %s" % (k,self.cataloguedata[k])) - # Write that string - f.write("\n".join(r)) + return "\n".join(r) + + def writeout_fh(self,f): + """Write a string representation of the user variables to the + filehandle. + f filehandle + """ + s=self._stringToWriteout() + f.write(s) return None -## def to_db(self,dBcsr,dBcnx,careful=True,associatedCatPkgs=None): -## """Put the file information to the database -## dBcsr, dBcnx database cursor and connection -## careful=True raise exception on error? -## associatedCatPkgs=None list of catalogue package id's that either -## are part of this texlive package of of which this texlive package -## is a part (None is the same as using what is now in the dB) -## """ -## tlId=self.name -## # Drop the file records if they are there -## sql="DELETE FROM tds_files WHERE tl_id=%(tlId)s" -## dct={'tlId':tlId} -## try: -## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='drop old texlive files record') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to drop existing tds files record' -## noteException(mesg,devel=mesg+' for package %(pkgId)s: %(err)s',log=self.log,debug=self.debug,pkgId=repr(pkgId),err=errorValue(err)) -## # Do the tl_cat records; see which ones to do -## if associatedCatPkgs is None: -## sql="SELECT pkg_id FROM tl_cat WHERE tl_id=%(tlId)s" -## dct['tlId']=tlId -## try: -## dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='get list of catalogue packages for this texlive package') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to get list of catalogue packages for this texlive packages ' -## noteException(mesg,devel=mesg+'%(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) -## associatedCatPkgs=[a for (a,) in dBres] -## if not(associatedCatPkgs): # try tlId -## possiblePkgId=tlId.lower() -## sql="SELECT id FROM package WHERE id=%(pkgId)s" -## dct['pkgId']=possiblePkgId -## try: -## dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='check for existence of a catalogue id') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to check for existence of a catalogue id' -## noteException(mesg,devel=mesg+'%(pkgId)s: %(err)s',log=self.log,debug=self.debug,pkgId=repr(possiblePkgId),err=errorValue(err)) -## if dBres: -## associatedCatPkgs.append(possiblePkgId) -## # drop any that are there now -## sql="DELETE FROM tl_cat WHERE tl_id=%(tlId)s" -## dct={'tlId':tlId} -## try: -## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='drop old texlive package-catalogue package association') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to drop existing texlive package record' -## noteException(mesg,devel=mesg+' for package %(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) -## # Drop the tl records if they are there -## sql="DELETE FROM tl WHERE tl_id=%(tlId)s" -## dct={'tlId':tlId} -## try: -## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='drop old texlive package record') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to drop existing texlive package record' -## noteException(mesg,devel=mesg+' for package %(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) -## # Insert new tl record -## sql="INSERT INTO tl (tl_id,shortdesc,longdesc) VALUES (%(tlId)s,%(shortdesc)s,%(longdesc)s)" -## dct['shortdesc']=self.shortdesc -## dct['longdesc']=self.longdesc -## try: -## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='insert new texlive package record') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to insert new texlive package record' -## noteException(mesg,devel=mesg+' for package %(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) -## # insert new tl_cat records -## if not(tlId.startswith('bin-') -## or tlId.startswith('collection-') -## or tlId.startswith('hyphen-') -## or tlId.startswith('lib-') -## or tlId.startswith('scheme-') -## or tlId.startswith('texlive-')): -## if (not(associatedCatPkgs) -## and careful): -## mesg=u'there are no catalogue packages associated with the TeX Live package %(tlId)s' -## warn(mesg,log=self.log,debug=self.debug,tlId=repr(tlId)) -## for pkgId in associatedCatPkgs: -## sql="INSERT INTO tl_cat (tl_id,pkg_id) VALUES (%(tlId)s,%(pkgId)s)" -## dct['pkgId']=pkgId -## try: -## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='insert new tl_cat package record') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to insert new association between a texlive package and a catalogue package' -## noteException(mesg,devel=mesg+' for texlive package %(tlId)s and catalgoue package %(pkgId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),pkgId=repr(pkgId),err=errorValue(err)) -## # Insert the file records -## # First build a list of pairs: (fileName,fileType) -## fileList=[] -## if self.srcfiles: -## for fn in self.srcfiles: -## fileList.append((fn,'src')) -## if self.runfiles: -## for fn in self.runfiles: -## fileList.append((fn,'run')) -## if self.docfiles: -## for fn in self.docfiles.keys(): -## fileList.append((fn,'doc')) -## if self.binfiles: -## for arch in self.binfiles.keys(): -## for fn in self.binfiles[arch]: -## fileList.append((fn,'bin')) -## # .. then put the info in the dB -## dct={'tlId':tlId} -## for (fn,fileType) in fileList: -## pathDir,pathBase=os.path.split(fn) -## sql="INSERT INTO tds_files (tl_id,type,path_dir,path_base) VALUES (%(tlId)s,%(fileType)s,%(pathDir)s,%(pathBase)s)" -## dct['fileType']=fileType -## dct['pathDir']=pathDir -## dct['pathBase']=pathBase -## try: -## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='insert new texlive files record') -## except dBUtils.dBUtilsError, err: -## mesg=u'unable to insert new tds files record' -## noteException(mesg,devel=mesg+' for package %(tlId)s with type=%(fileType)s, pathDir=%(pathDir)s, pathBase=%(pathBase)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),fileType=repr(fileType),pathDir=repr(pathDir),pathBase=repr(pathBase),err=errorValue(err)) -## # Commit -## try: -## dBcnx.commit() -## except Exception, err: -## mesg=u'unable to commit the insert of the new tds files record' -## noteException(mesg,devel=mesg+' for package %(pkgId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) -## return None + def get_associated_cat_ids(self,dBcsr,log=None,debug=False): + """Return the matching catalogue ids + """ + sql="""SELECT pkg_id FROM tl_cat + WHERE tl_id=%(tlId)s + AND pkg_id IS NOT NULL""" + dct={'tlId':self.name} + try: + dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get catalogue package ids related to this TeX Live package id') + except dBUtils.dBUtilsError, err: + mesg=u'unable to get texlive-catalogue packages record' + noteException(mesg,devel=mesg+' for TeX Live package %(tlId)s: %(err)s',exception=tlError,log=log,debug=debug,tlId=repr(tlId),err=errorValue(err)) + return [catId for (catId,) in dBres] + + def add_cat_desc(self,dBcsr,catId=None,log=None,debug=False): + """If there is not now a description, and there is a matching + Catalogue package, add its descriptions. (Also sets the catalogue + variable) + dBcsr database cursor + catId=None string or None if present, the name of the Catalogue + package whose descriptions will be added. + log=None,debug=False debugging + """ + if not(catId): + catIds=self.get_associated_cat_ids(dBcsr,log=log,debug=debug) + if len(catIds)==1: + catId=catIds[0] + if not(catId): + return False + sql="""SELECT caption, description FROM pkg_text + WHERE id=%(catId)s + AND iso_code='en'""" + dct={'catId':catId} + try: + dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get catalogue descriptions related to this TeX Live package id') + except dBUtils.dBUtilsError, err: + mesg=u'unable to get catalogue package descriptions for this TeX Live package' + noteException(mesg,devel=mesg+' for TeX Live package %(tlId)s and catalogue id %(catId)s: %(err)s',exception=tlError,log=log,debug=debug,tlId=repr(tlId),catId=repr(catId),err=errorValue(err)) + res=dBres[0] + if res: + (catCaption,catDesc)=dBres[0] + if (not(self.shortdesc) + and not(self.longdesc)): + self.catalogue=catId + self.shortdesc=catCaption + self.longdesc=catDesc + return True - def from_file(self,fn=None): - """Read a tlpobj from a file. You don't want to do this. + def from_file(self,fn=None,multi=False): + """Read a tlp object from a file. + fn=None string file name or None, which means sys.stdin + multi=False boolean whether to allow multiple pkgs in a file """ if fn is None: f=sys.stdin + if f.encoding!=DEFAULT_ENCODING: + warn('using STDIN but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING)) else: try: - f=open(fn,'rU') + if self.debug: + errors='strict' + else: + errors='replace' + f=codecs.open(fn,'rU',DEFAULT_ENCODING,errors) except Exception, err: - mesg='unable to open tlpobj file' + mesg='unable to open file for reading' noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err)) - r=self.from_fh(f,multi=True,lineNo=0) + r=self.from_fh(f,multi=multi,lineNo=0) return None initialState='init' @@ -722,298 +612,771 @@ class tlpobj(object): 'shortdesc':keyValueStates-set(['shortdesc']), 'longdesc':keyValueStates, 'depend':keyValueStates, - 'execute':keyValueStates, - 'revision':keyValueStates-set(['revision']), - 'srcfiles':keyValueStates | set(['srcfiles_file'])-set(['srcfiles']), - 'srcfiles_file':keyValueStates | set(['srcfiles_file']), - 'runfiles':keyValueStates | set(['runfiles_file'])-set(['runfiles']), - 'runfiles_file':keyValueStates | set(['runfiles_file']), - 'docfiles':keyValueStates | set(['docfiles_file'])-set(['docfiles']), - 'docfiles_file':keyValueStates | set(['docfiles_file'])-set(['docfiles']), - 'binfiles':keyValueStates | set(['binfiles_file']), - 'binfiles_file':keyValueStates | set(['binfiles_file']), - 'cataloguedata':keyValueStates} - def from_fh(self,f,multi=None,lineNo=None): - """Read a tlpobj from a file handle. Returns None if EOF read before - anything is found. + 'execute':keyValueStates} + def from_fh(self,f,multi=None,lineNo=0): + """Read tlp information from a file handle. Returns None if EOF is + read before anything is found. Note that files have been opened + with codecs.open() (except for stdin) f an opened file multi=None If not None then multiple tplobj's in the file are treated as an error - lineNo=None debugging + lineNo=0 debugging May raise tlError. """ + # Implementation note: I am putting the full parsing routine in + # this base class, except for some "handle" routines, to allow + # me to have the handle routines report error messages in the + # tplobj and tplsrc classes. state=self.initialState - currentArch=None # used for binfiles - # Parse the lines + # Look for blank lines that are delimeters between tlp parts anyNonblankLinesRead=False while True: line=f.readline() if not(line): # EOF? return False - # if self.debug: - # note('line is '+repr(line),self.log) + lineNo+=1 + if self.debug and self.verbose: + note('line is '+repr(line),self.log) line=line.rstrip() if not(anyNonblankLinesRead): if (not(line) - or line[0]=='#'): # throw away initial blank lines + or line[0]=='#'): # toss initial blank lines or comments continue else: anyNonblankLinesRead=True - # pkg=tplpkg(log=self.log,debug=self.debug) - if not(line): + elif not(line): if multi is None: mesg=u'unexpected multiple tpl objects' noteException(mesg,devel=mesg+'; trailing blank line found at line %(lineNo)s',exception=tlError,log=self.log,debug=self.debug,lineNo=repr(lineNo)) else: return self - # Find state under which this line will be read - fields=None - if (state=='srcfiles' - or state=='srcfiles_file'): - if line[0]==' ': + # Get the next state (the key) and the part of the line after key. + # The "srcfiles", etc., states need special handling as they + # put the reader in a position where the first thing on the line + # is not the next state. + restOfLine="" + if line[0]==' ': + if (state=='srcfiles' + or state=='srcfiles_file'): newState='srcfiles_file' - fields=line[1:].split() - if (state=='runfiles' - or state=='runfiles_file'): - if line[0]==' ': + restOfLine=line[1:] + elif (state=='runfiles' + or state=='runfiles_file'): newState='runfiles_file' - fields=line[1:].split() - elif (state=='docfiles' - or state=='docfiles_file'): - if line[0]==' ': + restOfLine=line[1:] + elif (state=='docfiles' + or state=='docfiles_file'): newState='docfiles_file' - fields=line[1:].split() - elif (state=='binfiles' - or state=='binfiles_file'): - if line[0]==' ': + restOfLine=line[1:] + elif (state=='binfiles' + or state=='binfiles_file'): newState='binfiles_file' - fields=line[1:].split() - if fields is None: + restOfLine=line[1:] + else: try: fields=line.split() - newState=fields[0] + newState=fields[0].lower() + restOfLine=line[len(newState):] # can't lstrip() if it is longdesc except: - mesg=u'no key unable in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; while in state %(state)s unable to get the new state in line=%(line)s',returnCode=10,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state)) + mesg=u'no key in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; while in state %(state)s unable to get the new state for line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state)) + # Reject illegal newState's if not(newState=='binfiles_file'): - currentArch=None + self.currentArch=None if not(newState in self.nextStates[state]): + # print"newState",repr(newState),"state",repr(state),'self.nextStates[state]=',repr(self.nextStates[state])," type",repr(type(self.nextStates[state])) mesg=u'unable to transition to the next state in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; while in state %(state)s the next state %(newState)s is not legal in line=%(line)s',returnCode=10,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state),newState=repr(newState)) - # Handle line state by state + noteException(mesg,devel=mesg+'; while in state %(state)s the next state %(newState)s is not legal in line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state),newState=repr(newState)) + # Main part of routine: handle the line depending on the newState. + # if self.debug: + # print "newState is ",repr(newState)," line is ",repr(line) if newState==self.initialState: mesg=u'unexpected initial state in line=%(lineNo)s' - noteException(mesg,devel=mesg+'; while in state %(state)s now in initial state reading line=%(line)s',returnCode=10,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state)) + noteException(mesg,devel=mesg+'; while in state %(state)s now in initial state reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state)) if newState=='name': try: - self.name=fields[1] + fields=restOfLine.strip().split() + self.name=fields[0] if self.debug: - print "package name is",repr(self.name) + note("package name is "+repr(self.name),self.log) except Exception, err: - mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err)) + mesg=u'no "value" text in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err)) elif newState=='category': try: - self.category=fields[1] + fields=restOfLine.strip().split() + self.category=fields[0] except: - mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + mesg=u'no "value" text in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) elif newState=='catalogue': try: - self.catalogue=fields[1] + fields=restOfLine.strip().split() + self.catalogue=fields[0] except: - mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + mesg=u'no "value" text in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) elif newState=='shortdesc': try: + # I'll allow multiline shortdesc's, although I'll regret it if self.shortdesc is None: self.shortdesc="" if self.shortdesc: self.shortdesc+=' ' - self.shortdesc+=" ".join(fields[1:]) + self.shortdesc+=restOfLine except: mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) elif newState=='longdesc': try: if self.longdesc is None: self.longdesc="" if self.longdesc: self.longdesc+="\n" - self.longdesc+=" ".join(fields[1:]) + self.longdesc+=restOfLine except Exception, err: mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValuse(err)) + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValuse(err)) elif newState=='depend': + fields=restOfLine.strip().split() if len(fields)>0: if self.depend is None: self.depend=[] - self.depend.append(fields[1]) + self.depend.append(restOfLine.strip()) else: mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) elif newState=='execute': + fields=restOfLine.strip().split() if len(fields)>0: if self.execute is None: self.execute=[] - self.execute.append(" ".join(fields[1:])) + self.execute.append(restOfLine.strip()) else: mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) elif newState=='revision': - try: - self.revision=fields[1] - except: - mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + self.handleRevision(newState,restOfLine,line,lineNo) elif newState=='srcfiles': - try: - sizeString=fields[1] - except: - mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - try: - sizeFields=sizeString.split('=') - if sizeFields[0].lower()!='size': - raise tlError - size=int(sizeFields[1]) - if self.srcsize is None: - self.srcsize=0 - self.srcsize+=size - except: - mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - if self.srcfiles is None: - self.srcfiles=[] + self.handleSrcfiles(newState,restOfLine,line,lineNo) elif newState=='srcfiles_file': - try: - self.srcfiles.append(fields[0].strip()) - except: - mesg=u'unable to understand file name in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + self.handleSrcfilesFile(newState,restOfLine,line,lineNo) elif newState=='docfiles': - try: - sizeString=fields[1] - except: - mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - try: - sizeFields=sizeString.split('=') - if sizeFields[0].lower()!='size': - raise tlError - size=int(sizeFields[1]) - if self.docsize is None: - self.docsize=0 - self.docsize+=size - except: - mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - if self.docfiles is None: - self.docfiles={} + self.handleDocfiles(newState,restOfLine,line,lineNo) elif newState=='docfiles_file': - kvDct={} - kvString=None - try: - for kvString in fields[1:]: - if kvString: - k,v=kvString.split("=") - kvDct[k]=v - except: - mesg=u'unable to understand file description in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s; key-value description string=%(kvString)s; reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),kvString=repr(kvString)) - try: - self.docfiles[fields[0].strip()]=kvDct - except Exception, err: - mesg=u'unable to understand file name in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err)) + self.handleDocfilesFile(newState,restOfLine,line,lineNo) elif newState=='runfiles': - try: - sizeString=fields[1] - except: - mesg=u'no value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - try: - sizeFields=sizeString.split('=') - if sizeFields[0].lower()!='size': - raise tlError - size=int(sizeFields[1]) - if self.runsize is None: - self.runsize=0 - self.runsize+=size - except: - mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - if self.runfiles is None: - self.runfiles=[] + self.handleRunfiles(newState,restOfLine,line,lineNo) elif newState=='runfiles_file': - try: - self.runfiles.append(fields[0].strip()) - except: - mesg=u'unable to understand file name in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + self.handleRunfilesFile(newState,restOfLine,line,lineNo) elif newState=='binfiles': - try: - firstString,secondString=fields[1],fields[2] - except: - mesg=u'expected an arch=XXXX size=NNNNNN in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - try: - firstStringFields=firstString.split('=') - secondStringFields=secondString.split('=') - if firstStringFields[0].lower()=='arch': - archFields=firstStringFields - sizeFields=secondStringFields - else: - archFields=secondStringFields - sizeFields=firstStringFields - if (archFields[0].lower()!='arch' - or sizeFields[0].lower()!='size'): - raise tlError - currentArch=archFields[1] - except: - mesg=u'expected two values of the form x=y in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - try: - if self.binsize is None: - self.binsize={} - self.binsize[currentArch]=int(sizeFields[1]) - except: - mesg=u'expected size to be an integer in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - if self.binfiles is None: - self.binfiles={} - self.binfiles[currentArch]=[] + self.handleBinfiles(newState,restOfLine,line,lineNo) elif newState=='binfiles_file': - if currentArch is None: - mesg=u'do not know the right architecture in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - try: - self.binfiles[currentArch].append(fields[0].strip()) - except: - mesg=u'unable to understand file path in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s with architecture %(currentArch)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),currentArch=repr(currentArch)) + self.handleBinfilesFile(newState,restOfLine,line,lineNo) + elif newState=='srcpattern': + self.handleSrcpattern(newState,restOfLine,line,lineNo) + elif newState=='runpattern': + self.handleRunpattern(newState,restOfLine,line,lineNo) + elif newState=='docpattern': + self.handleDocpattern(newState,restOfLine,line,lineNo) + elif newState=='binpattern': + self.handleBinpattern(newState,restOfLine,line,lineNo) elif newState.startswith('cataloguedata-'): - if self.cataloguedata is None: - self.cataloguedata={} - try: - if len(newState)<=len('cataloguedata-'): - raise tlError - self.cataloguedata[newState[len('cataloguedata-'):]]=fields[1] - except: - mesg=u'expected cataloguedata-xxx value to have a sensible xxx and value in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + self.handleCataloguedata(newState,restOfLine,line,lineNo) else: mesg=u'unknown key in line number=%(lineNo)s' - noteException(mesg,devel=mesg+'; key is %(newState)s reading line=%(line)s',returnCode=13,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) - # get ready to read next line + noteException(mesg,devel=mesg+'; key is %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=repr(lineNo),line=repr(line),newState=repr(newState)) + # Cycle back to read next line state=newState - lineNo+=1 return True - def is_arch_dependent(self): - """Returns True if there are binfiles. + def handleRevision(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'revision' """ - if self.binfiles: + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleSrcfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'srcfiles' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleSrcfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'srcfiles_file' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleDocfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'docfiles' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleDocfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'docfiles_file' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleRunfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'runfiles' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleRunfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'runfiles_file' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleBinfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'binfiles' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleBinfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'binfiles_file' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleSrcpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'srcpattern' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleRunpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'runpattern' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleDocpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'docpattern' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleBinpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'binpattern' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + def handleCataloguedata(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'cataloguedata-xxx' + """ + mesg=u'the new state is not allowed here, line number=%(lineNo)s' + noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line)) + return None + + +class tlp_db(object): + """Give information based on a tl file database. This class exists + for tlpobj_db and tlpsrc_db to inherit from + """ + def __init__(self,verbose=False,log=None,debug=False): + """Initialize an instance. + """ + self.verbose=verbose + self.log=log + self.debug=debug + self.value={} # map name --> tl pkg + return None + + def __unicode__(self): + names=self.value.keys() + names.sort() + return u" ".join(names) + + def get_package(self,name): + try: + return self.value[name] + except: + return None + + def _makeSingle(self): + """Helper fcn to make one of whatever self.value is holding multiple + of. Used in from_file. + """ + return tlp(log=self.log,debug=self.debug) + + def from_file(self,fn=None,multi=True): + """Read a tlp db file. Set self.value. + """ + if fn is None: + f=sys.stdin + if f.encoding!=DEFAULT_ENCODING: + warn('using STDIN but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING)) + else: + if self.debug: + errors='strict' + else: + errors='replace' + try: + f=codecs.open(fn,'rU',DEFAULT_ENCODING,errors) + except Exception, err: + mesg='unable to open tlpobj file' + noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err)) + r=self._makeSingle() + retVal=r.from_fh(f,multi=multi,lineNo=0) + self.value[r.name]=r + while retVal: + r=self._makeSingle() + retVal=r.from_fh(f,multi=multi,lineNo=0) + self.value[r.name]=r + return None + + def write_file(self,fn=None): + if fn is None: + f=sys.stdout + if f.encoding!=DEFAULT_ENCODING: + warn('using STDOUT but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING)) + else: + if self.debug: + errors='strict' + else: + errors='replace' + try: + f=codecs.open(fn,'w',DEFAULT_ENCODING,errors) + except Exception, err: + mesg=u'unable to write because unable to open the file' + noteException(mesg,devel=mesg+': %(err)s',exception=tlError,log=self.log,debug=self.debug,err=errorValue(err)) + pkgNames=self.value.keys() + pkgNames.sort() + pkgNo=1 + for k in pkgNames: + v=self.value[k] + v.writeout_fh(f) + f.write("\n") + if pkgNo (dct tagkey --> tagval) + self.binfiles=None # arch --> list of files + self.srcsize=None # int (or None) + self.runsize=None # int (or None) + self.docsize=None # int (or None) + self.binsize=None # arch --> size (or None) + self.cataloguedata=None # cataloguekey --> value + return None + + def __unicode__(self): + r=[] + r.append(super(tlpobj,self).__unicode__()) + r.append(" revision: %s" % (pprint.pformat(self.revision),)) + r.append(" srcsize: %s" % (pprint.pformat(self.srcsize),)) + r.append(" srcfiles: %s" % (pprint.pformat(self.srcfiles),)) + r.append(" docsize: %s" % (pprint.pformat(self.docsize),)) + r.append(" docfiles: %s" % (pprint.pformat(self.docfiles),)) + r.append(" runsize: %s" % (pprint.pformat(self.runsize),)) + r.append(" runfiles: %s" % (pprint.pformat(self.runfiles),)) + r.append(" binsize: %s" % (pprint.pformat(self.binsize),)) + r.append(" binfiles: %s" % (pprint.pformat(self.binfiles),)) + r.append(" cataloguedata: %s" % (pprint.pformat(self.cataloguedata),)) + return "\n".join(r) + + def _stringToWriteout(self): + """String that will be written out during writeout_fh + """ + r=[] + r.append(super(tlpobj,self)._stringToWriteout()) + if self.revision: + r.append("revision %s" % (self.revision,)) + if self.docfiles: + if self.docsize is None: + mesg='docsize is not there but there are docfiles' + noteException(mesg,devel=mesg,exception=tlError,log=self.log,debug=self.debug) + r.append("docfiles size=%s" % (self.docsize,)) + dFiles=self.docfiles.keys() + dFiles.sort() + for line in dFiles: + s=[] + for (k,v) in self.docfiles[line].items(): + s.append("%s=%s" % (k,v)) + dLine=" "+line + if s: + dLine+=" "+(" ".join(s)) + r.append(dLine) + if self.srcfiles: + if self.srcsize is None: + mesg='srcsize is not there but there are srcfiles' + noteException(mesg,devel=mesg,exception=tlError,log=self.log,debug=self.debug) + r.append("srcfiles size=%s" % (self.srcsize,)) + self.srcfiles.sort() + for line in self.srcfiles: + r.append(" "+line) + if self.runfiles: + if self.runsize is None: + mesg='runsize is not there but there are runfiles' + noteException(mesg,devel=mesg,exception=tlError,log=self.log,debug=self.debug) + r.append("runfiles size=%s" % (self.runsize,)) + self.runfiles.sort() + for line in self.runfiles: + r.append(" "+line) + if self.binfiles: + arches=self.binfiles.keys() + arches.sort() + for arch in arches: + if self.binsize[arch] is None: + mesg='binsize is not there but there are binfiles' + noteException(mesg,devel=mesg+" for %(arch)s",exception=tlError,log=self.log,debug=self.debug,arch=repr(arch)) + r.append("binfiles arch=%s size=%s" % (arch,self.binsize[arch],)) + sBinfiles=self.binfiles[arch] + sBinfiles.sort() + for line in sBinfiles: + r.append(" "+line) + if self.cataloguedata: + cdKeys=self.cataloguedata.keys() + for k in cdKeys: + r.append("catalogue-%s %s" % (k,self.cataloguedata[k])) + return "\n".join(r) + +## def to_db(self,dBcsr,dBcnx,careful=True,associatedCatPkgs=None,verbose=False): +## """Put the yyy-file information to a database +## dBcsr, dBcnx database cursor and connection +## careful=True raise exception on error? +## associatedCatPkgs=None list of catalogue package id's that either +## are part of this texlive package of of which this texlive package +## is a part (None is the same as using what is now in the dB) +## verbose=False Give warnings +## """ +## tlId=self.name +## # Drop the file records if they are there +## sql="DELETE FROM tds_files WHERE tl_id=%(tlId)s" +## dct={'tlId':tlId} +## try: +## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='drop old texlive files record') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to drop existing tds files record' +## noteException(mesg,devel=mesg+' for package %(pkgId)s: %(err)s',log=self.log,debug=self.debug,pkgId=repr(pkgId),err=errorValue(err)) +## # Do the tl_cat records; see which ones to do +## if associatedCatPkgs is None: +## sql="SELECT pkg_id FROM tl_cat WHERE tl_id=%(tlId)s" +## dct['tlId']=tlId +## try: +## dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='get list of catalogue packages for this texlive package') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to get list of catalogue packages for this texlive packages ' +## noteException(mesg,devel=mesg+'%(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) +## associatedCatPkgs=[a for (a,) in dBres] +## if not(associatedCatPkgs): # try tlId +## possiblePkgId=tlId.lower() +## sql="SELECT id FROM package WHERE id=%(pkgId)s" +## dct['pkgId']=possiblePkgId +## try: +## dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='check for existence of a catalogue id') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to check for existence of a catalogue id' +## noteException(mesg,devel=mesg+'%(pkgId)s: %(err)s',log=self.log,debug=self.debug,pkgId=repr(possiblePkgId),err=errorValue(err)) +## if dBres: +## associatedCatPkgs.append(possiblePkgId) +## # drop any that are there now +## sql="DELETE FROM tl_cat WHERE tl_id=%(tlId)s" +## dct={'tlId':tlId} +## try: +## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='drop old texlive package-catalogue package association') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to drop existing texlive package record' +## noteException(mesg,devel=mesg+' for package %(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) +## # Drop the tl records if they are there +## sql="DELETE FROM tl WHERE tl_id=%(tlId)s" +## dct={'tlId':tlId} +## try: +## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='drop old texlive package record') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to drop existing texlive package record' +## noteException(mesg,devel=mesg+' for package %(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) +## # Insert new tl record +## sql="INSERT INTO tl (tl_id,shortdesc,longdesc) VALUES (%(tlId)s,%(shortdesc)s,%(longdesc)s)" +## dct['shortdesc']=self.shortdesc +## dct['longdesc']=self.longdesc +## try: +## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='insert new texlive package record') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to insert new texlive package record' +## noteException(mesg,devel=mesg+' for package %(tlId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) +## # insert new tl_cat records +## if not(tlId.startswith('bin-') +## or tlId.startswith('collection-') +## or tlId.startswith('hyphen-') +## or tlId.startswith('lib-') +## or tlId.startswith('scheme-') +## or tlId.startswith('texlive-')): +## if (not(associatedCatPkgs) +## and careful +## and verbose): +## mesg=u'there are no catalogue packages associated with the TeX Live package %(tlId)s' +## warn(mesg,log=self.log,debug=self.debug,tlId=repr(tlId)) +## for pkgId in associatedCatPkgs: +## sql="INSERT INTO tl_cat (tl_id,pkg_id) VALUES (%(tlId)s,%(pkgId)s)" +## dct['pkgId']=pkgId +## try: +## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='insert new tl_cat package record') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to insert new association between a texlive package and a catalogue package' +## noteException(mesg,devel=mesg+' for texlive package %(tlId)s and catalgoue package %(pkgId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),pkgId=repr(pkgId),err=errorValue(err)) +## # Insert the file records +## # First build a list of pairs: (fileName,fileType) +## fileList=[] +## if self.srcfiles: +## for fn in self.srcfiles: +## fileList.append((fn,'src')) +## if self.runfiles: +## for fn in self.runfiles: +## fileList.append((fn,'run')) +## if self.docfiles: +## for fn in self.docfiles.keys(): +## fileList.append((fn,'doc')) +## if self.binfiles: +## for arch in self.binfiles.keys(): +## for fn in self.binfiles[arch]: +## fileList.append((fn,'bin')) +## # .. then put the info in the dB +## dct={'tlId':tlId} +## for (fn,fileType) in fileList: +## pathDir,pathBase=os.path.split(fn) +## sql="INSERT INTO tds_files (tl_id,type,path_dir,path_base) VALUES (%(tlId)s,%(fileType)s,%(pathDir)s,%(pathBase)s)" +## dct['fileType']=fileType +## dct['pathDir']=pathDir +## dct['pathBase']=pathBase +## try: +## dBUtils.putData(sql,dct=dct,dBcsr=dBcsr,log=self.log,debug=self.debug,listtype='insert new texlive files record') +## except dBUtils.dBUtilsError, err: +## mesg=u'unable to insert new tds files record' +## noteException(mesg,devel=mesg+' for package %(tlId)s with type=%(fileType)s, pathDir=%(pathDir)s, pathBase=%(pathBase)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),fileType=repr(fileType),pathDir=repr(pathDir),pathBase=repr(pathBase),err=errorValue(err)) +## # Commit +## try: +## dBcnx.commit() +## except Exception, err: +## mesg=u'unable to commit the insert of the new tds files record' +## noteException(mesg,devel=mesg+' for package %(pkgId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err)) +## return None + + initialState=tlp.initialState + nextStates=tlp.nextStates + nextStates['revision']=tlp.keyValueStates-set(['revision']) + nextStates['srcfiles']=tlp.keyValueStates | set(['srcfiles_file'])-set(['srcfiles']) + nextStates['srcfiles_file']=tlp.keyValueStates | set(['srcfiles_file']) + nextStates['runfiles']=tlp.keyValueStates | set(['runfiles_file'])-set(['runfiles']) + nextStates['runfiles_file']=tlp.keyValueStates | set(['runfiles_file']) + nextStates['docfiles']=tlp.keyValueStates | set(['docfiles_file'])-set(['docfiles']) + nextStates['docfiles_file']=tlp.keyValueStates | set(['docfiles_file'])-set(['docfiles']) + nextStates['binfiles']=tlp.keyValueStates | set(['binfiles_file']) + nextStates['binfiles_file']=tlp.keyValueStates | set(['binfiles_file']) + nextStates['cataloguedata']=tlp.keyValueStates + + def handleRevision(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'revision' + """ + try: + fields=restOfLine.strip().split() + self.revision=int(fields[0]) + except: + mesg=u'no "value" integer in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + return None + + def handleSrcfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'srcfiles' + """ + try: + fields=restOfLine.strip().split() + sizeString=fields[0] + except: + mesg=u'no value in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + try: + sizeFields=sizeString.split('=') + if sizeFields[0].lower()!='size': + raise tlError + size=int(sizeFields[1]) + if self.srcsize is None: + self.srcsize=0 + self.srcsize+=size + except: + mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + if self.srcfiles is None: + self.srcfiles=[] + return None + + def handleSrcfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'srcfiles_file' + """ + try: + fields=restOfLine.strip().split() # recall that restOfLine=line[1:] + self.srcfiles.append(fields[0]) + except: + mesg=u'unable to understand source file name in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + return None + + def handleDocfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'docfiles' + """ + try: + fields=restOfLine.strip().split() + sizeString=fields[0] + except: + mesg=u'no value in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + try: + sizeFields=sizeString.split('=',1) + if sizeFields[0].lower()!='size': + raise tlError + size=int(sizeFields[1]) + if self.docsize is None: + self.docsize=0 + self.docsize+=size + except: + mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + if self.docfiles is None: + self.docfiles={} + return None + + def handleDocfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'docfiles_file' + """ + fields=restOfLine.strip().split() # recall that restOfLine=line[1:] + kvDct={} + kvString=None + try: + for kvString in fields[1:]: + if kvString: + k,v=kvString.split("=",1) + kvDct[k]=v + except: + mesg=u'unable to understand doc file description in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s; key-value description string=%(kvString)s; reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),kvString=repr(kvString)) + try: + self.docfiles[fields[0].strip()]=kvDct + except Exception, err: + mesg=u'unable to understand doc file name in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err)) + return None + + def handleRunfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'runfiles' + """ + try: + fields=restOfLine.strip().split() + sizeString=fields[0] + except: + mesg=u'no value in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + try: + sizeFields=sizeString.split('=',1) + if sizeFields[0].lower()!='size': + raise tlError + size=int(sizeFields[1]) + if self.runsize is None: + self.runsize=0 + self.runsize+=size + except: + mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + if self.runfiles is None: + self.runfiles=[] + return None + + def handleRunfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'runfiles_file' + """ + try: + fields=restOfLine.strip().split() # recall that restOfLine=line[1:] + self.runfiles.append(fields[0].strip()) + except: + mesg=u'unable to understand run file name in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + return None + + def handleBinfiles(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'binfiles' + """ + try: + fields=restOfLine.strip().split() + firstString,secondString=fields[0],fields[1] + except: + mesg=u'expected an arch=XXXX size=NNNNNN in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + try: + firstStringFields=firstString.split('=',1) + secondStringFields=secondString.split('=',1) + if firstStringFields[0].lower()=='arch': + archFields=firstStringFields + sizeFields=secondStringFields + else: + archFields=secondStringFields + sizeFields=firstStringFields + if (archFields[0].lower()!='arch' + or sizeFields[0].lower()!='size'): + raise tlError + self.currentArch=archFields[1] + except: + mesg=u'expected two values of the form x=y in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + try: + if self.binsize is None: + self.binsize={} + self.binsize[self.currentArch]=int(sizeFields[1]) + except: + mesg=u'expected size to be an integer in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + if self.binfiles is None: + self.binfiles={} + self.binfiles[self.currentArch]=[] + return None + + def handleBinfilesFile(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'binfiles_file' + """ + if self.currentArch is None: + mesg=u'do not know the right architecture in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + try: + fields=restOfLine.strip().split() # recall that restOfLine=line[1:] + self.binfiles[self.currentArch].append(fields[0].strip()) + except: + mesg=u'unable to understand bin file path in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s with architecture %(currentArch)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),currentArch=repr(currentArch)) + return None + + def is_arch_dependent(self): + """Returns True if there are binfiles. + """ + if self.binfiles: return True else: return False @@ -1106,101 +1469,197 @@ class tlpobj(object): r.sort() return "\n".join(r) - def write_file(self,fn=None): - if fn: - try: - f=open(fn,'w') - except Exception, err: - mesg=u'unable to write because unable to open the file' - noteException(mesg,devel=mesg+': %(err)s',exception=tlError,log=self.log,debug=self.debug,err=errorValue(err)) - else: - f=None - self.writeout(f) - f.close() - return None - - def writeout(self,f=None): - if f is None: - f=sys.stdout - pkgs=self.value.keys() - pkgs.sort() - for k in pkgs: - v=self.value[k] - f.write(v.stringout()) - f.write("\n\n") - return None -class tlpobj_db(object): +class tlpobj_db(tlp_db): """Give information based on a TeXLive object file database """ + def _makeSingle(self): + """Helper fcn to make one of whatever self.value is holding multiple + of + f file handle + """ + return tlpobj(log=self.log,debug=self.debug) + +## def to_db(self,dBcsr,dBcnx,verbose=False): +## """Write all records to the database +## dBcsr,dBcnx database cursor and connection +## """ +## pkgNames=self.value.keys() +## pkgNames.sort() +## for k in pkgNames: +## v=self.value[k] +## v.to_db(dBcsr,dBcnx,verbose=verbose) +## return None + + +class tlpsrc(tlp): + """Give information based on a TeXLive source file + """ def __init__(self,verbose=False,log=None,debug=False): """Initialize an object. """ - self.verbose=verbose - self.log=log - self.debug=debug - self.value={} # map name --> tlpkg + super(tlpsrc,self).__init__(verbose=verbose,log=log,debug=debug) + self.srcpattern=None # list of strings (or None) + self.runpattern=None # list of strings (or None) + self.docpattern=None # list of strings (or None) + self.binpattern=None # list of strings (or None) return None def __unicode__(self): - names=self.value.keys() - names.sort() - return u" ".join(names) + r=[] + r.append(super(tlpsrc,self).__unicode__()) + r.append(" srcpattern: %s" % (pprint.pformat(self.srcpattern),)) + r.append(" runpattern: %s" % (pprint.pformat(self.runpattern),)) + r.append(" docpattern: %s" % (pprint.pformat(self.docpattern),)) + r.append(" binpattern: %s" % (pprint.pformat(self.binpattern),)) + return "\n".join(r) - def get_package(self,name): + def _stringToWriteout(self): + """String that will be written out during writeout_fh + """ + r=[] + r.append(super(tlpsrc,self)._stringToWriteout()) + if self.srcpattern: + for line in self.srcpattern: + r.append("srcpattern %s" % (line,)) + if self.runpattern: + for line in self.runpattern: + r.append("runpattern %s" % (line,)) + if self.docpattern: + for line in self.docpattern: + r.append("docpattern %s" % (line,)) + if self.binpattern: + for line in self.binpattern: + r.append("binpattern %s" % (line,)) + return "\n".join(r) + + initialState=tlp.initialState + nextStates=tlp.nextStates + nextStates['srcpattern']=tlp.keyValueStates + nextStates['runpattern']=tlp.keyValueStates + nextStates['docpattern']=tlp.keyValueStates + nextStates['binpattern']=tlp.keyValueStates + + def handleSrcpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'srcpattern' + """ try: - return self.value[name] + if self.srcpattern is None: + self.srcpattern=[] + self.srcpattern.append(restOfLine.rstrip()) except: - return None + mesg=u'no "value" text in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + return None - def from_file(self,fn=None): - """Read a tlpobj file. Set self.value. + def handleRunpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'runpattern' """ - if fn is None: - f=sys.stdin - else: - try: - f=open(fn,'rU') - except Exception, err: - mesg='unable to open tlpobj file' - noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err)) - r=tlpobj(log=self.log,debug=self.debug).from_fh(f,multi=True,lineNo=0) - while r: - self.value[r.name]=r - r=tlpobj(log=self.log,debug=self.debug).from_fh(f,multi=True,lineNo=0) + try: + if self.runpattern is None: + self.runpattern=[] + self.runpattern.append(restOfLine.rstrip()) + except: + mesg=u'no "value" text in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) return None - def write_file(self,fn=None): - if fn is None: - f=sys.stdout - else: - try: - f=open(fn,'w') - except Exception, err: - mesg=u'unable to write because unable to open the file' - noteException(mesg,devel=mesg+': %(err)s',exception=tlError,log=self.log,debug=self.debug,err=errorValue(err)) + def handleDocpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'docpattern' + """ + try: + if self.docpattern is None: + self.docpattern=[] + self.docpattern.append(restOfLine.rstrip()) + except: + mesg=u'no "value" text in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + return None + + def handleBinpattern(self,newState,restOfLine,line,lineNo): + """What to do if the newState is 'binpattern' + """ + try: + if self.binpattern is None: + self.binpattern=[] + self.binpattern.append(restOfLine.rstrip()) + except: + mesg=u'no "value" text in line number=%(lineNo)s' + noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState)) + return None + + +class tlpsrc_db(tlp_db): + """Give information based on a TeXLive src file list + """ + def from_dir(self,dn=None): + """Read a set of tlpsrc files. Set self.value. + dn=None directory name. If None, use the canonical name. + """ + if dn is None: + dn=os.path.join(os.path.dirname(TLPDB_FILENAME),'tlpkg/tlpsrc') + globPattern=os.path.join(dn,'*.tlpsrc') + for fn in glob.glob(globPattern): + self.from_file(fn=fn) + return None + + def _makeSingle(self): + """Helper fcn to make one of whatever self.value is holding multiple + of. + """ + return tlpsrc(log=self.log,debug=self.debug) + + def to_dir(self,dn=None): + """Write a set of tlpsrc files. + dn=None directory name. If None, use the canonical name. + """ + if dn is None: + dn=os.path.join(os.path.basename(TLPDB_FILENAME),'tlpsrc') + if not(os.path.exists(dn)): + os.makedirs(dn) pkgNames=self.value.keys() - pkgNames.sort() for k in pkgNames: v=self.value[k] - v.writeout_fh(f) - f.write("\n\n") - f.close() + fn=os.path.join(dn,v.name+'.tlpsrc') + v.writeout(fn=fn) return None -## def to_db(self,dBcsr,dBcnx): -## """Write all records to the database -## dBcsr,dBcnx database cursor and connection -## """ -## pkgNames=self.value.keys() -## pkgNames.sort() -## for k in pkgNames: -## ## if self.debug: -## ## print note("writing tds file information for %s" % (repr(k),),self.log) -## v=self.value[k] -## v.to_db(dBcsr,dBcnx) -## return None - + +def getRelatedCatPkg(tlId,dBcsr,log=None,debug=False): + """Get the list of catalogue packages that are related to the given + TeX Live package (does not return None values). + tlId string + dBcsr database cursor + log=None,debug=False debugging + """ + sql="""SELECT pkg_id FROM tl_cat + WHERE tl_id=%(tlId)s + AND pkg_id IS NOT NULL""" + dct={'tlId':tlId} + try: + dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get catalogue package ids related to this TeX Live package id') + except dBUtils.dBUtilsError, err: + mesg=u'unable to get texlive-catalogue packages record' + noteException(mesg,devel=mesg+' for TeX Live package %(tlId)s: %(err)s',log=log,debug=debug,tlId=repr(tlId),err=errorValue(err)) + return set([catId for (catId,) in dBres]) + +def getRelatedTlPkg(pkgId,dBcsr,log=None,debug=False): + """Get the list of TeX Live packages that are related to the given + Catalogue package. + pkgId string + dBcsr database cursor + log=None,debug=False debugging + """ + sql="""SELECT tl_id FROM tl_cat + WHERE pkg_id=%(pkgId)s""" + dct={'pkgId':pkgId} + try: + dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get TeX Live package ids related to this catalogue package id') + except dBUtils.dBUtilsError, err: + mesg=u'unable to get catalogue-texlive packages record' + noteException(mesg,devel=mesg+' for catalogue package %(pkgId)s: %(err)s',log=log,debug=debug,pkgId=repr(pkgId),err=errorValue(err)) + return set([tlId for (tlId,) in dBres]) + #...................................................................... def main(argv=None,log=None,debug=DEBUG): @@ -1211,11 +1670,13 @@ def main(argv=None,log=None,debug=DEBUG): if argv is None: argv=sys.argv # Parse the arguments - usage="""%prog: read a TeX Live dB obj file; typically write to the dB + usage="""%prog: read a TeX Live tlp src or obj file; typically write the obj filelists to the dB %prog [options]""" oP=optparse.OptionParser(usage=usage,version=__version__) - oP.add_option('--output','-O',action='store_true',default=False,dest='output',help='show all package records') + oP.add_option('--output','-o',action='store_true',default=False,dest='output',help='show all package records') oP.add_option('--no-database','-n',action='store_true',default=False,dest='noDb',help='do not put all package records to the database') + oP.add_option('--source-read','-s',action='store_true',default=False,dest='srcDir',help='read the source files') + oP.add_option('--add-catalogue-information','-a',action='store_true',default=False,dest='catInfo',help='add catalogue information to the source or object information') oP.add_option('--debug','-D',action='store_true',default=DEBUG,dest='debug',help='output debugging information') oP.add_option('--verbose','-V',action='store_true',default=VERBOSE,dest='verbose',help='talk a lot') opts, args=oP.parse_args(argv[1:]) @@ -1226,25 +1687,37 @@ def main(argv=None,log=None,debug=DEBUG): else: tlpdbFilename=TLPDB_FILENAME # Handle the various command line switches +## try: +## (dBcnx,dBcsr)=dBUtils.opendB() +## except dBUtils.dBUtilsError, err: +## fail("Unable to connect to the database: %(err)s",returnCode=10,log=log,debug=opts.debug,err=errorValue(err)) + if opts.srcDir: + try: + t=tlpsrc_db(log=log,debug=opts.debug) + t.from_dir(dn=None) + if opts.catInfo: + t.add_cat_descs(dBcsr) + t.to_dir(dn='tmpTexlive') + except Exception, err: + mesg="trouble reading the object file and writing: "+errorValue(err) + fail(mesg,returnCode=10,log=log,debug=opts.debug) + return None if opts.output: try: t=tlpobj_db(log=log,debug=opts.debug) t.from_file(tlpdbFilename) + if opts.catInfo: + t.add_cat_descs(dBcsr) t.write_file() except Exception, err: mesg="trouble reading the object file and writing: "+errorValue(err) fail(mesg,returnCode=10,log=log,debug=opts.debug) return None - if not(opts.noDb): - print "dB turned off; you need Python module psycopg for access to PostGreSQL" -## try: -## (dBcnx,dBcsr)=dBUtils.opendB() -## except dBUtils.dBUtilsError, err: -## fail("Unable to connect to the database: %(err)s",returnCode=10,log=log,debug=opts.debug,err=errorValue(err)) +## if not(opts.noDb): ## try: ## t=tlpobj_db(log=log,debug=opts.debug) ## t.from_file(tlpdbFilename) -## t.to_db(dBcsr,dBcnx) +## t.to_db(dBcsr,dBcnx,verbose=opts.verbose) ## except Exception, err: ## mesg="trouble reading the object file and sending the results to the database: "+errorValue(err) ## fail(mesg,returnCode=10,log=log,debug=opts.debug) -- cgit v1.2.3