update from jimh

git-svn-id: svn://tug.org/texlive/trunk@4700 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 926 insertions, 453 deletions

diff --git a/Master/tlpkg/lib/TeXLive.py b/Master/tlpkg/lib/TeXLive.py
index 67b64573c7c..fe302acee9b 100644
--- a/Master/tlpkg/lib/TeXLive.py
+++ b/Master/tlpkg/lib/TeXLive.py
@@ -2,40 +2,58 @@
 # TeXLive.py
 # -*- encoding: utf-8 -*-
 #  Usage:  call with argument of --help
-# 2007-Jul-28 Jim Hefferon  ftpmaint@tug.ctan.org
-""" Read a TeX Live package dB file.
+# 2007-Aug-05 Jim Hefferon  ftpmaint@tug.ctan.org
+"""Understand a TeX Live package and the associated dB file.
+
+TeX Live has a format for metadata located in trunk/Master/tlpkg (for
+the 'src' packages and documentation of the format) and
+trunk/Master/texlive.tlpdb (see also -withctan ?).  This module reads the
+two formats, can transform them by adding package documentation from the
+ctanWeb database, and can write them back out. 
+
 """
-__version__='0.9.9'
+__version__='1.0.0'
 __author__='Jim Hefferon'
-__date__='2007-Jul-15'
-__notes__="""This version has been stripped of utility routines for the texlive folks.
+__date__='2007-Aug-05'
+__notes__="""
+Bugs:
+1) Does not expand patterns for the src module, and does not
+  redo the sizes, etc., for obj files.
+2) You can't from the command line change the name of the db files.
+3) This version for the TeX Live svn tree has the dB stripped out, so
+  the option -a doesn't work, and the option -n has no effect.
 """
 
 import os, os.path, sys
 import re
 import optparse
 import codecs
+import tempfile
 import time
 import pprint
+import glob
 
-# import library
-# from util import *
-# import dBUtils
+## import library
+## from util import *
+## import dBUtils
+## from defaults import DEFAULT_ENCODING, DEFAULT_DATABASE_ENCODING, DEBUG, LOGGING
+DEFAULT_ENCODING='UTF-8'
 
 TLPDB_FILENAME='/home/svn/texlive/trunk/Master/texlive.tlpdb'
 
 THIS_SCRIPT=sys.argv[0]
-DEBUG=False
-LOGGING=False
-LOGFILE_NAME=os.path.splitext(THIS_SCRIPT)[0]+'.log'
-VERBOSE=False
+DEBUG=True
+LOGGING=True
+LOGFILE_NAME=os.path.splitext(os.path.basename(THIS_SCRIPT))[0]+'.log'
+VERBOSE=True
 
-import locale
 # What encoding for the terminal?
+import locale
 locale_language, locale_output_encoding = locale.getdefaultlocale()
 if locale_output_encoding is None:
     locale_output_encoding='UTF-8'
 
+
 #---------------------- stuff from util.py
 import StringIO # for msg
 import traceback # for msg
@@ -432,10 +450,9 @@ def exceptionName():
 class tlError(jhError):
     pass
 
-tlobjKeys=set(['name','category','catalogue','shortdesc','longdesc','depend','execute','revision','srcsize','srcfiles','docsize','docfiles','runsize','runfiles','binsize','binfiles','cataloguedata'])
+tlobjKeys=set(['name','category','catalogue','shortdesc','longdesc','depend','execute','revision','srcsize','srcfiles','docsize','docfiles','runsize','runfiles','binsize','binfiles','cataloguedata','srcpattern','runpattern','docpattern','binpattern'])
 
-
-class tlpobj(object):
+class tlp(object):
     """Give information based on a TeXLive object file
     """
     def __init__(self,verbose=False,log=None,debug=False):
@@ -444,28 +461,13 @@ class tlpobj(object):
         self.verbose=verbose
         self.log=log
         self.debug=debug
-        self.__initializeUserVars()
-        self.value={}  # map name --> tlpkg
-        return None
-
-    def __initializeUserVars(self):
-        self.name=None
-        self.category=None
-        self.catalogue=None
-        self.shortdesc=None
-        self.longdesc=None
-        self.depend=None
-        self.execute=None
-        self.revision=None
-        self.srcfiles=None # list of files
-        self.runfiles=None # list of files
-        self.docfiles=None # dct fileName--> (dct tagkey --> tagval)
-        self.binfiles=None # arch --> list of files
-        self.srcsize=None # int (or None)
-        self.runsize=None # int (or None)
-        self.docsize=None # int (or None)
-        self.binsize=None  # arch --> size  (or None)
-        self.cataloguedata=None # cataloguekey --> value
+        self.name=None   # string  id of package
+        self.category=None # string   category into which package falls 
+        self.catalogue=None # string  id of associated Catalogue package
+        self.shortdesc=None # string  one-liner
+        self.longdesc=None # string  may have newlines
+        self.depend=None  # list  of the form Category/Name
+        self.execute=None # list  free-form strings
         return None
 
     def __unicode__(self):
@@ -477,16 +479,6 @@ class tlpobj(object):
         r.append("  longdesc: %s" % (pprint.pformat(self.longdesc),))
         r.append("  depend: %s" % (pprint.pformat(self.depend),))
         r.append("  execute: %s" % (pprint.pformat(self.execute),))
-        r.append("  revision: %s" % (pprint.pformat(self.revision),))
-        r.append("  srcsize: %s" % (pprint.pformat(self.srcsize),))
-        r.append("  srcfiles: %s" % (pprint.pformat(self.srcfiles),))
-        r.append("  docsize: %s" % (pprint.pformat(self.docsize),))
-        r.append("  docfiles: %s" % (pprint.pformat(self.docfiles),))
-        r.append("  runsize: %s" % (pprint.pformat(self.runsize),))
-        r.append("  runfiles: %s" % (pprint.pformat(self.runfiles),))
-        r.append("  binsize: %s" % (pprint.pformat(self.binsize),))
-        r.append("  binfiles: %s" % (pprint.pformat(self.binfiles),))
-        r.append("  cataloguedata: %s" % (pprint.pformat(self.cataloguedata),))
         return "\n".join(r)
 
     def writeout(self,fn=None):
@@ -495,22 +487,25 @@ class tlpobj(object):
         """
         if fn is None:
             f=sys.stdout
+            if f.encoding!=DEFAULT_ENCODING:
+                warn('using STDOUT but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING))
         else:
+            if self.debug:
+                errors='strict'
+            else:
+                errors='replace'
             try:
-                f=open(fn,'w')
+                f=codecs.open(fn,'w',DEFAULT_ENCODING,errors)
             except Exception, err:
-                mesg='unable to open file for writing'
+                mesg='unable to open the file for writing'
                 noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err))
         self.writeout_fh(f)
         f.close()
         return None
 
-    def writeout_fh(self,f):
-        """Write a string representation of the user variables the filhandle.
-        Note that files are listed in sorted order.
-          f  filehandle
+    def _stringToWriteout(self):
+        """String that will be written out during writeout_fh
         """
-        # Make the string
         r=[]
         r.append("name %s" % (self.name,))
         if self.category:
@@ -518,16 +513,506 @@ class tlpobj(object):
         if self.shortdesc:
             r.append("shortdesc %s" % (self.shortdesc,))
         if self.longdesc:
-            for line in self.longdesc: 
+            for line in self.longdesc.split("\n"): 
                 r.append("longdesc %s" % (line,))
-        if self.revision:
-            r.append("revision %s" % (self.revision,))
         if self.depend:
             for line in self.depend:
                 r.append("depend %s" % (line,))
         if self.execute:
             for line in self.execute:
                 r.append("execute %s" % (line,))
+        return "\n".join(r)
+
+    def writeout_fh(self,f):
+        """Write a string representation of the user variables to the
+        filehandle.
+          f  filehandle
+        """
+        s=self._stringToWriteout()
+        f.write(s)
+        return None
+
+    def get_associated_cat_ids(self,dBcsr,log=None,debug=False):
+        """Return the matching catalogue ids
+        """
+        sql="""SELECT pkg_id FROM tl_cat
+                 WHERE tl_id=%(tlId)s
+                 AND pkg_id IS NOT NULL"""
+        dct={'tlId':self.name}
+        try:
+            dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get catalogue package ids related to this TeX Live package id')
+        except dBUtils.dBUtilsError, err:
+            mesg=u'unable to get texlive-catalogue packages record'
+            noteException(mesg,devel=mesg+' for TeX Live package %(tlId)s: %(err)s',exception=tlError,log=log,debug=debug,tlId=repr(tlId),err=errorValue(err))
+        return [catId for (catId,) in dBres]
+
+    def add_cat_desc(self,dBcsr,catId=None,log=None,debug=False):
+        """If there is not now a description, and there is a matching
+        Catalogue package, add its descriptions.  (Also sets the catalogue
+        variable)
+          dBcsr  database cursor
+          catId=None  string or None  if present, the name of the Catalogue
+            package whose descriptions will be added.
+          log=None,debug=False  debugging
+        """
+        if not(catId):
+            catIds=self.get_associated_cat_ids(dBcsr,log=log,debug=debug)
+            if len(catIds)==1:
+                catId=catIds[0]
+        if not(catId):
+            return False
+        sql="""SELECT caption, description FROM pkg_text
+                 WHERE id=%(catId)s
+                 AND iso_code='en'"""
+        dct={'catId':catId}
+        try:
+            dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get catalogue descriptions related to this TeX Live package id')
+        except dBUtils.dBUtilsError, err:
+            mesg=u'unable to get catalogue package descriptions for this TeX Live package'
+            noteException(mesg,devel=mesg+' for TeX Live package %(tlId)s and catalogue id %(catId)s: %(err)s',exception=tlError,log=log,debug=debug,tlId=repr(tlId),catId=repr(catId),err=errorValue(err))
+        res=dBres[0]
+        if res:
+            (catCaption,catDesc)=dBres[0]
+            if (not(self.shortdesc)
+                and not(self.longdesc)):
+                self.catalogue=catId
+                self.shortdesc=catCaption
+                self.longdesc=catDesc
+        return True
+
+    def from_file(self,fn=None,multi=False):
+        """Read a tlp object from a file.
+          fn=None  string  file name or None, which means sys.stdin
+          multi=False  boolean  whether to allow multiple pkgs in a file
+        """
+        if fn is None:
+            f=sys.stdin
+            if f.encoding!=DEFAULT_ENCODING:
+                warn('using STDIN but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING))
+        else:
+            try:
+                if self.debug:
+                    errors='strict'
+                else:
+                    errors='replace'
+                f=codecs.open(fn,'rU',DEFAULT_ENCODING,errors)
+            except Exception, err:
+                mesg='unable to open file for reading'
+                noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err))
+        r=self.from_fh(f,multi=multi,lineNo=0)
+        return None
+
+    initialState='init'
+    keyValueStates=tlobjKeys # line of form "key value"
+    filePathStates=set(['srcfiles_file','runfiles_file','docfiles_file','binfiles_file'])  # line contains file path indented by space
+    nextStates={initialState:tlobjKeys,
+                'name':keyValueStates-set(['name']),
+                'category':keyValueStates-set(['category']),
+                'catalogue':keyValueStates-set(['catalogue']),
+                'shortdesc':keyValueStates-set(['shortdesc']),
+                'longdesc':keyValueStates,
+                'depend':keyValueStates,
+                'execute':keyValueStates}
+    def from_fh(self,f,multi=None,lineNo=0):
+        """Read tlp information from a file handle.  Returns None if EOF is
+        read before anything is found.  Note that files have been opened
+        with codecs.open() (except for stdin)
+          f  an opened file
+          multi=None  If not None then multiple tplobj's in the file are
+            treated as an error
+          lineNo=0  debugging
+        May raise tlError.
+        """
+        # Implementation note: I am putting the full parsing routine in
+        # this base class, except for some "handle" routines, to allow
+        # me to have the handle routines report error messages in the
+        # tplobj and tplsrc classes.
+        state=self.initialState
+        # Look for blank lines that are delimeters between tlp parts
+        anyNonblankLinesRead=False
+        while True:
+            line=f.readline()
+            if not(line):  # EOF?
+                return False
+            lineNo+=1
+            if self.debug and self.verbose:
+                note('line is '+repr(line),self.log)
+            line=line.rstrip()
+            if not(anyNonblankLinesRead):
+                if (not(line)
+                    or line[0]=='#'): # toss initial blank lines or comments
+                    continue
+                else:
+                    anyNonblankLinesRead=True
+            elif not(line):
+                if multi is None:
+                    mesg=u'unexpected multiple tpl objects'
+                    noteException(mesg,devel=mesg+'; trailing blank line found at line %(lineNo)s',exception=tlError,log=self.log,debug=self.debug,lineNo=repr(lineNo))
+                else:
+                    return self
+            # Get the next state (the key) and the part of the line after key.
+            # The "srcfiles", etc., states need special handling as they
+            # put the reader in a position where the first thing on the line
+            # is not the next state.
+            restOfLine=""
+            if line[0]==' ':
+                if (state=='srcfiles'
+                    or state=='srcfiles_file'):
+                    newState='srcfiles_file'
+                    restOfLine=line[1:]
+                elif (state=='runfiles'
+                      or state=='runfiles_file'):
+                    newState='runfiles_file'
+                    restOfLine=line[1:]
+                elif (state=='docfiles'
+                      or state=='docfiles_file'):
+                    newState='docfiles_file'
+                    restOfLine=line[1:]
+                elif (state=='binfiles'
+                      or state=='binfiles_file'):
+                    newState='binfiles_file'
+                    restOfLine=line[1:]
+            else:
+                try:
+                    fields=line.split()
+                    newState=fields[0].lower()
+                    restOfLine=line[len(newState):] # can't lstrip() if it is longdesc
+                except:
+                    mesg=u'no key in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; while in state %(state)s unable to get the new state for line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state))
+            # Reject illegal newState's
+            if not(newState=='binfiles_file'):
+                self.currentArch=None
+            if not(newState in self.nextStates[state]):
+                # print"newState",repr(newState),"state",repr(state),'self.nextStates[state]=',repr(self.nextStates[state])," type",repr(type(self.nextStates[state]))
+                mesg=u'unable to transition to the next state in line number=%(lineNo)s'
+                noteException(mesg,devel=mesg+'; while in state %(state)s the next state %(newState)s is not legal in line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state),newState=repr(newState))
+            # Main part of routine: handle the line depending on the newState.
+            # if self.debug:
+            #     print "newState is ",repr(newState)," line is ",repr(line)
+            if newState==self.initialState:
+                mesg=u'unexpected initial state in line=%(lineNo)s'
+                noteException(mesg,devel=mesg+'; while in state %(state)s now in initial state reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state))
+            if newState=='name':
+                try:
+                    fields=restOfLine.strip().split()
+                    self.name=fields[0]
+                    if self.debug:
+                        note("package name is "+repr(self.name),self.log)
+                except Exception, err:
+                    mesg=u'no "value" text in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err))
+            elif newState=='category':
+                try:
+                    fields=restOfLine.strip().split()
+                    self.category=fields[0]
+                except:
+                    mesg=u'no "value" text in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+            elif newState=='catalogue':
+                try:
+                    fields=restOfLine.strip().split()
+                    self.catalogue=fields[0]
+                except:
+                    mesg=u'no "value" text in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+            elif newState=='shortdesc':
+                try:
+                    # I'll allow multiline shortdesc's, although I'll regret it
+                    if self.shortdesc is None:
+                        self.shortdesc=""
+                    if self.shortdesc:
+                        self.shortdesc+=' '
+                    self.shortdesc+=restOfLine
+                except:
+                    mesg=u'no value in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+            elif newState=='longdesc':
+                try:
+                    if self.longdesc is None:
+                        self.longdesc=""
+                    if self.longdesc:
+                        self.longdesc+="\n"
+                    self.longdesc+=restOfLine
+                except Exception, err:
+                    mesg=u'no value in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValuse(err))
+            elif newState=='depend':
+                fields=restOfLine.strip().split()
+                if len(fields)>0:
+                    if self.depend is None:
+                        self.depend=[]
+                    self.depend.append(restOfLine.strip())
+                else:
+                    mesg=u'no value in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+            elif newState=='execute':
+                fields=restOfLine.strip().split()
+                if len(fields)>0:
+                    if self.execute is None:
+                        self.execute=[]
+                    self.execute.append(restOfLine.strip())
+                else:
+                    mesg=u'no value in line number=%(lineNo)s'
+                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+            elif newState=='revision':
+                self.handleRevision(newState,restOfLine,line,lineNo)
+            elif newState=='srcfiles':
+                self.handleSrcfiles(newState,restOfLine,line,lineNo)
+            elif newState=='srcfiles_file':
+                self.handleSrcfilesFile(newState,restOfLine,line,lineNo)
+            elif newState=='docfiles':
+                self.handleDocfiles(newState,restOfLine,line,lineNo)
+            elif newState=='docfiles_file':
+                self.handleDocfilesFile(newState,restOfLine,line,lineNo)
+            elif newState=='runfiles':
+                self.handleRunfiles(newState,restOfLine,line,lineNo)
+            elif newState=='runfiles_file':
+                self.handleRunfilesFile(newState,restOfLine,line,lineNo)
+            elif newState=='binfiles':
+                self.handleBinfiles(newState,restOfLine,line,lineNo)
+            elif newState=='binfiles_file':
+                self.handleBinfilesFile(newState,restOfLine,line,lineNo)
+            elif newState=='srcpattern':
+                self.handleSrcpattern(newState,restOfLine,line,lineNo)
+            elif newState=='runpattern':
+                self.handleRunpattern(newState,restOfLine,line,lineNo)
+            elif newState=='docpattern':
+                self.handleDocpattern(newState,restOfLine,line,lineNo)
+            elif newState=='binpattern':
+                self.handleBinpattern(newState,restOfLine,line,lineNo)
+            elif newState.startswith('cataloguedata-'):
+                self.handleCataloguedata(newState,restOfLine,line,lineNo)
+            else:
+                mesg=u'unknown key in line number=%(lineNo)s'
+                noteException(mesg,devel=mesg+'; key is %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=repr(lineNo),line=repr(line),newState=repr(newState))
+            # Cycle back to read next line
+            state=newState
+        return True
+
+    def handleRevision(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'revision'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleSrcfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'srcfiles'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleSrcfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'srcfiles_file'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleDocfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'docfiles'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleDocfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'docfiles_file'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleRunfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'runfiles'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleRunfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'runfiles_file'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleBinfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'binfiles'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleBinfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'binfiles_file'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleSrcpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'srcpattern'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleRunpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'runpattern'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleDocpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'docpattern'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleBinpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'binpattern'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+    def handleCataloguedata(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'cataloguedata-xxx'
+        """
+        mesg=u'the new state is not allowed here, line number=%(lineNo)s'
+        noteException(mesg,devel=mesg+"; this object %(objName) cannot be in the state %(newState)s: line=%(line)s",exception=tlError,log=self.log,debug=self.debug,objName=repr(self.__class__.__name__),newState=repr(newState),lineNo=repr(lineNo),line=repr(line))
+        return None
+
+
+class tlp_db(object):
+    """Give information based on a tl file database.  This class exists
+    for tlpobj_db and tlpsrc_db to inherit from
+    """
+    def __init__(self,verbose=False,log=None,debug=False):
+        """Initialize an instance.
+        """
+        self.verbose=verbose
+        self.log=log
+        self.debug=debug
+        self.value={}  # map name --> tl pkg
+        return None
+
+    def __unicode__(self):
+        names=self.value.keys()
+        names.sort()
+        return u" ".join(names)
+
+    def get_package(self,name):
+        try:
+            return self.value[name]
+        except:
+            return None
+
+    def _makeSingle(self):
+        """Helper fcn to make one of whatever self.value is holding multiple
+        of.  Used in from_file.
+        """
+        return tlp(log=self.log,debug=self.debug)
+    
+    def from_file(self,fn=None,multi=True):
+        """Read a tlp db file.  Set self.value.
+        """
+        if fn is None:
+            f=sys.stdin
+            if f.encoding!=DEFAULT_ENCODING:
+                warn('using STDIN but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING))
+        else:
+            if self.debug:
+                errors='strict'
+            else:
+                errors='replace'
+            try:
+                f=codecs.open(fn,'rU',DEFAULT_ENCODING,errors)
+            except Exception, err:
+                mesg='unable to open tlpobj file'
+                noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err))
+        r=self._makeSingle()
+        retVal=r.from_fh(f,multi=multi,lineNo=0)
+        self.value[r.name]=r
+        while retVal:
+            r=self._makeSingle()
+            retVal=r.from_fh(f,multi=multi,lineNo=0)
+            self.value[r.name]=r
+        return None
+
+    def write_file(self,fn=None):
+        if fn is None:
+            f=sys.stdout
+            if f.encoding!=DEFAULT_ENCODING:
+                warn('using STDOUT but the encoding is %(fEncoding)s and not the expected %(defaultEncoding)s',log=self.log,debug=self.debug,fEncoding=repr(f.encoding),defaultEncoding=repr(DEFAULT_ENCODING))
+        else:
+            if self.debug:
+                errors='strict'
+            else:
+                errors='replace'
+            try:
+                f=codecs.open(fn,'w',DEFAULT_ENCODING,errors)
+            except Exception, err:
+                mesg=u'unable to write because unable to open the file'
+                noteException(mesg,devel=mesg+': %(err)s',exception=tlError,log=self.log,debug=self.debug,err=errorValue(err))
+        pkgNames=self.value.keys()
+        pkgNames.sort()
+        pkgNo=1
+        for k in pkgNames:
+            v=self.value[k]
+            v.writeout_fh(f)
+            f.write("\n")
+            if pkgNo<len(pkgNames): # keep off extra \n after end
+                f.write("\n")
+            pkgNo+=1
+        f.close()
+        return None
+
+    def add_cat_descs(self,dBcsr):
+        """Add catalogue descriptions to all matching TL packages that do not
+        already have a description.
+        """
+        pkgNames=self.value.keys()
+        pkgNames.sort()
+        for k in pkgNames:
+            self.value[k].add_cat_desc(dBcsr,log=self.log,debug=self.debug)
+        return None
+    
+
+
+class tlpobj(tlp):
+    """Give information based on a TeXLive object file
+    """
+    def __init__(self,verbose=False,log=None,debug=False):
+        """Initialize an object.
+        """
+        super(tlpobj,self).__init__(verbose=verbose,log=log,debug=debug)
+        self.revision=None # integer
+        self.srcfiles=None # list of files
+        self.runfiles=None # list of files
+        self.docfiles=None # dct fileName--> (dct tagkey --> tagval)
+        self.binfiles=None # arch --> list of files
+        self.srcsize=None # int (or None)
+        self.runsize=None # int (or None)
+        self.docsize=None # int (or None)
+        self.binsize=None  # arch --> size  (or None)
+        self.cataloguedata=None # cataloguekey --> value
+        return None
+
+    def __unicode__(self):
+        r=[]
+        r.append(super(tlpobj,self).__unicode__())
+        r.append("  revision: %s" % (pprint.pformat(self.revision),))
+        r.append("  srcsize: %s" % (pprint.pformat(self.srcsize),))
+        r.append("  srcfiles: %s" % (pprint.pformat(self.srcfiles),))
+        r.append("  docsize: %s" % (pprint.pformat(self.docsize),))
+        r.append("  docfiles: %s" % (pprint.pformat(self.docfiles),))
+        r.append("  runsize: %s" % (pprint.pformat(self.runsize),))
+        r.append("  runfiles: %s" % (pprint.pformat(self.runfiles),))
+        r.append("  binsize: %s" % (pprint.pformat(self.binsize),))
+        r.append("  binfiles: %s" % (pprint.pformat(self.binfiles),))
+        r.append("  cataloguedata: %s" % (pprint.pformat(self.cataloguedata),))
+        return "\n".join(r)
+
+    def _stringToWriteout(self):
+        """String that will be written out during writeout_fh
+        """
+        r=[]
+        r.append(super(tlpobj,self)._stringToWriteout())
+        if self.revision:
+            r.append("revision %s" % (self.revision,))
         if self.docfiles:
             if self.docsize is None:
                 mesg='docsize is not there but there are docfiles'
@@ -575,17 +1060,16 @@ class tlpobj(object):
             cdKeys=self.cataloguedata.keys()
             for k in cdKeys:
                 r.append("catalogue-%s %s" % (k,self.cataloguedata[k]))
-        # Write that string 
-        f.write("\n".join(r))
-        return None
+        return "\n".join(r)
 
-##     def to_db(self,dBcsr,dBcnx,careful=True,associatedCatPkgs=None):
-##         """Put the file information to the database
+##     def to_db(self,dBcsr,dBcnx,careful=True,associatedCatPkgs=None,verbose=False):
+##         """Put the yyy-file information to a database
 ##           dBcsr, dBcnx  database cursor and connection
 ##           careful=True  raise exception on error?
 ##           associatedCatPkgs=None  list of catalogue package id's that either 
 ##             are part of this texlive package of of which this texlive package
 ##             is a part (None is the same as using what is now in the dB)
+##           verbose=False  Give warnings
 ##         """
 ##         tlId=self.name
 ##         # Drop the file records if they are there
@@ -650,7 +1134,8 @@ class tlpobj(object):
 ##                or tlId.startswith('scheme-')
 ##                or tlId.startswith('texlive-')):
 ##             if (not(associatedCatPkgs)
-##                 and careful):
+##                 and careful
+##                 and verbose):
 ##                 mesg=u'there are no catalogue packages associated with the TeX Live package %(tlId)s'
 ##                 warn(mesg,log=self.log,debug=self.debug,tlId=repr(tlId))
 ##             for pkgId in associatedCatPkgs:
@@ -698,317 +1183,195 @@ class tlpobj(object):
 ##             noteException(mesg,devel=mesg+' for package %(pkgId)s: %(err)s',log=self.log,debug=self.debug,tlId=repr(tlId),err=errorValue(err))
 ##         return None
 
-    def from_file(self,fn=None):
-        """Read a tlpobj from a file.  You don't want to do this.
+    initialState=tlp.initialState
+    nextStates=tlp.nextStates
+    nextStates['revision']=tlp.keyValueStates-set(['revision'])
+    nextStates['srcfiles']=tlp.keyValueStates | set(['srcfiles_file'])-set(['srcfiles'])
+    nextStates['srcfiles_file']=tlp.keyValueStates | set(['srcfiles_file'])
+    nextStates['runfiles']=tlp.keyValueStates | set(['runfiles_file'])-set(['runfiles'])
+    nextStates['runfiles_file']=tlp.keyValueStates | set(['runfiles_file'])
+    nextStates['docfiles']=tlp.keyValueStates | set(['docfiles_file'])-set(['docfiles'])
+    nextStates['docfiles_file']=tlp.keyValueStates | set(['docfiles_file'])-set(['docfiles'])
+    nextStates['binfiles']=tlp.keyValueStates | set(['binfiles_file'])
+    nextStates['binfiles_file']=tlp.keyValueStates | set(['binfiles_file'])
+    nextStates['cataloguedata']=tlp.keyValueStates
+
+    def handleRevision(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'revision'
         """
-        if fn is None:
-            f=sys.stdin
-        else:
-            try:
-                f=open(fn,'rU')
-            except Exception, err:
-                mesg='unable to open tlpobj file'
-                noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err))
-        r=self.from_fh(f,multi=True,lineNo=0)
+        try:
+            fields=restOfLine.strip().split()
+            self.revision=int(fields[0])
+        except:
+            mesg=u'no "value" integer in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
         return None
 
-    initialState='init'
-    keyValueStates=tlobjKeys # line of form "key value"
-    filePathStates=set(['srcfiles_file','runfiles_file','docfiles_file','binfiles_file'])  # line contains file path indented by space
-    nextStates={initialState:tlobjKeys,
-                'name':keyValueStates-set(['name']),
-                'category':keyValueStates-set(['category']),
-                'catalogue':keyValueStates-set(['catalogue']),
-                'shortdesc':keyValueStates-set(['shortdesc']),
-                'longdesc':keyValueStates,
-                'depend':keyValueStates,
-                'execute':keyValueStates,
-                'revision':keyValueStates-set(['revision']),
-                'srcfiles':keyValueStates | set(['srcfiles_file'])-set(['srcfiles']),
-                'srcfiles_file':keyValueStates | set(['srcfiles_file']),
-                'runfiles':keyValueStates | set(['runfiles_file'])-set(['runfiles']),
-                'runfiles_file':keyValueStates | set(['runfiles_file']),
-                'docfiles':keyValueStates | set(['docfiles_file'])-set(['docfiles']),
-                'docfiles_file':keyValueStates | set(['docfiles_file'])-set(['docfiles']),
-                'binfiles':keyValueStates | set(['binfiles_file']),
-                'binfiles_file':keyValueStates | set(['binfiles_file']),
-                'cataloguedata':keyValueStates}
-    def from_fh(self,f,multi=None,lineNo=None):
-        """Read a tlpobj from a file handle.  Returns None if EOF read before
-        anything is found.
-          f  an opened file
-          multi=None  If not None then multiple tplobj's in the file are
-            treated as an error
-          lineNo=None  debugging
-        May raise tlError.
+    def handleSrcfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'srcfiles'
         """
-        state=self.initialState
-        currentArch=None  # used for binfiles
-        # Parse the lines
-        anyNonblankLinesRead=False
-        while True:
-            line=f.readline()
-            if not(line):  # EOF?
-                return False
-            # if self.debug:
-            #     note('line is '+repr(line),self.log)
-            line=line.rstrip()
-            if not(anyNonblankLinesRead):
-                if (not(line)
-                    or line[0]=='#'): # throw away initial blank lines
-                    continue
-                else:
-                    anyNonblankLinesRead=True
-                    # pkg=tplpkg(log=self.log,debug=self.debug)
-            if not(line):
-                if multi is None:
-                    mesg=u'unexpected multiple tpl objects'
-                    noteException(mesg,devel=mesg+'; trailing blank line found at line %(lineNo)s',exception=tlError,log=self.log,debug=self.debug,lineNo=repr(lineNo))
-                else:
-                    return self
-            # Find state under which this line will be read
-            fields=None
-            if (state=='srcfiles'
-                or state=='srcfiles_file'):
-                if line[0]==' ':
-                    newState='srcfiles_file'
-                    fields=line[1:].split()
-            if (state=='runfiles'
-                or state=='runfiles_file'):
-                if line[0]==' ':
-                    newState='runfiles_file'
-                    fields=line[1:].split()
-            elif (state=='docfiles'
-                  or state=='docfiles_file'):
-                if line[0]==' ':
-                    newState='docfiles_file'
-                    fields=line[1:].split()
-            elif (state=='binfiles'
-                  or state=='binfiles_file'):
-                if line[0]==' ':
-                    newState='binfiles_file'
-                    fields=line[1:].split()
-            if fields is None:
-                try:
-                    fields=line.split()
-                    newState=fields[0]
-                except:
-                    mesg=u'no key unable in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; while in state %(state)s unable to get the new state in line=%(line)s',returnCode=10,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state))
-            if not(newState=='binfiles_file'):
-                currentArch=None
-            if not(newState in self.nextStates[state]):
-                mesg=u'unable to transition to the next state in line number=%(lineNo)s'
-                noteException(mesg,devel=mesg+'; while in state %(state)s the next state %(newState)s is not legal in line=%(line)s',returnCode=10,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state),newState=repr(newState))
-            # Handle line state by state
-            if newState==self.initialState:
-                mesg=u'unexpected initial state in line=%(lineNo)s'
-                noteException(mesg,devel=mesg+'; while in state %(state)s now in initial state reading line=%(line)s',returnCode=10,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),state=repr(state))
-            if newState=='name':
-                try:
-                    self.name=fields[1]
-                    if self.debug:
-                        print "package name is",repr(self.name)
-                except Exception, err:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err))
-            elif newState=='category':
-                try:
-                    self.category=fields[1]
-                except:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='catalogue':
-                try:
-                    self.catalogue=fields[1]
-                except:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='shortdesc':
-                try:
-                    if self.shortdesc is None:
-                        self.shortdesc=""
-                    if self.shortdesc:
-                        self.shortdesc+=' '
-                    self.shortdesc+=" ".join(fields[1:])
-                except:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='longdesc':
-                try:
-                    if self.longdesc is None:
-                        self.longdesc=""
-                    if self.longdesc:
-                        self.longdesc+="\n"
-                    self.longdesc+=" ".join(fields[1:])
-                except Exception, err:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValuse(err))
-            elif newState=='depend':
-                if len(fields)>0:
-                    if self.depend is None:
-                        self.depend=[]
-                    self.depend.append(fields[1])
-                else:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='execute':
-                if len(fields)>0:
-                    if self.execute is None:
-                        self.execute=[]
-                    self.execute.append(" ".join(fields[1:]))
-                else:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='revision':
-                try:
-                    self.revision=fields[1]
-                except:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='srcfiles':
-                try:
-                    sizeString=fields[1]
-                except:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                try:
-                    sizeFields=sizeString.split('=')
-                    if sizeFields[0].lower()!='size':
-                        raise tlError
-                    size=int(sizeFields[1])
-                    if self.srcsize is None:
-                        self.srcsize=0
-                    self.srcsize+=size
-                except:
-                    mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                if self.srcfiles is None:
-                    self.srcfiles=[]
-            elif newState=='srcfiles_file':
-                try:
-                    self.srcfiles.append(fields[0].strip())
-                except:
-                    mesg=u'unable to understand file name in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='docfiles':
-                try:
-                    sizeString=fields[1]
-                except:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                try:
-                    sizeFields=sizeString.split('=')
-                    if sizeFields[0].lower()!='size':
-                        raise tlError
-                    size=int(sizeFields[1])
-                    if self.docsize is None:
-                        self.docsize=0
-                    self.docsize+=size
-                except:
-                    mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                if self.docfiles is None:
-                    self.docfiles={}
-            elif newState=='docfiles_file':
-                kvDct={}
-                kvString=None
-                try:
-                    for kvString in fields[1:]:
-                        if kvString:
-                            k,v=kvString.split("=")
-                            kvDct[k]=v
-                except:
-                    mesg=u'unable to understand file description in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s; key-value description string=%(kvString)s; reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),kvString=repr(kvString))                
-                try:
-                    self.docfiles[fields[0].strip()]=kvDct
-                except Exception, err:
-                    mesg=u'unable to understand file name in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err))
-            elif newState=='runfiles':
-                try:
-                    sizeString=fields[1]
-                except:
-                    mesg=u'no value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                try:
-                    sizeFields=sizeString.split('=')
-                    if sizeFields[0].lower()!='size':
-                        raise tlError
-                    size=int(sizeFields[1])
-                    if self.runsize is None:
-                        self.runsize=0
-                    self.runsize+=size
-                except:
-                    mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                if self.runfiles is None:
-                    self.runfiles=[]
-            elif newState=='runfiles_file':
-                try:
-                    self.runfiles.append(fields[0].strip())
-                except:
-                    mesg=u'unable to understand file name in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            elif newState=='binfiles':
-                try:
-                    firstString,secondString=fields[1],fields[2]
-                except:
-                    mesg=u'expected an arch=XXXX size=NNNNNN in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                try:
-                    firstStringFields=firstString.split('=')
-                    secondStringFields=secondString.split('=')
-                    if firstStringFields[0].lower()=='arch':
-                        archFields=firstStringFields
-                        sizeFields=secondStringFields
-                    else:
-                        archFields=secondStringFields
-                        sizeFields=firstStringFields
-                    if (archFields[0].lower()!='arch'
-                        or sizeFields[0].lower()!='size'):
-                        raise tlError
-                    currentArch=archFields[1]
-                except:
-                    mesg=u'expected two values of the form x=y in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                try:
-                    if self.binsize is None:
-                        self.binsize={}
-                    self.binsize[currentArch]=int(sizeFields[1])
-                except:
-                    mesg=u'expected size to be an integer in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-                if self.binfiles is None:
-                    self.binfiles={}
-                self.binfiles[currentArch]=[]
-            elif newState=='binfiles_file':
-                if currentArch is None:
-                    mesg=u'do not know the right architecture in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))                
-                try:
-                    self.binfiles[currentArch].append(fields[0].strip())
-                except:
-                    mesg=u'unable to understand file path in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s with architecture %(currentArch)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),currentArch=repr(currentArch))                
-            elif newState.startswith('cataloguedata-'):
-                if self.cataloguedata is None:
-                    self.cataloguedata={}
-                try:
-                    if len(newState)<=len('cataloguedata-'):
-                        raise tlError
-                    self.cataloguedata[newState[len('cataloguedata-'):]]=fields[1]
-                except:
-                    mesg=u'expected cataloguedata-xxx value to have a sensible xxx and value in line number=%(lineNo)s'
-                    noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',returnCode=14,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        try:
+            fields=restOfLine.strip().split()
+            sizeString=fields[0]
+        except:
+            mesg=u'no value in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        try:
+            sizeFields=sizeString.split('=')
+            if sizeFields[0].lower()!='size':
+                raise tlError
+            size=int(sizeFields[1])
+            if self.srcsize is None:
+                self.srcsize=0
+            self.srcsize+=size
+        except:
+            mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        if self.srcfiles is None:
+            self.srcfiles=[]
+        return None
+
+    def handleSrcfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'srcfiles_file'
+        """
+        try:
+            fields=restOfLine.strip().split() # recall that restOfLine=line[1:]
+            self.srcfiles.append(fields[0])
+        except:
+            mesg=u'unable to understand source file name in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        return None
+
+    def handleDocfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'docfiles'
+        """
+        try:
+            fields=restOfLine.strip().split()
+            sizeString=fields[0]
+        except:
+            mesg=u'no value in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        try:
+            sizeFields=sizeString.split('=',1)
+            if sizeFields[0].lower()!='size':
+                raise tlError
+            size=int(sizeFields[1])
+            if self.docsize is None:
+                self.docsize=0
+            self.docsize+=size
+        except:
+            mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        if self.docfiles is None:
+            self.docfiles={}
+        return None
+    
+    def handleDocfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'docfiles_file'
+        """
+        fields=restOfLine.strip().split() # recall that restOfLine=line[1:]
+        kvDct={}
+        kvString=None
+        try:
+            for kvString in fields[1:]:
+                if kvString:
+                    k,v=kvString.split("=",1)
+                    kvDct[k]=v
+        except:
+            mesg=u'unable to understand doc file description in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s; key-value description string=%(kvString)s; reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),kvString=repr(kvString))                
+        try:
+            self.docfiles[fields[0].strip()]=kvDct
+        except Exception, err:
+            mesg=u'unable to understand doc file name in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s: %(err)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),err=errorValue(err))
+        return None
+    
+    def handleRunfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'runfiles'
+        """
+        try:
+            fields=restOfLine.strip().split()
+            sizeString=fields[0]
+        except:
+            mesg=u'no value in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        try:
+            sizeFields=sizeString.split('=',1)
+            if sizeFields[0].lower()!='size':
+                raise tlError
+            size=int(sizeFields[1])
+            if self.runsize is None:
+                self.runsize=0
+            self.runsize+=size
+        except:
+            mesg=u'expected a value of size=NNNNNN in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        if self.runfiles is None:
+            self.runfiles=[]
+        return None
+
+    def handleRunfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'runfiles_file'
+        """
+        try:
+            fields=restOfLine.strip().split() # recall that restOfLine=line[1:]
+            self.runfiles.append(fields[0].strip())
+        except:
+            mesg=u'unable to understand run file name in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        return None
+    
+    def handleBinfiles(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'binfiles'
+        """
+        try:
+            fields=restOfLine.strip().split()
+            firstString,secondString=fields[0],fields[1]
+        except:
+            mesg=u'expected an arch=XXXX size=NNNNNN in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        try:
+            firstStringFields=firstString.split('=',1)
+            secondStringFields=secondString.split('=',1)
+            if firstStringFields[0].lower()=='arch':
+                archFields=firstStringFields
+                sizeFields=secondStringFields
             else:
-                mesg=u'unknown key in line number=%(lineNo)s'
-                noteException(mesg,devel=mesg+'; key is %(newState)s reading line=%(line)s',returnCode=13,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
-            # get ready to read next line
-            state=newState
-            lineNo+=1
-        return True
+                archFields=secondStringFields
+                sizeFields=firstStringFields
+            if (archFields[0].lower()!='arch'
+                or sizeFields[0].lower()!='size'):
+                raise tlError
+            self.currentArch=archFields[1]
+        except:
+            mesg=u'expected two values of the form x=y in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        try:
+            if self.binsize is None:
+                self.binsize={}
+            self.binsize[self.currentArch]=int(sizeFields[1])
+        except:
+            mesg=u'expected size to be an integer in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        if self.binfiles is None:
+            self.binfiles={}
+        self.binfiles[self.currentArch]=[]
+        return None
+    
+    def handleBinfilesFile(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'binfiles_file'
+        """
+        if self.currentArch is None:
+            mesg=u'do not know the right architecture in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))                
+        try:
+            fields=restOfLine.strip().split() # recall that restOfLine=line[1:]
+            self.binfiles[self.currentArch].append(fields[0].strip())
+        except:
+            mesg=u'unable to understand bin file path in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s with architecture %(currentArch)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState),currentArch=repr(currentArch))
+        return None
 
     def is_arch_dependent(self):
         """Returns True if there are binfiles.
@@ -1106,101 +1469,197 @@ class tlpobj(object):
         r.sort()
         return "\n".join(r)
 
-    def write_file(self,fn=None):
-        if fn:
-            try:
-                f=open(fn,'w')
-            except Exception, err:
-                mesg=u'unable to write because unable to open the file'
-                noteException(mesg,devel=mesg+': %(err)s',exception=tlError,log=self.log,debug=self.debug,err=errorValue(err))
-        else:
-            f=None
-        self.writeout(f)
-        f.close()
-        return None
-        
-    def writeout(self,f=None):
-        if f is None:
-            f=sys.stdout
-        pkgs=self.value.keys()
-        pkgs.sort()
-        for k in pkgs:
-            v=self.value[k]
-            f.write(v.stringout())
-            f.write("\n\n")
-        return None
 
-class tlpobj_db(object):
+class tlpobj_db(tlp_db):
     """Give information based on a TeXLive object file database
     """
+    def _makeSingle(self):
+        """Helper fcn to make one of whatever self.value is holding multiple
+        of
+          f  file handle
+        """
+        return tlpobj(log=self.log,debug=self.debug)
+
+##     def to_db(self,dBcsr,dBcnx,verbose=False):
+##         """Write all records to the database
+##           dBcsr,dBcnx  database cursor and connection
+##         """
+##         pkgNames=self.value.keys()
+##         pkgNames.sort()
+##         for k in pkgNames:
+##             v=self.value[k]
+##             v.to_db(dBcsr,dBcnx,verbose=verbose)
+##         return None
+    
+
+class tlpsrc(tlp):
+    """Give information based on a TeXLive source file
+    """
     def __init__(self,verbose=False,log=None,debug=False):
         """Initialize an object.
         """
-        self.verbose=verbose
-        self.log=log
-        self.debug=debug
-        self.value={}  # map name --> tlpkg
+        super(tlpsrc,self).__init__(verbose=verbose,log=log,debug=debug)
+        self.srcpattern=None  # list of strings (or None)
+        self.runpattern=None  # list of strings (or None)
+        self.docpattern=None  # list of strings (or None)
+        self.binpattern=None  # list of strings (or None)
         return None
 
     def __unicode__(self):
-        names=self.value.keys()
-        names.sort()
-        return u" ".join(names)
+        r=[]
+        r.append(super(tlpsrc,self).__unicode__())
+        r.append("  srcpattern: %s" % (pprint.pformat(self.srcpattern),))
+        r.append("  runpattern: %s" % (pprint.pformat(self.runpattern),))
+        r.append("  docpattern: %s" % (pprint.pformat(self.docpattern),))
+        r.append("  binpattern: %s" % (pprint.pformat(self.binpattern),))
+        return "\n".join(r)
 
-    def get_package(self,name):
+    def _stringToWriteout(self):
+        """String that will be written out during writeout_fh
+        """
+        r=[]
+        r.append(super(tlpsrc,self)._stringToWriteout())
+        if self.srcpattern:
+            for line in self.srcpattern:
+                r.append("srcpattern %s" % (line,))
+        if self.runpattern:
+            for line in self.runpattern:
+                r.append("runpattern %s" % (line,))
+        if self.docpattern:
+            for line in self.docpattern:
+                r.append("docpattern %s" % (line,))
+        if self.binpattern:
+            for line in self.binpattern:
+                r.append("binpattern %s" % (line,))
+        return "\n".join(r)
+
+    initialState=tlp.initialState
+    nextStates=tlp.nextStates
+    nextStates['srcpattern']=tlp.keyValueStates
+    nextStates['runpattern']=tlp.keyValueStates
+    nextStates['docpattern']=tlp.keyValueStates
+    nextStates['binpattern']=tlp.keyValueStates
+
+    def handleSrcpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'srcpattern'
+        """
         try:
-            return self.value[name]
+            if self.srcpattern is None:
+                self.srcpattern=[]
+            self.srcpattern.append(restOfLine.rstrip())
         except:
-            return None
+            mesg=u'no "value" text in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        return None
 
-    def from_file(self,fn=None):
-        """Read a tlpobj file.  Set self.value.
+    def handleRunpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'runpattern'
         """
-        if fn is None:
-            f=sys.stdin
-        else:
-            try:
-                f=open(fn,'rU')
-            except Exception, err:
-                mesg='unable to open tlpobj file'
-                noteException(mesg,devel=mesg+' %(fn)s: %(err)s',exception=tlError,log=log,debug=self.debug,fn=repr(fn),err=unicode(err))
-        r=tlpobj(log=self.log,debug=self.debug).from_fh(f,multi=True,lineNo=0)
-        while r:
-            self.value[r.name]=r
-            r=tlpobj(log=self.log,debug=self.debug).from_fh(f,multi=True,lineNo=0)
+        try:
+            if self.runpattern is None:
+                self.runpattern=[]
+            self.runpattern.append(restOfLine.rstrip())
+        except:
+            mesg=u'no "value" text in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
         return None
 
-    def write_file(self,fn=None):
-        if fn is None:
-            f=sys.stdout
-        else:
-            try:
-                f=open(fn,'w')
-            except Exception, err:
-                mesg=u'unable to write because unable to open the file'
-                noteException(mesg,devel=mesg+': %(err)s',exception=tlError,log=self.log,debug=self.debug,err=errorValue(err))
+    def handleDocpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'docpattern'
+        """
+        try:
+            if self.docpattern is None:
+                self.docpattern=[]
+            self.docpattern.append(restOfLine.rstrip())
+        except:
+            mesg=u'no "value" text in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        return None
+
+    def handleBinpattern(self,newState,restOfLine,line,lineNo):
+        """What to do if the newState is 'binpattern'
+        """
+        try:
+            if self.binpattern is None:
+                self.binpattern=[]
+            self.binpattern.append(restOfLine.rstrip())
+        except:
+            mesg=u'no "value" text in line number=%(lineNo)s'
+            noteException(mesg,devel=mesg+'; now in state %(newState)s reading line=%(line)s',exception=tlError,log=self.log,debug=self.debug,lineNo=lineNo,line=repr(line),newState=repr(newState))
+        return None
+
+
+class tlpsrc_db(tlp_db):
+    """Give information based on a TeXLive src file list
+    """
+    def from_dir(self,dn=None):
+        """Read a set of tlpsrc files.  Set self.value.
+          dn=None  directory name.  If None, use the canonical name.
+        """
+        if dn is None:
+            dn=os.path.join(os.path.dirname(TLPDB_FILENAME),'tlpkg/tlpsrc')
+        globPattern=os.path.join(dn,'*.tlpsrc')
+        for fn in glob.glob(globPattern):
+            self.from_file(fn=fn)
+        return None
+
+    def _makeSingle(self):
+        """Helper fcn to make one of whatever self.value is holding multiple
+        of.
+        """
+        return tlpsrc(log=self.log,debug=self.debug)
+    
+    def to_dir(self,dn=None):
+        """Write a set of tlpsrc files.
+          dn=None  directory name.  If None, use the canonical name.
+        """
+        if dn is None:
+            dn=os.path.join(os.path.basename(TLPDB_FILENAME),'tlpsrc')
+        if not(os.path.exists(dn)):
+            os.makedirs(dn)
         pkgNames=self.value.keys()
-        pkgNames.sort()
         for k in pkgNames:
             v=self.value[k]
-            v.writeout_fh(f)
-            f.write("\n\n")
-        f.close()
+            fn=os.path.join(dn,v.name+'.tlpsrc')
+            v.writeout(fn=fn)
         return None
 
-##     def to_db(self,dBcsr,dBcnx):
-##         """Write all records to the database
-##           dBcsr,dBcnx  database cursor and connection
-##         """
-##         pkgNames=self.value.keys()
-##         pkgNames.sort()
-##         for k in pkgNames:
-## ##             if self.debug:
-## ##                 print note("writing tds file information for %s" % (repr(k),),self.log)
-##             v=self.value[k]
-##             v.to_db(dBcsr,dBcnx)
-##         return None
-        
+
+def getRelatedCatPkg(tlId,dBcsr,log=None,debug=False):
+    """Get the list of catalogue packages that are related to the given
+    TeX Live package (does not return None values).
+      tlId  string
+      dBcsr  database cursor
+      log=None,debug=False  debugging
+    """
+    sql="""SELECT pkg_id FROM tl_cat
+             WHERE tl_id=%(tlId)s
+             AND pkg_id IS NOT NULL"""
+    dct={'tlId':tlId}
+    try:
+        dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get catalogue package ids related to this TeX Live package id')
+    except dBUtils.dBUtilsError, err:
+        mesg=u'unable to get texlive-catalogue packages record'
+        noteException(mesg,devel=mesg+' for TeX Live package %(tlId)s: %(err)s',log=log,debug=debug,tlId=repr(tlId),err=errorValue(err))
+    return set([catId for (catId,) in dBres])
+
+def getRelatedTlPkg(pkgId,dBcsr,log=None,debug=False):
+    """Get the list of TeX Live packages that are related to the given
+    Catalogue package.
+      pkgId  string
+      dBcsr  database cursor
+      log=None,debug=False  debugging
+    """
+    sql="""SELECT tl_id FROM tl_cat
+             WHERE pkg_id=%(pkgId)s"""
+    dct={'pkgId':pkgId}
+    try:
+        dBres=dBUtils.getData(sql,dct=dct,dBcsr=dBcsr,log=log,debug=debug,listtype='get TeX Live package ids related to this catalogue package id')
+    except dBUtils.dBUtilsError, err:
+        mesg=u'unable to get catalogue-texlive packages record'
+        noteException(mesg,devel=mesg+' for catalogue package %(pkgId)s: %(err)s',log=log,debug=debug,pkgId=repr(pkgId),err=errorValue(err))
+    return set([tlId for (tlId,) in dBres])
+
     
 #......................................................................
 def main(argv=None,log=None,debug=DEBUG):
@@ -1211,11 +1670,13 @@ def main(argv=None,log=None,debug=DEBUG):
     if argv is None:
         argv=sys.argv
     # Parse the arguments
-    usage="""%prog: read a TeX Live dB obj file; typically write to the dB
+    usage="""%prog: read a TeX Live tlp src or obj file; typically write the obj filelists to the dB
   %prog [options]"""
     oP=optparse.OptionParser(usage=usage,version=__version__)
-    oP.add_option('--output','-O',action='store_true',default=False,dest='output',help='show all package records')
+    oP.add_option('--output','-o',action='store_true',default=False,dest='output',help='show all package records')
     oP.add_option('--no-database','-n',action='store_true',default=False,dest='noDb',help='do not put all package records to the database')
+    oP.add_option('--source-read','-s',action='store_true',default=False,dest='srcDir',help='read the source files')
+    oP.add_option('--add-catalogue-information','-a',action='store_true',default=False,dest='catInfo',help='add catalogue information to the source or object information')
     oP.add_option('--debug','-D',action='store_true',default=DEBUG,dest='debug',help='output debugging information')
     oP.add_option('--verbose','-V',action='store_true',default=VERBOSE,dest='verbose',help='talk a lot')
     opts, args=oP.parse_args(argv[1:])
@@ -1226,25 +1687,37 @@ def main(argv=None,log=None,debug=DEBUG):
     else:
         tlpdbFilename=TLPDB_FILENAME
     # Handle the various command line switches
+##     try:
+##         (dBcnx,dBcsr)=dBUtils.opendB()
+##     except dBUtils.dBUtilsError, err:
+##         fail("Unable to connect to the database: %(err)s",returnCode=10,log=log,debug=opts.debug,err=errorValue(err))
+    if opts.srcDir:
+        try:
+            t=tlpsrc_db(log=log,debug=opts.debug)
+            t.from_dir(dn=None)
+            if opts.catInfo:
+                t.add_cat_descs(dBcsr)
+            t.to_dir(dn='tmpTexlive')
+        except Exception, err:
+            mesg="trouble reading the object file and writing: "+errorValue(err)
+            fail(mesg,returnCode=10,log=log,debug=opts.debug)
+        return None
     if opts.output:
         try:
             t=tlpobj_db(log=log,debug=opts.debug)
             t.from_file(tlpdbFilename)
+            if opts.catInfo:
+                t.add_cat_descs(dBcsr)
             t.write_file()
         except Exception, err:
             mesg="trouble reading the object file and writing: "+errorValue(err)
             fail(mesg,returnCode=10,log=log,debug=opts.debug)
         return None
-    if not(opts.noDb):
-        print "dB turned off; you need Python module psycopg for access to PostGreSQL"
-##         try:
-##             (dBcnx,dBcsr)=dBUtils.opendB()
-##         except dBUtils.dBUtilsError, err:
-##             fail("Unable to connect to the database: %(err)s",returnCode=10,log=log,debug=opts.debug,err=errorValue(err))
+##     if not(opts.noDb):
 ##         try:
 ##             t=tlpobj_db(log=log,debug=opts.debug)
 ##             t.from_file(tlpdbFilename)
-##             t.to_db(dBcsr,dBcnx)
+##             t.to_db(dBcsr,dBcnx,verbose=opts.verbose)
 ##         except Exception, err:
 ##             mesg="trouble reading the object file and sending the results to the database: "+errorValue(err)
 ##             fail(mesg,returnCode=10,log=log,debug=opts.debug)