de-macro (6jul20)

git-svn-id: svn://tug.org/texlive/trunk@55766 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 924 insertions, 917 deletions

diff --git a/Master/texmf-dist/scripts/de-macro/de-macro b/Master/texmf-dist/scripts/de-macro/de-macro
index 04d286d8ee9..c5795fe891f 100755
--- a/Master/texmf-dist/scripts/de-macro/de-macro
+++ b/Master/texmf-dist/scripts/de-macro/de-macro
@@ -1,17 +1,20 @@
 #!/usr/bin/python -O
 
-"""
-Copyright 2005 Peter Gacs
+r"""
+Copyright 2005-2020 Peter Gacs
 Licensed under the Academic Free Licence version 2.1
 
-                          DE-MACRO
+						  DE-MACRO
 
-Version 1.3 - this version is much more conservative about deleting
-              comments and inserting or deleting blank space: tries to
-              leave in all comments, adds space only when necessary, and
-              tries not to delete space in the main text.
-              The motivating comments came from Daniel Webb.
-Version 1.2 - a syntactical bug corrected, thanks Brian de Alwis!
+Version 1.4	 - Luca Citi made it python2.7 and python3 compatible.
+			   Peter Gacs improved the parsing of \input{<filename>},
+			   and made @ a letter in the style files.
+Version 1.3	 - this version is much more conservative about deleting
+			   comments and inserting or deleting blank space: tries to
+			   leave in all comments, adds space only when necessary, and
+			   tries not to delete space in the main text.
+			   The motivating comments came from Daniel Webb.
+Version 1.2	 - a syntactical bug corrected, thanks Brian de Alwis!
 
 
 PURPOSE
@@ -21,13 +24,17 @@ Applications:
   - your publisher has difficulty dealing with many private macros
   - you cooperate with colleagues who do not understand your macros
   - preprocessing before a system like latex2html, which is somewhat
-    unpredictable with private macros.
+	unpredictable with private macros.
+
+It cannot be used to eliminate more complex macros that rely on
+more programming-like constructs in style files.  In particular, it will
+not replace style files that have options.
 
 USAGE
 
 de-macro [--defs <defs-db>] <tex-file-1>[.tex] [<tex-file-2>[.tex] ...]
 
-Simplest example:    de-macro testament
+Simplest example:	 de-macro testament
 
 (As you see, the <> is used only in the notation of this documentation,
 you should not type it.)
@@ -77,105 +84,105 @@ import sys, os, re, shelve
 # Utilities
 
 class No_detail:
-    strerror = ""
+	strerror = ""
 
 no_detail = No_detail()
 
 
 class Error(Exception):
-    """Base class for exceptions in this module."""
-    pass
+	"""Base class for exceptions in this module."""
+	pass
 
 class Empty_text_error(Error):
-    """Exception raised for errors in the input.
+	"""Exception raised for errors in the input.
 
-    Attributes:
-        data -- data that was found empty
-        message
-    """
+	Attributes:
+		data -- data that was found empty
+		message
+	"""
 
-    def __init__(self, data, message):
-        self.data = data
-        self.message = message
+	def __init__(self, data, message):
+		self.data = data
+		self.message = message
 
 def warn(error_message, detail = no_detail):
-    sys.stderr.write(error_message + "\n")
-    if no_detail != detail:
-        sys.stderr.write(detail.strerror + "\n")
+	sys.stderr.write(error_message + "\n")
+	if no_detail != detail:
+		sys.stderr.write(detail.strerror + "\n")
 
 def die(error_message, detail = no_detail):
-    warn(error_message, detail = no_detail)
-    sys.exit(1)
+	warn(error_message, detail = no_detail)
+	sys.exit(1)
 
 def getopt_map(one_letter_opts, long_optlist):
-    "Turns long options into an option map, using getopt."
-    import getopt
-    optlist, args = getopt.getopt(sys.argv[1:],
-                                  one_letter_opts, long_optlist)
-    opt_map = {}
-    for pair in optlist: opt_map[pair[0]] = pair[1] or 1
-    return opt_map, args
+	"Turns long options into an option map, using getopt."
+	import getopt
+	optlist, args = getopt.getopt(sys.argv[1:],
+								  one_letter_opts, long_optlist)
+	opt_map = {}
+	for pair in optlist: opt_map[pair[0]] = pair[1] or 1
+	return opt_map, args
 
 def newer(file1, file2):
 
-    if not os.path.isfile(file1):
-        return False
+	if not os.path.isfile(file1):
+		return False
 
-    try:
-        stat_return = os.lstat(file1) 
-    except OSError, detail:
-	die("lstat " + file1 + " failed:", detail)
-    time1 = stat_return.st_mtime
+	try:
+		stat_return = os.lstat(file1) 
+	except OSError as detail:
+		die("lstat " + file1 + " failed:", detail)
+	time1 = stat_return.st_mtime
 
-    try:
-        stat_return = os.lstat(file2) 
-    except OSError, detail:
-	die("lstat " + file2 + " failed:", detail)
-    time2 = stat_return.st_mtime
+	try:
+		stat_return = os.lstat(file2) 
+	except OSError as detail:
+		die("lstat " + file2 + " failed:", detail)
+	time2 = stat_return.st_mtime
 
-    return time1 > time2    
+	return time1 > time2	
 
 def cut_extension(filename, ext):
-    """
-    If filename has extension ext (including the possible dot),
-    it will be cut off.
-    """
-    file = filename
-    index = filename.rfind(ext)
-    if 0 <= index and len(file)-len(ext) == index:
-        file = file[:index]
-    return file
+	"""
+	If filename has extension ext (including the possible dot),
+	it will be cut off.
+	"""
+	file = filename
+	index = filename.rfind(ext)
+	if 0 <= index and len(file)-len(ext) == index:
+		file = file[:index]
+	return file
 
 
 class Stream:
-    data = None
-    pos = None
-    item = None
-
-    def legal(self):
-        return 0 <= self.pos and self.pos < len(self.data)
-
-    def uplegal(self):
-        return self.pos < len(self.data)
-
-    def __init__(self, data_v = None):
-        self.data = data_v
-        if self.data:
-           self.pos = 0
-           self.item = self.data[self.pos]
-
-    def next(self):
-        self.pos += 1
-        if self.pos < len(self.data):
-            self.item = self.data[self.pos]
-            return self.item
-
-    def reset(self):
-        if self.data and 0 < len(self.data):
-            self.pos = 0
-            self.item = self.data[0]
-            return self.item
-        
+	data = None
+	pos = None
+	item = None
+
+	def legal(self):
+		return 0 <= self.pos and self.pos < len(self.data)
+
+	def uplegal(self):
+		return self.pos < len(self.data)
+
+	def __init__(self, data_v = None):
+		self.data = data_v
+		if self.data:
+		   self.pos = 0
+		   self.item = self.data[self.pos]
+
+	def next(self):
+		self.pos += 1
+		if self.pos < len(self.data):
+			self.item = self.data[self.pos]
+			return self.item
+
+	def reset(self):
+		if self.data and 0 < len(self.data):
+			self.pos = 0
+			self.item = self.data[0]
+			return self.item
+		
 
 # Basic classes
 
@@ -186,97 +193,97 @@ blank_or_rbrace_re = re.compile(r"[\s}]")
 pos_digit_re = re.compile(r"[1-9]")
 
 def isletter(c, isatletter=False):
-    if "@" == c:
-        return isatletter
-    else:
-        return c.isalpha()
+	if "@" == c:
+		return isatletter
+	else:
+		return c.isalpha()
 
 class Token:
-    """Type 0 means ordinary character, type 1 means escape sequence
-    (without the \ ), type 2 means comment.
-    """
-    simple_ty = 0
-    esc_symb_ty = 1
-    esc_str_ty = 2
-    comment_ty = 3
-    
-    type = simple_ty
-    val = " "
-
-    def __init__(self, type_v=simple_ty, val_v=" "):
-        self.type = type_v
-        self.val = val_v
-
-    def show(self, isatletter=False):
-        out = ""
-        if simple_ty == self.type or comment_ty == self.type:
-            out = self.val
-        else: 
-            out = "\\" + self.val
-        return out
+	"""Type 0 means ordinary character, types 1,2 mean escape sequence
+	(without the \ ), type 3 means comment.
+	"""
+	simple_ty = 0
+	esc_symb_ty = 1
+	esc_str_ty = 2
+	comment_ty = 3
+	
+	type = simple_ty
+	val = " "
+
+	def __init__(self, type_v=simple_ty, val_v=" "):
+		self.type = type_v
+		self.val = val_v
+
+	def show(self):
+		out = ""
+		if simple_ty == self.type or comment_ty == self.type:
+			out = self.val
+		else: 
+			out = "\\" + self.val
+		return out
 
 
 # Constants
 
-g_token = Token(0," ")  # generic token
+g_token = Token(0," ")	# generic token
 simple_ty = g_token.simple_ty
 comment_ty = g_token.comment_ty
 esc_symb_ty = g_token.esc_symb_ty
 esc_str_ty = g_token.esc_str_ty
 
 
+def detokenize(text, isatletter=False):
+	"""
+	Input is a list of tokens.
+	Output is a string.
+	"""
+	out = ""
+	if 0 == len(text):
+		return
+	pos = 0
+	out += text[pos].show()
+	pos += 1
+	while pos < len(text):
+		previtem = text[pos-1]
+		item = text[pos]
+		"""Insert a separating space after an escape sequence if it is a
+		string and is followed by a letter."""
+		if (esc_str_ty == previtem.type
+			and simple_ty == item.type and isletter(item.val[0], isatletter)):
+			out += " "
+		out += item.show()
+		pos += 1
+	return out
 
-def detokenize(text):
-    """
-    Input is a list of tokens.
-    Output is a string.
-    """
-    out = ""
-    if 0 == len(text):
-        return
-    pos = 0
-    out += text[pos].show()
-    pos += 1
-    while pos < len(text):
-        previtem = text[pos-1]
-        item = text[pos]
-        """Insert a separating space after an escape sequence if it is a
-        string and is followed by a letter."""
-        if (esc_str_ty == previtem.type
-            and simple_ty == item.type and isletter(item.val[0], False)):
-            out += " "
-        out += item.show()
-        pos += 1
-    return out
 
 def strip_comments(text):
-    """
-    Input is a list of tokens.
-    Output is the same list except the comment tokens.
-    """
-    out = []
-    for token in text:
-        if not comment_ty == token.type:
-            out.append(token)
-    return out
+	"""
+	Input is a list of tokens.
+	Output is the same list except the comment tokens.
+	"""
+	out = []
+	for token in text:
+		if not comment_ty == token.type:
+			out.append(token)
+	return out
 
 class Group:
-    """type 0 means a token, type 1 means contents of a group within {}
-    """
-    token_ty = 0
-    group_ty = 1
-    type = token_ty
-    val = [] # Value is a token list.
-
-    def __init__(self, type_v, val_v):
-        self.type = type_v
-        self.val = val_v
-
-    def show(self):
-        if token_ty == self.type:
-            return self.val.show()
-        else: 
-            return "{%s}" % detokenize(self.val)
+	"""type 0 means a token, type 1 means contents of a group within {}
+	"""
+	token_ty = 0
+	group_ty = 1
+	type = token_ty
+	val = [] # Value is a token list.
+
+	def __init__(self, type_v, val_v):
+		self.type = type_v
+		self.val = val_v
+
+	def show(self):
+		if token_ty == self.type:
+			return self.val.show()
+		else: 
+			return "{%s}" % detokenize(self.val)
 
 # Constants
 
@@ -285,779 +292,779 @@ token_ty = g_group.token_ty
 group_ty = g_group.group_ty
 
 
-def tokenize(in_str):
-    """Returns a list of tokens.
-    """
-    text = []
-    isatletter=False
-    cs = Char_stream(in_str)
-    cs.reset()
-    if not cs.legal():
-        raise "No string to tokenize."
-    while cs.uplegal():
-        if "%" == cs.item:
-            comment = cs.scan_comment_token()
-            text.append(Token(comment_ty, comment))
-        elif "\\" != cs.item:
-            text.append(Token(simple_ty, cs.item))
-            cs.next()
-        else:
-            cs.next()
-            name = cs.scan_escape_token(isatletter)
-            if isletter(name[0], isatletter):
-                token = Token(esc_str_ty, name)
-            else: 
-                token = Token(esc_symb_ty, name)
-            text.append(token)
-            if "makeatletter" == name:
-                isatletter=True
-            elif "makeatother" == name:
-                isatletter=False
-    return text
-
-
-        
+def tokenize(in_str, isatletter=False):
+	"""Returns a list of tokens.
+	"""
+	text = []
+	cs = Char_stream(in_str)
+	cs.reset()
+	if not cs.legal():
+		raise Error("No string to tokenize.")
+	while cs.uplegal():
+		if "%" == cs.item:
+			comment = cs.scan_comment_token()
+			text.append(Token(comment_ty, comment))
+		elif "\\" != cs.item:
+			text.append(Token(simple_ty, cs.item))
+			cs.next()
+		else:
+			cs.next()
+			name = cs.scan_escape_token(isatletter)
+			if isletter(name[0], isatletter):
+				token = Token(esc_str_ty, name)
+			else: 
+				token = Token(esc_symb_ty, name)
+			text.append(token)
+			if "makeatletter" == name:
+				isatletter=True
+			elif "makeatother" == name:
+				isatletter=False
+	return text
+
+
 class Command_def:
-    name = "1"
-    numargs = 0
-    body= ""
+	name = "1"
+	numargs = 0
+	body= ""
 
-    def __init__(self, name_v, numargs_v, body_v):
-        self.name = name_v
-        self.numargs = numargs_v
-        self.body = body_v
+	def __init__(self, name_v, numargs_v, body_v):
+		self.name = name_v
+		self.numargs = numargs_v
+		self.body = body_v
 
-    def show(self):
-        out = "\\newcommand{\\%s}" % (self.name)
-        if 0 < self.numargs:
-            out += "[%d]" % self.numargs
-        out += "{%s}" % detokenize(self.body)
-        return out
+	def show(self):
+		out = "\\newcommand{\\%s}" % (self.name)
+		if 0 < self.numargs:
+			out += "[%d]" % self.numargs
+		out += "{%s}" % detokenize(self.body)
+		return out
 
 
 class Env_def:
-    name = "1"
-    numargs = 0
-    begin = ""
-    end = ""
-
-    def __init__(self, name_v, numargs_v, begin_v, end_v):
-        self.name = name_v
-        self.numargs = numargs_v
-        self.begin = begin_v
-        self.end = end_v
-
-    def show(self):
-        out = "\\newenvironment{%s}" % self.name
-        if 0 < self.numargs:
-            out += "[%d]" % self.numargs
-        out += "{%s}" % detokenize(self.begin)
-        out += "{%s}" % detokenize(self.end)
-        return out
+	name = "1"
+	numargs = 0
+	begin = ""
+	end = ""
+
+	def __init__(self, name_v, numargs_v, begin_v, end_v):
+		self.name = name_v
+		self.numargs = numargs_v
+		self.begin = begin_v
+		self.end = end_v
+
+	def show(self):
+		out = "\\newenvironment{%s}" % self.name
+		if 0 < self.numargs:
+			out += "[%d]" % self.numargs
+		out += "{%s}" % detokenize(self.begin)
+		out += "{%s}" % detokenize(self.end)
+		return out
 
 
 class Command_instance:
-    name = "1"
-    args = []
+	name = "1"
+	args = []
 
-    def __init__(self, name_v, args_v):
-        self.name = name_v
-        self.args = args_v
+	def __init__(self, name_v, args_v):
+		self.name = name_v
+		self.args = args_v
 
-    def show(self):
-        out = "\\"+self.name
-        for arg in self.args:
-            out += "{%s}" % detokenize(arg)
-        return out
+	def show(self):
+		out = "\\"+self.name
+		for arg in self.args:
+			out += "{%s}" % detokenize(arg)
+		return out
 
 
 class Env_instance:
-    name = "1"
-    args = []
-
-    def __init__(self, name_v, args_v, body_v):
-        self.name = name_v
-        self.args = args_v
-        self.body = body_v
-
-    def show(self):
-        out = "\\begin{%s}" % self.name
-        for arg in self.args:
-            out += "{%s}" % detokenize(arg)
-        out += detokenize(self.body)
-        out += "\\end{%s}" % self.name
-        return out
+	name = "1"
+	args = []
+
+	def __init__(self, name_v, args_v, body_v):
+		self.name = name_v
+		self.args = args_v
+		self.body = body_v
+
+	def show(self):
+		out = "\\begin{%s}" % self.name
+		for arg in self.args:
+			out += "{%s}" % detokenize(arg)
+		out += detokenize(self.body)
+		out += "\\end{%s}" % self.name
+		return out
 
 class Char_stream(Stream):
 
-    def scan_escape_token(self, isatletter=False):
-        """
-        Starts after the escape sign, assumes that it is scanning a symbol.
-        Returns a token-string.
-        """
-        out = self.item # Continue only if this is a letter.
-        item = self.next()
-        if isletter(out, isatletter):
-            while self.uplegal() and isletter(item, isatletter):
-                out += item
-                item = self.next()
-        return out
-
-    def scan_comment_token(self):
-        """
-        Starts at the comment sign %, assumes that it is scanning a comment.
-        Returns the whole comment string,
-        including the % and all empty space after it.
-        """
-        comment = ""
-        while "\n" != self .item:
-            comment += self.item
-            self.next()
-        while self.uplegal() and blank_re.match(self.item):
-            comment += self.item
-            self.next() 
-        return comment
-
-    def scan_input_filename(self):
-        """We just read an \input token.  The next group or word will be
-        interpreted as a filename (possibly without .tex).
-        Return the filename.
-        """
-        item = self.item
-        while self.uplegal() and blank_re.match(self.item):
-            item = self.next()
-        if "{" == item:
-            item = self.next()
-        file = ""
-        while self.uplegal() and not blank_or_rbrace_re.match(item):
-            file += item
-            item = self.next()
-        self.next()
-        return file
-
-    def scan_package_filenames(self):
-        """We just read a \usepackage token.  The next group will be
-        interpreted as a list of filenames (without .sty) separated by commas.
-        Return the list.
-        """
-        item = self.item
-        while self.uplegal() and blank_re.match(item):
-            item = self.next()
-        file = ""
-        if not "{" == item:
-            raise "\\usepackage not followed by brace."
-        item = self.next()
-        while self.uplegal() and not blank_or_rbrace_re.match(item):
-            file += item
-            item = self.next()
-        self.next()
-        return file.split(",")
+	def scan_escape_token(self, isatletter=False):
+		"""
+		Starts after the escape sign, assumes that it is scanning a symbol.
+		Returns a token-string.
+		"""
+		out = self.item # Continue only if this is a letter.
+		item = self.next()
+		if isletter(out, isatletter):
+			while self.uplegal() and isletter(item, isatletter):
+				out += item
+				item = self.next()
+		return out
+
+	def scan_comment_token(self):
+		"""
+		Starts at the comment sign %, assumes that it is scanning a comment.
+		Returns the whole comment string,
+		including the % and all empty space after it.
+		"""
+		comment = ""
+		while self.uplegal() and "\n" != self .item:
+			comment += self.item
+			self.next()
+		while self.uplegal() and blank_re.match(self.item):
+			comment += self.item
+			self.next() 
+		return comment
+
+	def scan_input_filename(self):
+		"""We have just read an \input token.  The next group or word will be
+		interpreted as a filename (possibly without .tex).	Filenames should not begin with spaces.
+		Return the filename.
+		"""
+		item = self.item
+		file = ""
+		while self.uplegal() and blank_re.match(self.item):
+			item = self.next()
+		if "{" == item:
+			item = self.next()	
+			while self.uplegal() and not "}" == item:
+				file += item
+				item = self.next()
+			self.next()
+		else:
+			while self.uplegal() and not blank_re.match(item):
+				file += item
+				item = self.next()
+		return file
+
+	def scan_package_filenames(self):
+		r"""We just read a \usepackage token.  The next group will be
+		interpreted as a list of filenames (without .sty) separated by commas.
+		Return the list.
+		"""
+		item = self.item
+		while self.uplegal() and blank_re.match(item):
+			item = self.next()
+		file = ""
+		if not "{" == item:
+			raise Error("\\usepackage not followed by brace.")
+		item = self.next()
+		while self.uplegal() and not blank_or_rbrace_re.match(item):
+			file += item
+			item = self.next()
+		self.next()
+		return file.split(",")
 
 
 class Tex_stream(Stream):
 
-    defs = ({}, {})
-    defs_db = "x"
-    defs_db_file = "x.db"
-    debug = False
-
-    def smart_tokenize(self, in_str, handle_inputs=False):
-        """Returns a list of tokens.
-        It may interpret and carry out all \input commands.
-        """
-        self.data = []
-        text = self.data
-        isatletter=False
-        cs = Char_stream(in_str)
-        cs.reset()
-        if not cs.legal():
-            raise "No string to tokenize."
-        while cs.uplegal():
-            if "%" == cs.item:
-                comment = cs.scan_comment_token()
-                text.append(Token(comment_ty, comment))
-            elif "\\" != cs.item:
-                text.append(Token(simple_ty, cs.item))
-                cs.next()
-            else:
-                cs.next()
-                name = cs.scan_escape_token(isatletter)
-                if "input" == name and handle_inputs:
-                    file = cs.scan_input_filename()
-                    to_add = self.process_if_newer(file)
-                    text.extend(to_add)
-                elif "usepackage" == name:
-                    while cs.uplegal() and blank_re.match(cs.item):
-                        cs.next()
-                    if "[" == cs.item: # private packages have no options
-                        text.extend([Token(esc_str_ty, "usepackage"),
-                                     Token(simple_ty, "[")])
-                        cs.next()
-                        continue
-                    files = cs.scan_package_filenames()
-                    i = 0
-                    while i < len(files):  # process private packages
-                        file = files[i]
-                        p = file.rfind("-private")
-                        if p < 0 or not len(file) - len("-private") == p:
-                            i += 1
-                            continue
-                        defs_db_file = file+".db"
-                        self.add_defs(file)
-                        del files[i:(i+1)]
-                    if files: # non-private packages left
-                        group_content = ",".join(files)
-                        to_add_str = "\\usepackage{%s}" % (group_content)
-                        to_add = tokenize(to_add_str)
-                        text.extend(to_add)
-                else:
-                    if isletter(name[0], isatletter):
-                        token = Token(esc_str_ty, name)
-                    else: 
-                        token = Token(esc_symb_ty, name)
-                    text.append(token)
-                    if "makeatletter" == name:
-                        isatletter=True
-                    elif "makeatother" == name:
-                        isatletter=False
-        self.reset()
-        return self.data
-
-    def smart_detokenize(self):
-        """
-        Output is a string.
-        If the list contains an \input{file} then the content of file
-        file-clean.tex replaces it in the output.
-        """
-        self.reset()
-        if not self.legal():
-            return ""
-        out = ""
-        previtem = None
-        while self.uplegal():
-            item = self.item
-            """Insert a separating space after an escape sequence if it is a
-            string and is followed by a letter."""
-            if (None != previtem and esc_str_ty == previtem.type
-                and simple_ty == item.type and isletter(item.val[0], False)):
-                out += " "
-            previtem = item
-            if not (esc_str_ty == item.type and "input" == item.val):
-                out += item.show()
-                self.next()
-            else:
-                self.next()
-                group = self.scan_group()
-                file = detokenize(group.val)
-                clean_file = "%s-clean.tex" % (file)
-                print "Reading file %s" % (clean_file)
-                fp = open(clean_file,"r")
-                content = fp.read()
-                fp.close()
-                out += content
-        return out
-
-    # Basic tex scanning
-
-    def skip_blank_tokens(self): # we also skip comment tokens.
-        item = self.item
-        while (self.uplegal() and
-               (comment_ty == item.type or
-                (simple_ty == item.type and blank_re.match(item.val)))):
-            item = self.next()
-        return item
-
-    def scan_group(self):
-        """Returns group.
-        """
-        if not self.legal():
-            raise "No group to scan."
-        item = self.item
-        if not (simple_ty == item.type and "{" == item.val):
-            return Group(token_ty, [self.item])
-        count = 1
-        group = []
-        item = self.next()
-        while count and self.uplegal():
-            if simple_ty == item.type:
-                if "{" == item.val:
-                    count += 1
-                elif "}" == item.val:
-                    count -= 1
-            if count != 0:
-                group.append(item)
-            item = self.next()
-        return Group(group_ty, group)
-
-    # Command and environment definitions
-
-    def scan_command_name(self):
-        """Returns name.
-        """
-        if not self.legal():
-            raise "No command name to scan."
-        item = self.item
-        name = ""
-        if item.type in [esc_symb_ty, esc_str_ty]:
-            name = item.val
-        else:
-            if not "{" == item.val:
-                raise "Command definition misses first {."
-            self.next()
-            item = self.skip_blank_tokens()
-            if not item.type in [esc_symb_ty, esc_str_ty]:
-                raise "Command definition does not begin with control sequence."
-            name = item.val
-            self.next()
-            item = self.skip_blank_tokens()
-            if not "}" == item.val:
-                raise ("Definition for commmand %s misses first }., %s" %
-                       (name, item.val))
-        self.next()
-        self.skip_blank_tokens()
-        return name
-
-    def scan_numargs(self, name):
-        """
-        name is the name of the command or environment definition being
-        scanned.
-        Starts on a nonblank token.
-        Returns numargs
-        where numargs is the number of arguments in a command or environment
-        definition,
-        """
-        if not self.legal():
-            raise "No numargs to scan."
-        item = self.item
-        numargs = 0
-        if not simple_ty == item.type:
-            raise "Illegal command or environment definition: "+name
-        if "[" == item.val:
-            if not 4 < len(self.data):
-                raise "Command or environment definition is illegal: "+name
-            item = self.next()
-            if not simple_ty == item.type:
-                raise "Illegal command or environment definition: "+name
-            numargs = item.val
-            if not pos_digit_re.match(numargs):
-                raise "%s must be argument number after %s" % (numargs, name)
-            numargs = int(numargs)
-            self.next()
-            item = self.skip_blank_tokens()
-            if not simple_ty == item.type:
-                raise "Illegal command definition: "+name
-            if "]" != item.val:
-                raise "Illegal command definition: "+name
-            self.next()
-            self.skip_blank_tokens()
-        return numargs
-
-    def scan_command_def(self):
-        """Scan a command definition.
-        Return command_def.
-        Assumes that the number of arguments is at most 9.
-        """
-        if not self.legal():
-            raise "No command definition to scan."
-        item = self.item
-        if not 2 < len(self.data):
-            raise "Command definition is illegal."
-        # newcommand or renewcommand
-        if not item.type in [esc_symb_ty, esc_str_ty]:
-            raise "Command definition should begin with control sequence: "+item.val
-        if item.val not in ["newcommand", "renewcommand"]:
-            raise "Command definition should begin with control sequence."
-        self.next()
-        self.skip_blank_tokens()
-
-        cmd_name = self.scan_command_name()
-        numargs = self.scan_numargs(cmd_name)
-
-        body_group = self.scan_group()
-        if group_ty != body_group.type:
-            raise "Command body missing: "+cmd_name
-        body_val = strip_comments(body_group.val)
-        return Command_def(cmd_name, numargs, body_val)
-
-    def scan_env_name(self):
-        """Starts on a {.
-        Returns name.
-        """
-        if not self.legal():
-            raise "No environment name to scan."
-        item = self.item
-        if not "{" == item.val:
-            raise "Env. definition begins with %s, not with {" % (item.val)
-        self.next()
-        item = self.skip_blank_tokens()
-        name = ""
-        if not simple_ty == item.type:
-            raise ("1. Env. def. begins with cont. seq. %s, not with env.name."
-                 % (item.val))
-        while self.uplegal() and not blank_or_rbrace_re.match(item.val):
-            name += item.val
-            item = self.next()
-            if not simple_ty == item.type:
-                raise ("2. Env. def. begins with cont. seq. %s, not with env.name."
-                       % (item.val))
-        item = self.skip_blank_tokens()
-        if not "}" == item.val:
-            raise "Command definition does not begin with control sequence."
-        self.next()
-        self.skip_blank_tokens()
-        return name
-
-    def scan_env_def(self):
-        """Scan an environment definition.
-        Return env_def
-        Assumes that the number of arguments is at most 9.
-        """
-        if not self.legal():
-            raise "No environment definition to scan."
-        item = self.item
-        if not 7 < len(self.data):
-            raise "Environment definition is illegal."
-        pos = 0
-
-        if not item.type in [esc_symb_ty, esc_str_ty]:
-            raise ("Env. definition does not begin with control sequence:"+
-                   item.val)
-        if item.val not in ["newenvironment", "renewenvironment"]:
-            raise "Env. definition does not begin with control sequence."
-        self.next()
-        self.skip_blank_tokens()
-
-        env_name = self.scan_env_name()
-        numargs = self.scan_numargs(env_name)
-        self.skip_blank_tokens()
-
-        begin_group = self.scan_group()
-        if group_ty != begin_group.type:
-            raise "Begin body missing: "+env_name
-        begin_val = strip_comments(begin_group.val)
-
-        self.skip_blank_tokens()
-
-        end_group = self.scan_group()
-        if group_ty != end_group.type:
-            raise "End body missing:"+env_name
-        end_val = strip_comments(end_group.val)
-
-        return Env_def(env_name, numargs, begin_val, end_val)
-    
-    def scan_defs(self):
-        if not self.legal():
-            raise "No definitions to scan."
-        self.reset()
-        command_defs, env_defs = self.defs
-        while self.uplegal():
-            if (esc_str_ty == self.item.type
-                and self.item.val in ["newcommand", "renewcommand"]):
-                command_def = self.scan_command_def()
-                command_defs[command_def.name] = command_def
-            elif (esc_str_ty == self.item.type and self.item.val
-                  in ["newenvironment", "renewenvironment"]):
-                env_def = self.scan_env_def()
-                env_defs[env_def.name] = env_def
-            else:
-                self.next()
-
-    # Instances
-
-    def scan_args(self, command_or_env_def):
-        """Scan the arguments of a command or environment.
-        Return [args].
-        """
-        if not self.legal():
-            raise "No arguments to scan."
-        numargs = command_or_env_def.numargs
-        name = command_or_env_def.name
-
-        args = []
-        for i in range(numargs):
-            arg = []
-            if not (simple_ty == self.item.type and "{" == self.item.val):
-                arg = [self.item]
-                self.next()
-            else:
-                group = self.scan_group()
-                arg = group.val
-            args.append(arg)
-        return args
-
-    def scan_command(self, command_def):
-        """Scan the arguments of a command.
-        Return command_instance
-        """
-        if not self.legal():
-            raise "No command to scan."
-        if not self.item.type in [esc_symb_ty, esc_str_ty]:
-            raise "Command does not begin with control sequence."
-        name = self.item.val
-        self.next()
-        if 0 < command_def.numargs:
-            self.skip_blank_tokens()
-            args = self.scan_args(command_def)
-        else:
-            args = []
-        return Command_instance(name, args)
-
-    def test_env_boundary(self, item):
-        """Check whether an environment begin or end follows.
-        Return 1 if \begin, -1 if \end, 0 otherwise.
-        """
-        d = 0
-        if esc_str_ty == item.type:
-            if "begin"==item.val:
-                d = 1
-            elif "end"==item.val:
-                d = -1
-        return d
-
-    def scan_env_begin(self):
-        """Scan an environment name.
-        Return env_name.
-        """
-        if not self.legal():
-            raise "No environment begin to scan."
-        item = self.item
-        if not (esc_str_ty == item.type and "begin" == item.val):
-            raise "Environment does not begin with begin."
-        self.next()
-        name_group = self.scan_group()
-        name = detokenize(name_group.val)
-        return name
-
-    def scan_env_end(self):
-        """Scan an environment end.
-        Return env_name.
-        """
-        if not self.legal():
-            raise "No environment end to scan."
-        item = self.item
-        if not (esc_str_ty == item.type and "end" == item.val):
-            raise "Environment does not end with end."
-        self.next()
-        name_group = self.scan_group()
-        name = detokenize(name_group.val)
-        return name
-
-    def scan_env_rest(self, env_def):
-        """Scanning starts after \begin{envname}.
-        Returns env_instance.
-        """
-        if not self.legal():
-            raise "No environment rest to scan."
-        count = 1 # We are already within a boundary.
-        args = self.scan_args(env_def)
-        body = []
-        while count and self.uplegal():
-            old_pos = self.pos
-            d = self.test_env_boundary(self.item)
-            count += d
-            if 1 == d:
-                self.scan_env_begin()
-            elif -1 == d:
-                self.scan_env_end()
-            else:
-                self.next()
-            if 0 < count:
-                body.extend(self.data[old_pos : self.pos])
-        return Env_instance(env_def.name, args, body)
-
-    # Definitions
-
-    def restore_defs(self):
-        if os.path.isfile(self.defs_db_file):
-            print "Using defs db %s" % (self.defs_db_file)
-            db_h = shelve.open(self.defs_db)
-            self.defs = db_h["defs"]
-            db_h.close()
-
-    def save_defs(self):
-        db_h = shelve.open(self.defs_db)
-        if db_h.has_key("defs"):
-            del db_h["defs"]
-        db_h["defs"] = self.defs
-        db_h.close()
-
-    def add_defs(self, defs_file):
-        defs_file_compl = defs_file + ".sty"
-        if not os.path.isfile(defs_file_compl):
-            raise "%s does not exist" % (defs_file_compl)
-
-        defs_db_file = self.defs_db_file
-        if newer(defs_db_file, defs_file_compl):
-            print "Using defs db %s for %s" % (defs_db_file, defs_file)
-        else:
-            defs_fp = open(defs_file_compl, "r")
-            defs_str = defs_fp.read()
-            defs_fp.close()
-            ds = Tex_stream()
-            ds.defs = self.defs
-            defs_text = ds.smart_tokenize(defs_str)
-            # changing ds.defs will change self.defs
-            if self.debug:
-                defs_seen_file = "%s-seen.sty" % (defs_file)
-                defs_seen_fp = open(defs_seen_file, "w")
-                out = detokenize(defs_text)
-                defs_seen_fp.write(out)
-                defs_seen_fp.close()
-            ds.scan_defs()
-            if self.debug:
-                out = ""
-                command_defs, env_defs = self.defs
-                for def_name in command_defs.keys():
-                    out += command_defs[def_name].show() + "\n"
-                for def_name in env_defs.keys():
-                    out += env_defs[def_name].show() +"\n"
-                print "Definitions after reading %s:" % (defs_file)
-                print out
-
-    # Applying definitions, recursively
-    # (maybe not quite in Knuth order, so avoid tricks!)    
-
-    def subst_args(self, body, args):
-        out = []
-        pos = 0
-        while pos < len(body):
-            item = body[pos]
-            if not (simple_ty == item.type and "#" == item.val):
-                out.append(item)
-                pos += 1
-                continue
-            pos += 1
-            token = body[pos]
-            argnum = token.val
-            if not pos_digit_re.match(argnum):
-                raise "# is not followed by number."
-            argnum = int(argnum)
-            if argnum > len(args):
-                raise "Too large argument number."
-            arg = args[argnum-1]
-            out += arg
-            pos += 1
-        return out
-
-    def apply_command_recur(self, command_instance):
-        command_defs, env_defs = self.defs
-        name = command_instance.name
-        command_def = command_defs[name]
-
-        args = command_instance.args
-        body = command_def.body
-        result = self.subst_args(body, args)
-        try:
-            result = self.apply_all_recur(result)
-        except Empty_text_error, e:
-            raise "apply_all_recur fails on command instance %s: %s, %s" % \
-                  (command_instance.show(), detokenize(e.data), e.message)
-        return result
-
-    def apply_env_recur(self, env_instance):
-        command_defs, env_defs = self.defs
-        name = env_instance.name
-        env_def = env_defs[name]
-
-        begin, end = env_def.begin, env_def.end
-        body, args = env_instance.body, env_instance.args
-        out = self.subst_args(begin, args) + body + self.subst_args(end, args)
-        return self.apply_all_recur(out)
-        
-
-    def apply_all_recur(self, data, report=False):
-        ts = Tex_stream(data)
-        ts.defs = self.defs
-        command_defs, env_defs = self.defs
-        out = []
-        progress_step = 10000
-        progress = progress_step
-        if not ts.legal():
-            raise Empty_text_error(data, "No text to process.")
-        while ts.uplegal():
-            if self.pos > progress:
-                if report:
-                    print self.pos
-                progress += progress_step
-            if not ts.item.type in [esc_symb_ty, esc_str_ty]:
-                out.append(ts.item)
-                ts.next()
-                continue
-            if 1 == ts.test_env_boundary(ts.item):
-                old_pos = ts.pos
-                env_name = ts.scan_env_begin()
-                if not env_defs.has_key(env_name):
-                    out.extend(ts.data[old_pos : ts.pos])
-                    continue
-                else:
-                    env_def = env_defs[env_name]
-                    env_instance = ts.scan_env_rest(env_def)
-                    result = ts.apply_env_recur(env_instance)
-                    out.extend(result)
-            elif not command_defs.has_key(ts.item.val):
-                out.append(ts.item)
-                ts.next()
-                continue
-            else:
-                command_def = command_defs[ts.item.val]
-                command_inst = ts.scan_command(command_def)
-                result = ts.apply_command_recur(command_inst)
-                out.extend(result)
-        return out
-
-
-    # Processing files
-
-    def process_file(self, file):
-        """Returns the new defs.
-        """
-        file = cut_extension(file, ".tex")
-        source_file = "%s.tex" % (file)
-        print "File %s [" % (source_file)
-        source_fp = open(source_file, "r")
-        text_str = source_fp.read()
-        source_fp.close()
-
-        self.smart_tokenize(text_str, handle_inputs=True)
-        if not self.data:
-            raise "Empty tokenization result."
-        self.reset()
-
-        if self.debug:
-            source_seen_fname = "%s-seen.tex" % (file)
-            source_seen_fp = open(source_seen_fname, "w")
-            source_seen_fp.write(detokenize(self.data))
-            source_seen_fp.close()
-
-        self.data = self.apply_all_recur(self.data, report=True)
-
-        result_fname = "%s-clean.tex" % (file)
-        print "Writing %s [" % (result_fname)
-        result_fp = open(result_fname, "w")
-        result_fp.write(self.smart_detokenize())
-        result_fp.close()
-        print "] file %s" % (result_fname)
-        print "] file %s" % (source_file)
-
-    def process_if_newer(self, file):
-        """
-        \input{file} is be added to the token list.
-        If the input file is newer it is processed.
-        Returns tokenized \input{file}.
-        """
-        file = cut_extension(file, ".tex")
-        tex_file = file+".tex"
-        clean_tex_file = file+"-clean.tex"
-        if newer(clean_tex_file, tex_file):
-            print "Using %s." % (clean_tex_file)
-        else:
-            ts = Tex_stream()
-            ts.data = []
-            ts.defs = self.defs
-            ts.process_file(file)
-        to_add = "\\input{%s}" % (file)
-        return tokenize(to_add)
-
+	defs = ({}, {})
+	defs_db = "x"
+	defs_db_file = "x.db"
+	debug = False
+
+	def smart_tokenize(self, in_str, handle_inputs=False, isatletter=False):
+		"""Returns a list of tokens.
+		It may interpret and carry out all \input commands.
+		"""
+		self.data = []
+		text = self.data
+		cs = Char_stream(in_str)
+		cs.reset()
+		if not cs.legal():
+			raise Error("No string to tokenize.")
+		while cs.uplegal():
+			if "%" == cs.item:
+				comment = cs.scan_comment_token()
+				text.append(Token(comment_ty, comment))
+			elif "\\" != cs.item:
+				text.append(Token(simple_ty, cs.item))
+				cs.next()
+			else:
+				cs.next()
+				name = cs.scan_escape_token(isatletter)
+				if "input" == name and handle_inputs:
+					file = cs.scan_input_filename()
+					to_add = self.process_if_newer(file)
+					text.extend(to_add)
+				elif "usepackage" == name:
+					while cs.uplegal() and blank_re.match(cs.item):
+						cs.next()
+					if "[" == cs.item: # Packages with options will not be processed.
+						text.extend([Token(esc_str_ty, "usepackage"),
+									 Token(simple_ty, "[")])
+						cs.next()
+						continue
+					files = cs.scan_package_filenames()
+					i = 0
+					while i < len(files):  # process private packages
+						file = files[i]
+						p = file.rfind("-private")
+						if p < 0 or not len(file) - len("-private") == p:
+							i += 1
+							continue
+						defs_db_file = file+".db"
+						self.add_defs(file)
+						del files[i:(i+1)]
+					if files: # non-private packages left
+						group_content = ",".join(files)
+						to_add_str = "\\usepackage{%s}" % (group_content)
+						to_add = tokenize(to_add_str,isatletter)
+						text.extend(to_add)
+				else:
+					if isletter(name[0], isatletter):
+						token = Token(esc_str_ty, name)
+					else: 
+						token = Token(esc_symb_ty, name)
+					text.append(token)
+					if "makeatletter" == name:
+						isatletter=True
+					elif "makeatother" == name:
+						isatletter=False
+		self.reset()
+		return self.data
+
+	def smart_detokenize(self,isatletter=False):
+		"""
+		Output is a string.
+		If the list contains an \input{file} then the content of file
+		file-clean.tex replaces it in the output.
+		"""
+		self.reset()
+		if not self.legal():
+			return ""
+		out = ""
+		previtem = None
+		while self.uplegal():
+			item = self.item
+			"""Insert a separating space after an escape sequence if it is a
+			string and is followed by a letter."""
+			if (None != previtem and esc_str_ty == previtem.type
+				and simple_ty == item.type and isletter(item.val[0], isatletter)):
+				out += " "
+			previtem = item
+			if not (esc_str_ty == item.type and "input" == item.val):
+				out += item.show()
+				self.next()
+			else:
+				self.next()
+				group = self.scan_group()
+				file = detokenize(group.val)
+				clean_file = "%s-clean.tex" % (file)
+				print("Reading file %s" % (clean_file))
+				fp = open(clean_file,"r")
+				content = fp.read()
+				fp.close()
+				out += content
+		return out
+
+	# Basic tex scanning
+
+	def skip_blank_tokens(self): # we also skip comment tokens.
+		item = self.item
+		while (self.uplegal() and
+			   (comment_ty == item.type or
+				(simple_ty == item.type and blank_re.match(item.val)))):
+			item = self.next()
+		return item
+
+	def scan_group(self):
+		"""Returns group.
+		"""
+		if not self.legal():
+			raise Error("No group to scan.")
+		item = self.item
+		if not (simple_ty == item.type and "{" == item.val):
+			return Group(token_ty, [self.item])
+		count = 1
+		group = []
+		item = self.next()
+		while count and self.uplegal():
+			if simple_ty == item.type:
+				if "{" == item.val:
+					count += 1
+				elif "}" == item.val:
+					count -= 1
+			if count != 0:
+				group.append(item)
+			item = self.next()
+		return Group(group_ty, group)
+
+	# Command and environment definitions
+
+	def scan_command_name(self):
+		"""Returns name.
+		"""
+		if not self.legal():
+			raise Error("No command name to scan.")
+		item = self.item
+		name = ""
+		if item.type in [esc_symb_ty, esc_str_ty]:
+			name = item.val
+		else:
+			if not "{" == item.val:
+				raise Error("Command definition misses first {.")
+			self.next()
+			item = self.skip_blank_tokens()
+			if not item.type in [esc_symb_ty, esc_str_ty]:
+				raise Error("Command definition does not begin with control sequence.")
+			name = item.val
+			self.next()
+			item = self.skip_blank_tokens()
+			if not "}" == item.val:
+				raise Error("Definition for commmand %s misses first }., %s" %
+					   (name, item.val))
+		self.next()
+		self.skip_blank_tokens()
+		return name
+
+	def scan_numargs(self, name):
+		"""
+		name is the name of the command or environment definition being
+		scanned.
+		Starts on a nonblank token.
+		Returns numargs
+		where numargs is the number of arguments in a command or environment
+		definition,
+		"""
+		if not self.legal():
+			raise Error("No numargs to scan.")
+		item = self.item
+		numargs = 0
+		if not simple_ty == item.type:
+			raise Error("Illegal command or environment definition: "+name)
+		if "[" == item.val:
+			if not 4 < len(self.data):
+				raise Error("Command or environment definition is illegal: "+name)
+			item = self.next()
+			if not simple_ty == item.type:
+				raise Error("Illegal command or environment definition: "+name)
+			numargs = item.val
+			if not pos_digit_re.match(numargs):
+				raise Error("%s must be argument number after %s" % (numargs, name))
+			numargs = int(numargs)
+			self.next()
+			item = self.skip_blank_tokens()
+			if not simple_ty == item.type:
+				raise Error("Illegal command definition: "+name)
+			if "]" != item.val:
+				raise Error("Illegal command definition: "+name)
+			self.next()
+			self.skip_blank_tokens()
+		return numargs
+
+	def scan_command_def(self):
+		"""Scan a command definition.
+		Return command_def.
+		Assumes that the number of arguments is at most 9.
+		"""
+		if not self.legal():
+			raise Error("No command definition to scan.")
+		item = self.item
+		if not 2 < len(self.data):
+			raise Error("Command definition is illegal.")
+		# newcommand or renewcommand
+		if not item.type in [esc_symb_ty, esc_str_ty]:
+			raise Error("Command definition should begin with control sequence: "+item.val)
+		if item.val not in ["newcommand", "renewcommand"]:
+			raise Error("Command definition should begin with control sequence.")
+		self.next()
+		self.skip_blank_tokens()
+
+		cmd_name = self.scan_command_name()
+		numargs = self.scan_numargs(cmd_name)
+
+		body_group = self.scan_group()
+		if group_ty != body_group.type:
+			raise Error("Command body missing: "+cmd_name)
+		body_val = strip_comments(body_group.val)
+		return Command_def(cmd_name, numargs, body_val)
+
+	def scan_env_name(self):
+		"""Starts on a {.
+		Returns name.
+		"""
+		if not self.legal():
+			raise Error("No environment name to scan.")
+		item = self.item
+		if not "{" == item.val:
+			raise Error("Env. definition begins with %s, not with {" % (item.val))
+		self.next()
+		item = self.skip_blank_tokens()
+		name = ""
+		if not simple_ty == item.type:
+			raise Error("1. Env. def. begins with cont. seq. %s, not with env.name."
+				 % (item.val))
+		while self.uplegal() and not blank_or_rbrace_re.match(item.val):
+			name += item.val
+			item = self.next()
+			if not simple_ty == item.type:
+				raise Error("2. Env. def. begins with cont. seq. %s, not with env.name."
+					   % (item.val))
+		item = self.skip_blank_tokens()
+		if not "}" == item.val:
+			raise Error("Command definition does not begin with control sequence.")
+		self.next()
+		self.skip_blank_tokens()
+		return name
+
+	def scan_env_def(self):
+		"""Scan an environment definition.
+		Return env_def
+		Assumes that the number of arguments is at most 9.
+		"""
+		if not self.legal():
+			raise Error("No environment definition to scan.")
+		item = self.item
+		if not 7 < len(self.data):
+			raise Error("Environment definition is illegal.")
+		pos = 0
+
+		if not item.type in [esc_symb_ty, esc_str_ty]:
+			raise Error("Env. definition does not begin with control sequence:"+
+				   item.val)
+		if item.val not in ["newenvironment", "renewenvironment"]:
+			raise Error("Env. definition does not begin with control sequence.")
+		self.next()
+		self.skip_blank_tokens()
+
+		env_name = self.scan_env_name()
+		numargs = self.scan_numargs(env_name)
+		self.skip_blank_tokens()
+
+		begin_group = self.scan_group()
+		if group_ty != begin_group.type:
+			raise Error("Begin body missing: "+env_name)
+		begin_val = strip_comments(begin_group.val)
+
+		self.skip_blank_tokens()
+
+		end_group = self.scan_group()
+		if group_ty != end_group.type:
+			raise Error("End body missing:"+env_name)
+		end_val = strip_comments(end_group.val)
+
+		return Env_def(env_name, numargs, begin_val, end_val)
+	
+	def scan_defs(self):
+		if not self.legal():
+			raise Error("No definitions to scan.")
+		self.reset()
+		command_defs, env_defs = self.defs
+		while self.uplegal():
+			if (esc_str_ty == self.item.type
+				and self.item.val in ["newcommand", "renewcommand"]):
+				command_def = self.scan_command_def()
+				command_defs[command_def.name] = command_def
+			elif (esc_str_ty == self.item.type and self.item.val
+				  in ["newenvironment", "renewenvironment"]):
+				env_def = self.scan_env_def()
+				env_defs[env_def.name] = env_def
+			else:
+				self.next()
+
+	# Instances
+
+	def scan_args(self, command_or_env_def):
+		"""Scan the arguments of a command or environment.
+		Return [args].
+		"""
+		if not self.legal():
+			raise Error("No arguments to scan.")
+		numargs = command_or_env_def.numargs
+		name = command_or_env_def.name
+
+		args = []
+		for i in range(numargs):
+			arg = []
+			if not (simple_ty == self.item.type and "{" == self.item.val):
+				arg = [self.item]
+				self.next()
+			else:
+				group = self.scan_group()
+				arg = group.val
+			args.append(arg)
+		return args
+
+	def scan_command(self, command_def):
+		"""Scan the arguments of a command.
+		Return command_instance
+		"""
+		if not self.legal():
+			raise Error("No command to scan.")
+		if not self.item.type in [esc_symb_ty, esc_str_ty]:
+			raise Error("Command does not begin with control sequence.")
+		name = self.item.val
+		self.next()
+		if 0 < command_def.numargs:
+			self.skip_blank_tokens()
+			args = self.scan_args(command_def)
+		else:
+			args = []
+		return Command_instance(name, args)
+
+	def test_env_boundary(self, item):
+		"""Check whether an environment begin or end follows.
+		Return 1 if \begin, -1 if \end, 0 otherwise.
+		"""
+		d = 0
+		if esc_str_ty == item.type:
+			if "begin"==item.val:
+				d = 1
+			elif "end"==item.val:
+				d = -1
+		return d
+
+	def scan_env_begin(self):
+		"""Scan an environment name.
+		Return env_name.
+		"""
+		if not self.legal():
+			raise Error("No environment begin to scan.")
+		item = self.item
+		if not (esc_str_ty == item.type and "begin" == item.val):
+			raise Error("Environment does not begin with begin.")
+		self.next()
+		name_group = self.scan_group()
+		name = detokenize(name_group.val)
+		return name
+
+	def scan_env_end(self):
+		"""Scan an environment end.
+		Return env_name.
+		"""
+		if not self.legal():
+			raise Error("No environment end to scan.")
+		item = self.item
+		if not (esc_str_ty == item.type and "end" == item.val):
+			raise Error("Environment does not end with end.")
+		self.next()
+		name_group = self.scan_group()
+		name = detokenize(name_group.val)
+		return name
+
+	def scan_env_rest(self, env_def):
+		"""Scanning starts after \begin{envname}.
+		Returns env_instance.
+		"""
+		if not self.legal():
+			raise Error("No environment rest to scan.")
+		count = 1 # We are already within a boundary.
+		args = self.scan_args(env_def)
+		body = []
+		while count and self.uplegal():
+			old_pos = self.pos
+			d = self.test_env_boundary(self.item)
+			count += d
+			if 1 == d:
+				self.scan_env_begin()
+			elif -1 == d:
+				self.scan_env_end()
+			else:
+				self.next()
+			if 0 < count:
+				body.extend(self.data[old_pos : self.pos])
+		return Env_instance(env_def.name, args, body)
+
+	# Definitions
+
+	def restore_defs(self):
+		if os.path.isfile(self.defs_db_file):
+			print("Using defs db %s" % (self.defs_db_file))
+			db_h = shelve.open(self.defs_db)
+			self.defs = db_h["defs"]
+			db_h.close()
+
+	def save_defs(self):
+		db_h = shelve.open(self.defs_db)
+		if "defs" in db_h:
+			del db_h["defs"]
+		db_h["defs"] = self.defs
+		db_h.close()
+
+	def add_defs(self, defs_file):
+		defs_file_compl = defs_file + ".sty"
+		if not os.path.isfile(defs_file_compl):
+			raise Error("%s does not exist" % (defs_file_compl))
+
+		defs_db_file = self.defs_db_file
+		if newer(defs_db_file, defs_file_compl):
+			print("Using defs db %s for %s" % (defs_db_file, defs_file))
+		else:
+			defs_fp = open(defs_file_compl, "r")
+			defs_str = defs_fp.read()
+			defs_fp.close()
+			ds = Tex_stream()
+			ds.defs = self.defs
+			defs_text = ds.smart_tokenize(defs_str,isatletter=True)
+			# changing ds.defs will change self.defs
+			if self.debug:
+				defs_seen_file = "%s-seen.sty" % (defs_file)
+				defs_seen_fp = open(defs_seen_file, "w")
+				out = detokenize(defs_text,isatletter=True)
+				defs_seen_fp.write(out)
+				defs_seen_fp.close()
+			ds.scan_defs()
+			if self.debug:
+				out = ""
+				command_defs, env_defs = self.defs
+				for def_name in command_defs.keys():
+					out += command_defs[def_name].show() + "\n"
+				for def_name in env_defs.keys():
+					out += env_defs[def_name].show() +"\n"
+				print("Definitions after reading %s:" % (defs_file))
+				print(out)
+
+	# Applying definitions, recursively
+	# (maybe not quite in Knuth order, so avoid tricks!)	
+
+	def subst_args(self, body, args):
+		out = []
+		pos = 0
+		while pos < len(body):
+			item = body[pos]
+			if not (simple_ty == item.type and "#" == item.val):
+				out.append(item)
+				pos += 1
+				continue
+			pos += 1
+			token = body[pos]
+			argnum = token.val
+			if not pos_digit_re.match(argnum):
+				raise Error("# is not followed by number.")
+			argnum = int(argnum)
+			if argnum > len(args):
+				raise Error("Too large argument number.")
+			arg = args[argnum-1]
+			out += arg
+			pos += 1
+		return out
+
+	def apply_command_recur(self, command_instance):
+		command_defs, env_defs = self.defs
+		name = command_instance.name
+		command_def = command_defs[name]
+
+		args = command_instance.args
+		body = command_def.body
+		result = self.subst_args(body, args)
+		try:
+			result = self.apply_all_recur(result)
+		except Empty_text_error as e:
+			raise Error("apply_all_recur fails on command instance %s: %s, %s" % \
+				  (command_instance.show(), detokenize(e.data), e.message))
+		return result
+
+	def apply_env_recur(self, env_instance):
+		command_defs, env_defs = self.defs
+		name = env_instance.name
+		env_def = env_defs[name]
+
+		begin, end = env_def.begin, env_def.end
+		body, args = env_instance.body, env_instance.args
+		out = self.subst_args(begin, args) + body + self.subst_args(end, args)
+		return self.apply_all_recur(out)
+		
+
+	def apply_all_recur(self, data, report=False):
+		ts = Tex_stream(data)
+		ts.defs = self.defs
+		command_defs, env_defs = self.defs
+		out = []
+		progress_step = 10000
+		progress = progress_step
+		if not ts.legal():
+			raise Empty_text_error(data, "No text to process.")
+		while ts.uplegal():
+			if self.pos > progress:
+				if report:
+					print(self.pos)
+				progress += progress_step
+			if not ts.item.type in [esc_symb_ty, esc_str_ty]:
+				out.append(ts.item)
+				ts.next()
+				continue
+			if 1 == ts.test_env_boundary(ts.item):
+				old_pos = ts.pos
+				env_name = ts.scan_env_begin()
+				if env_name not in env_defs:
+					out.extend(ts.data[old_pos : ts.pos])
+					continue
+				else:
+					env_def = env_defs[env_name]
+					env_instance = ts.scan_env_rest(env_def)
+					result = ts.apply_env_recur(env_instance)
+					out.extend(result)
+			elif ts.item.val not in command_defs:
+				out.append(ts.item)
+				ts.next()
+				continue
+			else:
+				command_def = command_defs[ts.item.val]
+				command_inst = ts.scan_command(command_def)
+				result = ts.apply_command_recur(command_inst)
+				out.extend(result)
+		return out
+
+
+	# Processing files
+
+	def process_file(self, file):
+		"""Returns the new defs.
+		"""
+		file = cut_extension(file, ".tex")
+		source_file = "%s.tex" % (file)
+		print("File %s [" % (source_file))
+		source_fp = open(source_file, "r")
+		text_str = source_fp.read()
+		source_fp.close()
+
+		self.smart_tokenize(text_str, handle_inputs=True)
+		if not self.data:
+			raise Error("Empty tokenization result.")
+		self.reset()
+
+		if self.debug:
+			source_seen_fname = "%s-seen.tex" % (file)
+			source_seen_fp = open(source_seen_fname, "w")
+			source_seen_fp.write(detokenize(self.data))
+			source_seen_fp.close()
+
+		self.data = self.apply_all_recur(self.data, report=True)
+
+		result_fname = "%s-clean.tex" % (file)
+		print("Writing %s [" % (result_fname))
+		result_fp = open(result_fname, "w")
+		result_fp.write(self.smart_detokenize())
+		result_fp.close()
+		print("] file %s" % (result_fname))
+		print("] file %s" % (source_file))
+
+	def process_if_newer(self, file):
+		"""
+		\input{file} is added to the token list.
+		If the input file is newer it is processed.
+		Returns tokenized \input{file}.
+		"""
+		file = cut_extension(file, ".tex")
+		tex_file = file+".tex"
+		clean_tex_file = file+"-clean.tex"
+		if newer(clean_tex_file, tex_file):
+			print("Using %s." % (clean_tex_file))
+		else:
+			ts = Tex_stream()
+			ts.data = []
+			ts.defs = self.defs
+			ts.process_file(file)
+		to_add = "\\input{%s}" % (file)
+		return tokenize(to_add)
 
 # Main
 
@@ -1065,15 +1072,15 @@ long_optlist = ["debug","defs="]
 options, restargs = getopt_map("x", long_optlist)
 
 debug = False
-if options.has_key("--debug"):
-    debug = True
+if "--debug" in options:
+	debug = True
 
 root = restargs[0]
 root = cut_extension(root, ".tex")
-if options.has_key("--defs"):
-    defs_root = options["--defs"]
+if "--defs" in options:
+	defs_root = options["--defs"]
 else: 
-    defs_root = "%s" % (root)
+	defs_root = "%s" % (root)
 defs_db = defs_root
 defs_db_file = defs_root+".db"
 
@@ -1084,7 +1091,7 @@ ts.debug = debug
 
 ts.restore_defs()
 for root in restargs:
-    ts.process_file(root)
+	ts.process_file(root)
 
-print "(Re)creating defs db %s" % (defs_db)
+print("(Re)creating defs db %s" % (defs_db))
 ts.save_defs()