spelling (23may13)

git-svn-id: svn://tug.org/texlive/trunk@30665 c570f23f-e606-0410-a88d-b1316a301751

6 files changed, 501 insertions, 253 deletions

diff --git a/Master/texmf-dist/scripts/spelling/spelling.lua b/Master/texmf-dist/scripts/spelling/spelling-main.lua
index 3bcc0f6df36..937054732e2 100755
--- a/Master/texmf-dist/scripts/spelling/spelling.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-main.lua
@@ -1,4 +1,4 @@
---- spelling.lua
+--- spelling-main.lua
 --- Copyright 2012, 2013 Stephan Hennig
 --
 -- This work may be distributed and/or modified under the conditions of
@@ -16,10 +16,25 @@
 --
 -- @author Stephan Hennig
 -- @copyright 2012, 2013 Stephan Hennig
--- @release version 0.3
+-- @release version 0.4
 --
 
 
+-- Module identification.
+if luatexbase.provides_module then
+  luatexbase.provides_module(
+    {
+      name        = 'spelling',
+      date        = '2013/05/23',
+      version     = '0.4',
+      description = 'support for spell-checking of LuaTeX documents',
+      author      = 'Stephan Hennig',
+      licence     = 'LPPL ver. 1.3c',
+    }
+  )
+end
+
+
 --- Global table of modules.
 -- The work of the spelling package can be separated into four
 -- stages:<br />
@@ -30,6 +45,7 @@
 --   <dd><ul>
 --     <li>Load bad strings.</li>
 --     <li>Load good strings.</li>
+--     <li>Handle match rules.</li>
 --   </ul></dd>
 --
 -- <dt>Stage 2  (call-back <code>pre_linebreak_filter</code>)</dt>
@@ -56,34 +72,22 @@
 -- </dl>
 --
 -- The code of the spelling package is organized in modules reflecting
--- these stages.  References to modules are made available in a global
--- table so that module's public functions are accessible from within
--- external code.  Table indices correspond to the stages as shown
--- above.<br />
+-- these stages.  References to modules are stored in a table.  Table
+-- indices correspond to the stages as shown above.  The table of module
+-- references is shared in a global table (`PKG_spelling`) so that
+-- public module functions are accessible from within external code.<br
+-- />
 --
 -- <ul>
---   <li><code>spelling-stage-1.lua : pkg_spelling_stage[1]</code></li>
---   <li><code>spelling-stage-2.lua : pkg_spelling_stage[2]</code></li>
---   <li><code>spelling-stage-3.lua : pkg_spelling_stage[3]</code></li>
---   <li><code>spelling-stage-4.lua : pkg_spelling_stage[4]</code></li>
+--   <li><code>spelling-stage-1.lua : stage[1]</code></li>
+--   <li><code>spelling-stage-2.lua : stage[2]</code></li>
+--   <li><code>spelling-stage-3.lua : stage[3]</code></li>
+--   <li><code>spelling-stage-4.lua : stage[4]</code></li>
 -- </ul>
 --
 -- @class table
--- @name pkg_spelling_stage
-pkg_spelling_stage = {
-
-  -- bad and good string loading
-  [1] = require 'spelling-stage-1',
-  -- node list tagging
-  -- spell-checking
-  -- bad string highlighting
-  [2] = require 'spelling-stage-2',
-  -- text storage
-  [3] = require 'spelling-stage-3',
-  -- text output
-  [4] = require 'spelling-stage-4',
-
-}
+-- @name stage
+stage = {}
 
 
 --- Table of package-wide resources that are shared among several
@@ -92,15 +96,15 @@ pkg_spelling_stage = {
 -- @class table
 -- @name res
 --
--- @field is_bad  Table.<br />
+-- @field rules_bad  Table.<br />
 --
--- This table maps all strings known as bad spellings to the value
--- `true`.
+-- This table contains all bad rules.  Spellings can be matched against
+-- these rules.
 --
--- @field is_good  Table.<br />
+-- @field rules_good  Table.<br />
 --
--- This table maps all strings known as good spellings to the value
--- `true`.
+-- This table contains all good match rules.  Spellings can be matched
+-- against these rules.
 --
 -- @field text_document  Table.<br />
 --
@@ -141,29 +145,73 @@ pkg_spelling_stage = {
 --
 local res = {
 
-  is_bad,
-  is_good,
+  rules_bad,
+  rules_good,
   text_document,
-  whatsit_uid,
+  whatsit_ids,
 
 }
 
 
+--- Global package table.
+-- This global table provides access to package-wide variables from
+-- within other chunks.
+--
+-- @class table
+-- @name PKG_spelling
+PKG_spelling = {}
+
+
+--- Determine unique IDs for user-defined whatsit nodes used by this
+-- package.  Package luatexbase provides user-defined whatsit node ID
+-- allocation since version v0.6 (TL 2013).  For older package versions,
+-- we start allocating at an arbitrary hard-coded value of 35**8
+-- (ca. 2**41).
+--
+-- @return Table mapping names to IDs.
+local function __allocate_whatsit_ids()
+  local ids = {}
+  -- Allocation support present?
+  if luatexbase.new_user_whatsit_id then
+    ids.start_tag = luatexbase.new_user_whatsit_id('start_tag', 'spelling')
+    ids.end_tag = luatexbase.new_user_whatsit_id('end_tag', 'spelling')
+  else
+    local uid = 35^8
+    ids.start_tag = uid + 1
+    ids.end_tag = uid + 2
+  end
+  return ids
+end
+
+
 --- Package initialisation.
 --
 local function __init()
   -- Create resources.
-  res.is_bad = {}
-  res.is_good = {}
+  res.rules_bad = {}
+  res.rules_good = {}
   res.text_document = {}
-  res.whatsit_uid = 163
-  -- Make resources available to modules.
-  pkg_spelling_stage[1].set_resources(res)
-  pkg_spelling_stage[2].set_resources(res)
-  pkg_spelling_stage[3].set_resources(res)
-  pkg_spelling_stage[4].set_resources(res)
+  res.whatsit_ids = __allocate_whatsit_ids()
+  -- Provide global access to package ressources during module loading.
+  PKG_spelling.res = res
+  -- Load sub-modules:
+  -- * bad and good string loading
+  -- * match rule handling
+  stage[1] = require 'spelling-stage-1'
+  -- * node list tagging
+  -- * spell-checking
+  -- * bad string highlighting
+  stage[2] = require 'spelling-stage-2'
+  -- * text storage
+  stage[3] = require 'spelling-stage-3'
+  -- * text output
+  stage[4] = require 'spelling-stage-4'
+  -- Remove global reference to package ressources.
+  PKG_spelling.res = nil
+  -- Provide global access to module references.
+  PKG_spelling.stage = stage
   -- Enable text storage.
-  pkg_spelling_stage[3].enable_text_storage()
+  stage[3].enable_text_storage()
 end
 
 
diff --git a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua
index f07398d0483..4de903db36b 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua
@@ -18,7 +18,7 @@
 --
 -- @author Stephan Hennig
 -- @copyright 2012, 2013 Stephan Hennig
--- @release version 0.3
+-- @release version 0.4
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua
index f8b548a7ac5..0f4afe9bb69 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua
@@ -12,11 +12,11 @@
 --
 
 
---- Parse sources of bad and good strings.
+--- Handle lists of bad and good strings and match rules.
 --
 -- @author Stephan Hennig
 -- @copyright 2012, 2013 Stephan Hennig
--- @release version 0.3
+-- @release version 0.4
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
@@ -28,11 +28,19 @@ module(...)
 local M = {}
 
 
--- Load LuaXML module.
+-- Import external modules.
+local unicode = require('unicode')
 local xml = require('luaxml-mod-xml')
 
 
 -- Function short-cuts.
+local Sfind = string.find
+
+local tabinsert = table.insert
+
+local Ufind = unicode.utf8.find
+local Ugmatch = unicode.utf8.gmatch
+local Usub = unicode.utf8.sub
 
 
 -- Declare local variables to store references to resources that are
@@ -43,41 +51,50 @@ local __is_bad
 --
 -- Table of known good strings.
 local __is_good
+--
+-- Table of bad rules.
+local __rules_bad
+--
+-- Table of good rules.
+local __rules_good
 
 
---- Set module resources.
--- Make various resources, that are provided by external code, available
--- to this module.
+--- Generic function for reading bad or good spellings from a file.
+-- All data from the file is read into a string, which is then parsed by
+-- the given parse function.
 --
--- @param res  Ressource table.
-local function set_resources(res)
-  __is_bad = res.is_bad
-  __is_good = res.is_good
+-- @param fname  File name.
+-- @param parse_string  Custom parse function.
+-- @param t  Mapping table bad or good spellings should be added to.
+-- @param hint  String for info message.  Either `bad` or `good`.
+local function __parse_file(fname, parse_string, t, hint)
+  local total_c = 0
+  local new_c = 0
+  local f, err = io.open(fname, 'r')
+  if f then
+    local s = f:read('*all')
+    f:close()
+    total_c, new_c = parse_string(s, t)
+  else
+    texio.write_nl('package spelling: Warning! ' .. err)
+  end
+  texio.write_nl('package spelling: Info! ' .. total_c .. '/' .. new_c .. ' total/new ' .. hint .. ' strings read from file \'' .. fname .. '\'.')
 end
-M.set_resources = set_resources
 
 
---- Generic function for parsing a plain list of strings read from a
---- file.
--- All strings found are mapped to the boolean value `true`.  The format
--- of the input file is one string per line.
+--- Generic function for parsing a string containing a plain list of
+-- strings.  Input format are strings separated by new line or carriage
+-- return, i.e., one string per line.  All lines found in the list are
+-- mapped to the boolean value `true` in the given table.
 --
--- @param fname  File name.
+-- @param s  Input string (a list of strings).
 -- @param t  Table that maps strings to the value `true`.
 -- @return Number of total and new strings found.
-local function __parse_plain_list_file(fname, t)
-  local f, err = io.open(fname, 'r')
-  if not f then
-    texio.write_nl('package spelling: Error! Can\'t parse plain word list: file ' .. fname)
-    error(err)
-  end
-  -- Read complete plain file into string, to speed-up file operations.
-  local s = f:read('*all')
-  f:close()
+local function __parse_plain_list(s, t)
   local total_c = 0
   local new_c = 0
-  -- Iterate line-wise through file.
-  for l in s:gmatch('[^\r\n]+') do
+  -- Iterate line-wise through input string.
+  for l in Ugmatch(s, '[^\r\n]+') do
     -- Map string to boolean value `true`.
     if not t[l] then
       t[l] = true
@@ -96,9 +113,7 @@ end
 --
 -- @param fname  File name.
 local function parse_bad_plain_list_file(fname)
-  local total, new = __parse_plain_list_file(fname, __is_bad)
-  texio.write_nl('package spelling: ' .. total .. ' bad strings ('
-                 .. new .. ' new) read from file \'' .. fname .. '\'.')
+  __parse_file(fname, __parse_plain_list, __is_bad, 'bad')
 end
 M.parse_bad_plain_list_file = parse_bad_plain_list_file
 
@@ -110,25 +125,24 @@ M.parse_bad_plain_list_file = parse_bad_plain_list_file
 --
 -- @param fname  File name.
 local function parse_good_plain_list_file(fname)
-  local total, new = __parse_plain_list_file(fname, __is_good)
-  texio.write_nl('package spelling: ' .. total .. ' good strings ('
-                 .. new .. ' new) read from file \'' .. fname .. '\'.')
+  __parse_file(fname, __parse_plain_list, __is_good, 'good')
 end
 M.parse_good_plain_list_file = parse_good_plain_list_file
 
 
---- Parse LanguageTool XML data.
--- Currently, XML data is only scanned for incorrect spellings.  All
--- strings found in the given XML data (words with known incorrect
--- spelling) are mapped to the boolean value `true` in table `__is_bad`.
+--- Get a custom LanguageTool XML handler.
+-- The returned XML handler scans LanguageTool XML data for incorrect
+-- spellings.  For every incorrect spelling found, the given call-back
+-- function is called with the incorrect spelling string as argument.<br
+-- />
 --
--- @param s  String containing XML data.  XML data is checked for being
--- created by LanguageTool (via attribute <code>software</code> in tag
--- <code>matches</code>) and otherwise ignored.
--- @return Number of total and new incorrect spellings parsed.
-local function __parse_XML_LanguageTool(s)
-  local total_c = 0
-  local new_c = 0
+-- XML data is checked for being created by LanguageTool (via attribute
+-- <code>software</code> in tag <code>matches</code>).
+--
+-- @param cb  Call-back function handling incorrect spellings found in
+-- XML data.
+-- @return XML handler.
+local function __get_XML_handler_LanguageTool(cb)
 
   -- Some flags for checking validity of XML data.  LanguageTool XML
   -- data must declare as being UTF-8 encoded and advertise as being
@@ -152,6 +166,7 @@ local function __parse_XML_LanguageTool(s)
       -- Check XML encoding declaration.
       if attr.encoding == 'UTF-8' then
         is_XML_encoding_UTF_8 = true
+        is_XML_valid = is_XML_encoding_UTF_8 and is_XML_creator_LanguageTool
       else
         error('package spelling: Error! XML data not in the UTF-8 encoding.')
       end
@@ -165,21 +180,20 @@ local function __parse_XML_LanguageTool(s)
           is_XML_creator_LanguageTool = true
           is_XML_valid = is_XML_encoding_UTF_8 and is_XML_creator_LanguageTool
         end
+      -- Check XML data is valid.
+      elseif not is_XML_valid then
+        error('package spelling: Error! No valid LanguageTool XML data.')
       -- Process <error> tags.
-      elseif is_XML_valid and text == 'error' then
+      elseif text == 'error' then
         local ruleid = attr.ruleid
         if ruleid == 'HUNSPELL_RULE'
           or ruleid == 'HUNSPELL_NO_SUGGEST_RULE'
           or ruleid == 'GERMAN_SPELLER_RULE'
-          or string.find(ruleid, '^MORFOLOGIK_RULE_')
+          or Ufind(ruleid, '^MORFOLOGIK_RULE_')
         then
           -- Extract misspelled word from context attribute.
-          local word = unicode.utf8.sub(attr.context, attr.contextoffset + 1, attr.contextoffset + attr.errorlength)
-          if not __is_bad[word] then
-            __is_bad[word] = true
-            new_c = new_c + 1
-          end
-          total_c = total_c + 1
+          local word = Usub(attr.context, attr.contextoffset + 1, attr.contextoffset + attr.errorlength)
+          cb(word)
         end
       end
     end,
@@ -189,14 +203,37 @@ local function __parse_XML_LanguageTool(s)
 
   }
 
+  return XML_handler
+end
+
+
+--- Parse a string containing LanguageTool XML data.
+-- All incorrect spellings found in the given XML data are mapped to the
+-- boolean value `true` in the given table.
+--
+-- @param s  String containing XML data.
+-- @param t  Table mapping incorrect spellings to a boolean.
+-- @return Number of total and new incorrect spellings found.
+local function __parse_XML_LanguageTool(s, t)
+  local total_c = 0
+  local new_c = 0
+
+  -- Create call-back for custom LanguageTool XML handler that stores a
+  -- bad word in the given table and does some statistics.
+  local cb_incorrect_spelling = function(word)
+    if not t[word] then
+      t[word] = true
+      new_c = new_c + 1
+    end
+    total_c = total_c + 1
+  end
+
+  -- Create custom XML handler.
+  local XML_handler_LT = __get_XML_handler_LanguageTool(cb_incorrect_spelling)
   -- Create custom XML parser.
-  local x = xml.xmlParser(XML_handler)
+  local x = xml.xmlParser(XML_handler_LT)
   -- Parse XML data.
   x:parse(s)
-  -- Check XML data is created by LanguageTool.
-  if not is_XML_creator_LanguageTool then
-    error('package spelling: Error! XML data not created by LanguageTool.')
-  end
   return total_c, new_c
 end
 
@@ -207,22 +244,7 @@ end
 --
 -- @param fname  File name.
 local function parse_XML_LanguageTool_file(fname)
-  local f, err = io.open(fname, 'r')
-  if not f then
-    texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname)
-    error(err)
-  end
-  -- Read complete XML file into string, since LuaXML has no streaming
-  -- file reader.
-  local s = f:read('*all')
-  f:close()
-  local success, total, new = pcall(__parse_XML_LanguageTool, s)
-  if not success then
-    texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname)
-    error(total)
-  end
-  texio.write_nl('package spelling: ' .. total .. ' bad strings ('
-                 .. new .. ' new) read from file \'' .. fname .. '\'.')
+  __parse_file(fname, __parse_XML_LanguageTool, __is_bad, 'bad')
 end
 M.parse_XML_LanguageTool_file = parse_XML_LanguageTool_file
 
@@ -232,19 +254,12 @@ M.parse_XML_LanguageTool_file = parse_XML_LanguageTool_file
 -- spelling are mapped to the boolean value `true` in table `__is_bad`.
 -- All strings found in default sources for words with known correct
 -- spelling are mapped to the boolean value `true` in table `__is_good`.
--- Default sources for bad spellings are file `<jobname>.spell.bad` (a
--- plain list file).  Default sources for good spellings are file
--- `<jobname>.spell.good` (a plain list file).
+-- Default sources for bad spellings are files `<jobname>.spell.xml` (a
+-- LanguageTool XML file) and `<jobname>.spell.bad` (a plain list file).
+-- Default sources for good spellings are file `<jobname>.spell.good` (a
+-- plain list file).
 local function parse_default_bad_and_good()
   local fname, f
-  -- Try to read bad spellings from plain list file
-  -- '<jobname>.spell.bad'.
-  fname = tex.jobname .. '.spell.bad'
-  f = io.open(fname, 'r')
-  if f then
-     f:close()
-     parse_bad_plain_list_file(fname)
-  end
   -- Try to read bad spellings from LanguageTool XML file
   -- '<jobname>.spell.xml'.
   fname = tex.jobname .. '.spell.xml'
@@ -253,6 +268,14 @@ local function parse_default_bad_and_good()
      f:close()
      parse_XML_LanguageTool_file(fname)
   end
+  -- Try to read bad spellings from plain list file
+  -- '<jobname>.spell.bad'.
+  fname = tex.jobname .. '.spell.bad'
+  f = io.open(fname, 'r')
+  if f then
+     f:close()
+     parse_bad_plain_list_file(fname)
+  end
   -- Try to read good spellings from plain list file
   -- '<jobname>.spell.good'.
   fname = tex.jobname .. '.spell.good'
@@ -265,5 +288,83 @@ end
 M.parse_default_bad_and_good = parse_default_bad_and_good
 
 
+--- Default bad dictionary look-up match rule.
+-- This function looks-up both arguments in the list of bad spellings.
+-- It returns `true` if either of the arguments is found in the list of
+-- bad spellings, otherwise `false`.
+--
+-- @param raw  Raw string to check.
+-- @param stripped  Same as `raw`, but with stripped surrounding
+-- punctuation.
+-- @return A boolean value indicating a match.
+local function __bad_rule_bad_dictionary_lookup(raw, stripped)
+  return __is_bad[stripped] or __is_bad[raw]
+end
+
+
+--- Default good dictionary look-up match rule.
+-- This function looks-up both arguments in the list of good spellings.
+-- It returns `true` if either of the arguments is found in the list of
+-- good spellings, otherwise `false`.
+--
+-- @param raw  Raw string to check.
+-- @param stripped  Same as `raw`, but with stripped surrounding
+-- punctuation.
+-- @return A boolean value indicating a match.
+local function __good_rule_good_dictionary_lookup(raw, stripped)
+  return __is_good[stripped] or __is_good[raw]
+end
+
+
+--- Load match rule module.
+-- Match rule modules are loaded using `require`.  The module table must
+-- follow the following convention: Indentifiers of bad match rules
+-- start `bad_rule_`.  Indentifiers of good match rules start
+-- `good_rule_`.  Other and non-function identifiers are ignore.
+--
+-- All match rules found in a module are added to the table of bad and
+-- good match rules.  Arguments of a match rule function are a raw
+-- string and the same string with stripped surrounding punctuation.
+--
+-- @param fname  Module file name.
+local function read_match_rules(fname)
+  local bad_c = 0
+  local good_c = 0
+  local rules = require(fname)
+  for k,v in pairs(rules) do
+    if type(v) == 'function' then
+      if Sfind(k, '^bad_rule_') then
+        tabinsert(__rules_bad, v)
+        bad_c = bad_c + 1
+      elseif Sfind(k, '^good_rule_') then
+        tabinsert(__rules_good, v)
+        good_c = good_c + 1
+      end
+    end
+  end
+  texio.write_nl('package spelling: Info! ' .. bad_c .. '/' .. good_c .. ' bad/good match rules read from module \'' .. fname .. '\'.')
+end
+M.read_match_rules = read_match_rules
+
+
+--- Module initialisation.
+--
+local function __init()
+  -- Get local references to package ressources.
+  __rules_bad = PKG_spelling.res.rules_bad
+  __rules_good = PKG_spelling.res.rules_good
+  -- Add default dictionary look-up match rules.
+  tabinsert(__rules_bad, __bad_rule_bad_dictionary_lookup)
+  tabinsert(__rules_good, __good_rule_good_dictionary_lookup)
+  -- Create emtpy lists of known spellings.
+  __is_bad = {}
+  __is_good = {}
+end
+
+
+-- Initialize module.
+__init()
+
+
 -- Return module table.
 return M
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua
index 5db44d41634..1a397217a16 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua
@@ -21,7 +21,7 @@
 --
 -- @author Stephan Hennig
 -- @copyright 2012, 2013 Stephan Hennig
--- @release version 0.3
+-- @release version 0.4
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
@@ -33,20 +33,29 @@ module(...)
 local M = {}
 
 
--- Import helper module.
-local __recurse = require 'spelling-recurse'
-local __recurse_node_list = __recurse.recurse_node_list
+-- Import external modules.
+local recurse = require('spelling-recurse')
+local unicode = require('unicode')
 
 
 -- Function short-cuts.
 local tabconcat = table.concat
 local tabinsert = table.insert
 local tabremove = table.remove
-local utf8char = unicode.utf8.char
+
 local node_new = node.new
 local node_insert_after = node.insert_after
 local node_insert_before = node.insert_before
 
+local recurse_node_list = recurse.recurse_node_list
+
+local Sfind = string.find
+local Sgmatch = string.gmatch
+local Smatch = string.match
+
+local Uchar = unicode.utf8.char
+local Umatch = unicode.utf8.match
+
 
 -- Short-cuts for constants.
 local DISC = node.id('disc')
@@ -61,27 +70,17 @@ local PDF_COLORSTACK = node.subtype('pdf_colorstack')
 -- Declare local variables to store references to resources that are
 -- provided by external code.
 --
--- Table of known bad spellings.
-local __is_bad
+-- Table of bad rules.
+local __rules_bad
 --
--- Table of known good spellings.
-local __is_good
+-- Table of good rules.
+local __rules_good
 --
--- ID of user-defined whatsit nodes.
-local __whatsit_uid
-
-
---- Set module resources.
--- Make various resources, that are provided by external code, available
--- to this module.
+-- ID of user-defined whatsit nodes marking the start of a word.
+local __uid_start_tag
 --
--- @param res  Ressource table.
-local function set_resources(res)
-  __is_bad = res.is_bad
-  __is_good = res.is_good
-  __whatsit_uid = res.whatsit_uid
-end
-M.set_resources = set_resources
+-- ID of user-defined whatsit nodes marking the end of a word.
+local __uid_end_tag
 
 
 --- Module options.
@@ -98,18 +97,94 @@ local __opts = {
 
 
 --- Set colour used for highlighting.
--- Set colourused for highlighting bad spellings in PDF output.
+-- Set colour used for highlighting bad spellings in PDF output.  The
+-- argument is checked for a valid PDF colour statement.  As an example,
+-- the string `1 0 0 rg` represents a red colour in the RGB colour
+-- space.  A similar colour in the CMYK colour space would be
+-- represented by the string '0 1 1 0 k'.
 --
--- @param col New colour.  This must be a colour statement in PDF format
--- given as string.  As an example, the string `1 0 0 rg` represents a
--- red colour in the RGB colour space.  A similar colour in the CMYK
--- colour space would be represented by the string '0 1 1 0 k'.
+-- @param col  New colour.
 local function set_highlight_color(col)
-  __opts.hl_color = col
+  -- Extract all colour components.
+  local components = Smatch(col, '^(%S+ %S+ %S+) rg$') or Smatch(col, '^(%S+ %S+ %S+ %S+) k$')
+  local is_valid_arg = components
+  if is_valid_arg then
+    -- Validate colour components.
+    for comp in Sgmatch(components, '%S+') do
+      -- Check number syntax.
+      local is_valid_comp = Sfind(comp, '^%d+%.?%d*$') or Sfind(comp, '^%d*%.?%d+$')
+      if is_valid_comp then
+        -- Check number range.
+        comp = tonumber(comp)
+        is_valid_comp = comp >= 0 and comp <= 1
+      end
+      is_valid_arg = is_valid_arg and is_valid_comp
+    end
+  end
+  if is_valid_arg then
+    __opts.hl_color = col
+  else
+    error('package spelling: Error! Invalid PDF colour statement: ' .. tostring(col))
+  end
 end
 M.set_highlight_color = set_highlight_color
 
 
+--- Highlighting status cache table.
+-- Determining the highlighting status of a string can be an expensive
+-- operation.  To reduce average run-time penalty per string,
+-- highlighting status of all strings found in a document is cached in
+-- this table, so that determining the highlighting status of a known
+-- string requires only one table look-up.<br />
+--
+-- This table needs an `__index` meta method calculating the
+-- highlighting status of unknown keys (strings).
+--
+-- @class table
+-- @name __is_highlighting_needed
+local __is_highlighting_needed = {}
+
+
+--- Calculate and cache the highlighting status of a string.
+-- First, surrounding punctuation is stripped from the string argument.
+-- Then, the given raw as well as the stripped string are checked
+-- against all rules.  Highlighting of the string is required, if any
+-- bad rule matches, but no good rule matches.  That is, good rules take
+-- precedence over bad rules.
+--
+-- @param t  Original table.
+-- @param raw  Raw string to check.
+-- @return True, if highlighting is required.  False, otherwise.
+local function __calc_is_highlighting_needed(t, raw)
+  -- Strip surrounding punctuation from string.
+  local stripped = Umatch(raw, '^%p*(.-)%p*$')
+  -- Check for a bad match.
+  local is_bad = false
+  for _,matches_bad in ipairs(__rules_bad) do
+    is_bad = is_bad or matches_bad(raw, stripped)
+    if is_bad then break end
+  end
+  -- Check for a good match.
+  local is_good = false
+  for _,matches_good in ipairs(__rules_good) do
+    is_good = is_good or matches_good(raw, stripped)
+    if is_good then break end
+  end
+  -- Calculate highlighting status.
+  local status = (is_bad and not is_good) or false
+  -- Store status in cache table.
+  rawset(t, raw, status)
+  -- Return status.
+  return status
+end
+
+
+-- Set-up meta table for highlighting status cache table.
+setmetatable(__is_highlighting_needed, {
+  __index = __calc_is_highlighting_needed,
+})
+
+
 --- Convert a Unicode code point to a regular UTF-8 encoded string.
 -- This function can be used as an `__index` meta method.
 --
@@ -117,7 +192,7 @@ M.set_highlight_color = set_highlight_color
 -- @param cp  originl key, a Unicode code point
 -- @return UTF-8 encoded string corresponding to the Unicode code point.
 local function __meta_cp2utf8(t, cp)
-  return utf8char(cp)
+  return Uchar(cp)
 end
 
 
@@ -146,8 +221,6 @@ local __codepoint_map = {
   [0x0153] = 'oe',-- LATIN SMALL LIGATURE OE
   [0x017f] = 's',-- LATIN SMALL LETTER LONG S
 
-  [0x1e9e] = 'SS',-- LATIN CAPITAL LETTER SHARP S
-
   [0xfb00] = 'ff',-- LATIN SMALL LIGATURE FF
   [0xfb01] = 'fi',-- LATIN SMALL LIGATURE FI
   [0xfb02] = 'fl',-- LATIN SMALL LIGATURE FL
@@ -250,7 +323,7 @@ local function __tag_word(word)
     local start_tag = node_new(WHATSIT, USER_DEFINED)
     -- Mark whatsit node with module ID, so that we can recognize it
     -- later.
-    start_tag.user_id = __whatsit_uid
+    start_tag.user_id = __uid_start_tag
     -- Value is an empty string.
     start_tag.type = 115
     start_tag.value = ''
@@ -261,7 +334,7 @@ local function __tag_word(word)
   local end_tag = node_new(WHATSIT, USER_DEFINED)
   -- Mark whatsit node with module ID, so that we can recognize it
   -- later.
-  end_tag.user_id = __whatsit_uid
+  end_tag.user_id = __uid_end_tag
   -- Value of end tag is an index (a number).
   end_tag.type = 115
   end_tag.value = word
@@ -283,6 +356,29 @@ local function __highlight_by_color()
      local pop = node_new(WHATSIT, PDF_COLORSTACK)
      push.stack = 0
      pop.stack = 0
+     push.command = 1
+     pop.command = 2
+     push.data = __opts.hl_color
+     node_insert_before(__curr_word_start_head, __curr_word_start, push)
+     node_insert_after(__curr_word_end_head, __curr_word_end, pop)
+  end
+end
+
+
+--- Highlight bad spelling by colour (using node field `cmd`).
+-- Insert colour whatsits before and after the first and last nodes
+-- known to belong to the current word.
+-- @see function __highlight_by_color
+local function __highlight_by_color_cmd()
+  -- Check, if start node of current word is a head node.  Inserting
+  -- before head nodes needs special attention.  This is not yet
+  -- implemented.
+  if (__curr_word_start ~= __curr_word_start_head) then
+    -- Create pdf_colorstack whatsit nodes.
+     local push = node_new(WHATSIT, PDF_COLORSTACK)
+     local pop = node_new(WHATSIT, PDF_COLORSTACK)
+     push.stack = 0
+     pop.stack = 0
      push.cmd = 1
      pop.cmd = 2
      push.data = __opts.hl_color
@@ -332,13 +428,10 @@ local function __finish_current_word()
     if start_prev and end_next
     and (start_prev.id == WHATSIT)
     and (start_prev.subtype == USER_DEFINED)
-    and (start_prev.user_id == __whatsit_uid)
-    and (start_prev.type == 115)
-    and (start_prev.value == '')
+    and (start_prev.user_id == __uid_start_tag)
     and (end_next.id == WHATSIT)
     and (end_next.subtype == USER_DEFINED)
-    and (end_next.user_id == __whatsit_uid)
-    and (end_next.type == 115)
+    and (end_next.user_id == __uid_end_tag)
     and (end_next.value == word) then
       __curr_word = nil
       __curr_word_start_head = nil
@@ -351,8 +444,8 @@ local function __finish_current_word()
     if __is_active_tagging then
       __tag_word(word)
     end
-    -- Test for bad spelling.
-    if __is_active_highlighting and __is_bad[word] and not __is_good[word] then
+    -- Highlighting needed?
+    if __is_highlighting_needed[word] and __is_active_highlighting then
       __highlight_bad_word()
     end
     __curr_word = nil
@@ -475,7 +568,7 @@ local function __process_node_list(head)
   __curr_word_start = nil
   __curr_word_end_head = nil
   __curr_word_end = nil
-  __recurse_node_list(head, __cb_tag_words)
+  recurse_node_list(head, __cb_tag_words)
   -- Clean-up left-over word and/or paragraph.
   __finish_current_paragraph()
 end
@@ -526,9 +619,40 @@ end
 M.disable_word_highlighting = disable_word_highlighting
 
 
+--- Try to maintain compatibility with older LuaTeX versions.
+-- Between LuaTeX 0.70.2 and 0.76.0 node field `cmd` of `whatsits` nodes
+-- of subtype `pdf_colorstack` has been renamed to `command`.  This
+-- function checks which node field is the correct one and activates a
+-- fall-back function in case.  Due to a bug in LuaTeX 0.76.0 (shipped
+-- with TL2013) function `node.has_field()` doesn't return correct
+-- results.  It is therefore tested if an assignment to field `command`
+-- raises an error or not.
+local function __maintain_compatibility()
+  -- Create pdf_colorstack whatsit node.
+  local n = node.new(WHATSIT, PDF_COLORSTACK)
+  -- Function that assigns a value to node field 'command'.
+  local f = function()
+    n.command = 1
+  end
+  -- If the assignment is not successful, fall-back to node field 'cmd'.
+  if not pcall(f) then
+    __highlight_by_color = __highlight_by_color_cmd
+  end
+  -- Delete test node.
+  node.free(n)
+end
+
+
 --- Module initialisation.
 --
 local function __init()
+  -- Try to maintain compatibility with older LuaTeX versions.
+  __maintain_compatibility()
+  -- Get local references to package ressources.
+  __rules_bad = PKG_spelling.res.rules_bad
+  __rules_good = PKG_spelling.res.rules_good
+  __uid_start_tag = PKG_spelling.res.whatsit_ids.start_tag
+  __uid_end_tag = PKG_spelling.res.whatsit_ids.end_tag
   -- Create empty paragraph management stack.
   __is_vlist_paragraph = {}
   -- Remember tagging status.
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua
index dfee2fed283..436fe6422fd 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua
@@ -25,7 +25,7 @@
 --
 -- @author Stephan Hennig
 -- @copyright 2012, 2013 Stephan Hennig
--- @release version 0.3
+-- @release version 0.4
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
@@ -37,12 +37,13 @@ module(...)
 local M = {}
 
 
--- Import helper module.
-local __recurse = require 'spelling-recurse'
-local __recurse_node_list = __recurse.recurse_node_list
+-- Import external modules.
+local recurse = require('spelling-recurse')
 
 
 -- Function short-cuts.
+local recurse_node_list = recurse.recurse_node_list
+
 local tabinsert = table.insert
 local tabremove = table.remove
 
@@ -59,20 +60,11 @@ local USER_DEFINED = node.subtype('user_defined')
 -- Text document data structure.
 local __text_document
 --
--- ID of user-defined whatsit nodes.
-local __whatsit_uid
-
-
---- Set module resources.
--- Make various resources, that are provided by external code, available
--- to this module.
+-- ID of user-defined whatsit nodes marking the start of a word.
+local __uid_start_tag
 --
--- @param res  Ressource table.
-local function set_resources(res)
-  __text_document = res.text_document
-  __whatsit_uid = res.whatsit_uid
-end
-M.set_resources = set_resources
+-- ID of user-defined whatsit nodes marking the end of a word.
+local __uid_end_tag
 
 
 --- Module options.
@@ -199,11 +191,8 @@ local function __visit_node(head, n)
   -- Test for node containing a word string.
   if nid == WHATSIT then
     -- Test for word string tag.
-    if (n.subtype == USER_DEFINED) and (n.user_id == __whatsit_uid) then
-      -- End tag?
-      if n.value ~= '' then
-        __finish_current_word(n)
-      end
+    if (n.subtype == USER_DEFINED) and (n.user_id == __uid_end_tag) then
+      __finish_current_word(n)
     -- Test for paragraph start.
     elseif n.subtype == LOCAL_PAR then
       __finish_current_paragraph()
@@ -247,7 +236,7 @@ local __cb_store_words = {
 --
 -- @param head  Node list.
 local function __process_node_list(head)
-  __recurse_node_list(head, __cb_store_words)
+  recurse_node_list(head, __cb_store_words)
   -- Clean-up left-over word and/or paragraph.
   __finish_current_paragraph()
 end
@@ -258,10 +247,11 @@ local __is_active_storage
 
 
 --- Call-back function that processes the node list.
--- <i>This is a global function!</i>
+-- <i>This function is not made available in the module table, but in
+-- the global package table!</i>
 --
 -- @param head  Node list.
-function cb_AtBeginShipout_pkg_spelling(box)
+local function cb_AtBeginShipout(box)
   if __is_active_storage then
     __process_node_list(tex.box[box])
   end
@@ -288,6 +278,12 @@ M.disable_text_storage = disable_text_storage
 --- Module initialisation.
 --
 local function __init()
+  -- Get local references to package ressources.
+  __text_document = PKG_spelling.res.text_document
+  __uid_start_tag = PKG_spelling.res.whatsit_ids.start_tag
+  __uid_end_tag = PKG_spelling.res.whatsit_ids.end_tag
+  -- Make \AtBeginShipout function available in package table.
+  PKG_spelling.cb_AtBeginShipout = cb_AtBeginShipout
   -- Create empty paragraph management stack.
   __is_vlist_paragraph = {}
   -- Remember call-back status.
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua
index 734d3006098..ab4fabb22f7 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua
@@ -19,7 +19,7 @@
 --
 -- @author Stephan Hennig
 -- @copyright 2012, 2013 Stephan Hennig
--- @release version 0.3
+-- @release version 0.4
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
@@ -31,9 +31,15 @@ module(...)
 local M = {}
 
 
+-- Import external modules.
+local unicode = require('unicode')
+
+
 -- Function short-cuts.
 local tabconcat = table.concat
-local utf8len = unicode.utf8.len
+local tabinsert = table.insert
+
+local Ulen = unicode.utf8.len
 
 
 -- Declare local variables to store references to resources that are
@@ -43,43 +49,20 @@ local utf8len = unicode.utf8.len
 local __text_document
 
 
---- Set module resources.
--- Make various resources, that are provided by external code, available
--- to this module.
---
--- @param res  Ressource table.
-local function set_resources(res)
-  __text_document = res.text_document
-end
-M.set_resources = set_resources
-
-
 --- Module options.
 -- This table contains all module options.  User functions to set
 -- options are provided.
 --
 -- @class table
 -- @name __opts
--- @field output_eol  End-of-line  character in output.
 -- @field output_file_name  Output file name.
 -- @field output_line_length  Line length in output.
 local __opts = {
-  output_eol,
   output_file_name,
   output_line_lenght,
 }
 
 
---- Set output EOL character.
--- Text output will be written with the given end-of-line character.
---
--- @param eol  New output EOL character.
-local function set_output_eol(eol)
-  __opts.output_eol = eol
-end
-M.set_output_eol = set_output_eol
-
-
 --- Set output file name.
 -- Text output will be written to a file with the given name.
 --
@@ -106,37 +89,39 @@ end
 M.set_output_line_length = set_output_line_length
 
 
---- Break a paragraph into lines of a fixed length and write the lines
---- to a file.
+--- Break a text paragraph into lines.
+-- Lines are broken at spaces only.  Lines containing only one word may
+-- exceed maximum line length.
 --
--- @param f  A file handle.
 -- @param par  A text paragraph (an array of words).
-local function __write_text_paragraph(f, par)
-  local maxlinelength = __opts.output_line_length
-  local eol = __opts.output_eol
+-- @param max_line_len Maximum length of lines in wrapped paragraph.  If
+-- the argument is less then 1, paragraph isn't wrapped at all.
+-- @return Table containing the lines of the paragraph.
+local function __wrap_text_paragraph(par, max_line_len)
+  local wrapped_par = {}
   -- Index of first word on current line.  Initialize current line with
   -- first word of paragraph.
-  local lstart = 1
+  local line_start = 1
   -- Track current line length.
-  local llen = utf8len(par[1])
+  local line_len = Ulen(par[line_start])
   -- Iterate over remaining words in paragraph.
   for i = 2,#par do
-    local wlen = utf8len(par[i])
-    -- Does word fit onto current line?
-    if (llen + 1 + wlen <= maxlinelength) or (maxlinelength < 1) then
-      -- Append word to current line.
-      llen = llen + 1 + wlen
-    else
-      -- Write the current line up to the preceeding word to file (words
-      -- separated by spaces and with a trailing newline).
-      f:write(tabconcat(par, ' ', lstart, i-1), eol)
+    local word_len = Ulen(par[i])
+    local new_line_len = line_len + 1 + word_len
+    -- Maximum line length exceeded?
+    if new_line_len > max_line_len and max_line_len >= 1 then
+      -- Insert current line into wrapped paragraph.
+      tabinsert(wrapped_par, tabconcat(par, ' ', line_start, i-1))
       -- Initialize new current line.
-      lstart = i
-      llen = wlen
+      line_start = i
+      new_line_len = word_len
     end
+    -- Append word to current line.
+    line_len = new_line_len
   end
-  -- Write last line of paragraph.
-  f:write(tabconcat(par, ' ', lstart), eol)
+  -- Insert last line of paragraph.
+  tabinsert(wrapped_par, tabconcat(par, ' ', line_start))
+  return wrapped_par
 end
 
 
@@ -145,14 +130,12 @@ end
 local function __write_text_document()
   -- Open output file.
   local fname = __opts.output_file_name or (tex.jobname .. '.spell.txt')
-  local f = assert(io.open(fname, 'wb'))
-  local eol = __opts.output_eol
+  local f = assert(io.open(fname, 'w'))
+  local max_line_len = __opts.output_line_length
   -- Iterate through document paragraphs.
   for _,par in ipairs(__text_document) do
-    -- Separate paragraphs by a blank line.
-    f:write(eol)
-    -- Write paragraph to file.
-    __write_text_paragraph(f, par)
+    -- Write wrapped paragraph to file with a leading empty line.
+    f:write('\n', tabconcat(__wrap_text_paragraph(par, max_line_len), '\n'), '\n')
     -- Delete paragraph from memory.
     __text_document[_] = nil
   end
@@ -200,16 +183,12 @@ M.disable_text_output = disable_text_output
 --- Module initialisation.
 --
 local function __init()
+  -- Get local references to package ressources.
+  __text_document = PKG_spelling.res.text_document
   -- Set default output file name.
   set_output_file_name(nil)
   -- Set default output line length.
   set_output_line_length(72)
-  -- Set default output EOL character.
-  if (os.type == 'windows') or (os.type == 'msdos') then
-    set_output_eol('\r\n')
-  else
-    set_output_eol('\n')
-  end
   -- Remember call-back status.
   __is_active_output = false
 end