spelling (13feb13)

git-svn-id: svn://tug.org/texlive/trunk@29102 c570f23f-e606-0410-a88d-b1316a301751

6 files changed, 219 insertions, 64 deletions

diff --git a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua
index f7df992bb74..f07398d0483 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua
@@ -1,5 +1,5 @@
 --- spelling-recurse.lua
---- Copyright 2012 Stephan Hennig
+--- Copyright 2012, 2013 Stephan Hennig
 --
 -- This work may be distributed and/or modified under the conditions of
 -- the LaTeX Project Public License, either version 1.3 of this license
@@ -17,8 +17,8 @@
 -- user-provided call-back functions upon certain events.
 --
 -- @author Stephan Hennig
--- @copyright 2012 Stephan Hennig
--- @release version 0.2
+-- @copyright 2012, 2013 Stephan Hennig
+-- @release version 0.3
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua
index fba40b195e4..f8b548a7ac5 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua
@@ -1,5 +1,5 @@
 --- spelling-stage-1.lua
---- Copyright 2012 Stephan Hennig
+--- Copyright 2012, 2013 Stephan Hennig
 --
 -- This work may be distributed and/or modified under the conditions of
 -- the LaTeX Project Public License, either version 1.3 of this license
@@ -12,11 +12,11 @@
 --
 
 
---- Read lists of bad and good strings.
+--- Parse sources of bad and good strings.
 --
 -- @author Stephan Hennig
--- @copyright 2012 Stephan Hennig
--- @release version 0.2
+-- @copyright 2012, 2013 Stephan Hennig
+-- @release version 0.3
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
@@ -28,6 +28,10 @@ module(...)
 local M = {}
 
 
+-- Load LuaXML module.
+local xml = require('luaxml-mod-xml')
+
+
 -- Function short-cuts.
 
 
@@ -53,69 +57,212 @@ end
 M.set_resources = set_resources
 
 
---- Generic function for reading a list of strings from a file.
--- All strings read from the given file are mapped to the boolean value
--- `true`.  The format of the file is simple: one string per file.
+--- Generic function for parsing a plain list of strings read from a
+--- file.
+-- All strings found are mapped to the boolean value `true`.  The format
+-- of the input file is one string per line.
 --
--- @param fname  Name of file containing strings.
+-- @param fname  File name.
 -- @param t  Table that maps strings to the value `true`.
--- @return Number of total and new strings read.
-local function __read_strings(fname, t)
+-- @return Number of total and new strings found.
+local function __parse_plain_list_file(fname, t)
+  local f, err = io.open(fname, 'r')
+  if not f then
+    texio.write_nl('package spelling: Error! Can\'t parse plain word list: file ' .. fname)
+    error(err)
+  end
+  -- Read complete plain file into string, to speed-up file operations.
+  local s = f:read('*all')
+  f:close()
   local total_c = 0
   local new_c = 0
-  local f, msg = io.open(fname, 'r')
-  if f then
-    -- Iterate line-wise through file.
-    for l in f:lines() do
-      -- Map string to boolean value `true`.
-      if not t[l] then
-        t[l] = true
-        new_c = new_c + 1
-      end
-      total_c = total_c + 1
+  -- Iterate line-wise through file.
+  for l in s:gmatch('[^\r\n]+') do
+    -- Map string to boolean value `true`.
+    if not t[l] then
+      t[l] = true
+      new_c = new_c + 1
     end
-  else
-    texio.write_nl('package spelling: Warning! Could not open file: ' .. msg)
+    total_c = total_c + 1
   end
   return total_c, new_c
 end
 
 
---- Read a list of bad strings from a file.
--- All strings read from the given file (words with known incorrect
--- spelling) are mapped to the boolean value `true` in table `__is_bad`.
+--- Parse a plain list of bad strings read from a file.
+-- All strings found (words with known incorrect spelling) are mapped to
+-- the boolean value `true` in table `__is_bad`.  The format of the
+-- input file is one string per line.
 --
--- @param fname  Name of file containing bad strings.  If an empty string
--- is provided, strings are read from file `<jobname>.bad`.
-local function read_bad_strings(fname)
-  -- If file name is empty, set default file name.
-  if fname == '' then
-    fname = tex.jobname .. '.bad'
-  end
-  local total, new = __read_strings(fname, __is_bad)
+-- @param fname  File name.
+local function parse_bad_plain_list_file(fname)
+  local total, new = __parse_plain_list_file(fname, __is_bad)
   texio.write_nl('package spelling: ' .. total .. ' bad strings ('
                  .. new .. ' new) read from file \'' .. fname .. '\'.')
 end
-M.read_bad_strings = read_bad_strings
+M.parse_bad_plain_list_file = parse_bad_plain_list_file
 
 
---- Read a list of good strings from a file.
--- All strings read from the given file (words with known correct
--- spelling) are mapped to the boolean value `true` in table
--- `__is_good`.
+--- Parse a plain list of good strings read from a file.
+-- All strings found (words with known correct spelling) are mapped to
+-- the boolean value `true` in table `__is_good`.  The format of the
+-- input file is one string per line.
 --
--- @param fname  Name of file containing good strings.  If an empty
--- string is provided, strings are read from file `<jobname>.good`.
-local function read_good_strings(fname)
-  -- If file name is empty, set default file name.
-  if fname == '' then
-    fname = tex.jobname .. '.good'
-  end
-  local total, new = __read_strings(fname, __is_good)
+-- @param fname  File name.
+local function parse_good_plain_list_file(fname)
+  local total, new = __parse_plain_list_file(fname, __is_good)
   texio.write_nl('package spelling: ' .. total .. ' good strings ('
                  .. new .. ' new) read from file \'' .. fname .. '\'.')
 end
-M.read_good_strings = read_good_strings
+M.parse_good_plain_list_file = parse_good_plain_list_file
+
+
+--- Parse LanguageTool XML data.
+-- Currently, XML data is only scanned for incorrect spellings.  All
+-- strings found in the given XML data (words with known incorrect
+-- spelling) are mapped to the boolean value `true` in table `__is_bad`.
+--
+-- @param s  String containing XML data.  XML data is checked for being
+-- created by LanguageTool (via attribute <code>software</code> in tag
+-- <code>matches</code>) and otherwise ignored.
+-- @return Number of total and new incorrect spellings parsed.
+local function __parse_XML_LanguageTool(s)
+  local total_c = 0
+  local new_c = 0
+
+  -- Some flags for checking validity of XML data.  LanguageTool XML
+  -- data must declare as being UTF-8 encoded and advertise as being
+  -- created by LanguageTool.
+  local is_XML_encoding_UTF_8 = false
+  local is_XML_creator_LanguageTool = false
+  local is_XML_valid = false
+
+  --- Handler object for parsing LanguageTool XML data.
+  -- This table contains call-backs used by LuaXML when parsing XML
+  -- data.
+  --
+  -- @class table
+  -- @name XML_handler
+  -- @field decl  Handle XML declaration.
+  -- @field starttag  Handle all relevant tags.
+  -- @field endtag  Not used, but mandatory.
+  local XML_handler = {
+
+    decl = function(self, text, attr)
+      -- Check XML encoding declaration.
+      if attr.encoding == 'UTF-8' then
+        is_XML_encoding_UTF_8 = true
+      else
+        error('package spelling: Error! XML data not in the UTF-8 encoding.')
+      end
+    end,
+
+    starttag = function(self, text, attr)
+      -- Process <matches> tag.
+      if text == 'matches' then
+        -- Check XML creator is LanguageTool.
+        if attr and attr.software == 'LanguageTool' then
+          is_XML_creator_LanguageTool = true
+          is_XML_valid = is_XML_encoding_UTF_8 and is_XML_creator_LanguageTool
+        end
+      -- Process <error> tags.
+      elseif is_XML_valid and text == 'error' then
+        local ruleid = attr.ruleid
+        if ruleid == 'HUNSPELL_RULE'
+          or ruleid == 'HUNSPELL_NO_SUGGEST_RULE'
+          or ruleid == 'GERMAN_SPELLER_RULE'
+          or string.find(ruleid, '^MORFOLOGIK_RULE_')
+        then
+          -- Extract misspelled word from context attribute.
+          local word = unicode.utf8.sub(attr.context, attr.contextoffset + 1, attr.contextoffset + attr.errorlength)
+          if not __is_bad[word] then
+            __is_bad[word] = true
+            new_c = new_c + 1
+          end
+          total_c = total_c + 1
+        end
+      end
+    end,
+
+    endtag = function(self, text)
+    end,
+
+  }
+
+  -- Create custom XML parser.
+  local x = xml.xmlParser(XML_handler)
+  -- Parse XML data.
+  x:parse(s)
+  -- Check XML data is created by LanguageTool.
+  if not is_XML_creator_LanguageTool then
+    error('package spelling: Error! XML data not created by LanguageTool.')
+  end
+  return total_c, new_c
+end
+
+
+--- Parse LanguageTool XML data read from a file.
+-- All strings found in the file (words with known incorrect spelling)
+-- are mapped to the boolean value `true` in table `__is_bad`.
+--
+-- @param fname  File name.
+local function parse_XML_LanguageTool_file(fname)
+  local f, err = io.open(fname, 'r')
+  if not f then
+    texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname)
+    error(err)
+  end
+  -- Read complete XML file into string, since LuaXML has no streaming
+  -- file reader.
+  local s = f:read('*all')
+  f:close()
+  local success, total, new = pcall(__parse_XML_LanguageTool, s)
+  if not success then
+    texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname)
+    error(total)
+  end
+  texio.write_nl('package spelling: ' .. total .. ' bad strings ('
+                 .. new .. ' new) read from file \'' .. fname .. '\'.')
+end
+M.parse_XML_LanguageTool_file = parse_XML_LanguageTool_file
+
+
+--- Parse default sources for bad and good strings.
+-- All strings found in default sources for words with known incorrect
+-- spelling are mapped to the boolean value `true` in table `__is_bad`.
+-- All strings found in default sources for words with known correct
+-- spelling are mapped to the boolean value `true` in table `__is_good`.
+-- Default sources for bad spellings are file `<jobname>.spell.bad` (a
+-- plain list file).  Default sources for good spellings are file
+-- `<jobname>.spell.good` (a plain list file).
+local function parse_default_bad_and_good()
+  local fname, f
+  -- Try to read bad spellings from plain list file
+  -- '<jobname>.spell.bad'.
+  fname = tex.jobname .. '.spell.bad'
+  f = io.open(fname, 'r')
+  if f then
+     f:close()
+     parse_bad_plain_list_file(fname)
+  end
+  -- Try to read bad spellings from LanguageTool XML file
+  -- '<jobname>.spell.xml'.
+  fname = tex.jobname .. '.spell.xml'
+  f = io.open(fname, 'r')
+  if f then
+     f:close()
+     parse_XML_LanguageTool_file(fname)
+  end
+  -- Try to read good spellings from plain list file
+  -- '<jobname>.spell.good'.
+  fname = tex.jobname .. '.spell.good'
+  f = io.open(fname, 'r')
+  if f then
+     f:close()
+     parse_good_plain_list_file(fname)
+  end
+end
+M.parse_default_bad_and_good = parse_default_bad_and_good
 
 
 -- Return module table.
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua
index 6f46654cb41..5db44d41634 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua
@@ -1,5 +1,5 @@
 --- spelling-stage-2.lua
---- Copyright 2012 Stephan Hennig
+--- Copyright 2012, 2013 Stephan Hennig
 --
 -- This work may be distributed and/or modified under the conditions of
 -- the LaTeX Project Public License, either version 1.3 of this license
@@ -20,8 +20,8 @@
 -- string.  Tagging is applied before hyphenation takes place.
 --
 -- @author Stephan Hennig
--- @copyright 2012 Stephan Hennig
--- @release version 0.2
+-- @copyright 2012, 2013 Stephan Hennig
+-- @release version 0.3
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua
index bfb0c183e7c..dfee2fed283 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua
@@ -1,5 +1,5 @@
 --- spelling-stage-3.lua
---- Copyright 2012 Stephan Hennig
+--- Copyright 2012, 2013 Stephan Hennig
 --
 -- This work may be distributed and/or modified under the conditions of
 -- the LaTeX Project Public License, either version 1.3 of this license
@@ -24,8 +24,8 @@
 -- letters.
 --
 -- @author Stephan Hennig
--- @copyright 2012 Stephan Hennig
--- @release version 0.2
+-- @copyright 2012, 2013 Stephan Hennig
+-- @release version 0.3
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua
index ee71c03a081..734d3006098 100755
--- a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua
@@ -1,5 +1,5 @@
 --- spelling-stage-4.lua
---- Copyright 2012 Stephan Hennig
+--- Copyright 2012, 2013 Stephan Hennig
 --
 -- This work may be distributed and/or modified under the conditions of
 -- the LaTeX Project Public License, either version 1.3 of this license
@@ -18,8 +18,8 @@
 -- document data structure to a file at the end of a LuaTeX run.
 --
 -- @author Stephan Hennig
--- @copyright 2012 Stephan Hennig
--- @release version 0.2
+-- @copyright 2012, 2013 Stephan Hennig
+-- @release version 0.3
 --
 -- @trick Prevent LuaDoc from looking past here for module description.
 --[[ Trick LuaDoc into entering 'module' mode without using that command.
@@ -144,7 +144,7 @@ end
 --
 local function __write_text_document()
   -- Open output file.
-  local fname = __opts.output_file_name or (tex.jobname .. '.txt')
+  local fname = __opts.output_file_name or (tex.jobname .. '.spell.txt')
   local f = assert(io.open(fname, 'wb'))
   local eol = __opts.output_eol
   -- Iterate through document paragraphs.
diff --git a/Master/texmf-dist/scripts/spelling/spelling.lua b/Master/texmf-dist/scripts/spelling/spelling.lua
index d16688ce0be..3bcc0f6df36 100755
--- a/Master/texmf-dist/scripts/spelling/spelling.lua
+++ b/Master/texmf-dist/scripts/spelling/spelling.lua
@@ -1,5 +1,5 @@
 --- spelling.lua
---- Copyright 2012 Stephan Hennig
+--- Copyright 2012, 2013 Stephan Hennig
 --
 -- This work may be distributed and/or modified under the conditions of
 -- the LaTeX Project Public License, either version 1.3 of this license
@@ -12,6 +12,14 @@
 --
 
 
+--- Main Lua file.
+--
+-- @author Stephan Hennig
+-- @copyright 2012, 2013 Stephan Hennig
+-- @release version 0.3
+--
+
+
 --- Global table of modules.
 -- The work of the spelling package can be separated into four
 -- stages:<br />
@@ -20,8 +28,8 @@
 --
 -- <dt>Stage 1</dt>
 --   <dd><ul>
---     <li>Load list(s) of bad strings.</li>
---     <li>Load list(s) of good strings.</li>
+--     <li>Load bad strings.</li>
+--     <li>Load good strings.</li>
 --   </ul></dd>
 --
 -- <dt>Stage 2  (call-back <code>pre_linebreak_filter</code>)</dt>
@@ -64,7 +72,7 @@
 -- @name pkg_spelling_stage
 pkg_spelling_stage = {
 
-  -- string list loading
+  -- bad and good string loading
   [1] = require 'spelling-stage-1',
   -- node list tagging
   -- spell-checking