From 469e41303044d048cec1de7cf5d814b9f2b5e52e Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Wed, 13 Feb 2013 23:22:28 +0000 Subject: spelling (13feb13) git-svn-id: svn://tug.org/texlive/trunk@29102 c570f23f-e606-0410-a88d-b1316a301751 --- .../scripts/spelling/spelling-recurse.lua | 6 +- .../scripts/spelling/spelling-stage-1.lua | 241 +++++++++++++++++---- .../scripts/spelling/spelling-stage-2.lua | 6 +- .../scripts/spelling/spelling-stage-3.lua | 6 +- .../scripts/spelling/spelling-stage-4.lua | 8 +- Master/texmf-dist/scripts/spelling/spelling.lua | 16 +- 6 files changed, 219 insertions(+), 64 deletions(-) (limited to 'Master/texmf-dist/scripts') diff --git a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua index f7df992bb74..f07398d0483 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua @@ -1,5 +1,5 @@ --- spelling-recurse.lua ---- Copyright 2012 Stephan Hennig +--- Copyright 2012, 2013 Stephan Hennig -- -- This work may be distributed and/or modified under the conditions of -- the LaTeX Project Public License, either version 1.3 of this license @@ -17,8 +17,8 @@ -- user-provided call-back functions upon certain events. -- -- @author Stephan Hennig --- @copyright 2012 Stephan Hennig --- @release version 0.2 +-- @copyright 2012, 2013 Stephan Hennig +-- @release version 0.3 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua index fba40b195e4..f8b548a7ac5 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua @@ -1,5 +1,5 @@ --- spelling-stage-1.lua ---- Copyright 2012 Stephan Hennig +--- Copyright 2012, 2013 Stephan Hennig -- -- This work may be distributed and/or modified under the conditions of -- the LaTeX Project Public License, either version 1.3 of this license @@ -12,11 +12,11 @@ -- ---- Read lists of bad and good strings. +--- Parse sources of bad and good strings. -- -- @author Stephan Hennig --- @copyright 2012 Stephan Hennig --- @release version 0.2 +-- @copyright 2012, 2013 Stephan Hennig +-- @release version 0.3 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. @@ -28,6 +28,10 @@ module(...) local M = {} +-- Load LuaXML module. +local xml = require('luaxml-mod-xml') + + -- Function short-cuts. @@ -53,69 +57,212 @@ end M.set_resources = set_resources ---- Generic function for reading a list of strings from a file. --- All strings read from the given file are mapped to the boolean value --- `true`. The format of the file is simple: one string per file. +--- Generic function for parsing a plain list of strings read from a +--- file. +-- All strings found are mapped to the boolean value `true`. The format +-- of the input file is one string per line. -- --- @param fname Name of file containing strings. +-- @param fname File name. -- @param t Table that maps strings to the value `true`. --- @return Number of total and new strings read. -local function __read_strings(fname, t) +-- @return Number of total and new strings found. +local function __parse_plain_list_file(fname, t) + local f, err = io.open(fname, 'r') + if not f then + texio.write_nl('package spelling: Error! Can\'t parse plain word list: file ' .. fname) + error(err) + end + -- Read complete plain file into string, to speed-up file operations. + local s = f:read('*all') + f:close() local total_c = 0 local new_c = 0 - local f, msg = io.open(fname, 'r') - if f then - -- Iterate line-wise through file. - for l in f:lines() do - -- Map string to boolean value `true`. - if not t[l] then - t[l] = true - new_c = new_c + 1 - end - total_c = total_c + 1 + -- Iterate line-wise through file. + for l in s:gmatch('[^\r\n]+') do + -- Map string to boolean value `true`. + if not t[l] then + t[l] = true + new_c = new_c + 1 end - else - texio.write_nl('package spelling: Warning! Could not open file: ' .. msg) + total_c = total_c + 1 end return total_c, new_c end ---- Read a list of bad strings from a file. --- All strings read from the given file (words with known incorrect --- spelling) are mapped to the boolean value `true` in table `__is_bad`. +--- Parse a plain list of bad strings read from a file. +-- All strings found (words with known incorrect spelling) are mapped to +-- the boolean value `true` in table `__is_bad`. The format of the +-- input file is one string per line. -- --- @param fname Name of file containing bad strings. If an empty string --- is provided, strings are read from file `.bad`. -local function read_bad_strings(fname) - -- If file name is empty, set default file name. - if fname == '' then - fname = tex.jobname .. '.bad' - end - local total, new = __read_strings(fname, __is_bad) +-- @param fname File name. +local function parse_bad_plain_list_file(fname) + local total, new = __parse_plain_list_file(fname, __is_bad) texio.write_nl('package spelling: ' .. total .. ' bad strings (' .. new .. ' new) read from file \'' .. fname .. '\'.') end -M.read_bad_strings = read_bad_strings +M.parse_bad_plain_list_file = parse_bad_plain_list_file ---- Read a list of good strings from a file. --- All strings read from the given file (words with known correct --- spelling) are mapped to the boolean value `true` in table --- `__is_good`. +--- Parse a plain list of good strings read from a file. +-- All strings found (words with known correct spelling) are mapped to +-- the boolean value `true` in table `__is_good`. The format of the +-- input file is one string per line. -- --- @param fname Name of file containing good strings. If an empty --- string is provided, strings are read from file `.good`. -local function read_good_strings(fname) - -- If file name is empty, set default file name. - if fname == '' then - fname = tex.jobname .. '.good' - end - local total, new = __read_strings(fname, __is_good) +-- @param fname File name. +local function parse_good_plain_list_file(fname) + local total, new = __parse_plain_list_file(fname, __is_good) texio.write_nl('package spelling: ' .. total .. ' good strings (' .. new .. ' new) read from file \'' .. fname .. '\'.') end -M.read_good_strings = read_good_strings +M.parse_good_plain_list_file = parse_good_plain_list_file + + +--- Parse LanguageTool XML data. +-- Currently, XML data is only scanned for incorrect spellings. All +-- strings found in the given XML data (words with known incorrect +-- spelling) are mapped to the boolean value `true` in table `__is_bad`. +-- +-- @param s String containing XML data. XML data is checked for being +-- created by LanguageTool (via attribute software in tag +-- matches) and otherwise ignored. +-- @return Number of total and new incorrect spellings parsed. +local function __parse_XML_LanguageTool(s) + local total_c = 0 + local new_c = 0 + + -- Some flags for checking validity of XML data. LanguageTool XML + -- data must declare as being UTF-8 encoded and advertise as being + -- created by LanguageTool. + local is_XML_encoding_UTF_8 = false + local is_XML_creator_LanguageTool = false + local is_XML_valid = false + + --- Handler object for parsing LanguageTool XML data. + -- This table contains call-backs used by LuaXML when parsing XML + -- data. + -- + -- @class table + -- @name XML_handler + -- @field decl Handle XML declaration. + -- @field starttag Handle all relevant tags. + -- @field endtag Not used, but mandatory. + local XML_handler = { + + decl = function(self, text, attr) + -- Check XML encoding declaration. + if attr.encoding == 'UTF-8' then + is_XML_encoding_UTF_8 = true + else + error('package spelling: Error! XML data not in the UTF-8 encoding.') + end + end, + + starttag = function(self, text, attr) + -- Process tag. + if text == 'matches' then + -- Check XML creator is LanguageTool. + if attr and attr.software == 'LanguageTool' then + is_XML_creator_LanguageTool = true + is_XML_valid = is_XML_encoding_UTF_8 and is_XML_creator_LanguageTool + end + -- Process tags. + elseif is_XML_valid and text == 'error' then + local ruleid = attr.ruleid + if ruleid == 'HUNSPELL_RULE' + or ruleid == 'HUNSPELL_NO_SUGGEST_RULE' + or ruleid == 'GERMAN_SPELLER_RULE' + or string.find(ruleid, '^MORFOLOGIK_RULE_') + then + -- Extract misspelled word from context attribute. + local word = unicode.utf8.sub(attr.context, attr.contextoffset + 1, attr.contextoffset + attr.errorlength) + if not __is_bad[word] then + __is_bad[word] = true + new_c = new_c + 1 + end + total_c = total_c + 1 + end + end + end, + + endtag = function(self, text) + end, + + } + + -- Create custom XML parser. + local x = xml.xmlParser(XML_handler) + -- Parse XML data. + x:parse(s) + -- Check XML data is created by LanguageTool. + if not is_XML_creator_LanguageTool then + error('package spelling: Error! XML data not created by LanguageTool.') + end + return total_c, new_c +end + + +--- Parse LanguageTool XML data read from a file. +-- All strings found in the file (words with known incorrect spelling) +-- are mapped to the boolean value `true` in table `__is_bad`. +-- +-- @param fname File name. +local function parse_XML_LanguageTool_file(fname) + local f, err = io.open(fname, 'r') + if not f then + texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname) + error(err) + end + -- Read complete XML file into string, since LuaXML has no streaming + -- file reader. + local s = f:read('*all') + f:close() + local success, total, new = pcall(__parse_XML_LanguageTool, s) + if not success then + texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname) + error(total) + end + texio.write_nl('package spelling: ' .. total .. ' bad strings (' + .. new .. ' new) read from file \'' .. fname .. '\'.') +end +M.parse_XML_LanguageTool_file = parse_XML_LanguageTool_file + + +--- Parse default sources for bad and good strings. +-- All strings found in default sources for words with known incorrect +-- spelling are mapped to the boolean value `true` in table `__is_bad`. +-- All strings found in default sources for words with known correct +-- spelling are mapped to the boolean value `true` in table `__is_good`. +-- Default sources for bad spellings are file `.spell.bad` (a +-- plain list file). Default sources for good spellings are file +-- `.spell.good` (a plain list file). +local function parse_default_bad_and_good() + local fname, f + -- Try to read bad spellings from plain list file + -- '.spell.bad'. + fname = tex.jobname .. '.spell.bad' + f = io.open(fname, 'r') + if f then + f:close() + parse_bad_plain_list_file(fname) + end + -- Try to read bad spellings from LanguageTool XML file + -- '.spell.xml'. + fname = tex.jobname .. '.spell.xml' + f = io.open(fname, 'r') + if f then + f:close() + parse_XML_LanguageTool_file(fname) + end + -- Try to read good spellings from plain list file + -- '.spell.good'. + fname = tex.jobname .. '.spell.good' + f = io.open(fname, 'r') + if f then + f:close() + parse_good_plain_list_file(fname) + end +end +M.parse_default_bad_and_good = parse_default_bad_and_good -- Return module table. diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua index 6f46654cb41..5db44d41634 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua @@ -1,5 +1,5 @@ --- spelling-stage-2.lua ---- Copyright 2012 Stephan Hennig +--- Copyright 2012, 2013 Stephan Hennig -- -- This work may be distributed and/or modified under the conditions of -- the LaTeX Project Public License, either version 1.3 of this license @@ -20,8 +20,8 @@ -- string. Tagging is applied before hyphenation takes place. -- -- @author Stephan Hennig --- @copyright 2012 Stephan Hennig --- @release version 0.2 +-- @copyright 2012, 2013 Stephan Hennig +-- @release version 0.3 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua index bfb0c183e7c..dfee2fed283 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua @@ -1,5 +1,5 @@ --- spelling-stage-3.lua ---- Copyright 2012 Stephan Hennig +--- Copyright 2012, 2013 Stephan Hennig -- -- This work may be distributed and/or modified under the conditions of -- the LaTeX Project Public License, either version 1.3 of this license @@ -24,8 +24,8 @@ -- letters. -- -- @author Stephan Hennig --- @copyright 2012 Stephan Hennig --- @release version 0.2 +-- @copyright 2012, 2013 Stephan Hennig +-- @release version 0.3 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua index ee71c03a081..734d3006098 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua @@ -1,5 +1,5 @@ --- spelling-stage-4.lua ---- Copyright 2012 Stephan Hennig +--- Copyright 2012, 2013 Stephan Hennig -- -- This work may be distributed and/or modified under the conditions of -- the LaTeX Project Public License, either version 1.3 of this license @@ -18,8 +18,8 @@ -- document data structure to a file at the end of a LuaTeX run. -- -- @author Stephan Hennig --- @copyright 2012 Stephan Hennig --- @release version 0.2 +-- @copyright 2012, 2013 Stephan Hennig +-- @release version 0.3 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. @@ -144,7 +144,7 @@ end -- local function __write_text_document() -- Open output file. - local fname = __opts.output_file_name or (tex.jobname .. '.txt') + local fname = __opts.output_file_name or (tex.jobname .. '.spell.txt') local f = assert(io.open(fname, 'wb')) local eol = __opts.output_eol -- Iterate through document paragraphs. diff --git a/Master/texmf-dist/scripts/spelling/spelling.lua b/Master/texmf-dist/scripts/spelling/spelling.lua index d16688ce0be..3bcc0f6df36 100755 --- a/Master/texmf-dist/scripts/spelling/spelling.lua +++ b/Master/texmf-dist/scripts/spelling/spelling.lua @@ -1,5 +1,5 @@ --- spelling.lua ---- Copyright 2012 Stephan Hennig +--- Copyright 2012, 2013 Stephan Hennig -- -- This work may be distributed and/or modified under the conditions of -- the LaTeX Project Public License, either version 1.3 of this license @@ -12,6 +12,14 @@ -- +--- Main Lua file. +-- +-- @author Stephan Hennig +-- @copyright 2012, 2013 Stephan Hennig +-- @release version 0.3 +-- + + --- Global table of modules. -- The work of the spelling package can be separated into four -- stages:
@@ -20,8 +28,8 @@ -- --
Stage 1
--
    ---
  • Load list(s) of bad strings.
  • ---
  • Load list(s) of good strings.
  • +--
  • Load bad strings.
  • +--
  • Load good strings.
  • --
-- --
Stage 2 (call-back pre_linebreak_filter)
@@ -64,7 +72,7 @@ -- @name pkg_spelling_stage pkg_spelling_stage = { - -- string list loading + -- bad and good string loading [1] = require 'spelling-stage-1', -- node list tagging -- spell-checking -- cgit v1.2.3