From fed22753adeb474e7470dbe7c96326fd6bcfbc27 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Fri, 24 May 2013 00:18:09 +0000 Subject: spelling (23may13) git-svn-id: svn://tug.org/texlive/trunk@30665 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/luatex/spelling/CHANGES | 51 ++- Master/texmf-dist/doc/luatex/spelling/README | 19 +- .../doc/luatex/spelling/spelling-doc-lst-lua.tex | 84 ++++ .../doc/luatex/spelling/spelling-doc.bad | 4 +- .../doc/luatex/spelling/spelling-doc.pdf | Bin 106933 -> 127173 bytes .../doc/luatex/spelling/spelling-doc.tex | 465 ++++++++++++++------- .../texmf-dist/scripts/spelling/spelling-main.lua | 219 ++++++++++ .../scripts/spelling/spelling-recurse.lua | 2 +- .../scripts/spelling/spelling-stage-1.lua | 269 ++++++++---- .../scripts/spelling/spelling-stage-2.lua | 208 +++++++-- .../scripts/spelling/spelling-stage-3.lua | 46 +- .../scripts/spelling/spelling-stage-4.lua | 95 ++--- Master/texmf-dist/scripts/spelling/spelling.lua | 171 -------- Master/texmf-dist/tex/luatex/spelling/spelling.sty | 89 ++-- 14 files changed, 1115 insertions(+), 607 deletions(-) create mode 100644 Master/texmf-dist/doc/luatex/spelling/spelling-doc-lst-lua.tex create mode 100755 Master/texmf-dist/scripts/spelling/spelling-main.lua delete mode 100755 Master/texmf-dist/scripts/spelling/spelling.lua (limited to 'Master') diff --git a/Master/texmf-dist/doc/luatex/spelling/CHANGES b/Master/texmf-dist/doc/luatex/spelling/CHANGES index 24dd549ca4b..383a64dcb0f 100644 --- a/Master/texmf-dist/doc/luatex/spelling/CHANGES +++ b/Master/texmf-dist/doc/luatex/spelling/CHANGES @@ -2,21 +2,56 @@ This material is subject to the LaTeX Project Public License. See for the details of that license. +### v0.4 (2013-05-23) + +New features: + +* In addition to lists of bad and good spellings, words can be checked + against user-defined match rules to determine highlighting status. + +Changes: + +* Removed means to configure EOL character of text output file. + Standard Lua EOL character is always used, which is platform + dependent. + +* Improved compatibility with recent LuaTeX versions (v0.74 and newer). + +* File `.spell.xml` is loaded before file `.spell.bad` + (if both files exist). + +Fixes: + +* Bad words with surrounding punctuation haven't been highlighted. + Determining whether a word needs to be highlighted is now done by + checking against the lists of bad and good spellings words as they + appear in the document with possible punctuation as well as with all + surrounding punctuation stripped ([GitHub issue 8][ghi#8]). + +* Macro `\spellingoutputlinelength` was broken. + +* Raising an error when a file cannot be opened for reading causes + problems when compiling a document for the first time. Now, only a + warning is written to the console and log file in that case. + +[ghi#8]: https://github.com/sh2d/spelling/issues/8 + + ### v0.3 (2013-02-12) New: - * [LanguageTool][lt] support: LanguageTool error reports in the XML - format can be parsed for spelling errors (with the help of the - [LuaXML][luaxml] package). LanguageTool is a cross-platform style - and grammar checker. +* [LanguageTool][lt] support: LanguageTool error reports in the XML + format can be parsed for spelling errors (with the help of the + [LuaXML][luaxml] package). LanguageTool is a cross-platform style and + grammar checker. [lt]: http://www.languagetool.org/ [luaxml]: http://www.ctan.org/pkg/luaxml Changes: - * Default file names used by the package have been changed: +* Default file names used by the package have been changed: .spb => .spell.bad .spg => .spell.good @@ -27,10 +62,10 @@ Changes: Fixes: - * File `spelling.lua` could not be found by LaTeX style file - ([GitHub issue 14][#14]). +* File `spelling.lua` could not be found by LaTeX style file + ([GitHub issue 14][ghi#14]). -[#14]: https://github.com/sh2d/spelling/issues/14 +[ghi#14]: https://github.com/sh2d/spelling/issues/14 ### v0.1 (2012-11-30) diff --git a/Master/texmf-dist/doc/luatex/spelling/README b/Master/texmf-dist/doc/luatex/spelling/README index 6489095d30c..fa268331678 100644 --- a/Master/texmf-dist/doc/luatex/spelling/README +++ b/Master/texmf-dist/doc/luatex/spelling/README @@ -6,9 +6,9 @@ This material is subject to the LaTeX Project Public License. See ### Package information Package name: spelling -Summary description: Aid spell-checking LuaTeX documents. -Version: v0.3 -Date: 2013-02-12 +Summary description: support for spell-checking of LuaTeX documents +Version: v0.4 +Date: 2013-05-23 License: [LPPL v1.3c](http://www.latex-project.org/lppl/lppl-1-3c.html) Maintenance status: maintained Current maintainer: Stephan Hennig, @@ -17,11 +17,11 @@ Current maintainer: Stephan Hennig, ### Description -This package aids spell-checking of TeX documents compiled with the +This package supports spell-checking of TeX documents compiled with the LuaTeX engine. It can give visual feedback in PDF output similar to WYSIWYG word processors. The package relies on an external -spell-checker application to check spelling of a text file and to output -a list of bad spellings. The package should work with most +spell-checker application that can check a plain text file and output a +list of bad spellings. The package should work with most spell-checkers, even dumb, TeX-unaware ones. @@ -36,8 +36,8 @@ generated via luadoc -d API *.lua Bugs and a wish list can be found in the -[issue tracker](https://github.com/sh2d/spelling/issues/). -Contributions of any kind are warmly welcome! +[issue tracker](https://github.com/sh2d/spelling/issues/). Patches +welcome! _Happy TeXing!_ @@ -49,10 +49,11 @@ _Happy TeXing!_ CHANGES LICENSE README - spelling.lua spelling.sty spelling-doc.bad spelling-doc.tex + spelling-doc-lst-lua.tex + spelling-main.lua spelling-recurse.lua spelling-stage-1.lua spelling-stage-2.lua diff --git a/Master/texmf-dist/doc/luatex/spelling/spelling-doc-lst-lua.tex b/Master/texmf-dist/doc/luatex/spelling/spelling-doc-lst-lua.tex new file mode 100644 index 00000000000..4fee01d300a --- /dev/null +++ b/Master/texmf-dist/doc/luatex/spelling/spelling-doc-lst-lua.tex @@ -0,0 +1,84 @@ +%%% spelling-doc-lst-lua.tex +%%% Copyright 2013 Stephan Hennig +%% +%% This work may be distributed and/or modified under the conditions of +%% the LaTeX Project Public License, either version 1.3 of this license +%% or (at your option) any later version. The latest version of this +%% license is in http://www.latex-project.org/lppl.txt +%% and version 1.3 or later is part of all distributions of LaTeX +%% version 2005/12/01 or later. +%% +\lstdefinelanguage[5.2]{Lua}{% + alsoletter={.},% + % language keywords + morekeywords=[1]{% + and,break,do,else,elseif,end,% + false,for,function,goto,if,in,% + local,nil,not,or,repeat,return,% + then,true,until,while,% + },% + % standard library identifiers + morekeywords=[2]{% + % basic library + assert,collectgarbage,dofile,error,_G,getmetatable,ipairs,% + load,loadfile,next,pairs,pcall,print,rawequal,rawget,rawlen,rawset,% + select,setmetatable,tonumber,tostring,type,_VERSION,xpcall,% + % coroutine library + coroutine.create,coroutine.resume,coroutine.running,% + coroutine.status,coroutine.wrap,coroutine.yield,% + % package library + require,package.config,package.cpath,package.loaded,% + package.loadlib,package.path,package.preload,package.searchers,% + package.searchpath,% + % string library + string.byte,string.char,string.dump,string.find,string.format,% + string.gmatch,string.gsub,string.len,string.lower,string.match,% + string.rep,string.reverse,string.sub,string.upper,% + % table library + table.concat,table.insert,table.pack,table.remove,table.sort,% + table.unpack,% + % mathematical library + math.abs,math.acos,math.asin,math.atan,math.atan2,math.ceil,% + math.cos,math.cosh,math.deg,math.exp,math.floor,math.fmod,% + math.frexp,math.huge,math.ldexp,math.log,math.max,math.min,% + math.modf,math.pi,math.pow,math.rad,math.random,math.randomseed,% + math.sin,math.sinh,math.sqrt,math.tan,math.tanh,% + % bit library + bit32.arshift,bit32.band,bit32.bnot,bit32.bor,bit32.btest,% + bit32.bxor,bit32.extract,bit32.replace,bit32.lrotate,bit32.lshift,% + bit32.rrotate,bit32.rshift,% + % io library + io.close,io.flush,io.input,io.lines,io.open,io.output,io.popen,% + io.read,io.stderr,io.stdin,io.stdout,io.tmpfile,io.type,io.write,% + % os library + os.clock,os.date,os.difftime,os.execute,os.exit,os.getenv,% + os.remove,os.rename,os.setlocale,os.time,os.tmpname,% + % debug library + debug.debug,debug.gethook,debug.getinfo,debug.getlocal,% + debug.getmetatable,debug.getregistry,debug.getupvalue,% + debug.getuservalue,debug.sethook,debug.setlocal,debug.setmetatable,% + debug.setupvalue,debug.setuservalue,debug.traceback,% + debug.upvalueid,debug.upvaluejoin,% + },% + % add environment + morekeywords=[2]{_ENV},% + % + sensitive=true,% + % single line comments + morecomment=[l]{--},% + % multi line comments + morecomment=[s]{--[[}{]]},% + morecomment=[s]{--[=[}{]=]},% + morecomment=[s]{--[==[}{]==]},% + morecomment=[s]{--[===[}{]===]},% + % backslash escaped strings + morestring=[b]",% + morestring=[b]',% + % multi line strings + morestring=[s]{[[}{]]},% + morestring=[s]{[=[}{]=]},% + morestring=[s]{[==[}{]==]},% + morestring=[s]{[===[}{]===]},% + % labels + moredelim=[s][keywordstyle3]{::}{::},% +}[keywords,comments,strings]% diff --git a/Master/texmf-dist/doc/luatex/spelling/spelling-doc.bad b/Master/texmf-dist/doc/luatex/spelling/spelling-doc.bad index c5cc1638119..9e344863074 100644 --- a/Master/texmf-dist/doc/luatex/spelling/spelling-doc.bad +++ b/Master/texmf-dist/doc/luatex/spelling/spelling-doc.bad @@ -1,6 +1,4 @@ Ther -fotenoot -misspelllings +mispellings spellling foo’s -incorect diff --git a/Master/texmf-dist/doc/luatex/spelling/spelling-doc.pdf b/Master/texmf-dist/doc/luatex/spelling/spelling-doc.pdf index c1cacbf3c7f..442195e87a6 100644 Binary files a/Master/texmf-dist/doc/luatex/spelling/spelling-doc.pdf and b/Master/texmf-dist/doc/luatex/spelling/spelling-doc.pdf differ diff --git a/Master/texmf-dist/doc/luatex/spelling/spelling-doc.tex b/Master/texmf-dist/doc/luatex/spelling/spelling-doc.tex index 7a0c1bf3af6..88b9a74c5c7 100644 --- a/Master/texmf-dist/doc/luatex/spelling/spelling-doc.tex +++ b/Master/texmf-dist/doc/luatex/spelling/spelling-doc.tex @@ -1,4 +1,4 @@ -%%% spelling-doc.sty +%%% spelling-doc.tex %%% Copyright 2012, 2013 Stephan Hennig %% %% This work may be distributed and/or modified under the conditions of @@ -14,7 +14,9 @@ \usepackage{fontspec} \defaultfontfeatures{Ligatures=TeX} \usepackage{multicol} +\usepackage[rgb, x11names]{xcolor} \usepackage{listings} +\input{\jobname-lst-lua.tex} \lstset{ basicstyle=\ttfamily, columns=spaceflexible, @@ -26,11 +28,18 @@ language={[LaTeX]TeX}, basicstyle=\ttfamily, ]° +\lstdefinestyle{Lua}{ + language=[5.2]Lua, + keywordstyle=\bfseries\color{Blue4}, + keywordstyle=[2]\bfseries\color{RoyalBlue3}, + keywordstyle=[3]\bfseries\color{Purple3}, + stringstyle=\bfseries\color{Coral4}, + commentstyle=\itshape\color{Green4}, +} \usepackage{xspace} \usepackage{array} \usepackage{booktabs} \usepackage[latin, UKenglish]{babel} -\usepackage[rgb, x11names]{xcolor} \usepackage{hyperref} \hypersetup{ pdftitle={spelling}, @@ -59,17 +68,17 @@ basicstyle=\ttfamily, \newcommand*{\lpie}{\latinphrase{i.\,e.}\xspace} \begin{document} \author{Stephan Hennig\thanks{sh2d@arcor.de}} -\title{\pkg\thanks{This document describes the \pkg\ package v0.3.}} +\title{\pkg\thanks{This document describes the \pkg\ package v0.4.}} \maketitle \begin{abstract} - This package aids spell-checking of \TeX\ documents compiled with the - Lua\TeX\ engine. It can give visual feedback in \acr{pdf} output + This package supports spell-checking of \TeX\ documents compiled with + the Lua\TeX\ engine. It can give visual feedback in \acr{pdf} output similar to \acr{wysiwyg} word processors. The package relies on an - external spell-checker application to check spelling of a text file - and to output a list of bad spellings. The package should work with - most spell-checkers, even dumb, \TeX-unaware ones. + external spell-checker application that can check a plain text file + and output a list of bad spellings. The package should work with most + spell-checkers, even dumb, \TeX-unaware ones. \emph{Warning! This package is in a very early state. Everything may change!} @@ -89,9 +98,7 @@ basicstyle=\ttfamily, \label{sec:intro} Ther% -\footnote{A fotenoot containing misspelllings. But note how - `misspelllings' currently slips through due to punctuation. Some - annoying bugs are listed in \autoref{sec:bugs}.} +\footnote{A footnote containing mispellings.} % are three main approaches to spell-checking \TeX\ documents: @@ -113,9 +120,9 @@ package follows the third approach, providing some unique features: \item In traditional solutions, text is extracted from typeset \acr{dvi}, \acr{ps} or \acr{pdf} files, including hyphenated words. - Therefore, to avoid lots of false positives being reported by the - spell-checker, hyphenation has to be switched off during the \TeX\ - run. So, one doesn't work on the original document any more. + To avoid (lots of) false positives being reported by the + spell-checker, hyphenation needs to be switched off during the \TeX\ + run. That is, one doesn't work on the original document any more. In contrast to that, the \pkg\ package works transparently on the original |.tex| source file. Text is extracted \emph{during} @@ -140,94 +147,254 @@ described below, is effectively a wrapper around the Lua interface. \emph{Implementing such wrappers for other formats shouldn't be too difficult. The author is a \LaTeX-only user, though, and therefore grateful for contributions. By the way, the \LaTeX\ package needs - some polishing, too, \lpeg, a key-value interface is desirable.} + some polishing, too, \lpeg, a key-value interface is desirable. + Patches welcome!} \subsection{Work-flow} \label{sec:work-flow} -The work-flow of the \pkg\ package is as follows: +Here's a short outline of how using the \pkg\ package fits into the +general process of compiling a document with Lua\TeX: \begin{enumerate} -\item After the package is loaded in the preamble of a |.tex| source - file via °\usepackage{spelling}°, a list of bad spellings is read from - a file named \descr{jobname}|.spell.bad|, if that file exists. +\item After loading the package in the preamble of a |.tex| source file, + a list of bad spellings is read from a file (if that file exists). \item During the Lua\TeX\ run, text is extracted from pages and all words are checked against the list of bad spellings. Words with a known incorrect spelling are highlighted in \acr{pdf} output. \item At the end of the Lua\TeX\ run, in addition to the \acr{pdf} file, - a text file is written, named \descr{jobname}|.spell.txt|, by default. - The text file should contain most of the text of the original + a text file is written, containing most of the text of the typeset document. -\item The text file is then checked by your favourite spell-checker - application, \lpeg, Aspell or Hunspell. The spell-checker should be - able to write a list of bad spellings to a file. Otherwise, visual - feedback in \acr{pdf} output won't work. Preferably, the file is - named \descr{jobname}|.spell.bad|, but any other file name works just - as well. - -\item Now, there are two ways to proceed: - - \begin{enumerate} +\item The text file is then checked by your favourite external + spell-checker application, \lpeg, Aspell or Hunspell. The + spell-checker should be able to write a list of bad spellings to a + file. Otherwise, visual feedback in \acr{pdf} output won't work. - \item Visually minded people may just compile their document a second - time. This time, file \descr{jobname}|.spell.bad| is read-in again - and the words with incorrect spelling found by the spell-checker - should now be highlighted in \acr{pdf} output. Checking the - \acr{pdf} file, the necessary corrections to the |.tex| source file - can be applied. - - \item If you're not interested in visual feedback or if your - spell-checker doesn't provide a non-interactive mode, you can as - well apply the necessary corrections directly to the |.tex| source - file(s), either interactively, during the spell-checker run, or by - looking at the final list of bad spellings in an editor (whatever - file it was saved to). That way, the benefit of this package is, - that spell-checker input has already been processed by Lua\TeX, but - contains no hyphenated words. - - \end{enumerate} +\item Visually minded people may now compile their document a second + time. This time, the new list of bad spellings is read-in and words + with incorrect spelling found by the spell-checker should now be + highlighted in \acr{pdf} output. Users can then apply the necessary + corrections to the |.tex| source file. \end{enumerate} +Whatever way spell-checker output is employed, users not interested in +visual feedback (because their spell-checker has an interactive mode +only or because they prefer grabbing bad spellings from a file directly) +can also benefit from this package. Using it, Lua\TeX\ writes a pure +text file that is particularly well suited as spell-checker input, +because it contains no hyphenated words (and neither macros, nor active +characters). That way, any spell-checker application, even \TeX-unaware +ones, can be used to check spelling of \TeX\ documents. + \subsection{Word lists} \label{sec:wordlists} -As described above, if a file \descr{jobname}|.spell.bad| exists, it is -loaded by the \pkg\ package. The words found in the file are stored in -an internal list of bad spellings and are later used for highlighting -spelling mistakes in \acr{pdf} output. - -Additionally, a second file \descr{jobname}|.spell.good| is read, if -that file exists. The words found in that file are stored in an -internal list of good spellings. Words in the list of good spellings -are never highlighted in \acr{pdf} output. That is, words in the -Lua\TeX\ document are only considered spelling mistakes if they occur in -the list of bad spellings, but not in the list of good spellings. The -list of good spellings can be used to deal with false positives (words -incorrectly reported as bad spellings by the spell-checker). - -Words from additional files can be appended to the internal lists of bad -and good spellings with the \macro{spellingreadbad} and -\macro{spellingreadgood} commands. Argument to both macros is a file -name. File format is one word per line. Letter case is significant. -The file must be in the \acr{utf-8} encoding. As an example, the -command +As described above, after loading the \pkg\ package, a list of bad +spellings is read from a file \descr{jobname}.|spell.bad|, if that file +exists. Words found in this file are stored in an internal list of bad +spellings and are later used for highlighting spelling mistakes in +\acr{pdf} output. Additionally, a list of good spellings is read from a +file \descr{jobname}|.spell.good|, if that file exists. Words found in +the latter file are stored in an internal list of good spellings. File +format for both files is one word per line. Files must be in the +\acr{utf-8} encoding. Letter case is significant. + +A word in the document is highlighted, if it occurs in the internal list +of bad spellings, but not in the internal list of good spellings. That +is, known good spellings take precedence over known bad spellings. + +Users can load additional files containing lists of bad or good +spellings with macros \macro{spellingreadbad} and +\macro{spellingreadgood}. Argument to both macros is a file name. If a +file cannot be found, a warning is written to the console and |log| file +and compilation continues. As an example, the command \begin{lstlisting}[language={[LaTeX]TeX}] \spellingreadgood{myproject.whitelist} \end{lstlisting} % reads words from a file |myproject.whitelist| and adds them to the list -of good spellings. Note, most spell-checkers provide means to deal with -unknown words via additional dictionaries. It is recommended to -configure your spell-checker to report as few false positives as -possible. +of good spellings. + +Known good spellings can be used to deal with words wrongly reported as +bad spellings by the spell-checker (false positives). But note, most +spell-checkers also provide means to deal with unknown words via +additional dictionaries. It is recommended to configure your +spell-checker to report as few false positives as possible. + + +\subsection{Match rules} +\label{sec:matchrules} + +\emph{This section describes an advanced feature. You may safely skip + this section upon first reading.} + +The \pkg\ package provides an additional way to deal with bad and good +spellings, match rules. Match rules can be used to employ regular +patterns within certain ‘words’. A typical example are bibliographic +references like \emph{Lin86}, which are often flagged by spell-checkers, +but need not be highlighted as they are generated by \TeX. + +There are two kinds of rules, bad and good rules. A rule is a Lua +function whose boolean return value indicates whether a word +\emph{matches} the rule. A bad rule should return a true value for all +strings identified as bad spellings, otherwise a false value. A good +rule should return a true value for all strings identified as good +spellings, otherwise a false value. A word in the document is +highlighted if it matches any bad rule, but no good rule. + +Function arguments are a \emph{raw} string and a \emph{stripped} string. +The raw string is a string representing a word as it is found in the +document possibly surrounded by punctuation characters. The stripped +string is the same string with surrounding punctuation already stripped. + +As an example, the rule in \autoref{lst:mr-three-letter-words} matches +all words consisting of exactly three letters. The function matches the +stripped string against the Lua string pattern |^%a%a%a$| via function +|unicode.utf8.find| from the Selene Unicode library. The latter +function is a \acr{utf-8} capable version of Lua's built-in function +|string.find|. It returns |nil| (a false value) if there has been no +match and a number (a true value) if there has been a match. The string +|%a| represents a string class matching a single letter. Characters |^| +and |$| are anchors for the beginning and the end of the string in +question. Note, the pattern |%a%a%a| without anchors would match +any string which contains \emph{at least} three letters in a row. More +information about Lua string patterns can be found in the Lua reference +manual% +\footnote{\url{http://www.lua.org/manual/5.2/manual.html\#6.4}}% +% +, the Selene Unicode library documentation% +\footnote{\url{https://github.com/LuaDist/slnunicode/blob/master/unitest}} +% +and in the Unicode standard% +\footnote{\url{http://www.unicode.org/Public/4.0-Update1/UCD-4.0.1.html\#General_Category_Values}}% +. + +\suppressfloats[b] + +\begin{lstlisting}[style=Lua, float, label=lst:mr-three-letter-words, caption={Matching three-letter words.}] +function three_letter_words(raw, stripped) + return unicode.utf8.find(stripped, '^%a%a%a$') +end +\end{lstlisting} + +\autoref{lst:mr-double-punctuation} shows a rule matching all ‘words’ +containing double punctuation. + +\begin{lstlisting}[style=Lua, float, label=lst:mr-double-punctuation, caption={Matching double punctuation.}] +function double_punctuation(raw, stripped) + return unicode.utf8.find(raw, '%p%p') +end +\end{lstlisting} + +The rule in \autoref{lst:mr-bibtex-alpha} combines the results of three +string searches to match bibliographic references as generated by the +Bib\TeX\ \emph{alpha} style. + +\begin{lstlisting}[style=Lua, float, label=lst:mr-bibtex-alpha, caption={Matching references generated by the Bib\TeX\ alpha style.}] +function bibtex_alpha(raw, stripped) + return unicode.utf8.find(stripped, '^%u%l%l%d%d$') + or unicode.utf8.find(stripped, '^%u%u%u?%u?%d%d$') + or unicode.utf8.find(stripped, '^%u%u%u%+%d%d$') +end +\end{lstlisting} + +Match rules have to be provided by means of a Lua module. Such modules +can be loaded with the \macro{spellingmatchrules} command. Argument is +a module name. To tell bad rules from good rules, the table returned by +the module must follow this convention: Function identifiers +representing bad and good match rules are prefixed |bad_rule_| and +|good_rule_|, resp. The rest of an identifier is actually irrelevant. +Other and non-function identifiers are ignored. + +\autoref{lst:mr-module} shows an example module declaring the rules from +\autoref{lst:mr-three-letter-words} and +\autoref{lst:mr-double-punctuation} as \emph{bad} match rules and the +rule from \autoref{lst:mr-bibtex-alpha} as a \emph{good} match rule. +Note, how function identifiers are made local and how exported function +identifiers are prefixed |bad_rule_| and |good_rule_|, while local +function identifiers have no prefixes. When the module resides in a +file named |myproject.rules.lua|, it can be loaded in the preamble of a +document via +\begin{lstlisting}[language={[LaTeX]TeX}] +\spellingmatchrules{myproject.rules} +\end{lstlisting} + +\begin{lstlisting}[style=Lua, float=p, label=lst:mr-module, caption={A module containing two bad and one good match rule.}] +-- Module table. +local M = {} + +-- Import Selene Unicode library. +local unicode = require('unicode') +-- Add short-cut. +local Ufind = unicode.utf8.find + +local function three_letter_words(raw, stripped) + return Ufind(stripped, '^%a%a%a$') +end +M.bad_rule_three_letter_words = three_letter_words + +local function double_punctuation(raw, stripped) + return Ufind(raw, '%p%p') +end +M.bad_rule_double_punctuation = double_punctuation + +local function bibtex_alpha(raw, stripped) + return Ufind(stripped, '^%u%l%l%d%d$') + or Ufind(stripped, '^%u%u%u?%u?%d%d$') + or Ufind(stripped, '^%u%u%u%+%d%d$') +end +M.good_rule_bibtex_alpha = bibtex_alpha + +return M +\end{lstlisting} + +How are match rules and lists of bad and good spellings related? +Internally, the lists of bad and good spellings are referred to by two +special default match rules, that look-up raw and stripped strings and +return a true value if either argument has been found in the +corresponding list. Since good rules take precedence over bad rules, an +entry in the list of good spellings takes precedence over any +user-supplied bad rule. Likewise, any user-supplied good rule takes +precedence over an entry in the list of bad spellings. + +Some final remarks on match rules: + +\begin{itemize} + +\item It must be stressed that the boolean return value of a match rule + \emph{does not} indicate whether a spelling is bad or good, but + whether a word matches a certain rule or not. Whether it's a bad or a + good spelling, depends on the name of the match rule in the module + table. + +\item When written without care, match rules can easily produce false + positives as well as false negatives. While false positives in bad + rules and false negatives in good rules can easily be spotted due to + the unexpected highlighting of words, the other cases are more + problematic. To avoid all kinds of false results, match rules should + be stated as specific as possible. + +\item Match rules are only called upon the first occurrence of a + spelling in a document. The information, whether a spelling needs to + be highlighted, is stored in a cache table. Subsequent occurrences of + a spelling just need a table look-up to determine highlighting status. + For that reason, it is safe to do relatively expensive operations + within a match rule without affecting compilation time too much. + Nevertheless, match rules should be stated as efficient as possible.% + \footnote{Some Lua performance tips can be found in the book \emph{Lua + Programming Gems} by Figueiredo, Celes and Ierusalimschy + \emph{(eds.)}, 2008, ch.~2. That chapter is also available online + at \url{http://www.lua.org/gems/}.} + +\end{itemize} \subsection{Highlighting spellling mistakes} @@ -245,9 +412,7 @@ colour statement in the \acr{pdf} language. As an example, the colour red in the \acr{rgb} colour space is represented by the statement % |1 0 0 rg|. In the \acr{cmyk} colour space, a reddish colour is represented by |0 1 1 0 k|. Default colour used for highlighting is % -|1 0 0 rg|, \lpie, red in the \acr{rgb} colour space. \emph{Warning: - There's currently no error checking. Make sure, you're applying a - valid PDF colour statement!} +|1 0 0 rg|, \lpie, red in the \acr{rgb} colour space. \subsection{Text output} @@ -301,40 +466,11 @@ file name. Default text output file name is \paragraph{Line length} In text output, paragraphs can either be put on a single line or broken into lines of a fixed length. The behaviour can be controlled via command \macro{spellingoutputlinelength}. Argument is -a number. If the number is~0 or less, paragraphs are put on a single -line. For larger arguments, the number specifies the maximum line -length. Note, lines are broken at spaces only. Words longer than -maximum line length are put on a single line exceeding maximum line -length. Default line length is~72. - -\paragraph{Line ending convention} The end-of-line (\acr{eol}) character -in text output can be configured via command \macro{spellingoutputeol}. -Argument is an arbitrary sequence of characters in the \acr{utf-8} -encoding. - -Well, things are a bit more complicated, because in Lua\TeX, as in the -original \TeX, some characters are treated special. To keep Lua\TeX\ -from messing with our \acr{eol} characters in the input, we need to set -their category codes accordingly. As an example, to set \acr{eol} -character to follow \acr{dos} line ending convention (carriage return -followed by line feed, \TeX\ notation |^^M^^J|), the following code can -be used: - -\begin{lstlisting}[language={[LaTeX]TeX}] -\begingroup -\catcode`\^^J=12% make line feed and carriage return -\catcode`\^^M=12% of category Other -\spellingoutputeol{^^M^^J} -\endgroup -\end{lstlisting} -% -For the \acr{unix} line ending convention (a single line feed), just -leave out |^^M| in the argument to |\spellingoutputeol|. - -Default line ending convention depends on the operating system -determined by Lua\TeX. If |os.type| is either |windows| or |msdos|, -\acr{dos} line ending convention is used. Otherwise \acr{unix} line -ending convention is used. +a number. If the number is less than~1, paragraphs are put on a single +line. For larger arguments, the number specifies maximum line length. +Note, lines are broken at spaces only. Words longer than maximum line +length are put on a single line exceeding maximum line length. Default +line length is~72. \subsection{Text extraction} @@ -370,32 +506,45 @@ characters corresponding to their constituting letters. The default mappings applied can be found in \autoref{tab:cp-mapping}. \begin{table} - \centering - - \begin{tabular}{>{\ttfamily}l>{\ttfamily}l>{\ttfamily}l} - \normalfont Unicode name & \normalfont code point & \normalfont target characters\\ + \begin{minipage}{1.0\linewidth} + \centering + + \newcommand*{\coltitle}[2]{% + \normalfont% + \vbox{ + \hbox{\strut#1} + \hbox{\strut#2} + }% + } + + \begin{tabular}{>{\ttfamily}l>{\fontspec{Linux Libertine + O}}l>{\ttfamily}l>{\ttfamily}l} + \normalfont Unicode name & \coltitle{sample}{glyph\footnote{Sample + glyphs are taken from font \emph{Linux Libertine~O}.}} & + \coltitle{code}{point} & \coltitle{target}{characters}\\ \addlinespace \toprule \addlinespace - LATIN CAPITAL LIGATURE IJ & 0x0132 & IJ\\ - LATIN SMALL LIGATURE IJ & 0x0133 & ij\\ - LATIN CAPITAL LIGATURE OE & 0x0152 & OE\\ - LATIN SMALL LIGATURE OE & 0x0153 & oe\\ - LATIN SMALL LETTER LONG S & 0x017f & s\\ - LATIN CAPITAL LETTER SHARP S & 0x1e9e & SS\\ - LATIN SMALL LIGATURE FF & 0xfb00 & ff\\ - LATIN SMALL LIGATURE FI & 0xfb01 & fi\\ - LATIN SMALL LIGATURE FL & 0xfb02 & fl\\ - LATIN SMALL LIGATURE FFI & 0xfb03 & ffi\\ - LATIN SMALL LIGATURE FFL & 0xfb04 & ffl\\ - LATIN SMALL LIGATURE LONG S T & 0xfb05 & st\\ - LATIN SMALL LIGATURE ST & 0xfb06 & st\\ + LATIN CAPITAL LIGATURE IJ & ^^^^0132 & 0x0132 & IJ \\ + LATIN SMALL LIGATURE IJ & ^^^^0133 & 0x0133 & ij \\ + LATIN CAPITAL LIGATURE OE & ^^^^0152 & 0x0152 & OE \\ + LATIN SMALL LIGATURE OE & ^^^^0153 & 0x0153 & oe \\ + LATIN SMALL LETTER LONG S & ^^^^017f & 0x017f & s \\ + \addlinespace + LATIN SMALL LIGATURE FF & ^^^^fb00 & 0xfb00 & ff \\ + LATIN SMALL LIGATURE FI & ^^^^fb01 & 0xfb01 & fi \\ + LATIN SMALL LIGATURE FL & ^^^^fb02 & 0xfb02 & fl \\ + LATIN SMALL LIGATURE FFI & ^^^^fb03 & 0xfb03 & ffi \\ + LATIN SMALL LIGATURE FFL & ^^^^fb04 & 0xfb04 & ffl \\ + LATIN SMALL LIGATURE LONG S T & ^^^^fb05 & 0xfb05 & st \\ + LATIN SMALL LIGATURE ST & ^^^^fb06 & 0xfb06 & st \\ \end{tabular} \caption{Default code point mappings.} \label{tab:cp-mapping} + \end{minipage} \end{table} Additional mappings can be declared by command \macro{spellingmapping}. @@ -444,33 +593,33 @@ Installing spell-checkers and dictionaries can be a difficult task if there are no pre-built packages available for an architecture. That's one reason the \pkg\ package is rather spell-checker agnostic and the manual doesn't recommend a particular spell-checker application. -Another reason is, there's no best spell-checker. The only +Another reason is, there is no best spell-checker. The only recommendation the author makes is not to trust in one spell-checker, but to use multiple spell-checkers at the same time, with different dictionaries or, better yet, different checking engines under the hood. -Among the set of options available, LanguageTool, a style and grammar -checker% -\footnote{\url{http://www.languagetool.org/}} +Among the set of options available, LanguageTool% +\footnote{\url{http://www.languagetool.org/}}% % -that can also check spelling since version~1.8, deserves some notice for -its portability, ease of installation and active development. For these -reasons, the \pkg\ package provides explicit LanguageTool support. -LanguageTool uses Hunspell as the spell-checking engine, augmenting -results with a rule based engine and a morphological analyser (depending -on the language). The \pkg\ package can parse LanguageTool's error -reports in the \acr{xml} format, pick those errors that are spelling -related and use them to highlight bad spellings.% -\footnote{Support for style and grammar errors found by LanguageTool +, a style and grammar checker that can also check spelling since +version~1.8, deserves some notice for its portability, ease of +installation and active development. For these reasons, the \pkg\ +package provides explicit LanguageTool support. LanguageTool uses +Hunspell as the spell-checking engine, augmenting results with a rule +based engine and a morphological analyser (depending on the language). +The \pkg\ package can parse LanguageTool's error reports in the +\acr{xml} format, pick those errors that are spelling related and use +them to highlight bad spellings.% +\footnote{Highlighting style and grammar errors found by LanguageTool should be possible, but requires major restructuring of the \pkg\ - package that won't happen any time soon.} + package.} \subsection{Installation} \label{sec:lt-installation} Here are some brief installation instructions for the stand-alone -version of LanguageTool (tested with LanguageTool~2.0). The stand-alone +version of LanguageTool (tested with LanguageTool~2.1). The stand-alone version contains a \acr{gui} as well as a command-line interface. For the \pkg\ package, the latter is needed. @@ -494,10 +643,10 @@ java -version \item Uncompress the downloaded archive to a location of your choice. \item Open a command-line in the directory containing file - |LanguageTool.jar| and type + |languagetool-commandline.jar| and type \begin{lstlisting}[escapeinside=°°] -°\descr{path to}°/java -jar LanguageTool.jar --help +°\descr{path to}°/java -jar languagetool-commandline.jar --help \end{lstlisting} % Prepending the path to the Java executable is optional, depending on @@ -509,17 +658,17 @@ java -version \begin{itemize} - \item For users of Unixoid systems, the script might look like + \item For users of unixoide systems, the script might look like \begin{lstlisting}[escapeinside=°°] #!/bin/sh -°\descr{path to}°/java -jar °\descr{path to}°/LanguageTool.jar $* +°\descr{path to}°/java -jar °\descr{path to}°/languagetool-commandline.jar $* \end{lstlisting} % where \texttt{\descr{path to}} should point to the Java executable - (optional) and file |LanguageTool.jar| (mandatory). If the script - is named |lt.sh|, you should be able to run LanguageTool on the - command shell by typing, \lpeg, + (optional) and file |languagetool-commandline.jar| (mandatory). If + the script is named |lt.sh|, you should be able to run LanguageTool + on the command shell by typing, \lpeg, \begin{lstlisting} sh lt.sh --version @@ -533,13 +682,13 @@ sh lt.sh --version \begin{lstlisting}[escapeinside=°°] @echo off -°\descr{path to}°\java -jar °\descr{path to}°\LanguageTool.jar %* +°\descr{path to}°\java -jar °\descr{path to}°\languagetool-commandline.jar %* \end{lstlisting} % where \texttt{\descr{path to}} should point to the Java executable - (optional) and file |LanguageTool.jar| (mandatory). If the script - is named |lt.bat|, you should be able to run LanguageTool on the - command-line by typing, \lpeg, + (optional) and file |languagetool-commandline.jar| (mandatory). If + the script is named |lt.bat|, you should be able to run LanguageTool + on the command-line by typing, \lpeg, \begin{lstlisting} lt --version @@ -615,9 +764,9 @@ bad spellings and to use LanguageTool as a spell-checker, the following commands should be typed on the command-line: \begin{lstlisting} -lualatex myproject +lualatex myletter lt --api -c utf-8 -l fr myletter.spell.txt > myletter.spell.xml -lualatex myproject +lualatex myletter \end{lstlisting} @@ -652,12 +801,6 @@ The most user-visible issues are listed below: \item Text in head and foot lines is neither extracted nor highlighted (\href{https://github.com/sh2d/spelling/issues/7}{issue~7}). - \item Punctuation characters are currently not stripped from words. - For that reason, misspellings of words with leading or trailing - punctuation will currently slip through. This affects at least one - word per sentence, the last one - (\href{https://github.com/sh2d/spelling/issues/8}{issue~8}). - \item The first word starting right after an |hlist|, \lpeg, the first word within an |\mbox|, is never highlighted. It is extracted and written to the text file, though. This might affect acronyms, names @@ -672,7 +815,7 @@ The most user-visible issues are listed below: \end{itemize} -Any contributions are warmly welcome! +Patches welcome! \bigskip \emph{Happy \TeX ing!} diff --git a/Master/texmf-dist/scripts/spelling/spelling-main.lua b/Master/texmf-dist/scripts/spelling/spelling-main.lua new file mode 100755 index 00000000000..937054732e2 --- /dev/null +++ b/Master/texmf-dist/scripts/spelling/spelling-main.lua @@ -0,0 +1,219 @@ +--- spelling-main.lua +--- Copyright 2012, 2013 Stephan Hennig +-- +-- This work may be distributed and/or modified under the conditions of +-- the LaTeX Project Public License, either version 1.3 of this license +-- or (at your option) any later version. The latest version of this +-- license is in http://www.latex-project.org/lppl.txt +-- and version 1.3 or later is part of all distributions of LaTeX +-- version 2005/12/01 or later. +-- +-- See file README for more information. +-- + + +--- Main Lua file. +-- +-- @author Stephan Hennig +-- @copyright 2012, 2013 Stephan Hennig +-- @release version 0.4 +-- + + +-- Module identification. +if luatexbase.provides_module then + luatexbase.provides_module( + { + name = 'spelling', + date = '2013/05/23', + version = '0.4', + description = 'support for spell-checking of LuaTeX documents', + author = 'Stephan Hennig', + licence = 'LPPL ver. 1.3c', + } + ) +end + + +--- Global table of modules. +-- The work of the spelling package can be separated into four +-- stages:
+-- +--
+-- +--
Stage 1
+--
    +--
  • Load bad strings.
  • +--
  • Load good strings.
  • +--
  • Handle match rules.
  • +--
+-- +--
Stage 2 (call-back pre_linebreak_filter)
+--
    +--
  • Tag word strings in node lists before paragraph breaking +-- takes place.
  • +--
  • Check spelling of strings.
  • +--
  • Highlight strings with known incorrect spelling in PDF +-- output.
  • +--
+-- +--
Stage 3 (\AtBeginShipout)
+--
    +--
  • Store all strings found on built page via tag nodes in text +-- document data structure.
  • +--
+-- +--
Stage 4 (call-back stop_run)
+--
    +--
  • Output text stored in text document data structure to a +-- file.
  • +--
+-- +--
+-- +-- The code of the spelling package is organized in modules reflecting +-- these stages. References to modules are stored in a table. Table +-- indices correspond to the stages as shown above. The table of module +-- references is shared in a global table (`PKG_spelling`) so that +-- public module functions are accessible from within external code.
+-- +--
    +--
  • spelling-stage-1.lua : stage[1]
  • +--
  • spelling-stage-2.lua : stage[2]
  • +--
  • spelling-stage-3.lua : stage[3]
  • +--
  • spelling-stage-4.lua : stage[4]
  • +--
+-- +-- @class table +-- @name stage +stage = {} + + +--- Table of package-wide resources that are shared among several +--- modules. +-- +-- @class table +-- @name res +-- +-- @field rules_bad Table.
+-- +-- This table contains all bad rules. Spellings can be matched against +-- these rules. +-- +-- @field rules_good Table.
+-- +-- This table contains all good match rules. Spellings can be matched +-- against these rules. +-- +-- @field text_document Table.
+-- +-- Data structure that stores the text of a document. The text document +-- data structure stores the text of a document. The data structure is +-- quite simple. A text document is an ordered list (an array) of +-- paragraphs. A paragraph is an ordered list (an array) of words. A +-- word is a single UTF-8 encoded string.
+-- +-- During the LuTeX run, node lists are scanned for strings before +-- hyphenation takes place. The strings found in a node list are stored +-- in the current paragraph. After finishing scanning a node list, the +-- current paragraph is inserted into the text document. At the end of +-- the LuaTeX run, all paragraphs of the text document are broken into +-- lines of a fixed length and the lines are written to a file.
+-- +-- Here's the rationale of this approach: +-- +--
    +-- +--
  • It reduces file access during the LuaTeX run by delaying +-- write operations until the end. +-- +--
  • It saves space. In Lua, strings are internalized. Since in a +-- document, the same words are used over and over again, relatively +-- few strings are actually stored in memory. +-- +--
  • It allows for pre-processing the text document before writing it +-- to a file. +-- +--
+-- +-- @field whatsit_uid Number.
+-- +-- Unique ID for marking user-defined whatsit nodes created by this +-- package. The ID is generated at run-time. See this GitHub issue. +-- +local res = { + + rules_bad, + rules_good, + text_document, + whatsit_ids, + +} + + +--- Global package table. +-- This global table provides access to package-wide variables from +-- within other chunks. +-- +-- @class table +-- @name PKG_spelling +PKG_spelling = {} + + +--- Determine unique IDs for user-defined whatsit nodes used by this +-- package. Package luatexbase provides user-defined whatsit node ID +-- allocation since version v0.6 (TL 2013). For older package versions, +-- we start allocating at an arbitrary hard-coded value of 35**8 +-- (ca. 2**41). +-- +-- @return Table mapping names to IDs. +local function __allocate_whatsit_ids() + local ids = {} + -- Allocation support present? + if luatexbase.new_user_whatsit_id then + ids.start_tag = luatexbase.new_user_whatsit_id('start_tag', 'spelling') + ids.end_tag = luatexbase.new_user_whatsit_id('end_tag', 'spelling') + else + local uid = 35^8 + ids.start_tag = uid + 1 + ids.end_tag = uid + 2 + end + return ids +end + + +--- Package initialisation. +-- +local function __init() + -- Create resources. + res.rules_bad = {} + res.rules_good = {} + res.text_document = {} + res.whatsit_ids = __allocate_whatsit_ids() + -- Provide global access to package ressources during module loading. + PKG_spelling.res = res + -- Load sub-modules: + -- * bad and good string loading + -- * match rule handling + stage[1] = require 'spelling-stage-1' + -- * node list tagging + -- * spell-checking + -- * bad string highlighting + stage[2] = require 'spelling-stage-2' + -- * text storage + stage[3] = require 'spelling-stage-3' + -- * text output + stage[4] = require 'spelling-stage-4' + -- Remove global reference to package ressources. + PKG_spelling.res = nil + -- Provide global access to module references. + PKG_spelling.stage = stage + -- Enable text storage. + stage[3].enable_text_storage() +end + + +-- Initialize package. +__init() diff --git a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua index f07398d0483..4de903db36b 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-recurse.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-recurse.lua @@ -18,7 +18,7 @@ -- -- @author Stephan Hennig -- @copyright 2012, 2013 Stephan Hennig --- @release version 0.3 +-- @release version 0.4 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua index f8b548a7ac5..0f4afe9bb69 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-1.lua @@ -12,11 +12,11 @@ -- ---- Parse sources of bad and good strings. +--- Handle lists of bad and good strings and match rules. -- -- @author Stephan Hennig -- @copyright 2012, 2013 Stephan Hennig --- @release version 0.3 +-- @release version 0.4 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. @@ -28,11 +28,19 @@ module(...) local M = {} --- Load LuaXML module. +-- Import external modules. +local unicode = require('unicode') local xml = require('luaxml-mod-xml') -- Function short-cuts. +local Sfind = string.find + +local tabinsert = table.insert + +local Ufind = unicode.utf8.find +local Ugmatch = unicode.utf8.gmatch +local Usub = unicode.utf8.sub -- Declare local variables to store references to resources that are @@ -43,41 +51,50 @@ local __is_bad -- -- Table of known good strings. local __is_good +-- +-- Table of bad rules. +local __rules_bad +-- +-- Table of good rules. +local __rules_good ---- Set module resources. --- Make various resources, that are provided by external code, available --- to this module. +--- Generic function for reading bad or good spellings from a file. +-- All data from the file is read into a string, which is then parsed by +-- the given parse function. -- --- @param res Ressource table. -local function set_resources(res) - __is_bad = res.is_bad - __is_good = res.is_good +-- @param fname File name. +-- @param parse_string Custom parse function. +-- @param t Mapping table bad or good spellings should be added to. +-- @param hint String for info message. Either `bad` or `good`. +local function __parse_file(fname, parse_string, t, hint) + local total_c = 0 + local new_c = 0 + local f, err = io.open(fname, 'r') + if f then + local s = f:read('*all') + f:close() + total_c, new_c = parse_string(s, t) + else + texio.write_nl('package spelling: Warning! ' .. err) + end + texio.write_nl('package spelling: Info! ' .. total_c .. '/' .. new_c .. ' total/new ' .. hint .. ' strings read from file \'' .. fname .. '\'.') end -M.set_resources = set_resources ---- Generic function for parsing a plain list of strings read from a ---- file. --- All strings found are mapped to the boolean value `true`. The format --- of the input file is one string per line. +--- Generic function for parsing a string containing a plain list of +-- strings. Input format are strings separated by new line or carriage +-- return, i.e., one string per line. All lines found in the list are +-- mapped to the boolean value `true` in the given table. -- --- @param fname File name. +-- @param s Input string (a list of strings). -- @param t Table that maps strings to the value `true`. -- @return Number of total and new strings found. -local function __parse_plain_list_file(fname, t) - local f, err = io.open(fname, 'r') - if not f then - texio.write_nl('package spelling: Error! Can\'t parse plain word list: file ' .. fname) - error(err) - end - -- Read complete plain file into string, to speed-up file operations. - local s = f:read('*all') - f:close() +local function __parse_plain_list(s, t) local total_c = 0 local new_c = 0 - -- Iterate line-wise through file. - for l in s:gmatch('[^\r\n]+') do + -- Iterate line-wise through input string. + for l in Ugmatch(s, '[^\r\n]+') do -- Map string to boolean value `true`. if not t[l] then t[l] = true @@ -96,9 +113,7 @@ end -- -- @param fname File name. local function parse_bad_plain_list_file(fname) - local total, new = __parse_plain_list_file(fname, __is_bad) - texio.write_nl('package spelling: ' .. total .. ' bad strings (' - .. new .. ' new) read from file \'' .. fname .. '\'.') + __parse_file(fname, __parse_plain_list, __is_bad, 'bad') end M.parse_bad_plain_list_file = parse_bad_plain_list_file @@ -110,25 +125,24 @@ M.parse_bad_plain_list_file = parse_bad_plain_list_file -- -- @param fname File name. local function parse_good_plain_list_file(fname) - local total, new = __parse_plain_list_file(fname, __is_good) - texio.write_nl('package spelling: ' .. total .. ' good strings (' - .. new .. ' new) read from file \'' .. fname .. '\'.') + __parse_file(fname, __parse_plain_list, __is_good, 'good') end M.parse_good_plain_list_file = parse_good_plain_list_file ---- Parse LanguageTool XML data. --- Currently, XML data is only scanned for incorrect spellings. All --- strings found in the given XML data (words with known incorrect --- spelling) are mapped to the boolean value `true` in table `__is_bad`. +--- Get a custom LanguageTool XML handler. +-- The returned XML handler scans LanguageTool XML data for incorrect +-- spellings. For every incorrect spelling found, the given call-back +-- function is called with the incorrect spelling string as argument.
-- --- @param s String containing XML data. XML data is checked for being --- created by LanguageTool (via attribute software in tag --- matches) and otherwise ignored. --- @return Number of total and new incorrect spellings parsed. -local function __parse_XML_LanguageTool(s) - local total_c = 0 - local new_c = 0 +-- XML data is checked for being created by LanguageTool (via attribute +-- software in tag matches). +-- +-- @param cb Call-back function handling incorrect spellings found in +-- XML data. +-- @return XML handler. +local function __get_XML_handler_LanguageTool(cb) -- Some flags for checking validity of XML data. LanguageTool XML -- data must declare as being UTF-8 encoded and advertise as being @@ -152,6 +166,7 @@ local function __parse_XML_LanguageTool(s) -- Check XML encoding declaration. if attr.encoding == 'UTF-8' then is_XML_encoding_UTF_8 = true + is_XML_valid = is_XML_encoding_UTF_8 and is_XML_creator_LanguageTool else error('package spelling: Error! XML data not in the UTF-8 encoding.') end @@ -165,21 +180,20 @@ local function __parse_XML_LanguageTool(s) is_XML_creator_LanguageTool = true is_XML_valid = is_XML_encoding_UTF_8 and is_XML_creator_LanguageTool end + -- Check XML data is valid. + elseif not is_XML_valid then + error('package spelling: Error! No valid LanguageTool XML data.') -- Process tags. - elseif is_XML_valid and text == 'error' then + elseif text == 'error' then local ruleid = attr.ruleid if ruleid == 'HUNSPELL_RULE' or ruleid == 'HUNSPELL_NO_SUGGEST_RULE' or ruleid == 'GERMAN_SPELLER_RULE' - or string.find(ruleid, '^MORFOLOGIK_RULE_') + or Ufind(ruleid, '^MORFOLOGIK_RULE_') then -- Extract misspelled word from context attribute. - local word = unicode.utf8.sub(attr.context, attr.contextoffset + 1, attr.contextoffset + attr.errorlength) - if not __is_bad[word] then - __is_bad[word] = true - new_c = new_c + 1 - end - total_c = total_c + 1 + local word = Usub(attr.context, attr.contextoffset + 1, attr.contextoffset + attr.errorlength) + cb(word) end end end, @@ -189,14 +203,37 @@ local function __parse_XML_LanguageTool(s) } + return XML_handler +end + + +--- Parse a string containing LanguageTool XML data. +-- All incorrect spellings found in the given XML data are mapped to the +-- boolean value `true` in the given table. +-- +-- @param s String containing XML data. +-- @param t Table mapping incorrect spellings to a boolean. +-- @return Number of total and new incorrect spellings found. +local function __parse_XML_LanguageTool(s, t) + local total_c = 0 + local new_c = 0 + + -- Create call-back for custom LanguageTool XML handler that stores a + -- bad word in the given table and does some statistics. + local cb_incorrect_spelling = function(word) + if not t[word] then + t[word] = true + new_c = new_c + 1 + end + total_c = total_c + 1 + end + + -- Create custom XML handler. + local XML_handler_LT = __get_XML_handler_LanguageTool(cb_incorrect_spelling) -- Create custom XML parser. - local x = xml.xmlParser(XML_handler) + local x = xml.xmlParser(XML_handler_LT) -- Parse XML data. x:parse(s) - -- Check XML data is created by LanguageTool. - if not is_XML_creator_LanguageTool then - error('package spelling: Error! XML data not created by LanguageTool.') - end return total_c, new_c end @@ -207,22 +244,7 @@ end -- -- @param fname File name. local function parse_XML_LanguageTool_file(fname) - local f, err = io.open(fname, 'r') - if not f then - texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname) - error(err) - end - -- Read complete XML file into string, since LuaXML has no streaming - -- file reader. - local s = f:read('*all') - f:close() - local success, total, new = pcall(__parse_XML_LanguageTool, s) - if not success then - texio.write_nl('package spelling: Error! Can\'t parse LanguageTool XML error report: file ' .. fname) - error(total) - end - texio.write_nl('package spelling: ' .. total .. ' bad strings (' - .. new .. ' new) read from file \'' .. fname .. '\'.') + __parse_file(fname, __parse_XML_LanguageTool, __is_bad, 'bad') end M.parse_XML_LanguageTool_file = parse_XML_LanguageTool_file @@ -232,19 +254,12 @@ M.parse_XML_LanguageTool_file = parse_XML_LanguageTool_file -- spelling are mapped to the boolean value `true` in table `__is_bad`. -- All strings found in default sources for words with known correct -- spelling are mapped to the boolean value `true` in table `__is_good`. --- Default sources for bad spellings are file `.spell.bad` (a --- plain list file). Default sources for good spellings are file --- `.spell.good` (a plain list file). +-- Default sources for bad spellings are files `.spell.xml` (a +-- LanguageTool XML file) and `.spell.bad` (a plain list file). +-- Default sources for good spellings are file `.spell.good` (a +-- plain list file). local function parse_default_bad_and_good() local fname, f - -- Try to read bad spellings from plain list file - -- '.spell.bad'. - fname = tex.jobname .. '.spell.bad' - f = io.open(fname, 'r') - if f then - f:close() - parse_bad_plain_list_file(fname) - end -- Try to read bad spellings from LanguageTool XML file -- '.spell.xml'. fname = tex.jobname .. '.spell.xml' @@ -253,6 +268,14 @@ local function parse_default_bad_and_good() f:close() parse_XML_LanguageTool_file(fname) end + -- Try to read bad spellings from plain list file + -- '.spell.bad'. + fname = tex.jobname .. '.spell.bad' + f = io.open(fname, 'r') + if f then + f:close() + parse_bad_plain_list_file(fname) + end -- Try to read good spellings from plain list file -- '.spell.good'. fname = tex.jobname .. '.spell.good' @@ -265,5 +288,83 @@ end M.parse_default_bad_and_good = parse_default_bad_and_good +--- Default bad dictionary look-up match rule. +-- This function looks-up both arguments in the list of bad spellings. +-- It returns `true` if either of the arguments is found in the list of +-- bad spellings, otherwise `false`. +-- +-- @param raw Raw string to check. +-- @param stripped Same as `raw`, but with stripped surrounding +-- punctuation. +-- @return A boolean value indicating a match. +local function __bad_rule_bad_dictionary_lookup(raw, stripped) + return __is_bad[stripped] or __is_bad[raw] +end + + +--- Default good dictionary look-up match rule. +-- This function looks-up both arguments in the list of good spellings. +-- It returns `true` if either of the arguments is found in the list of +-- good spellings, otherwise `false`. +-- +-- @param raw Raw string to check. +-- @param stripped Same as `raw`, but with stripped surrounding +-- punctuation. +-- @return A boolean value indicating a match. +local function __good_rule_good_dictionary_lookup(raw, stripped) + return __is_good[stripped] or __is_good[raw] +end + + +--- Load match rule module. +-- Match rule modules are loaded using `require`. The module table must +-- follow the following convention: Indentifiers of bad match rules +-- start `bad_rule_`. Indentifiers of good match rules start +-- `good_rule_`. Other and non-function identifiers are ignore. +-- +-- All match rules found in a module are added to the table of bad and +-- good match rules. Arguments of a match rule function are a raw +-- string and the same string with stripped surrounding punctuation. +-- +-- @param fname Module file name. +local function read_match_rules(fname) + local bad_c = 0 + local good_c = 0 + local rules = require(fname) + for k,v in pairs(rules) do + if type(v) == 'function' then + if Sfind(k, '^bad_rule_') then + tabinsert(__rules_bad, v) + bad_c = bad_c + 1 + elseif Sfind(k, '^good_rule_') then + tabinsert(__rules_good, v) + good_c = good_c + 1 + end + end + end + texio.write_nl('package spelling: Info! ' .. bad_c .. '/' .. good_c .. ' bad/good match rules read from module \'' .. fname .. '\'.') +end +M.read_match_rules = read_match_rules + + +--- Module initialisation. +-- +local function __init() + -- Get local references to package ressources. + __rules_bad = PKG_spelling.res.rules_bad + __rules_good = PKG_spelling.res.rules_good + -- Add default dictionary look-up match rules. + tabinsert(__rules_bad, __bad_rule_bad_dictionary_lookup) + tabinsert(__rules_good, __good_rule_good_dictionary_lookup) + -- Create emtpy lists of known spellings. + __is_bad = {} + __is_good = {} +end + + +-- Initialize module. +__init() + + -- Return module table. return M diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua index 5db44d41634..1a397217a16 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua @@ -21,7 +21,7 @@ -- -- @author Stephan Hennig -- @copyright 2012, 2013 Stephan Hennig --- @release version 0.3 +-- @release version 0.4 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. @@ -33,20 +33,29 @@ module(...) local M = {} --- Import helper module. -local __recurse = require 'spelling-recurse' -local __recurse_node_list = __recurse.recurse_node_list +-- Import external modules. +local recurse = require('spelling-recurse') +local unicode = require('unicode') -- Function short-cuts. local tabconcat = table.concat local tabinsert = table.insert local tabremove = table.remove -local utf8char = unicode.utf8.char + local node_new = node.new local node_insert_after = node.insert_after local node_insert_before = node.insert_before +local recurse_node_list = recurse.recurse_node_list + +local Sfind = string.find +local Sgmatch = string.gmatch +local Smatch = string.match + +local Uchar = unicode.utf8.char +local Umatch = unicode.utf8.match + -- Short-cuts for constants. local DISC = node.id('disc') @@ -61,27 +70,17 @@ local PDF_COLORSTACK = node.subtype('pdf_colorstack') -- Declare local variables to store references to resources that are -- provided by external code. -- --- Table of known bad spellings. -local __is_bad +-- Table of bad rules. +local __rules_bad -- --- Table of known good spellings. -local __is_good +-- Table of good rules. +local __rules_good -- --- ID of user-defined whatsit nodes. -local __whatsit_uid - - ---- Set module resources. --- Make various resources, that are provided by external code, available --- to this module. +-- ID of user-defined whatsit nodes marking the start of a word. +local __uid_start_tag -- --- @param res Ressource table. -local function set_resources(res) - __is_bad = res.is_bad - __is_good = res.is_good - __whatsit_uid = res.whatsit_uid -end -M.set_resources = set_resources +-- ID of user-defined whatsit nodes marking the end of a word. +local __uid_end_tag --- Module options. @@ -98,18 +97,94 @@ local __opts = { --- Set colour used for highlighting. --- Set colourused for highlighting bad spellings in PDF output. +-- Set colour used for highlighting bad spellings in PDF output. The +-- argument is checked for a valid PDF colour statement. As an example, +-- the string `1 0 0 rg` represents a red colour in the RGB colour +-- space. A similar colour in the CMYK colour space would be +-- represented by the string '0 1 1 0 k'. -- --- @param col New colour. This must be a colour statement in PDF format --- given as string. As an example, the string `1 0 0 rg` represents a --- red colour in the RGB colour space. A similar colour in the CMYK --- colour space would be represented by the string '0 1 1 0 k'. +-- @param col New colour. local function set_highlight_color(col) - __opts.hl_color = col + -- Extract all colour components. + local components = Smatch(col, '^(%S+ %S+ %S+) rg$') or Smatch(col, '^(%S+ %S+ %S+ %S+) k$') + local is_valid_arg = components + if is_valid_arg then + -- Validate colour components. + for comp in Sgmatch(components, '%S+') do + -- Check number syntax. + local is_valid_comp = Sfind(comp, '^%d+%.?%d*$') or Sfind(comp, '^%d*%.?%d+$') + if is_valid_comp then + -- Check number range. + comp = tonumber(comp) + is_valid_comp = comp >= 0 and comp <= 1 + end + is_valid_arg = is_valid_arg and is_valid_comp + end + end + if is_valid_arg then + __opts.hl_color = col + else + error('package spelling: Error! Invalid PDF colour statement: ' .. tostring(col)) + end end M.set_highlight_color = set_highlight_color +--- Highlighting status cache table. +-- Determining the highlighting status of a string can be an expensive +-- operation. To reduce average run-time penalty per string, +-- highlighting status of all strings found in a document is cached in +-- this table, so that determining the highlighting status of a known +-- string requires only one table look-up.
+-- +-- This table needs an `__index` meta method calculating the +-- highlighting status of unknown keys (strings). +-- +-- @class table +-- @name __is_highlighting_needed +local __is_highlighting_needed = {} + + +--- Calculate and cache the highlighting status of a string. +-- First, surrounding punctuation is stripped from the string argument. +-- Then, the given raw as well as the stripped string are checked +-- against all rules. Highlighting of the string is required, if any +-- bad rule matches, but no good rule matches. That is, good rules take +-- precedence over bad rules. +-- +-- @param t Original table. +-- @param raw Raw string to check. +-- @return True, if highlighting is required. False, otherwise. +local function __calc_is_highlighting_needed(t, raw) + -- Strip surrounding punctuation from string. + local stripped = Umatch(raw, '^%p*(.-)%p*$') + -- Check for a bad match. + local is_bad = false + for _,matches_bad in ipairs(__rules_bad) do + is_bad = is_bad or matches_bad(raw, stripped) + if is_bad then break end + end + -- Check for a good match. + local is_good = false + for _,matches_good in ipairs(__rules_good) do + is_good = is_good or matches_good(raw, stripped) + if is_good then break end + end + -- Calculate highlighting status. + local status = (is_bad and not is_good) or false + -- Store status in cache table. + rawset(t, raw, status) + -- Return status. + return status +end + + +-- Set-up meta table for highlighting status cache table. +setmetatable(__is_highlighting_needed, { + __index = __calc_is_highlighting_needed, +}) + + --- Convert a Unicode code point to a regular UTF-8 encoded string. -- This function can be used as an `__index` meta method. -- @@ -117,7 +192,7 @@ M.set_highlight_color = set_highlight_color -- @param cp originl key, a Unicode code point -- @return UTF-8 encoded string corresponding to the Unicode code point. local function __meta_cp2utf8(t, cp) - return utf8char(cp) + return Uchar(cp) end @@ -146,8 +221,6 @@ local __codepoint_map = { [0x0153] = 'oe',-- LATIN SMALL LIGATURE OE [0x017f] = 's',-- LATIN SMALL LETTER LONG S - [0x1e9e] = 'SS',-- LATIN CAPITAL LETTER SHARP S - [0xfb00] = 'ff',-- LATIN SMALL LIGATURE FF [0xfb01] = 'fi',-- LATIN SMALL LIGATURE FI [0xfb02] = 'fl',-- LATIN SMALL LIGATURE FL @@ -250,7 +323,7 @@ local function __tag_word(word) local start_tag = node_new(WHATSIT, USER_DEFINED) -- Mark whatsit node with module ID, so that we can recognize it -- later. - start_tag.user_id = __whatsit_uid + start_tag.user_id = __uid_start_tag -- Value is an empty string. start_tag.type = 115 start_tag.value = '' @@ -261,7 +334,7 @@ local function __tag_word(word) local end_tag = node_new(WHATSIT, USER_DEFINED) -- Mark whatsit node with module ID, so that we can recognize it -- later. - end_tag.user_id = __whatsit_uid + end_tag.user_id = __uid_end_tag -- Value of end tag is an index (a number). end_tag.type = 115 end_tag.value = word @@ -274,6 +347,29 @@ end -- Insert colour whatsits before and after the first and last nodes -- known to belong to the current word. local function __highlight_by_color() + -- Check, if start node of current word is a head node. Inserting + -- before head nodes needs special attention. This is not yet + -- implemented. + if (__curr_word_start ~= __curr_word_start_head) then + -- Create pdf_colorstack whatsit nodes. + local push = node_new(WHATSIT, PDF_COLORSTACK) + local pop = node_new(WHATSIT, PDF_COLORSTACK) + push.stack = 0 + pop.stack = 0 + push.command = 1 + pop.command = 2 + push.data = __opts.hl_color + node_insert_before(__curr_word_start_head, __curr_word_start, push) + node_insert_after(__curr_word_end_head, __curr_word_end, pop) + end +end + + +--- Highlight bad spelling by colour (using node field `cmd`). +-- Insert colour whatsits before and after the first and last nodes +-- known to belong to the current word. +-- @see function __highlight_by_color +local function __highlight_by_color_cmd() -- Check, if start node of current word is a head node. Inserting -- before head nodes needs special attention. This is not yet -- implemented. @@ -332,13 +428,10 @@ local function __finish_current_word() if start_prev and end_next and (start_prev.id == WHATSIT) and (start_prev.subtype == USER_DEFINED) - and (start_prev.user_id == __whatsit_uid) - and (start_prev.type == 115) - and (start_prev.value == '') + and (start_prev.user_id == __uid_start_tag) and (end_next.id == WHATSIT) and (end_next.subtype == USER_DEFINED) - and (end_next.user_id == __whatsit_uid) - and (end_next.type == 115) + and (end_next.user_id == __uid_end_tag) and (end_next.value == word) then __curr_word = nil __curr_word_start_head = nil @@ -351,8 +444,8 @@ local function __finish_current_word() if __is_active_tagging then __tag_word(word) end - -- Test for bad spelling. - if __is_active_highlighting and __is_bad[word] and not __is_good[word] then + -- Highlighting needed? + if __is_highlighting_needed[word] and __is_active_highlighting then __highlight_bad_word() end __curr_word = nil @@ -475,7 +568,7 @@ local function __process_node_list(head) __curr_word_start = nil __curr_word_end_head = nil __curr_word_end = nil - __recurse_node_list(head, __cb_tag_words) + recurse_node_list(head, __cb_tag_words) -- Clean-up left-over word and/or paragraph. __finish_current_paragraph() end @@ -526,9 +619,40 @@ end M.disable_word_highlighting = disable_word_highlighting +--- Try to maintain compatibility with older LuaTeX versions. +-- Between LuaTeX 0.70.2 and 0.76.0 node field `cmd` of `whatsits` nodes +-- of subtype `pdf_colorstack` has been renamed to `command`. This +-- function checks which node field is the correct one and activates a +-- fall-back function in case. Due to a bug in LuaTeX 0.76.0 (shipped +-- with TL2013) function `node.has_field()` doesn't return correct +-- results. It is therefore tested if an assignment to field `command` +-- raises an error or not. +local function __maintain_compatibility() + -- Create pdf_colorstack whatsit node. + local n = node.new(WHATSIT, PDF_COLORSTACK) + -- Function that assigns a value to node field 'command'. + local f = function() + n.command = 1 + end + -- If the assignment is not successful, fall-back to node field 'cmd'. + if not pcall(f) then + __highlight_by_color = __highlight_by_color_cmd + end + -- Delete test node. + node.free(n) +end + + --- Module initialisation. -- local function __init() + -- Try to maintain compatibility with older LuaTeX versions. + __maintain_compatibility() + -- Get local references to package ressources. + __rules_bad = PKG_spelling.res.rules_bad + __rules_good = PKG_spelling.res.rules_good + __uid_start_tag = PKG_spelling.res.whatsit_ids.start_tag + __uid_end_tag = PKG_spelling.res.whatsit_ids.end_tag -- Create empty paragraph management stack. __is_vlist_paragraph = {} -- Remember tagging status. diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua index dfee2fed283..436fe6422fd 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-3.lua @@ -25,7 +25,7 @@ -- -- @author Stephan Hennig -- @copyright 2012, 2013 Stephan Hennig --- @release version 0.3 +-- @release version 0.4 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. @@ -37,12 +37,13 @@ module(...) local M = {} --- Import helper module. -local __recurse = require 'spelling-recurse' -local __recurse_node_list = __recurse.recurse_node_list +-- Import external modules. +local recurse = require('spelling-recurse') -- Function short-cuts. +local recurse_node_list = recurse.recurse_node_list + local tabinsert = table.insert local tabremove = table.remove @@ -59,20 +60,11 @@ local USER_DEFINED = node.subtype('user_defined') -- Text document data structure. local __text_document -- --- ID of user-defined whatsit nodes. -local __whatsit_uid - - ---- Set module resources. --- Make various resources, that are provided by external code, available --- to this module. +-- ID of user-defined whatsit nodes marking the start of a word. +local __uid_start_tag -- --- @param res Ressource table. -local function set_resources(res) - __text_document = res.text_document - __whatsit_uid = res.whatsit_uid -end -M.set_resources = set_resources +-- ID of user-defined whatsit nodes marking the end of a word. +local __uid_end_tag --- Module options. @@ -199,11 +191,8 @@ local function __visit_node(head, n) -- Test for node containing a word string. if nid == WHATSIT then -- Test for word string tag. - if (n.subtype == USER_DEFINED) and (n.user_id == __whatsit_uid) then - -- End tag? - if n.value ~= '' then - __finish_current_word(n) - end + if (n.subtype == USER_DEFINED) and (n.user_id == __uid_end_tag) then + __finish_current_word(n) -- Test for paragraph start. elseif n.subtype == LOCAL_PAR then __finish_current_paragraph() @@ -247,7 +236,7 @@ local __cb_store_words = { -- -- @param head Node list. local function __process_node_list(head) - __recurse_node_list(head, __cb_store_words) + recurse_node_list(head, __cb_store_words) -- Clean-up left-over word and/or paragraph. __finish_current_paragraph() end @@ -258,10 +247,11 @@ local __is_active_storage --- Call-back function that processes the node list. --- This is a global function! +-- This function is not made available in the module table, but in +-- the global package table! -- -- @param head Node list. -function cb_AtBeginShipout_pkg_spelling(box) +local function cb_AtBeginShipout(box) if __is_active_storage then __process_node_list(tex.box[box]) end @@ -288,6 +278,12 @@ M.disable_text_storage = disable_text_storage --- Module initialisation. -- local function __init() + -- Get local references to package ressources. + __text_document = PKG_spelling.res.text_document + __uid_start_tag = PKG_spelling.res.whatsit_ids.start_tag + __uid_end_tag = PKG_spelling.res.whatsit_ids.end_tag + -- Make \AtBeginShipout function available in package table. + PKG_spelling.cb_AtBeginShipout = cb_AtBeginShipout -- Create empty paragraph management stack. __is_vlist_paragraph = {} -- Remember call-back status. diff --git a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua index 734d3006098..ab4fabb22f7 100755 --- a/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua +++ b/Master/texmf-dist/scripts/spelling/spelling-stage-4.lua @@ -19,7 +19,7 @@ -- -- @author Stephan Hennig -- @copyright 2012, 2013 Stephan Hennig --- @release version 0.3 +-- @release version 0.4 -- -- @trick Prevent LuaDoc from looking past here for module description. --[[ Trick LuaDoc into entering 'module' mode without using that command. @@ -31,9 +31,15 @@ module(...) local M = {} +-- Import external modules. +local unicode = require('unicode') + + -- Function short-cuts. local tabconcat = table.concat -local utf8len = unicode.utf8.len +local tabinsert = table.insert + +local Ulen = unicode.utf8.len -- Declare local variables to store references to resources that are @@ -43,43 +49,20 @@ local utf8len = unicode.utf8.len local __text_document ---- Set module resources. --- Make various resources, that are provided by external code, available --- to this module. --- --- @param res Ressource table. -local function set_resources(res) - __text_document = res.text_document -end -M.set_resources = set_resources - - --- Module options. -- This table contains all module options. User functions to set -- options are provided. -- -- @class table -- @name __opts --- @field output_eol End-of-line character in output. -- @field output_file_name Output file name. -- @field output_line_length Line length in output. local __opts = { - output_eol, output_file_name, output_line_lenght, } ---- Set output EOL character. --- Text output will be written with the given end-of-line character. --- --- @param eol New output EOL character. -local function set_output_eol(eol) - __opts.output_eol = eol -end -M.set_output_eol = set_output_eol - - --- Set output file name. -- Text output will be written to a file with the given name. -- @@ -106,37 +89,39 @@ end M.set_output_line_length = set_output_line_length ---- Break a paragraph into lines of a fixed length and write the lines ---- to a file. +--- Break a text paragraph into lines. +-- Lines are broken at spaces only. Lines containing only one word may +-- exceed maximum line length. -- --- @param f A file handle. -- @param par A text paragraph (an array of words). -local function __write_text_paragraph(f, par) - local maxlinelength = __opts.output_line_length - local eol = __opts.output_eol +-- @param max_line_len Maximum length of lines in wrapped paragraph. If +-- the argument is less then 1, paragraph isn't wrapped at all. +-- @return Table containing the lines of the paragraph. +local function __wrap_text_paragraph(par, max_line_len) + local wrapped_par = {} -- Index of first word on current line. Initialize current line with -- first word of paragraph. - local lstart = 1 + local line_start = 1 -- Track current line length. - local llen = utf8len(par[1]) + local line_len = Ulen(par[line_start]) -- Iterate over remaining words in paragraph. for i = 2,#par do - local wlen = utf8len(par[i]) - -- Does word fit onto current line? - if (llen + 1 + wlen <= maxlinelength) or (maxlinelength < 1) then - -- Append word to current line. - llen = llen + 1 + wlen - else - -- Write the current line up to the preceeding word to file (words - -- separated by spaces and with a trailing newline). - f:write(tabconcat(par, ' ', lstart, i-1), eol) + local word_len = Ulen(par[i]) + local new_line_len = line_len + 1 + word_len + -- Maximum line length exceeded? + if new_line_len > max_line_len and max_line_len >= 1 then + -- Insert current line into wrapped paragraph. + tabinsert(wrapped_par, tabconcat(par, ' ', line_start, i-1)) -- Initialize new current line. - lstart = i - llen = wlen + line_start = i + new_line_len = word_len end + -- Append word to current line. + line_len = new_line_len end - -- Write last line of paragraph. - f:write(tabconcat(par, ' ', lstart), eol) + -- Insert last line of paragraph. + tabinsert(wrapped_par, tabconcat(par, ' ', line_start)) + return wrapped_par end @@ -145,14 +130,12 @@ end local function __write_text_document() -- Open output file. local fname = __opts.output_file_name or (tex.jobname .. '.spell.txt') - local f = assert(io.open(fname, 'wb')) - local eol = __opts.output_eol + local f = assert(io.open(fname, 'w')) + local max_line_len = __opts.output_line_length -- Iterate through document paragraphs. for _,par in ipairs(__text_document) do - -- Separate paragraphs by a blank line. - f:write(eol) - -- Write paragraph to file. - __write_text_paragraph(f, par) + -- Write wrapped paragraph to file with a leading empty line. + f:write('\n', tabconcat(__wrap_text_paragraph(par, max_line_len), '\n'), '\n') -- Delete paragraph from memory. __text_document[_] = nil end @@ -200,16 +183,12 @@ M.disable_text_output = disable_text_output --- Module initialisation. -- local function __init() + -- Get local references to package ressources. + __text_document = PKG_spelling.res.text_document -- Set default output file name. set_output_file_name(nil) -- Set default output line length. set_output_line_length(72) - -- Set default output EOL character. - if (os.type == 'windows') or (os.type == 'msdos') then - set_output_eol('\r\n') - else - set_output_eol('\n') - end -- Remember call-back status. __is_active_output = false end diff --git a/Master/texmf-dist/scripts/spelling/spelling.lua b/Master/texmf-dist/scripts/spelling/spelling.lua deleted file mode 100755 index 3bcc0f6df36..00000000000 --- a/Master/texmf-dist/scripts/spelling/spelling.lua +++ /dev/null @@ -1,171 +0,0 @@ ---- spelling.lua ---- Copyright 2012, 2013 Stephan Hennig --- --- This work may be distributed and/or modified under the conditions of --- the LaTeX Project Public License, either version 1.3 of this license --- or (at your option) any later version. The latest version of this --- license is in http://www.latex-project.org/lppl.txt --- and version 1.3 or later is part of all distributions of LaTeX --- version 2005/12/01 or later. --- --- See file README for more information. --- - - ---- Main Lua file. --- --- @author Stephan Hennig --- @copyright 2012, 2013 Stephan Hennig --- @release version 0.3 --- - - ---- Global table of modules. --- The work of the spelling package can be separated into four --- stages:
--- ---
--- ---
Stage 1
---
    ---
  • Load bad strings.
  • ---
  • Load good strings.
  • ---
--- ---
Stage 2 (call-back pre_linebreak_filter)
---
    ---
  • Tag word strings in node lists before paragraph breaking --- takes place.
  • ---
  • Check spelling of strings.
  • ---
  • Highlight strings with known incorrect spelling in PDF --- output.
  • ---
--- ---
Stage 3 (\AtBeginShipout)
---
    ---
  • Store all strings found on built page via tag nodes in text --- document data structure.
  • ---
--- ---
Stage 4 (call-back stop_run)
---
    ---
  • Output text stored in text document data structure to a --- file.
  • ---
--- ---
--- --- The code of the spelling package is organized in modules reflecting --- these stages. References to modules are made available in a global --- table so that module's public functions are accessible from within --- external code. Table indices correspond to the stages as shown --- above.
--- ---
    ---
  • spelling-stage-1.lua : pkg_spelling_stage[1]
  • ---
  • spelling-stage-2.lua : pkg_spelling_stage[2]
  • ---
  • spelling-stage-3.lua : pkg_spelling_stage[3]
  • ---
  • spelling-stage-4.lua : pkg_spelling_stage[4]
  • ---
--- --- @class table --- @name pkg_spelling_stage -pkg_spelling_stage = { - - -- bad and good string loading - [1] = require 'spelling-stage-1', - -- node list tagging - -- spell-checking - -- bad string highlighting - [2] = require 'spelling-stage-2', - -- text storage - [3] = require 'spelling-stage-3', - -- text output - [4] = require 'spelling-stage-4', - -} - - ---- Table of package-wide resources that are shared among several ---- modules. --- --- @class table --- @name res --- --- @field is_bad Table.
--- --- This table maps all strings known as bad spellings to the value --- `true`. --- --- @field is_good Table.
--- --- This table maps all strings known as good spellings to the value --- `true`. --- --- @field text_document Table.
--- --- Data structure that stores the text of a document. The text document --- data structure stores the text of a document. The data structure is --- quite simple. A text document is an ordered list (an array) of --- paragraphs. A paragraph is an ordered list (an array) of words. A --- word is a single UTF-8 encoded string.
--- --- During the LuTeX run, node lists are scanned for strings before --- hyphenation takes place. The strings found in a node list are stored --- in the current paragraph. After finishing scanning a node list, the --- current paragraph is inserted into the text document. At the end of --- the LuaTeX run, all paragraphs of the text document are broken into --- lines of a fixed length and the lines are written to a file.
--- --- Here's the rationale of this approach: --- ---
    --- ---
  • It reduces file access during the LuaTeX run by delaying --- write operations until the end. --- ---
  • It saves space. In Lua, strings are internalized. Since in a --- document, the same words are used over and over again, relatively --- few strings are actually stored in memory. --- ---
  • It allows for pre-processing the text document before writing it --- to a file. --- ---
--- --- @field whatsit_uid Number.
--- --- Unique ID for marking user-defined whatsit nodes created by this --- package. The ID is generated at run-time. See this GitHub issue. --- -local res = { - - is_bad, - is_good, - text_document, - whatsit_uid, - -} - - ---- Package initialisation. --- -local function __init() - -- Create resources. - res.is_bad = {} - res.is_good = {} - res.text_document = {} - res.whatsit_uid = 163 - -- Make resources available to modules. - pkg_spelling_stage[1].set_resources(res) - pkg_spelling_stage[2].set_resources(res) - pkg_spelling_stage[3].set_resources(res) - pkg_spelling_stage[4].set_resources(res) - -- Enable text storage. - pkg_spelling_stage[3].enable_text_storage() -end - - --- Initialize package. -__init() diff --git a/Master/texmf-dist/tex/luatex/spelling/spelling.sty b/Master/texmf-dist/tex/luatex/spelling/spelling.sty index d73661ee1ea..7b06b097c23 100644 --- a/Master/texmf-dist/tex/luatex/spelling/spelling.sty +++ b/Master/texmf-dist/tex/luatex/spelling/spelling.sty @@ -11,7 +11,7 @@ %% See file README for more information. %% \ProvidesPackage{spelling} - [2013/02/12 v0.3 Aid spell-checking (SH)] + [2013/05/23 v0.4 Support for spell-checking of LuaTeX documents (SH)] \NeedsTeXFormat{LaTeX2e}[1999/12/01] % Test for the LuaTeX engine. \RequirePackage{ifluatex} @@ -20,104 +20,103 @@ \PackageError{spelling}{LuaTeX engine required}{You could try with the `lualatex' command.} \fi -\RequirePackage{ifthen} -\RequirePackage{luatexbase} +% Lua module version management. +\RequirePackage{luatexbase-modutils} +% User-defined whatsit node ID allocation. +\RequirePackage{luatexbase-attr} % Load main Lua file. \directlua name {spelling}{ % The main Lua file is not a module, so we must pass a full path to % `dofile`. - local f = kpse.find_file('spelling.lua', 'lua') + local f = kpse.find_file('spelling-main.lua', 'lua') dofile(f) } % Words are extracted after pages have been built. In lack of a proper % Lua call-back, we're hooking into shipout from the LaTeX side. \RequirePackage{atbegshi} \AtBeginShipout{% - \directlua name {spelling-atbeginshipout}{cb_AtBeginShipout_pkg_spelling(\the\AtBeginShipoutBox)}% + \directlua name {spelling-atbeginshipout}{ + PKG_spelling.cb_AtBeginShipout(\the\AtBeginShipoutBox) + }% } % Provide command for reading-in a list of bad spellings. \newcommand*{\spellingreadbad}[1]{% \directlua{ - pkg_spelling_stage[1].parse_bad_plain_list_file('\luaescapestring{#1}') + PKG_spelling.stage[1].parse_bad_plain_list_file('\luaescapestring{#1}') }% } % Provide command for reading-in a list of good spellings. \newcommand*{\spellingreadgood}[1]{% \directlua{ - pkg_spelling_stage[1].parse_good_plain_list_file('\luaescapestring{#1}') + PKG_spelling.stage[1].parse_good_plain_list_file('\luaescapestring{#1}') }% } % Provide command for reading bad spellings from a LanguageTool XML % file. \newcommand*{\spellingreadLT}[1]{% \directlua{ - pkg_spelling_stage[1].parse_XML_LanguageTool_file('\luaescapestring{#1}') + PKG_spelling.stage[1].parse_XML_LanguageTool_file('\luaescapestring{#1}') + }% +} +% Provide command for reading match rules from a file. Argument must be +% a file name. +\newcommand*{\spellingmatchrules}[1]{% + \directlua{ + PKG_spelling.stage[1].read_match_rules('\luaescapestring{#1}') }% } % Provide command for enabling/disabling visual feedback. \newcommand*{\spellinghighlight}[1]{% - \ifthenelse{\equal{#1}{on}}{% - \directlua{ - pkg_spelling_stage[2].enable_word_highlighting() - }% - }{% - \directlua{ - pkg_spelling_stage[2].disable_word_highlighting() - }% + \directlua{ + if '\luaescapestring{#1}' == 'on' then + PKG_spelling.stage[2].enable_word_highlighting() + else + PKG_spelling.stage[2].disable_word_highlighting() + end }% } % Provide command for setting visual feedback colour. \newcommand*{\spellinghighlightcolor}[1]{% \directlua{ - pkg_spelling_stage[2].set_highlight_color('\luaescapestring{#1}') + PKG_spelling.stage[2].set_highlight_color('\luaescapestring{#1}') }% } % Provide command for enabling/disabling text ouput. \newcommand*{\spellingoutput}[1]{% - \ifthenelse{\equal{#1}{on}}{% - \directlua{ - pkg_spelling_stage[4].enable_text_output() - }% - }{% - \directlua{ - pkg_spelling_stage[4].disable_text_output() - }% + \directlua{ + if '\luaescapestring{#1}' == 'on' then + PKG_spelling.stage[4].enable_text_output() + else + PKG_spelling.stage[4].disable_text_output() + end }% } % Provide command for setting text output file name. \newcommand*{\spellingoutputname}[1]{% \directlua{ - pkg_spelling_stage[4].set_output_file_name('\luaescapestring{#1}') + PKG_spelling.stage[4].set_output_file_name('\luaescapestring{#1}') }% } % Provide command for setting text output file line length. \newcommand*{\spellingoutputlinelength}[1]{% \directlua{ - pkg_spelling_stage[4].set_output_line_length('\luaescapestring{#1}') - }% -} -% Provide command for setting text output EOL character. -\newcommand*{\spellingoutputeol}[1]{% - \directlua{ - pkg_spelling_stage[4].set_output_eol('\luaescapestring{#1}') + PKG_spelling.stage[4].set_output_line_length(\luaescapestring{#1}) }% } % Provide command for enabling/disabling text extraction. \newcommand*{\spellingextract}[1]{% - \ifthenelse{\equal{#1}{on}}{% - \directlua{ - pkg_spelling_stage[2].enable_text_tagging() - }% - }{% - \directlua{ - pkg_spelling_stage[2].disable_text_tagging() - }% + \directlua{ + if '\luaescapestring{#1}' == 'on' then + PKG_spelling.stage[2].enable_text_tagging() + else + PKG_spelling.stage[2].disable_text_tagging() + end }% } % Provide command to declare code point mappings. \newcommand*{\spellingmapping}[2]{% \directlua{ - local r = pkg_spelling_stage[2].set_mapping(\luaescapestring{#1}, '\luaescapestring{#2}') + local r = PKG_spelling.stage[2].set_mapping(\luaescapestring{#1}, '\luaescapestring{#2}') if r == false then texio.write_nl('package spelling: invalid mapping: \luaescapestring{#1} => \luaescapestring{#2}') end @@ -126,19 +125,19 @@ % Provide command to clear all existing code point mappings. \newcommand*{\spellingclearallmappings}{% \directlua{ - pkg_spelling_stage[2].clear_all_mappings() + PKG_spelling.stage[2].clear_all_mappings() }% } % Provide command to specify table paragraph behaviour. \newcommand*{\spellingtablepar}[1]{% \directlua{ - pkg_spelling_stage[3].set_table_paragraphs(\luaescapestring{#1}) + PKG_spelling.stage[3].set_table_paragraphs(\luaescapestring{#1}) }% } % % Read bad and good spellings from default sources. \directlua{ - pkg_spelling_stage[1].parse_default_bad_and_good() + PKG_spelling.stage[1].parse_default_bad_and_good() }% % % Enable visual feedback. -- cgit v1.2.3