--- spelling-stage-2.lua
--- Copyright 2012 Stephan Hennig
--
-- This work may be distributed and/or modified under the conditions of
-- the LaTeX Project Public License, either version 1.3 of this license
-- or (at your option) any later version. The latest version of this
-- license is in http://www.latex-project.org/lppl.txt
-- and version 1.3 or later is part of all distributions of LaTeX
-- version 2005/12/01 or later.
--
-- See file README for more information.
--
--- Tag node lists with word strings before hyphenation takes place.
-- This module provides means to tag node lists by inserting
-- user-defined whatsit nodes before and after first and last node
-- belonging to a chain representing a string in the node list. The
-- final tag node contains a reference to a string containing the word
-- string. Tagging is applied before hyphenation takes place.
--
-- @author Stephan Hennig
-- @copyright 2012 Stephan Hennig
-- @release version 0.2
--
-- @trick Prevent LuaDoc from looking past here for module description.
--[[ Trick LuaDoc into entering 'module' mode without using that command.
module(...)
--]]
-- Module table.
local M = {}
-- Import helper module.
local __recurse = require 'spelling-recurse'
local __recurse_node_list = __recurse.recurse_node_list
-- Function short-cuts.
local tabconcat = table.concat
local tabinsert = table.insert
local tabremove = table.remove
local utf8char = unicode.utf8.char
local node_new = node.new
local node_insert_after = node.insert_after
local node_insert_before = node.insert_before
-- Short-cuts for constants.
local DISC = node.id('disc')
local GLYPH = node.id('glyph')
local KERN = node.id('kern')
local WHATSIT = node.id('whatsit')
local LOCAL_PAR = node.subtype('local_par')
local USER_DEFINED = node.subtype('user_defined')
local PDF_COLORSTACK = node.subtype('pdf_colorstack')
-- Declare local variables to store references to resources that are
-- provided by external code.
--
-- Table of known bad spellings.
local __is_bad
--
-- Table of known good spellings.
local __is_good
--
-- ID of user-defined whatsit nodes.
local __whatsit_uid
--- Set module resources.
-- Make various resources, that are provided by external code, available
-- to this module.
--
-- @param res Ressource table.
local function set_resources(res)
__is_bad = res.is_bad
__is_good = res.is_good
__whatsit_uid = res.whatsit_uid
end
M.set_resources = set_resources
--- Module options.
-- This table contains all module options. User functions to set
-- options are provided.
--
-- @class table
-- @name __opts
-- @field hl_color Colour used for highlighting bad spellings in PDF
-- output.
local __opts = {
hl_color,
}
--- Set colour used for highlighting.
-- Set colourused for highlighting bad spellings in PDF output.
--
-- @param col New colour. This must be a colour statement in PDF format
-- given as string. As an example, the string `1 0 0 rg` represents a
-- red colour in the RGB colour space. A similar colour in the CMYK
-- colour space would be represented by the string '0 1 1 0 k'.
local function set_highlight_color(col)
__opts.hl_color = col
end
M.set_highlight_color = set_highlight_color
--- Convert a Unicode code point to a regular UTF-8 encoded string.
-- This function can be used as an `__index` meta method.
--
-- @param t original table
-- @param cp originl key, a Unicode code point
-- @return UTF-8 encoded string corresponding to the Unicode code point.
local function __meta_cp2utf8(t, cp)
return utf8char(cp)
end
--- Table of Unicode code point mappings.
-- This table maps Unicode code point to strings. The mappings are used
-- during text extraction to translate certain Unicode code points to an
-- arbitrary string instead of the corresponding UTF-8 encoded
-- character.
--
-- As an example, by adding an appropriate entry to this table, the
-- single Unicode code point U-fb00 (LATIN SMALL LIGATURE FF) can be
-- resolved into the multi character string 'ff' instead of being
-- converted to the single character string 'ff' ('ff').
--
-- Keys are Unicode code points. Values must be strings in the UTF-8
-- encoding. If a key is not present in this table, the regular UTF-8
-- character is returned by means of a meta table.
--
-- @class table
-- @name __codepoint_map
local __codepoint_map = {
[0x0132] = 'IJ',-- LATIN CAPITAL LIGATURE IJ
[0x0133] = 'ij',-- LATIN SMALL LIGATURE IJ
[0x0152] = 'OE',-- LATIN CAPITAL LIGATURE OE
[0x0153] = 'oe',-- LATIN SMALL LIGATURE OE
[0x017f] = 's',-- LATIN SMALL LETTER LONG S
[0x1e9e] = 'SS',-- LATIN CAPITAL LETTER SHARP S
[0xfb00] = 'ff',-- LATIN SMALL LIGATURE FF
[0xfb01] = 'fi',-- LATIN SMALL LIGATURE FI
[0xfb02] = 'fl',-- LATIN SMALL LIGATURE FL
[0xfb03] = 'ffi',-- LATIN SMALL LIGATURE FFI
[0xfb04] = 'ffl',-- LATIN SMALL LIGATURE FFL
[0xfb05] = 'st',-- LATIN SMALL LIGATURE LONG S T
[0xfb06] = 'st',-- LATIN SMALL LIGATURE ST
}
--- Meta table for code point mapping table.
--
-- @class table
-- @name __meta_codepoint_map
-- @field __index Index operator.
local __meta_codepoint_map = {
__index = __meta_cp2utf8,
}
-- Set meta table for code point mapping table.
setmetatable(__codepoint_map, __meta_codepoint_map)
--- Clear all code point mappings.
-- After calling this function, there are no known code point mappings
-- and no code point mapping takes place during text extraction.
local function clear_all_mappings()
__codepoint_map = {}
setmetatable(__codepoint_map, __meta_codepoint_map)
end
M.clear_all_mappings = clear_all_mappings
--- Manage Unicode code point mappings.
-- This function can be used to set-up code point mappings. First
-- argument must be a number, second argument must be a string in the
-- UTF-8 encoding or `nil`.
--
-- If the second argument is a string, after calling this function, the
-- Unicode code point given as first argument, when found in a node list
-- during text extraction, is mapped to the string given as second
-- argument instead of being converted to a UTF-8 encoded character
-- corresponding to the code point.
--
-- If the second argument is `nil`, a mapping for the given code point,
-- if existing, is deleted.
--
-- @param cp A Unicode code point, e.g., 0xfb00 for the code point LATIN
-- SMALL LIGATURE FF.
-- @param newt New target string to map the code point to or `nil`.
-- @return Old target string the code point was mapped to before
-- (possibly `nil`). If any arguments are invalid, return value is
-- `false`. Arguments are invalid if code point is not of type `number`
-- or not in the range 0 to 0x10ffff or if new target string is neither
-- of type `string` nor `nil`).
local function set_mapping(cp, newt)
-- Prevent from invalid entries in mapping table.
if (type(cp) ~= 'number') or
(cp < 0) or
(cp > 0x10ffff) or
((type(newt) ~= 'string') and (type(newt) ~= 'nil')) then
return false
end
-- Retrieve old mapping.
local oldt = rawget(__codepoint_map, cp)
-- Set new mapping.
__codepoint_map[cp] = newt
-- Return old mapping.
return oldt
end
M.set_mapping = set_mapping
-- First and last nodes known to belong to the current word and their
-- head nodes. These nodes are logged, so that after recognizing the
-- end of a word, they can be tagged by inserting new user-defined
-- whatsit nodes before and after them.
local __curr_word_start_head
local __curr_word_start
local __curr_word_end_head
local __curr_word_end
--- Tag the current word in the node list.
-- Insert tag nodes (user-defined whatsit nodes) into the node list
-- before and after the first and last nodes belonging to the current
-- word. The tag marking the start of a word contains as value an empty
-- string. The tag marking the end of a word contains as value a
-- reference to the word string.
--
-- @param word Word string.
local function __tag_word(word)
-- Check, if start node of current word is a head node. Inserting
-- before head nodes needs special attention. This is not yet
-- implemented.
if (__curr_word_start ~= __curr_word_start_head) then
-- Create new start tag node.
local start_tag = node_new(WHATSIT, USER_DEFINED)
-- Mark whatsit node with module ID, so that we can recognize it
-- later.
start_tag.user_id = __whatsit_uid
-- Value is an empty string.
start_tag.type = 115
start_tag.value = ''
-- Insert start tag before first node belonging to current word.
node_insert_before(__curr_word_start_head, __curr_word_start, start_tag)
end
-- Create new end tag node.
local end_tag = node_new(WHATSIT, USER_DEFINED)
-- Mark whatsit node with module ID, so that we can recognize it
-- later.
end_tag.user_id = __whatsit_uid
-- Value of end tag is an index (a number).
end_tag.type = 115
end_tag.value = word
-- Insert end tag after last node belonging to current word.
node_insert_after(__curr_word_end_head, __curr_word_end, end_tag)
end
--- Highlight bad spelling by colour.
-- Insert colour whatsits before and after the first and last nodes
-- known to belong to the current word.
local function __highlight_by_color()
-- Check, if start node of current word is a head node. Inserting
-- before head nodes needs special attention. This is not yet
-- implemented.
if (__curr_word_start ~= __curr_word_start_head) then
-- Create pdf_colorstack whatsit nodes.
local push = node_new(WHATSIT, PDF_COLORSTACK)
local pop = node_new(WHATSIT, PDF_COLORSTACK)
push.stack = 0
pop.stack = 0
push.cmd = 1
pop.cmd = 2
push.data = __opts.hl_color
node_insert_before(__curr_word_start_head, __curr_word_start, push)
node_insert_after(__curr_word_end_head, __curr_word_end, pop)
end
end
--- Generic function for highlighting bad spellings.
local function __highlight_bad_word()
__highlight_by_color()
end
-- Tagging status.
local __is_active_tagging
-- Highlighting status.
local __is_active_highlighting
--- Data structure that stores the characters of a word string.
-- The current word data structure is an ordered list (an array) of the
-- characters of the word. The characters are collected while scanning
-- a node list. They are concatenated to a single string only after the
-- end of a word is detected, before inserting the current word into the
-- current paragraph data structure.
--
-- @class table
-- @name __curr_word
local __curr_word
--- Act upon detection of end of current word string.
-- If the current word contains visible characters, store the current
-- word in the current tag.
local function __finish_current_word()
-- Finish a word?
if __curr_word then
local word = tabconcat(__curr_word)
-- Check, if the current word has already been tagged. This is only
-- a quick hack.
local start_prev = __curr_word_start.prev
local end_next = __curr_word_end.next
if start_prev and end_next
and (start_prev.id == WHATSIT)
and (start_prev.subtype == USER_DEFINED)
and (start_prev.user_id == __whatsit_uid)
and (start_prev.type == 115)
and (start_prev.value == '')
and (end_next.id == WHATSIT)
and (end_next.subtype == USER_DEFINED)
and (end_next.user_id == __whatsit_uid)
and (end_next.type == 115)
and (end_next.value == word) then
__curr_word = nil
__curr_word_start_head = nil
__curr_word_start = nil
__curr_word_end_head = nil
__curr_word_end = nil
return
end
-- Tag node list with word string.
if __is_active_tagging then
__tag_word(word)
end
-- Test for bad spelling.
if __is_active_highlighting and __is_bad[word] and not __is_good[word] then
__highlight_bad_word()
end
__curr_word = nil
end
__curr_word_start_head = nil
__curr_word_start = nil
__curr_word_end_head = nil
__curr_word_end = nil
end
--- Act upon detection of end of current paragraph.
-- If the current paragraph contains words, store the current paragraph
-- in the text document.
local function __finish_current_paragraph()
-- Finish current word.
__finish_current_word()
end
--- Paragraph management stack.
-- Stack of boolean flags, that are used for logging the occurence of a
-- new paragraph within nested vlists.
local __is_vlist_paragraph
--- Paragraph management.
-- This function puts a new boolean flag onto a stack that is used to
-- log the occurence of a new paragraph, while recursing into the coming
-- vlist. After finishing recursing into the vlist, the flag needs to
-- be removed from the stack. Depending on the flag, the then current
-- paragraph can be finished.
local function __vlist_pre_recurse()
tabinsert(__is_vlist_paragraph, false)
end
--- Paragraph management.
-- Remove flag from stack after recursing into a vlist. If necessary,
-- finish the current paragraph.
local function __vlist_post_recurse()
local p = tabremove(__is_vlist_paragraph)
if p then
__finish_current_paragraph()
end
end
--- Find paragraphs and strings.
-- While scanning a node list, this call-back function finds nodes
-- representing the start of a paragraph (local_par whatsit nodes) and
-- strings (chains of nodes of type glyph, kern, disc).
--
-- @param head Head node of current branch.
-- @param n The current node.
local function __visit_node(head, n)
local nid = n.id
-- Test for word string component node.
if nid == GLYPH then
-- Save first node belonging to current word and its head for later
-- reference.
if not __curr_word_start then
__curr_word_start_head = head
__curr_word_start = n
end
-- Save latest node belonging to current word and its head for later
-- reference.
__curr_word_end_head = head
__curr_word_end = n
-- Provide new empty word, if necessary.
if not __curr_word then
__curr_word = {}
end
-- Append character to current word string.
tabinsert(__curr_word, __codepoint_map[n.char])
-- Test for other word string component nodes.
elseif (nid == DISC) or (nid == KERN) then
-- We're still within the current word string. Do nothing.
-- Test for paragraph start.
elseif (nid == WHATSIT) and (n.subtype == LOCAL_PAR) then
__finish_current_paragraph()
__is_vlist_paragraph[#__is_vlist_paragraph] = true
else
-- End of current word string detected.
__finish_current_word()
end
end
--- Table of call-back functions for node list recursion: store the
--- word strings found in a node list.
-- The call-back functions in this table identify chains of nodes
-- representing word strings in a node list and stores the strings in
-- the text document. Local_par whatsit nodes are word boundaries.
-- Nodes of type `hlist` are recursed into as if they were non-existent.
-- As an example, the LaTeX input `a\mbox{a b}b` is recognized as two
-- strings `aa` and `bb`.
--
-- @class table
-- @name __cb_tag_words
-- @field vlist_pre_recurse Paragraph management.
-- @field vlist_post_recurse Paragraph management.
-- @field visit_node Find nodes representing paragraphs and words.
local __cb_tag_words = {
vlist_pre_recurse = __vlist_pre_recurse,
vlist_post_recurse = __vlist_post_recurse,
visit_node = __visit_node,
}
--- Process node list according to this stage.
-- This function recurses into the given node list, extracts all text
-- and stores it in the text document.
--
-- @param head Node list.
local function __process_node_list(head)
__curr_word_start_head = nil
__curr_word_start = nil
__curr_word_end_head = nil
__curr_word_end = nil
__recurse_node_list(head, __cb_tag_words)
-- Clean-up left-over word and/or paragraph.
__finish_current_paragraph()
end
--- Call-back function that processes the node list.
--
-- @param head Node list.
local function __cb_pre_linebreak_filter_pkg_spelling(head)
__process_node_list(head)
return head
end
--- Start tagging text.
-- After calling this function, words are tagged in node lists before
-- hyphenation takes place.
local function enable_text_tagging()
__is_active_tagging = true
end
M.enable_text_tagging = enable_text_tagging
--- Stop tagging text.
-- After calling this function, no more word tagging in node lists takes
-- place.
local function disable_text_tagging()
__is_active_tagging = false
end
M.disable_text_tagging = disable_text_tagging
--- Start highlighting bad spellings.
-- After calling this function, bad spellings are highlighted in PDF
-- output.
local function enable_word_highlighting()
__is_active_highlighting = true
end
M.enable_word_highlighting = enable_word_highlighting
--- Stop highlighting bad spellings.
-- After calling this function, no more bad spellings are highlighted in
-- PDF output.
local function disable_word_highlighting()
__is_active_highlighting = false
end
M.disable_word_highlighting = disable_word_highlighting
--- Module initialisation.
--
local function __init()
-- Create empty paragraph management stack.
__is_vlist_paragraph = {}
-- Remember tagging status.
__is_active_tagging = false
-- Remember highlighting status.
__is_active_highlighting = false
-- Set default highlighting colour.
set_highlight_color('1 0 0 rg')
-- Register call-back: Before TeX breaks a paragraph into lines, tag
-- and highlight strings.
luatexbase.add_to_callback('pre_linebreak_filter', __cb_pre_linebreak_filter_pkg_spelling, '__cb_pre_linebreak_filter_pkg_spelling')
end
-- Initialize module.
__init()
-- Return module table.
return M