summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts/spelling/spelling-stage-2.lua
blob: 6f46654cb41485fe0832e699a5e2365433ca4676 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
--- spelling-stage-2.lua
--- Copyright 2012 Stephan Hennig
--
-- This work may be distributed and/or modified under the conditions of
-- the LaTeX Project Public License, either version 1.3 of this license
-- or (at your option) any later version.  The latest version of this
-- license is in http://www.latex-project.org/lppl.txt
-- and version 1.3 or later is part of all distributions of LaTeX
-- version 2005/12/01 or later.
--
-- See file README for more information.
--


--- Tag node lists with word strings before hyphenation takes place.
-- This module provides means to tag node lists by inserting
-- user-defined whatsit nodes before and after first and last node
-- belonging to a chain representing a string in the node list.  The
-- final tag node contains a reference to a string containing the word
-- string.  Tagging is applied before hyphenation takes place.
--
-- @author Stephan Hennig
-- @copyright 2012 Stephan Hennig
-- @release version 0.2
--
-- @trick Prevent LuaDoc from looking past here for module description.
--[[ Trick LuaDoc into entering 'module' mode without using that command.
module(...)
--]]


-- Module table.
local M = {}


-- Import helper module.
local __recurse = require 'spelling-recurse'
local __recurse_node_list = __recurse.recurse_node_list


-- Function short-cuts.
local tabconcat = table.concat
local tabinsert = table.insert
local tabremove = table.remove
local utf8char = unicode.utf8.char
local node_new = node.new
local node_insert_after = node.insert_after
local node_insert_before = node.insert_before


-- Short-cuts for constants.
local DISC = node.id('disc')
local GLYPH = node.id('glyph')
local KERN = node.id('kern')
local WHATSIT = node.id('whatsit')
local LOCAL_PAR = node.subtype('local_par')
local USER_DEFINED = node.subtype('user_defined')
local PDF_COLORSTACK = node.subtype('pdf_colorstack')


-- Declare local variables to store references to resources that are
-- provided by external code.
--
-- Table of known bad spellings.
local __is_bad
--
-- Table of known good spellings.
local __is_good
--
-- ID of user-defined whatsit nodes.
local __whatsit_uid


--- Set module resources.
-- Make various resources, that are provided by external code, available
-- to this module.
--
-- @param res  Ressource table.
local function set_resources(res)
  __is_bad = res.is_bad
  __is_good = res.is_good
  __whatsit_uid = res.whatsit_uid
end
M.set_resources = set_resources


--- Module options.
-- This table contains all module options.  User functions to set
-- options are provided.
--
-- @class table
-- @name __opts
-- @field hl_color  Colour used for highlighting bad spellings in PDF
-- output.
local __opts = {
  hl_color,
}


--- Set colour used for highlighting.
-- Set colourused for highlighting bad spellings in PDF output.
--
-- @param col New colour.  This must be a colour statement in PDF format
-- given as string.  As an example, the string `1 0 0 rg` represents a
-- red colour in the RGB colour space.  A similar colour in the CMYK
-- colour space would be represented by the string '0 1 1 0 k'.
local function set_highlight_color(col)
  __opts.hl_color = col
end
M.set_highlight_color = set_highlight_color


--- Convert a Unicode code point to a regular UTF-8 encoded string.
-- This function can be used as an `__index` meta method.
--
-- @param t  original table
-- @param cp  originl key, a Unicode code point
-- @return UTF-8 encoded string corresponding to the Unicode code point.
local function __meta_cp2utf8(t, cp)
  return utf8char(cp)
end


--- Table of Unicode code point mappings.
-- This table maps Unicode code point to strings.  The mappings are used
-- during text extraction to translate certain Unicode code points to an
-- arbitrary string instead of the corresponding UTF-8 encoded
-- character.<br />
--
-- As an example, by adding an appropriate entry to this table, the
-- single Unicode code point U-fb00 (LATIN SMALL LIGATURE FF) can be
-- resolved into the multi character string 'ff' instead of being
-- converted to the single character string 'ff' ('&#xfb00;').<br />
--
-- Keys are Unicode code points.  Values must be strings in the UTF-8
-- encoding.  If a key is not present in this table, the regular UTF-8
-- character is returned by means of a meta table.<br />
--
-- @class table
-- @name __codepoint_map
local __codepoint_map = {

  [0x0132] = 'IJ',-- LATIN CAPITAL LIGATURE IJ
  [0x0133] = 'ij',-- LATIN SMALL LIGATURE IJ
  [0x0152] = 'OE',-- LATIN CAPITAL LIGATURE OE
  [0x0153] = 'oe',-- LATIN SMALL LIGATURE OE
  [0x017f] = 's',-- LATIN SMALL LETTER LONG S

  [0x1e9e] = 'SS',-- LATIN CAPITAL LETTER SHARP S

  [0xfb00] = 'ff',-- LATIN SMALL LIGATURE FF
  [0xfb01] = 'fi',-- LATIN SMALL LIGATURE FI
  [0xfb02] = 'fl',-- LATIN SMALL LIGATURE FL
  [0xfb03] = 'ffi',-- LATIN SMALL LIGATURE FFI
  [0xfb04] = 'ffl',-- LATIN SMALL LIGATURE FFL
  [0xfb05] = 'st',-- LATIN SMALL LIGATURE LONG S T
  [0xfb06] = 'st',-- LATIN SMALL LIGATURE ST

}


--- Meta table for code point mapping table.
--
-- @class table
-- @name __meta_codepoint_map
-- @field __index  Index operator.
local __meta_codepoint_map = {
   __index = __meta_cp2utf8,
}


-- Set meta table for code point mapping table.
setmetatable(__codepoint_map, __meta_codepoint_map)


--- Clear all code point mappings.
-- After calling this function, there are no known code point mappings
-- and no code point mapping takes place during text extraction.
local function clear_all_mappings()
  __codepoint_map = {}
  setmetatable(__codepoint_map, __meta_codepoint_map)
end
M.clear_all_mappings = clear_all_mappings


--- Manage Unicode code point mappings.
-- This function can be used to set-up code point mappings.  First
-- argument must be a number, second argument must be a string in the
-- UTF-8 encoding or `nil`.<br />
--
-- If the second argument is a string, after calling this function, the
-- Unicode code point given as first argument, when found in a node list
-- during text extraction, is mapped to the string given as second
-- argument instead of being converted to a UTF-8 encoded character
-- corresponding to the code point.<br />
--
-- If the second argument is `nil`, a mapping for the given code point,
-- if existing, is deleted.
--
-- @param cp A Unicode code point, e.g., 0xfb00 for the code point LATIN
-- SMALL LIGATURE FF.
-- @param newt  New target string to map the code point to or `nil`.
-- @return Old target string the code point was mapped to before
-- (possibly `nil`).  If any arguments are invalid, return value is
-- `false`.  Arguments are invalid if code point is not of type `number`
-- or not in the range 0 to 0x10ffff or if new target string is neither
-- of type `string` nor `nil`).
local function set_mapping(cp, newt)
  -- Prevent from invalid entries in mapping table.
  if (type(cp) ~= 'number') or
     (cp < 0) or
     (cp > 0x10ffff) or
     ((type(newt) ~= 'string') and (type(newt) ~= 'nil')) then
    return false
  end
  -- Retrieve old mapping.
  local oldt = rawget(__codepoint_map, cp)
  -- Set new mapping.
  __codepoint_map[cp] = newt
  -- Return old mapping.
  return oldt
end
M.set_mapping = set_mapping


-- First and last nodes known to belong to the current word and their
-- head nodes.  These nodes are logged, so that after recognizing the
-- end of a word, they can be tagged by inserting new user-defined
-- whatsit nodes before and after them.
local __curr_word_start_head
local __curr_word_start
local __curr_word_end_head
local __curr_word_end


--- Tag the current word in the node list.
-- Insert tag nodes (user-defined whatsit nodes) into the node list
-- before and after the first and last nodes belonging to the current
-- word.  The tag marking the start of a word contains as value an empty
-- string.  The tag marking the end of a word contains as value a
-- reference to the word string.
--
-- @param word  Word string.
local function __tag_word(word)
  -- Check, if start node of current word is a head node.  Inserting
  -- before head nodes needs special attention.  This is not yet
  -- implemented.
  if (__curr_word_start ~= __curr_word_start_head) then
    -- Create new start tag node.
    local start_tag = node_new(WHATSIT, USER_DEFINED)
    -- Mark whatsit node with module ID, so that we can recognize it
    -- later.
    start_tag.user_id = __whatsit_uid
    -- Value is an empty string.
    start_tag.type = 115
    start_tag.value = ''
    -- Insert start tag before first node belonging to current word.
    node_insert_before(__curr_word_start_head, __curr_word_start, start_tag)
  end
  -- Create new end tag node.
  local end_tag = node_new(WHATSIT, USER_DEFINED)
  -- Mark whatsit node with module ID, so that we can recognize it
  -- later.
  end_tag.user_id = __whatsit_uid
  -- Value of end tag is an index (a number).
  end_tag.type = 115
  end_tag.value = word
  -- Insert end tag after last node belonging to current word.
  node_insert_after(__curr_word_end_head, __curr_word_end, end_tag)
end


--- Highlight bad spelling by colour.
-- Insert colour whatsits before and after the first and last nodes
-- known to belong to the current word.
local function __highlight_by_color()
  -- Check, if start node of current word is a head node.  Inserting
  -- before head nodes needs special attention.  This is not yet
  -- implemented.
  if (__curr_word_start ~= __curr_word_start_head) then
    -- Create pdf_colorstack whatsit nodes.
     local push = node_new(WHATSIT, PDF_COLORSTACK)
     local pop = node_new(WHATSIT, PDF_COLORSTACK)
     push.stack = 0
     pop.stack = 0
     push.cmd = 1
     pop.cmd = 2
     push.data = __opts.hl_color
     node_insert_before(__curr_word_start_head, __curr_word_start, push)
     node_insert_after(__curr_word_end_head, __curr_word_end, pop)
  end
end


--- Generic function for highlighting bad spellings.
local function __highlight_bad_word()
  __highlight_by_color()
end


-- Tagging status.
local __is_active_tagging


-- Highlighting status.
local __is_active_highlighting


--- Data structure that stores the characters of a word string.
-- The current word data structure is an ordered list (an array) of the
-- characters of the word.  The characters are collected while scanning
-- a node list.  They are concatenated to a single string only after the
-- end of a word is detected, before inserting the current word into the
-- current paragraph data structure.
--
-- @class table
-- @name __curr_word
local __curr_word


--- Act upon detection of end of current word string.
-- If the current word contains visible characters, store the current
-- word in the current tag.
local function __finish_current_word()
  -- Finish a word?
  if __curr_word then
    local word = tabconcat(__curr_word)
    -- Check, if the current word has already been tagged.  This is only
    -- a quick hack.
    local start_prev = __curr_word_start.prev
    local end_next = __curr_word_end.next
    if start_prev and end_next
    and (start_prev.id == WHATSIT)
    and (start_prev.subtype == USER_DEFINED)
    and (start_prev.user_id == __whatsit_uid)
    and (start_prev.type == 115)
    and (start_prev.value == '')
    and (end_next.id == WHATSIT)
    and (end_next.subtype == USER_DEFINED)
    and (end_next.user_id == __whatsit_uid)
    and (end_next.type == 115)
    and (end_next.value == word) then
      __curr_word = nil
      __curr_word_start_head = nil
      __curr_word_start = nil
      __curr_word_end_head = nil
      __curr_word_end = nil
      return
    end
    -- Tag node list with word string.
    if __is_active_tagging then
      __tag_word(word)
    end
    -- Test for bad spelling.
    if __is_active_highlighting and __is_bad[word] and not __is_good[word] then
      __highlight_bad_word()
    end
    __curr_word = nil
  end
  __curr_word_start_head = nil
  __curr_word_start = nil
  __curr_word_end_head = nil
  __curr_word_end = nil
end


--- Act upon detection of end of current paragraph.
-- If the current paragraph contains words, store the current paragraph
-- in the text document.
local function __finish_current_paragraph()
  -- Finish current word.
  __finish_current_word()
end


--- Paragraph management stack.
-- Stack of boolean flags, that are used for logging the occurence of a
-- new paragraph within nested vlists.
local __is_vlist_paragraph


--- Paragraph management.
-- This function puts a new boolean flag onto a stack that is used to
-- log the occurence of a new paragraph, while recursing into the coming
-- vlist.  After finishing recursing into the vlist, the flag needs to
-- be removed from the stack.  Depending on the flag, the then current
-- paragraph can be finished.
local function __vlist_pre_recurse()
  tabinsert(__is_vlist_paragraph, false)
end


--- Paragraph management.
-- Remove flag from stack after recursing into a vlist.  If necessary,
-- finish the current paragraph.
local function __vlist_post_recurse()
  local p = tabremove(__is_vlist_paragraph)
  if p then
    __finish_current_paragraph()
  end
end


--- Find paragraphs and strings.
-- While scanning a node list, this call-back function finds nodes
-- representing the start of a paragraph (local_par whatsit nodes) and
-- strings (chains of nodes of type glyph, kern, disc).
--
-- @param head  Head node of current branch.
-- @param n  The current node.
local function __visit_node(head, n)
  local nid = n.id
  -- Test for word string component node.
  if nid == GLYPH then
    -- Save first node belonging to current word and its head for later
    -- reference.
    if not __curr_word_start then
      __curr_word_start_head = head
      __curr_word_start = n
    end
    -- Save latest node belonging to current word and its head for later
    -- reference.
    __curr_word_end_head = head
    __curr_word_end = n
    -- Provide new empty word, if necessary.
    if not __curr_word then
      __curr_word = {}
    end
    -- Append character to current word string.
    tabinsert(__curr_word, __codepoint_map[n.char])
  -- Test for other word string component nodes.
  elseif (nid == DISC) or (nid == KERN) then
    -- We're still within the current word string.  Do nothing.
  -- Test for paragraph start.
  elseif (nid == WHATSIT) and (n.subtype == LOCAL_PAR) then
    __finish_current_paragraph()
    __is_vlist_paragraph[#__is_vlist_paragraph] = true
  else
    -- End of current word string detected.
    __finish_current_word()
  end
end


--- Table of call-back functions for node list recursion: store the
--- word strings found in a node list.
-- The call-back functions in this table identify chains of nodes
-- representing word strings in a node list and stores the strings in
-- the text document.  Local_par whatsit nodes are word boundaries.
-- Nodes of type `hlist` are recursed into as if they were non-existent.
-- As an example, the LaTeX input `a\mbox{a b}b` is recognized as two
-- strings `aa` and `bb`.
--
-- @class table
-- @name __cb_tag_words
-- @field vlist_pre_recurse  Paragraph management.
-- @field vlist_post_recurse  Paragraph management.
-- @field visit_node  Find nodes representing paragraphs and words.
local __cb_tag_words = {

  vlist_pre_recurse = __vlist_pre_recurse,
  vlist_post_recurse = __vlist_post_recurse,
  visit_node = __visit_node,

}


--- Process node list according to this stage.
-- This function recurses into the given node list, extracts all text
-- and stores it in the text document.
--
-- @param head  Node list.
local function __process_node_list(head)
  __curr_word_start_head = nil
  __curr_word_start = nil
  __curr_word_end_head = nil
  __curr_word_end = nil
  __recurse_node_list(head, __cb_tag_words)
  -- Clean-up left-over word and/or paragraph.
  __finish_current_paragraph()
end


--- Call-back function that processes the node list.
--
-- @param head  Node list.
local function __cb_pre_linebreak_filter_pkg_spelling(head)
  __process_node_list(head)
  return head
end


--- Start tagging text.
-- After calling this function, words are tagged in node lists before
-- hyphenation takes place.
local function enable_text_tagging()
  __is_active_tagging = true
end
M.enable_text_tagging = enable_text_tagging


--- Stop tagging text.
-- After calling this function, no more word tagging in node lists takes
-- place.
local function disable_text_tagging()
  __is_active_tagging = false
end
M.disable_text_tagging = disable_text_tagging


--- Start highlighting bad spellings.
-- After calling this function, bad spellings are highlighted in PDF
-- output.
local function enable_word_highlighting()
  __is_active_highlighting = true
end
M.enable_word_highlighting = enable_word_highlighting


--- Stop highlighting bad spellings.
-- After calling this function, no more bad spellings are highlighted in
-- PDF output.
local function disable_word_highlighting()
  __is_active_highlighting = false
end
M.disable_word_highlighting = disable_word_highlighting


--- Module initialisation.
--
local function __init()
  -- Create empty paragraph management stack.
  __is_vlist_paragraph = {}
  -- Remember tagging status.
  __is_active_tagging = false
  -- Remember highlighting status.
  __is_active_highlighting = false
  -- Set default highlighting colour.
  set_highlight_color('1 0 0 rg')
  -- Register call-back: Before TeX breaks a paragraph into lines, tag
  -- and highlight strings.
  luatexbase.add_to_callback('pre_linebreak_filter', __cb_pre_linebreak_filter_pkg_spelling, '__cb_pre_linebreak_filter_pkg_spelling')
end


-- Initialize module.
__init()


-- Return module table.
return M