summaryrefslogtreecommitdiff
path: root/macros/luatex/generic/lua-uni-algos/lua-uni-graphemes.lua
blob: 5c7efc28c171df9a74d1a9bdf529f6c04fcfded6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
-- lua-uni-graphemes.lua
-- Copyright 2020 Marcel Krüger
--
-- This work may be distributed and/or modified under the
-- conditions of the LaTeX Project Public License, either version 1.3
-- of this license or (at your option) any later version.
-- The latest version of this license is in
--   http://www.latex-project.org/lppl.txt
-- and version 1.3 or later is part of all distributions of LaTeX
-- version 2005/12/01 or later.
--
-- This work has the LPPL maintenance status `maintained'.
-- 
-- The Current Maintainer of this work is Marcel Krüger

local property do
  local p = require'lua-uni-parse'
  local l = lpeg or require'lpeg'

  property = p.parse_file('emoji-data',
    l.Cg(p.fields(p.codepoint_range, l.C'Extended_Pictographic')) + p.ignore_line,
    p.multiset)

  property = p.parse_file('GraphemeBreakProperty', l.Cf(
      l.Carg(1)
    * (l.Cg(p.fields(p.codepoint_range, l.C(l.R('az', 'AZ', '__')^1))) + p.ignore_line)^0
    * -1, p.multiset),
    nil,
    property)
  if not property then
    error[[Break Property matching failed]]
  end
end

local controls = { CR = true, LF = true, Control = true, }
local precore_lookup = {
  Prepend = "PRECORE",
  L = "L",
  V = "V",
  LV = "V",
  LVT = "T",
  T = "T",
  Regional_Indicator = "RI",
  Extended_Pictographic = "POST_PICTO",
}
local l_lookup = {
  L = "L",
  V = "V",
  LV = "V",
  LVT = "T",
}
local postcore_map = { Extend = true, ZWJ = true, SpacingMark = true, }
local state_map state_map = {
  START = function(prop)
    if prop == 'CR' then
      return 'CR', true
    end
    if prop == 'LF' or prop == 'Control' then
      return 'START', true
    end
    return state_map.PRECORE(prop), true
  end,
  PRECORE = function(prop)
    if controls[prop] then
      return state_map.START(prop)
    end
    return precore_lookup[prop] or 'POSTCORE'
  end,
  POSTCORE = function(prop)
    if postcore_map[prop] then
      return 'POSTCORE'
    end
    return state_map.START(prop)
  end,
  RI = function(prop)
    if prop == 'Regional_Indicator' then
      return 'POSTCORE'
    end
    return state_map.POSTCORE(prop)
  end,
  PRE_PICTO = function(prop)
    if prop == "Extended_Pictographic" then
      return "POST_PICTO"
    end
    return state_map.POSTCORE(prop)
  end,
  POST_PICTO = function(prop)
    if prop == "Extend" then
      return "POST_PICTO"
    end
    if prop == "ZWJ" then
      return "PRE_PICTO"
    end
    return state_map.POSTCORE(prop)
  end,
  L = function(prop)
    local nextstate = l_lookup[prop]
    if nextstate then
      return nextstate
    end
    return state_map.POSTCORE(prop)
  end,
  V = function(prop)
    if prop == 'V' then
      return 'V'
    end
    return state_map.T(prop)
  end,
  T = function(prop)
    if prop == 'T' then
      return 'T'
    end
    return state_map.POSTCORE(prop)
  end,
  CR = function(prop)
    if prop == 'LF' then
      return 'START'
    else
      return state_map.START(prop)
    end
  end,
}

-- The value of "state" is considered internal and should not be relied upon.
-- Just pass it to the function as is or pass nil. `nil` should only be passed when the passed codepoint starts a new cluster
function read_codepoint(cp, state)
  local new_cluster
  state, new_cluster = state_map[state or 'START'](property[cp])
  return new_cluster, state
end

-- A Lua iterator for strings -- Only reporting the beginning of every grapheme cluster
local function graphemes_start(str)
  local nextcode, str, i = utf8.codes(str)
  local state = "START"
  return function()
    local new_cluster, code
    repeat
      i, code = nextcode(str, i)
      if not i then return end
      new_cluster, state = read_codepoint(code, state)
    until new_cluster
    return i, code
  end
end
-- A more useful iterator: returns the byterange of the graphemecluster in reverse order followed by a string with te cluster
local function graphemes(str)
  local iter = graphemes_start(str)
  return function(_, cur)
    if cur == #str then return end
    local new = iter()
    if not new then return #str, cur + 1, str:sub(cur + 1) end
    return new - 1, cur + 1, str:sub(cur + 1, new - 1)
  end, nil, iter() - 1
end
return {
  read_codepoint = read_codepoint,
  graphemes_start = graphemes_start,
  graphemes = graphemes,
}
--[[
for i, c in graphemes_start'äbcdef' do
  print(i, utf8.char(c))
end
for i, j, s in graphemes'Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞' do
  print(j, i, s)
end
]]