summaryrefslogtreecommitdiff
path: root/macros/luatex/generic/lua-uni-algos/lua-uni-graphemes.lua
diff options
context:
space:
mode:
Diffstat (limited to 'macros/luatex/generic/lua-uni-algos/lua-uni-graphemes.lua')
-rw-r--r--macros/luatex/generic/lua-uni-algos/lua-uni-graphemes.lua168
1 files changed, 168 insertions, 0 deletions
diff --git a/macros/luatex/generic/lua-uni-algos/lua-uni-graphemes.lua b/macros/luatex/generic/lua-uni-algos/lua-uni-graphemes.lua
new file mode 100644
index 0000000000..5c7efc28c1
--- /dev/null
+++ b/macros/luatex/generic/lua-uni-algos/lua-uni-graphemes.lua
@@ -0,0 +1,168 @@
+-- lua-uni-graphemes.lua
+-- Copyright 2020 Marcel Krüger
+--
+-- This work may be distributed and/or modified under the
+-- conditions of the LaTeX Project Public License, either version 1.3
+-- of this license or (at your option) any later version.
+-- The latest version of this license is in
+-- http://www.latex-project.org/lppl.txt
+-- and version 1.3 or later is part of all distributions of LaTeX
+-- version 2005/12/01 or later.
+--
+-- This work has the LPPL maintenance status `maintained'.
+--
+-- The Current Maintainer of this work is Marcel Krüger
+
+local property do
+ local p = require'lua-uni-parse'
+ local l = lpeg or require'lpeg'
+
+ property = p.parse_file('emoji-data',
+ l.Cg(p.fields(p.codepoint_range, l.C'Extended_Pictographic')) + p.ignore_line,
+ p.multiset)
+
+ property = p.parse_file('GraphemeBreakProperty', l.Cf(
+ l.Carg(1)
+ * (l.Cg(p.fields(p.codepoint_range, l.C(l.R('az', 'AZ', '__')^1))) + p.ignore_line)^0
+ * -1, p.multiset),
+ nil,
+ property)
+ if not property then
+ error[[Break Property matching failed]]
+ end
+end
+
+local controls = { CR = true, LF = true, Control = true, }
+local precore_lookup = {
+ Prepend = "PRECORE",
+ L = "L",
+ V = "V",
+ LV = "V",
+ LVT = "T",
+ T = "T",
+ Regional_Indicator = "RI",
+ Extended_Pictographic = "POST_PICTO",
+}
+local l_lookup = {
+ L = "L",
+ V = "V",
+ LV = "V",
+ LVT = "T",
+}
+local postcore_map = { Extend = true, ZWJ = true, SpacingMark = true, }
+local state_map state_map = {
+ START = function(prop)
+ if prop == 'CR' then
+ return 'CR', true
+ end
+ if prop == 'LF' or prop == 'Control' then
+ return 'START', true
+ end
+ return state_map.PRECORE(prop), true
+ end,
+ PRECORE = function(prop)
+ if controls[prop] then
+ return state_map.START(prop)
+ end
+ return precore_lookup[prop] or 'POSTCORE'
+ end,
+ POSTCORE = function(prop)
+ if postcore_map[prop] then
+ return 'POSTCORE'
+ end
+ return state_map.START(prop)
+ end,
+ RI = function(prop)
+ if prop == 'Regional_Indicator' then
+ return 'POSTCORE'
+ end
+ return state_map.POSTCORE(prop)
+ end,
+ PRE_PICTO = function(prop)
+ if prop == "Extended_Pictographic" then
+ return "POST_PICTO"
+ end
+ return state_map.POSTCORE(prop)
+ end,
+ POST_PICTO = function(prop)
+ if prop == "Extend" then
+ return "POST_PICTO"
+ end
+ if prop == "ZWJ" then
+ return "PRE_PICTO"
+ end
+ return state_map.POSTCORE(prop)
+ end,
+ L = function(prop)
+ local nextstate = l_lookup[prop]
+ if nextstate then
+ return nextstate
+ end
+ return state_map.POSTCORE(prop)
+ end,
+ V = function(prop)
+ if prop == 'V' then
+ return 'V'
+ end
+ return state_map.T(prop)
+ end,
+ T = function(prop)
+ if prop == 'T' then
+ return 'T'
+ end
+ return state_map.POSTCORE(prop)
+ end,
+ CR = function(prop)
+ if prop == 'LF' then
+ return 'START'
+ else
+ return state_map.START(prop)
+ end
+ end,
+}
+
+-- The value of "state" is considered internal and should not be relied upon.
+-- Just pass it to the function as is or pass nil. `nil` should only be passed when the passed codepoint starts a new cluster
+function read_codepoint(cp, state)
+ local new_cluster
+ state, new_cluster = state_map[state or 'START'](property[cp])
+ return new_cluster, state
+end
+
+-- A Lua iterator for strings -- Only reporting the beginning of every grapheme cluster
+local function graphemes_start(str)
+ local nextcode, str, i = utf8.codes(str)
+ local state = "START"
+ return function()
+ local new_cluster, code
+ repeat
+ i, code = nextcode(str, i)
+ if not i then return end
+ new_cluster, state = read_codepoint(code, state)
+ until new_cluster
+ return i, code
+ end
+end
+-- A more useful iterator: returns the byterange of the graphemecluster in reverse order followed by a string with te cluster
+local function graphemes(str)
+ local iter = graphemes_start(str)
+ return function(_, cur)
+ if cur == #str then return end
+ local new = iter()
+ if not new then return #str, cur + 1, str:sub(cur + 1) end
+ return new - 1, cur + 1, str:sub(cur + 1, new - 1)
+ end, nil, iter() - 1
+end
+return {
+ read_codepoint = read_codepoint,
+ graphemes_start = graphemes_start,
+ graphemes = graphemes,
+}
+--[[
+for i, c in graphemes_start'äbcdef' do
+ print(i, utf8.char(c))
+end
+for i, j, s in graphemes'Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞' do
+ print(j, i, s)
+end
+]]