summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/mkiv/toks-ini.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/mkiv/toks-ini.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/mkiv/toks-ini.lua233
1 files changed, 139 insertions, 94 deletions
diff --git a/Master/texmf-dist/tex/context/base/mkiv/toks-ini.lua b/Master/texmf-dist/tex/context/base/mkiv/toks-ini.lua
index eef26574e0e..7120b47c3cd 100644
--- a/Master/texmf-dist/tex/context/base/mkiv/toks-ini.lua
+++ b/Master/texmf-dist/tex/context/base/mkiv/toks-ini.lua
@@ -18,63 +18,53 @@ local printtable = table.print
local concat = table.concat
local format = string.format
-if token.commands then
-
- local commands = token.commands()
-
- tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
-
-else
-
- tokens.commands = { }
-
-end
-
-local scan_toks = token.scan_toks
-local scan_string = token.scan_string
-local scan_argument = token.scan_argument
-local scan_tokenlist = token.scan_tokenlist
-local scan_int = token.scan_int
-local scan_code = token.scan_code
-local scan_token_code = token.scan_token_code
-local scan_dimen = token.scan_dimen
-local scan_glue = token.scan_glue
-local scan_keyword = token.scan_keyword
-local scan_keyword_cs = token.scan_keyword_cs or scan_keyword
-local scan_token = token.scan_token
-local scan_box = token.scan_box
-local scan_word = token.scan_word
-local scan_key = token.scan_key
-local scan_value = token.scan_value
-local scan_char = token.scan_char
-local scan_number = token.scan_number
-local scan_csname = token.scan_csname
-local scan_real = token.scan_real
-local scan_float = token.scan_float
-
-local get_next = token.get_next
-local get_next_token = token.get_next_token
-local skip_next = token.skip_next
-local peek_next_char = token.peek_next_char
-local is_next_char = token.is_next_char
-
-local set_macro = token.set_macro
-local get_macro = token.get_macro
-local get_meaning = token.get_meaning
-local get_cmdname = token.get_cmdname
-local set_char = token.set_char
-local set_lua = token.set_lua
-
-local create_token = token.create
-local new_token = token.new
-local is_defined = token.is_defined
-local is_token = token.is_token
-
-tokens.new = new_token
-tokens.create = create_token
-tokens.istoken = is_token
-tokens.isdefined = is_defined
-tokens.defined = is_defined
+local commands = token.commands()
+tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
+tokens.values = { }
+
+local scan_toks = token.scan_toks
+local scan_string = token.scan_string
+local scan_argument = token.scan_argument
+local scan_delimited = token.scan_delimited
+local scan_tokenlist = token.scan_tokenlist or scan_string
+local scan_integer = token.scan_integer or token.scan_int
+local scan_cardinal = token.scan_cardinal
+local scan_code = token.scan_code
+local scan_token_code = token.scan_token_code
+local scan_dimen = token.scan_dimen
+local scan_glue = token.scan_glue
+local scan_skip = token.scan_skip
+local scan_keyword = token.scan_keyword
+local scan_keyword_cs = token.scan_keyword_cs or scan_keyword
+local scan_token = token.scan_token
+local scan_box = token.scan_box
+local scan_word = token.scan_word
+local scan_letters = token.scan_letters or scan_word -- lmtx
+local scan_key = token.scan_key
+local scan_value = token.scan_value
+local scan_char = token.scan_char
+local scan_number = token.scan_number -- not defined
+local scan_csname = token.scan_csname
+local scan_real = token.scan_real
+local scan_float = token.scan_float
+local scan_luanumber = token.scan_luanumber or scan_float -- only lmtx
+local scan_luainteger = token.scan_luainteger or scan_integer -- only lmtx
+local scan_luacardinal = token.scan_luacardinal or scan_cardinal -- only lmtx
+
+local set_macro = token.set_macro
+local set_char = token.set_char
+local set_lua = token.set_lua
+
+local create_token = token.create
+local new_token = token.new
+local is_defined = token.is_defined
+local is_token = token.is_token
+
+tokens.new = new_token
+tokens.create = create_token
+tokens.istoken = is_token
+tokens.isdefined = is_defined
+tokens.defined = is_defined
local bits = {
escape = 0x00000001, -- 2^00
@@ -147,6 +137,7 @@ if not scan_box then
if s == "hbox" or s == "vbox" or s == "vtop" then
put_next(create_token(s))
end
+ return scan_list()
end
token.scan_box = scan_box
@@ -154,45 +145,65 @@ if not scan_box then
end
tokens.scanners = { -- these expand
- token = scan_token,
- toks = scan_toks,
- tokens = scan_toks,
- box = scan_box,
- hbox = function() return scan_box("hbox") end,
- vbox = function() return scan_box("vbox") end,
- vtop = function() return scan_box("vtop") end,
- dimen = scan_dimen,
- dimension = scan_dimen,
- glue = scan_glue,
- skip = scan_glue,
- integer = scan_int,
- real = scan_real,
- float = scan_float,
- count = scan_int,
- string = scan_string,
- argument = scan_argument,
- tokenlist = scan_tokenlist,
- verbatim = scan_verbatim,
- code = scan_code,
- tokencode = scan_token_code,
- word = scan_word,
- key = scan_key,
- value = scan_value,
- char = scan_char,
- number = scan_number,
- boolean = scan_boolean,
- keyword = scan_keyword,
- keywordcs = scan_keyword_cs,
- csname = scan_csname,
- peek = peek_next_char,
- skip = skip_next,
- ischar = is_next_char,
+ token = scan_token,
+ toks = scan_toks,
+ tokens = scan_toks,
+ box = scan_box,
+ hbox = function() return scan_box("hbox") end,
+ vbox = function() return scan_box("vbox") end,
+ vtop = function() return scan_box("vtop") end,
+ dimen = scan_dimen,
+ dimension = scan_dimen,
+ glue = scan_glue,
+ gluevalues = function() return scan_glue(false,false,true) end,
+ gluespec = scan_skip,
+ integer = scan_integer,
+ cardinal = scan_cardinal,
+ real = scan_real,
+ float = scan_float,
+ luanumber = scan_luanumber,
+ luainteger = scan_luainteger,
+ luacardinal = scan_luacardinal,
+ count = scan_integer,
+ string = scan_string,
+ argument = scan_argument,
+ delimited = scan_delimited,
+ tokenlist = scan_tokenlist,
+ verbatim = scan_verbatim, -- detokenize
+ code = scan_code,
+ tokencode = scan_token_code,
+ word = scan_word,
+ letters = scan_letters,
+ key = scan_key,
+ value = scan_value,
+ char = scan_char,
+ number = scan_number,
+ boolean = scan_boolean,
+ keyword = scan_keyword,
+ keywordcs = scan_keyword_cs,
+ csname = scan_csname,
+
+ next = token.scan_next,
+ nextexpanded = token.scan_next_expanded,
+
+ peek = token.peek_next,
+ peekexpanded = token.peek_next_expanded,
+ peekchar = token.peek_next_char,
+
+ skip = token.skip_next,
+ skipexpanded = token.skip_next_expanded,
+
+ cmdchr = token.scan_cmdchr,
+ cmdchrexpanded = token.scan_cmdchr_expanded,
+
+ ischar = token.is_next_char,
}
tokens.getters = { -- these don't expand
- meaning = get_meaning,
- macro = get_macro,
- token = get_next,
+ meaning = token.get_meaning,
+ macro = token.get_macro,
+ token = token.scan_next or token.get_next, -- not here, use scanners.next or token
+ cstoken = token.get_cstoken,
count = tex.getcount,
dimen = tex.getdimen,
skip = tex.getglue,
@@ -215,6 +226,23 @@ tokens.setters = {
box = tex.setbox,
}
+token.accessors = {
+ command = token.get_command,
+ cmd = token.get_command,
+ cmdname = token.get_cmdname,
+ name = token.get_cmdname,
+ csname = token.get_csname,
+ index = token.get_index,
+ active = token.get_active,
+ frozen = token.get_frozen,
+ protected = token.get_protected,
+ expandable = token.get_protected,
+ user = token.get_user,
+ cmdchrcs = token.get_cmdchrcs,
+ active = token.get_active,
+ range = token.get_range,
+}
+
-- static int run_scan_token(lua_State * L)
-- {
-- saved_tex_scanner texstate;
@@ -240,12 +268,13 @@ if setinspector then
local simple = { letter = "letter", other_char = "other" }
- local function astable(t)
+ local astable = function(t)
if t and is_token(t) then
local cmdname = t.cmdname
local simple = simple[cmdname]
if simple then
return {
+ id = t.id,
category = simple,
character = utfchar(t.mode) or nil,
}
@@ -261,6 +290,7 @@ if setinspector then
frozen = t.frozen,
mode = t.mode,
index = t.index,
+ user = t.user,
cmdname = cmdname,
}
end
@@ -281,3 +311,18 @@ tokens.cache = table.setmetatableindex(function(t,k)
t[k] = v
return v
end)
+
+if LUATEXVERSION < 114 then
+
+ local d = tokens.defined
+ local c = tokens.create
+
+ function tokens.defined(s,b)
+ if b then
+ return d(s)
+ else
+ return c(s).cmd_name == "undefined_cmd"
+ end
+ end
+
+end