summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/context/data/scite/lexers
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/context/data/scite/lexers')
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/archive/scite-context-lexer-pre-3-3-1.lua1100
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-context.lua4
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metafun.lua2
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metapost.lua4
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-tex.lua4
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/lexer.lua3
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-bibtex.lua176
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cld.lua11
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp-web.lua23
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp.lua188
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua19
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua.lua242
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-mps.lua132
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-object.lua105
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua42
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf.lua204
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex-web.lua23
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex.lua161
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-txt.lua28
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web-snippets.lua133
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web.lua166
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua21
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-comment.lua35
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-script.lua27
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml.lua213
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua1790
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme-keep.lua233
-rw-r--r--Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme.lua319
28 files changed, 2941 insertions, 2467 deletions
diff --git a/Master/texmf-dist/context/data/scite/lexers/archive/scite-context-lexer-pre-3-3-1.lua b/Master/texmf-dist/context/data/scite/lexers/archive/scite-context-lexer-pre-3-3-1.lua
deleted file mode 100644
index 7883177b41e..00000000000
--- a/Master/texmf-dist/context/data/scite/lexers/archive/scite-context-lexer-pre-3-3-1.lua
+++ /dev/null
@@ -1,1100 +0,0 @@
-local info = {
- version = 1.324,
- comment = "basics for scintilla lpeg lexer for context/metafun",
- author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
- copyright = "PRAGMA ADE / ConTeXt Development Team",
- license = "see context related readme files",
- comment = "contains copyrighted code from mitchell.att.foicica.com",
-
-}
-
--- todo: move all code here
--- todo: explore adapted dll ... properties + init
-
--- The fold and lex functions are copied and patched from original code by Mitchell (see
--- lexer.lua). All errors are mine.
---
--- Starting with SciTE version 3.20 there is an issue with coloring. As we still lack
--- a connection with scite itself (properties as well as printing to the log pane) we
--- cannot trace this (on windows). As far as I can see, there are no fundamental
--- changes in lexer.lua or LexLPeg.cxx so it must be in scintilla itself. So for the
--- moment I stick to 3.10. Indicators are: no lexing of 'next' and 'goto <label>' in the
--- Lua lexer and no brace highlighting either. Interesting is that it does work ok in
--- the cld lexer (so the Lua code is okay). Also the fact that char-def.lua lexes fast
--- is a signal that the lexer quits somewhere halfway.
---
--- After checking 3.24 and adapting to the new lexer tables things are okay again. So,
--- this version assumes 3.24 or higher. In 3.24 we have a different token result, i.e. no
--- longer a { tag, pattern } but just two return values. I didn't check other changes but
--- will do that when I run into issues.
---
--- I've considered making a whole copy and patch the other functions too as we need
--- an extra nesting model. However, I don't want to maintain too much. An unfortunate
--- change in 3.03 is that no longer a script can be specified. This means that instead
--- of loading the extensions via the properties file, we now need to load them in our
--- own lexers, unless of course we replace lexer.lua completely (which adds another
--- installation issue).
---
--- Another change has been that _LEXERHOME is no longer available. It looks like more and
--- more functionality gets dropped so maybe at some point we need to ship our own dll/so
--- files. For instance, I'd like to have access to the current filename and other scite
--- properties. For instance, we could cache some info with each file, if only we had
--- knowledge of what file we're dealing with.
---
--- For huge files folding can be pretty slow and I do have some large ones that I keep
--- open all the time. Loading is normally no ussue, unless one has remembered the status
--- and the cursor is at the last line of a 200K line file. Optimizing the fold function
--- brought down loading of char-def.lua from 14 sec => 8 sec. Replacing the word_match
--- function and optimizing the lex function gained another 2+ seconds. A 6 second load
--- is quite ok for me. The changed lexer table structure (no subtables) brings loading
--- down to a few seconds.
---
--- When the lexer path is copied to the textadept lexer path, and the theme definition to
--- theme path (as lexer.lua), the lexer works there as well. When I have time and motive
--- I will make a proper setup file to tune the look and feel a bit and associate suffixes
--- with the context lexer. The textadept editor has a nice style tracing option but lacks
--- the tabs for selecting files that scite has. It also has no integrated run that pipes
--- to the log pane (I wonder if it could borrow code from the console2 project). Interesting
--- is that the jit version of textadept crashes on lexing large files (and does not feel
--- faster either).
---
--- Function load(lexer_name) starts with _M.WHITESPACE = lexer_name..'_whitespace' which
--- means that we need to have it frozen at the moment we load another lexer. Because spacing
--- is used to revert to a parent lexer we need to make sure that we load children as late
--- as possible in order not to get the wrong whitespace trigger. This took me quite a while
--- to figure out (not being that familiar with the internals). The lex and fold functions
--- have been optimized. It is a pitty that there is no proper print available. Another thing
--- needed is a default style in ourown theme style definition, as otherwise we get wrong
--- nested lexers, especially if they are larger than a view. This is the hardest part of
--- getting things right.
---
--- Eventually it might be safer to copy the other methods from lexer.lua here as well so
--- that we have no dependencies, apart from the c library (for which at some point the api
--- will be stable I hope).
---
--- It's a pitty that there is no scintillua library for the OSX version of scite. Even
--- better would be to have the scintillua library as integral part of scite as that way I
--- could use OSX alongside windows and linux (depending on needs). Also nice would be to
--- have a proper interface to scite then because currently the lexer is rather isolated and the
--- lua version does not provide all standard libraries. It would also be good to have lpeg
--- support in the regular scite lua extension (currently you need to pick it up from someplace
--- else).
-
-local lpeg = require 'lpeg'
-
-local R, P, S, C, V, Cp, Cs, Ct, Cmt, Cc, Cf, Cg, Carg = lpeg.R, lpeg.P, lpeg.S, lpeg.C, lpeg.V, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Carg
-local lpegmatch = lpeg.match
-local find, gmatch, match, lower, upper, gsub = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub
-local concat = table.concat
-local global = _G
-local type, next, setmetatable, rawset = type, next, setmetatable, rawset
-
-if lexer then
- -- in recent c++ code the lexername and loading is hard coded
-elseif _LEXERHOME then
- dofile(_LEXERHOME .. '/lexer.lua') -- pre 3.03 situation
-else
- dofile('lexer.lua') -- whatever
-end
-
-lexer.context = lexer.context or { }
-local context = lexer.context
-
-context.patterns = context.patterns or { }
-local patterns = context.patterns
-
-lexer._CONTEXTEXTENSIONS = true
-
-local locations = {
- -- lexer.context.path,
- "data", -- optional data directory
- "..", -- regular scite directory
-}
-
-local function collect(name)
--- local definitions = loadfile(name .. ".luc") or loadfile(name .. ".lua")
- local okay, definitions = pcall(function () return require(name) end)
- if okay then
- if type(definitions) == "function" then
- definitions = definitions()
- end
- if type(definitions) == "table" then
- return definitions
- end
- end
-end
-
-function context.loaddefinitions(name)
- for i=1,#locations do
- local data = collect(locations[i] .. "/" .. name)
- if data then
- return data
- end
- end
-end
-
--- maybe more efficient:
-
-function context.word_match(words,word_chars,case_insensitive)
- local chars = '%w_' -- maybe just "" when word_chars
- if word_chars then
- chars = '^([' .. chars .. gsub(word_chars,'([%^%]%-])', '%%%1') ..']+)'
- else
- chars = '^([' .. chars ..']+)'
- end
- if case_insensitive then
- local word_list = { }
- for i=1,#words do
- word_list[lower(words[i])] = true
- end
- return P(function(input, index)
- local s, e, word = find(input,chars,index)
- return word and word_list[lower(word)] and e + 1 or nil
- end)
- else
- local word_list = { }
- for i=1,#words do
- word_list[words[i]] = true
- end
- return P(function(input, index)
- local s, e, word = find(input,chars,index)
- return word and word_list[word] and e + 1 or nil
- end)
- end
-end
-
-local idtoken = R("az","AZ","\127\255","__")
-local digit = R("09")
-local sign = S("+-")
-local period = P(".")
-local space = S(" \n\r\t\f\v")
-
-patterns.idtoken = idtoken
-
-patterns.digit = digit
-patterns.sign = sign
-patterns.period = period
-
-patterns.cardinal = digit^1
-patterns.integer = sign^-1 * digit^1
-
-patterns.real =
- sign^-1 * ( -- at most one
- digit^1 * period * digit^0 -- 10.0 10.
- + digit^0 * period * digit^1 -- 0.10 .10
- + digit^1 -- 10
- )
-
-patterns.restofline = (1-S("\n\r"))^1
-patterns.space = space
-patterns.spacing = space^1
-patterns.nospacing = (1-space)^1
-patterns.anything = P(1)
-
-local endof = S("\n\r\f")
-
-patterns.startofline = P(function(input,index)
- return (index == 1 or lpegmatch(endof,input,index-1)) and index
-end)
-
-function context.exact_match(words,word_chars,case_insensitive)
- local characters = concat(words)
- local pattern -- the concat catches _ etc
- if word_chars == true or word_chars == false or word_chars == nil then
- word_chars = ""
- end
- if type(word_chars) == "string" then
- pattern = S(characters) + idtoken
- if case_insensitive then
- pattern = pattern + S(upper(characters)) + S(lower(characters))
- end
- if word_chars ~= "" then
- pattern = pattern + S(word_chars)
- end
- elseif word_chars then
- pattern = word_chars
- end
- if case_insensitive then
- local list = { }
- for i=1,#words do
- list[lower(words[i])] = true
- end
- return Cmt(pattern^1, function(_,i,s)
- return list[lower(s)] -- and i or nil
- end)
- else
- local list = { }
- for i=1,#words do
- list[words[i]] = true
- end
- return Cmt(pattern^1, function(_,i,s)
- return list[s] -- and i or nil
- end)
- end
-end
-
--- spell checking (we can only load lua files)
---
--- return {
--- min = 3,
--- max = 40,
--- n = 12345,
--- words = {
--- ["someword"] = "someword",
--- ["anotherword"] = "Anotherword",
--- },
--- }
-
-local lists = { }
-
-function context.setwordlist(tag,limit) -- returns hash (lowercase keys and original values)
- if not tag or tag == "" then
- return false, 3
- end
- local list = lists[tag]
- if not list then
- list = context.loaddefinitions("spell-" .. tag)
- if not list or type(list) ~= "table" then
- list = { words = false, min = 3 }
- else
- list.words = list.words or false
- list.min = list.min or 3
- end
- lists[tag] = list
- end
- return list.words, list.min
-end
-
-patterns.wordtoken = R("az","AZ","\127\255")
-patterns.wordpattern = patterns.wordtoken^3 -- todo: if limit and #s < limit then
-
--- -- pre 3.24:
---
--- function context.checkedword(validwords,validminimum,s,i) -- ,limit
--- if not validwords then -- or #s < validminimum then
--- return true, { "text", i } -- { "default", i }
--- else
--- -- keys are lower
--- local word = validwords[s]
--- if word == s then
--- return true, { "okay", i } -- exact match
--- elseif word then
--- return true, { "warning", i } -- case issue
--- else
--- local word = validwords[lower(s)]
--- if word == s then
--- return true, { "okay", i } -- exact match
--- elseif word then
--- return true, { "warning", i } -- case issue
--- elseif upper(s) == s then
--- return true, { "warning", i } -- probably a logo or acronym
--- else
--- return true, { "error", i }
--- end
--- end
--- end
--- end
-
-function context.checkedword(validwords,validminimum,s,i) -- ,limit
- if not validwords then -- or #s < validminimum then
- return true, "text", i -- { "default", i }
- else
- -- keys are lower
- local word = validwords[s]
- if word == s then
- return true, "okay", i -- exact match
- elseif word then
- return true, "warning", i -- case issue
- else
- local word = validwords[lower(s)]
- if word == s then
- return true, "okay", i -- exact match
- elseif word then
- return true, "warning", i -- case issue
- elseif upper(s) == s then
- return true, "warning", i -- probably a logo or acronym
- else
- return true, "error", i
- end
- end
- end
-end
-
-function context.styleofword(validwords,validminimum,s) -- ,limit
- if not validwords or #s < validminimum then
- return "text"
- else
- -- keys are lower
- local word = validwords[s]
- if word == s then
- return "okay" -- exact match
- elseif word then
- return "warning" -- case issue
- else
- local word = validwords[lower(s)]
- if word == s then
- return "okay" -- exact match
- elseif word then
- return "warning" -- case issue
- elseif upper(s) == s then
- return "warning" -- probably a logo or acronym
- else
- return "error"
- end
- end
- end
-end
-
--- overloaded functions
-
-local FOLD_BASE = SC_FOLDLEVELBASE
-local FOLD_HEADER = SC_FOLDLEVELHEADERFLAG
-local FOLD_BLANK = SC_FOLDLEVELWHITEFLAG
-
-local get_style_at = GetStyleAt
-local get_property = GetProperty
-local get_indent_amount = GetIndentAmount
-
-local h_table, b_table, n_table = { }, { }, { }
-
-setmetatable(h_table, { __index = function(t,level) local v = { level, FOLD_HEADER } t[level] = v return v end })
-setmetatable(b_table, { __index = function(t,level) local v = { level, FOLD_BLANK } t[level] = v return v end })
-setmetatable(n_table, { __index = function(t,level) local v = { level } t[level] = v return v end })
-
--- -- todo: move the local functions outside (see below) .. old variant < 3.24
---
--- local newline = P("\r\n") + S("\r\n")
--- local p_yes = Cp() * Cs((1-newline)^1) * newline^-1
--- local p_nop = newline
---
--- local function fold_by_parsing(text,start_pos,start_line,start_level,lexer)
--- local foldsymbols = lexer._foldsymbols
--- if not foldsymbols then
--- return { }
--- end
--- local patterns = foldsymbols._patterns
--- if not patterns then
--- return { }
--- end
--- local nofpatterns = #patterns
--- if nofpatterns == 0 then
--- return { }
--- end
--- local folds = { }
--- local line_num = start_line
--- local prev_level = start_level
--- local current_level = prev_level
--- local validmatches = foldsymbols._validmatches
--- if not validmatches then
--- validmatches = { }
--- for symbol, matches in next, foldsymbols do -- whatever = { start = 1, stop = -1 }
--- if not find(symbol,"^_") then -- brrr
--- for s, _ in next, matches do
--- validmatches[s] = true
--- end
--- end
--- end
--- foldsymbols._validmatches = validmatches
--- end
--- -- of course we could instead build a nice lpeg checker .. something for
--- -- a rainy day with a stack of new cd's at hand
--- local function action_y(pos,line)
--- for i=1,nofpatterns do
--- for s, m in gmatch(line,patterns[i]) do
--- if validmatches[m] then
--- local symbols = foldsymbols[get_style_at(start_pos + pos + s - 1)]
--- if symbols then
--- local action = symbols[m]
--- if action then
--- if type(action) == 'number' then -- we could store this in validmatches if there was only one symbol category
--- current_level = current_level + action
--- else
--- current_level = current_level + action(text,pos,line,s,m)
--- end
--- if current_level < FOLD_BASE then
--- current_level = FOLD_BASE
--- end
--- end
--- end
--- end
--- end
--- end
--- if current_level > prev_level then
--- folds[line_num] = h_table[prev_level] -- { prev_level, FOLD_HEADER }
--- else
--- folds[line_num] = n_table[prev_level] -- { prev_level }
--- end
--- prev_level = current_level
--- line_num = line_num + 1
--- end
--- local function action_n()
--- folds[line_num] = b_table[prev_level] -- { prev_level, FOLD_BLANK }
--- line_num = line_num + 1
--- end
--- if lexer._reset_parser then
--- lexer._reset_parser()
--- end
--- local lpegpattern = (p_yes/action_y + p_nop/action_n)^0 -- not too efficient but indirect function calls are neither but
--- lpegmatch(lpegpattern,text) -- keys are not pressed that fast ... large files are slow anyway
--- return folds
--- end
-
--- The 3.24 variant; no longer subtable optimization is needed:
-
-local newline = P("\r\n") + S("\r\n")
-local p_yes = Cp() * Cs((1-newline)^1) * newline^-1
-local p_nop = newline
-
-local folders = { }
-
-local function fold_by_parsing(text,start_pos,start_line,start_level,lexer)
- local folder = folders[lexer]
- if not folder then
- --
- local pattern, folds, text, start_pos, line_num, prev_level, current_level
- --
- local fold_symbols = lexer._foldsymbols
- local fold_pattern = lexer._foldpattern -- use lpeg instead (context extension)
- --
- if fold_pattern then
- -- if no functions are found then we could have a faster one
-
- -- fold_pattern = Cp() * C(fold_pattern) * Carg(1) / function(s,match,pos)
- -- local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)]
- -- local l = symbols and symbols[match]
- -- if l then
- -- local t = type(l)
- -- if t == 'number' then
- -- current_level = current_level + l
- -- elseif t == 'function' then
- -- current_level = current_level + l(text, pos, line, s, match)
- -- end
- -- end
- -- end
- -- fold_pattern = (fold_pattern + P(1))^0
- -- local action_y = function(pos,line)
- -- lpegmatch(fold_pattern,line,1,pos)
- -- folds[line_num] = prev_level
- -- if current_level > prev_level then
- -- folds[line_num] = prev_level + FOLD_HEADER
- -- end
- -- if current_level < FOLD_BASE then
- -- current_level = FOLD_BASE
- -- end
- -- prev_level = current_level
- -- line_num = line_num + 1
- -- end
- -- local action_n = function()
- -- folds[line_num] = prev_level + FOLD_BLANK
- -- line_num = line_num + 1
- -- end
- -- pattern = (p_yes/action_y + p_nop/action_n)^0
-
- fold_pattern = Cp() * C(fold_pattern) / function(s,match)
- local symbols = fold_symbols[get_style_at(start_pos + s)]
- if symbols then
- local l = symbols[match]
- if l then
- current_level = current_level + l
- end
- end
- end
- local action_y = function()
- folds[line_num] = prev_level
- if current_level > prev_level then
- folds[line_num] = prev_level + FOLD_HEADER
- end
- if current_level < FOLD_BASE then
- current_level = FOLD_BASE
- end
- prev_level = current_level
- line_num = line_num + 1
- end
- local action_n = function()
- folds[line_num] = prev_level + FOLD_BLANK
- line_num = line_num + 1
- end
- pattern = ((fold_pattern + (1-newline))^1 * newline / action_y + newline/action_n)^0
-
- else
- -- the traditional one but a bit optimized
- local fold_symbols_patterns = fold_symbols._patterns
- local action_y = function(pos,line)
- for j = 1, #fold_symbols_patterns do
- for s, match in gmatch(line,fold_symbols_patterns[j]) do -- '()('..patterns[i]..')'
- local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)]
- local l = symbols and symbols[match]
- local t = type(l)
- if t == 'number' then
- current_level = current_level + l
- elseif t == 'function' then
- current_level = current_level + l(text, pos, line, s, match)
- end
- end
- end
- folds[line_num] = prev_level
- if current_level > prev_level then
- folds[line_num] = prev_level + FOLD_HEADER
- end
- if current_level < FOLD_BASE then
- current_level = FOLD_BASE
- end
- prev_level = current_level
- line_num = line_num + 1
- end
- local action_n = function()
- folds[line_num] = prev_level + FOLD_BLANK
- line_num = line_num + 1
- end
- pattern = (p_yes/action_y + p_nop/action_n)^0
- end
- --
- local reset_parser = lexer._reset_parser
- --
- folder = function(_text_,_start_pos_,_start_line_,_start_level_)
- if reset_parser then
- reset_parser()
- end
- folds = { }
- text = _text_
- start_pos = _start_pos_
- line_num = _start_line_
- prev_level = _start_level_
- current_level = prev_level
- lpegmatch(pattern,text)
--- return folds
-local t = folds
-folds = nil
-return t -- so folds can be collected
- end
- folders[lexer] = folder
- end
- return folder(text,start_pos,start_line,start_level,lexer)
-end
-
--- local function fold_by_indentation(text,start_pos,start_line,start_level)
--- local folds = { }
--- local current_line = start_line
--- local prev_level = start_level
--- for line in gmatch(text,'[\t ]*(.-)\r?\n') do
--- if line ~= "" then
--- local current_level = FOLD_BASE + get_indent_amount(current_line)
--- if current_level > prev_level then -- next level
--- local i = current_line - 1
--- while true do
--- local f = folds[i]
--- if f and f[2] == FOLD_BLANK then
--- i = i - 1
--- else
--- break
--- end
--- end
--- local f = folds[i]
--- if f then
--- f[2] = FOLD_HEADER
--- end -- low indent
--- folds[current_line] = n_table[current_level] -- { current_level } -- high indent
--- elseif current_level < prev_level then -- prev level
--- local f = folds[current_line - 1]
--- if f then
--- f[1] = prev_level -- high indent
--- end
--- folds[current_line] = n_table[current_level] -- { current_level } -- low indent
--- else -- same level
--- folds[current_line] = n_table[prev_level] -- { prev_level }
--- end
--- prev_level = current_level
--- else
--- folds[current_line] = b_table[prev_level] -- { prev_level, FOLD_BLANK }
--- end
--- current_line = current_line + 1
--- end
--- return folds
--- end
-
--- local function fold_by_indentation(text,start_pos,start_line,start_level)
--- local folds = { }
--- local current_line = start_line
--- local prev_level = start_level
--- for line in gmatch(text,'[\t ]*(.-)\r?\n') do
--- if line ~= '' then
--- local current_level = FOLD_BASE + get_indent_amount(current_line)
--- if current_level > prev_level then -- next level
--- local i = current_line - 1
--- local f
--- while true do
--- f = folds[i]
--- if not f then
--- break
--- elseif f[2] == FOLD_BLANK then
--- i = i - 1
--- else
--- f[2] = FOLD_HEADER -- low indent
--- break
--- end
--- end
--- folds[current_line] = { current_level } -- high indent
--- elseif current_level < prev_level then -- prev level
--- local f = folds[current_line - 1]
--- if f then
--- f[1] = prev_level -- high indent
--- end
--- folds[current_line] = { current_level } -- low indent
--- else -- same level
--- folds[current_line] = { prev_level }
--- end
--- prev_level = current_level
--- else
--- folds[current_line] = { prev_level, FOLD_BLANK }
--- end
--- current_line = current_line + 1
--- end
--- for line, level in next, folds do
--- folds[line] = level[1] + (level[2] or 0)
--- end
--- return folds
--- end
-
-local folds, current_line, prev_level
-
-local function action_y()
- local current_level = FOLD_BASE + get_indent_amount(current_line)
- if current_level > prev_level then -- next level
- local i = current_line - 1
- local f
- while true do
- f = folds[i]
- if not f then
- break
- elseif f[2] == FOLD_BLANK then
- i = i - 1
- else
- f[2] = FOLD_HEADER -- low indent
- break
- end
- end
- folds[current_line] = { current_level } -- high indent
- elseif current_level < prev_level then -- prev level
- local f = folds[current_line - 1]
- if f then
- f[1] = prev_level -- high indent
- end
- folds[current_line] = { current_level } -- low indent
- else -- same level
- folds[current_line] = { prev_level }
- end
- prev_level = current_level
- current_line = current_line + 1
-end
-
-local function action_n()
- folds[current_line] = { prev_level, FOLD_BLANK }
- current_line = current_line + 1
-end
-
-local pattern = ( S("\t ")^0 * ( (1-S("\n\r"))^1 / action_y + P(true) / action_n) * newline )^0
-
-local function fold_by_indentation(text,start_pos,start_line,start_level)
- -- initialize
- folds = { }
- current_line = start_line
- prev_level = start_level
- -- define
- -- -- not here .. pattern binds and local functions are not frozen
- -- analyze
- lpegmatch(pattern,text)
- -- flatten
- for line, level in next, folds do
- folds[line] = level[1] + (level[2] or 0)
- end
- -- done
--- return folds
-local t = folds
-folds = nil
-return t -- so folds can be collected
-end
-
-local function fold_by_line(text,start_pos,start_line,start_level)
- local folds = { }
- -- can also be lpeg'd
- for _ in gmatch(text,".-\r?\n") do
- folds[start_line] = n_table[start_level] -- { start_level }
- start_line = start_line + 1
- end
- return folds
-end
-
-local threshold_by_lexer = 512 * 1024 -- we don't know the filesize yet
-local threshold_by_parsing = 512 * 1024 -- we don't know the filesize yet
-local threshold_by_indentation = 512 * 1024 -- we don't know the filesize yet
-local threshold_by_line = 512 * 1024 -- we don't know the filesize yet
-
-function context.fold(text,start_pos,start_line,start_level) -- hm, we had size thresholds .. where did they go
- if text == '' then
- return { }
- end
- local lexer = global._LEXER
- local fold_by_lexer = lexer._fold
- local fold_by_symbols = lexer._foldsymbols
- local filesize = 0 -- we don't know that
- if fold_by_lexer then
- if filesize <= threshold_by_lexer then
- return fold_by_lexer(text,start_pos,start_line,start_level,lexer)
- end
- elseif fold_by_symbols then -- and get_property('fold.by.parsing',1) > 0 then
- if filesize <= threshold_by_parsing then
- return fold_by_parsing(text,start_pos,start_line,start_level,lexer)
- end
- elseif get_property('fold.by.indentation',1) > 0 then
- if filesize <= threshold_by_indentation then
- return fold_by_indentation(text,start_pos,start_line,start_level,lexer)
- end
- elseif get_property('fold.by.line',1) > 0 then
- if filesize <= threshold_by_line then
- return fold_by_line(text,start_pos,start_line,start_level,lexer)
- end
- end
- return { }
-end
-
--- The following code is mostly unchanged:
-
-local function add_rule(lexer, id, rule)
- if not lexer._RULES then
- lexer._RULES = {}
- lexer._RULEORDER = {}
- end
- lexer._RULES[id] = rule
- lexer._RULEORDER[#lexer._RULEORDER + 1] = id
-end
-
-local function add_style(lexer, token_name, style)
- local len = lexer._STYLES.len
- if len == 32 then
- len = len + 8
- end
- if len >= 128 then
- print('Too many styles defined (128 MAX)')
- end
- lexer._TOKENS[token_name] = len
- lexer._STYLES[len] = style
- lexer._STYLES.len = len + 1
-end
-
-local function join_tokens(lexer)
- local patterns, order = lexer._RULES, lexer._RULEORDER
- local token_rule = patterns[order[1]]
- for i=2,#order do
- token_rule = token_rule + patterns[order[i]]
- end
- lexer._TOKENRULE = token_rule
- return lexer._TOKENRULE
-end
-
-local function add_lexer(grammar, lexer, token_rule)
- local token_rule = join_tokens(lexer)
- local lexer_name = lexer._NAME
- local children = lexer._CHILDREN
- for i=1,#children do
- local child = children[i]
- if child._CHILDREN then
- add_lexer(grammar, child)
- end
- local child_name = child._NAME
- local rules = child._EMBEDDEDRULES[lexer_name]
- local rules_token_rule = grammar['__'..child_name] or rules.token_rule
- grammar[child_name] = (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1 * V(lexer_name)
- local embedded_child = '_' .. child_name
- grammar[embedded_child] = rules.start_rule * (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1
- token_rule = V(embedded_child) + token_rule
- end
- grammar['__' .. lexer_name] = token_rule
- grammar[lexer_name] = token_rule^0
-end
-
-local function build_grammar(lexer, initial_rule)
- local children = lexer._CHILDREN
- if children then
- local lexer_name = lexer._NAME
- if not initial_rule then
- initial_rule = lexer_name
- end
- local grammar = { initial_rule }
- add_lexer(grammar, lexer)
- lexer._INITIALRULE = initial_rule
- lexer._GRAMMAR = Ct(P(grammar))
- else
- lexer._GRAMMAR = Ct(join_tokens(lexer)^0)
- end
-end
-
--- so far. We need these local functions in the next one.
---
--- Before 3.24 we had tokens[..] = { category, position }, now it's a two values.
-
-local lineparsers = { }
-
-function context.lex(text,init_style)
- local lexer = global._LEXER
- local grammar = lexer._GRAMMAR
- if not grammar then
- return { }
- elseif lexer._LEXBYLINE then -- we could keep token
- local tokens = { }
- local offset = 0
- local noftokens = 0
- -- -- pre 3.24
- --
- -- for line in gmatch(text,'[^\r\n]*\r?\n?') do -- could be an lpeg
- -- local line_tokens = lpegmatch(grammar,line)
- -- if line_tokens then
- -- for i=1,#line_tokens do
- -- local token = line_tokens[i]
- -- token[2] = token[2] + offset
- -- noftokens = noftokens + 1
- -- tokens[noftokens] = token
- -- end
- -- end
- -- offset = offset + #line
- -- if noftokens > 0 and tokens[noftokens][2] ~= offset then
- -- noftokens = noftokens + 1
- -- tokens[noftokens] = { 'default', offset + 1 }
- -- end
- -- end
-
- -- for line in gmatch(text,'[^\r\n]*\r?\n?') do
- -- local line_tokens = lpegmatch(grammar,line)
- -- if line_tokens then
- -- for i=1,#line_tokens,2 do
- -- noftokens = noftokens + 1
- -- tokens[noftokens] = line_tokens[i]
- -- noftokens = noftokens + 1
- -- tokens[noftokens] = line_tokens[i + 1] + offset
- -- end
- -- end
- -- offset = offset + #line
- -- if noftokens > 0 and tokens[noftokens] ~= offset then
- -- noftokens = noftokens + 1
- -- tokens[noftokens] = 'default'
- -- noftokens = noftokens + 1
- -- tokens[noftokens] = offset + 1
- -- end
- -- end
-
- local lineparser = lineparsers[lexer]
- if not lineparser then -- probably a cmt is more efficient
- lineparser = C((1-newline)^0 * newline) / function(line)
- local length = #line
- local line_tokens = length > 0 and lpegmatch(grammar,line)
- if line_tokens then
- for i=1,#line_tokens,2 do
- noftokens = noftokens + 1
- tokens[noftokens] = line_tokens[i]
- noftokens = noftokens + 1
- tokens[noftokens] = line_tokens[i + 1] + offset
- end
- end
- offset = offset + length
- if noftokens > 0 and tokens[noftokens] ~= offset then
- noftokens = noftokens + 1
- tokens[noftokens] = 'default'
- noftokens = noftokens + 1
- tokens[noftokens] = offset + 1
- end
- end
- lineparser = lineparser^0
- lineparsers[lexer] = lineparser
- end
- lpegmatch(lineparser,text)
- return tokens
-
- elseif lexer._CHILDREN then
- -- as we cannot print, tracing is not possible ... this might change as we can as well
- -- generate them all in one go (sharing as much as possible)
- local hash = lexer._HASH -- hm, was _hash
- if not hash then
- hash = { }
- lexer._HASH = hash
- end
- grammar = hash[init_style]
- if grammar then
- lexer._GRAMMAR = grammar
- else
- for style, style_num in next, lexer._TOKENS do
- if style_num == init_style then
- -- the name of the lexers is filtered from the whitespace
- -- specification
- local lexer_name = match(style,'^(.+)_whitespace') or lexer._NAME
- if lexer._INITIALRULE ~= lexer_name then
- grammar = hash[lexer_name]
- if not grammar then
- build_grammar(lexer,lexer_name)
- grammar = lexer._GRAMMAR
- hash[lexer_name] = grammar
- end
- end
- break
- end
- end
- grammar = grammar or lexer._GRAMMAR
- hash[init_style] = grammar
- end
- return lpegmatch(grammar,text)
- else
- return lpegmatch(grammar,text)
- end
-end
-
--- todo: keywords: one lookup and multiple matches
-
--- function context.token(name, patt)
--- return Ct(patt * Cc(name) * Cp())
--- end
---
--- -- hm, changed in 3.24 .. no longer a table
-
-function context.token(name, patt)
- return patt * Cc(name) * Cp()
-end
-
-lexer.fold = context.fold
-lexer.lex = context.lex
-lexer.token = context.token
-lexer.exact_match = context.exact_match
-
--- helper .. alas ... the lexer's lua instance is rather crippled .. not even
--- math is part of it
-
-local floor = math and math.floor
-local char = string.char
-
-if not floor then
-
- floor = function(n)
- return tonumber(string.format("%d",n))
- end
-
- math = math or { }
-
- math.floor = floor
-
-end
-
-local function utfchar(n)
- if n < 0x80 then
- return char(n)
- elseif n < 0x800 then
- return char(
- 0xC0 + floor(n/0x40),
- 0x80 + (n % 0x40)
- )
- elseif n < 0x10000 then
- return char(
- 0xE0 + floor(n/0x1000),
- 0x80 + (floor(n/0x40) % 0x40),
- 0x80 + (n % 0x40)
- )
- elseif n < 0x40000 then
- return char(
- 0xF0 + floor(n/0x40000),
- 0x80 + floor(n/0x1000),
- 0x80 + (floor(n/0x40) % 0x40),
- 0x80 + (n % 0x40)
- )
- else
- -- return char(
- -- 0xF1 + floor(n/0x1000000),
- -- 0x80 + floor(n/0x40000),
- -- 0x80 + floor(n/0x1000),
- -- 0x80 + (floor(n/0x40) % 0x40),
- -- 0x80 + (n % 0x40)
- -- )
- return "?"
- end
-end
-
-context.utfchar = utfchar
-
--- a helper from l-lpeg:
-
-local gmatch = string.gmatch
-
-local function make(t)
- local p
- for k, v in next, t do
- if not p then
- if next(v) then
- p = P(k) * make(v)
- else
- p = P(k)
- end
- else
- if next(v) then
- p = p + P(k) * make(v)
- else
- p = p + P(k)
- end
- end
- end
- return p
-end
-
-function lpeg.utfchartabletopattern(list)
- local tree = { }
- for i=1,#list do
- local t = tree
- for c in gmatch(list[i],".") do
- if not t[c] then
- t[c] = { }
- end
- t = t[c]
- end
- end
- return make(tree)
-end
-
--- patterns.invisibles =
--- P(utfchar(0x00A0)) -- nbsp
--- + P(utfchar(0x2000)) -- enquad
--- + P(utfchar(0x2001)) -- emquad
--- + P(utfchar(0x2002)) -- enspace
--- + P(utfchar(0x2003)) -- emspace
--- + P(utfchar(0x2004)) -- threeperemspace
--- + P(utfchar(0x2005)) -- fourperemspace
--- + P(utfchar(0x2006)) -- sixperemspace
--- + P(utfchar(0x2007)) -- figurespace
--- + P(utfchar(0x2008)) -- punctuationspace
--- + P(utfchar(0x2009)) -- breakablethinspace
--- + P(utfchar(0x200A)) -- hairspace
--- + P(utfchar(0x200B)) -- zerowidthspace
--- + P(utfchar(0x202F)) -- narrownobreakspace
--- + P(utfchar(0x205F)) -- math thinspace
-
-patterns.invisibles = lpeg.utfchartabletopattern {
- utfchar(0x00A0), -- nbsp
- utfchar(0x2000), -- enquad
- utfchar(0x2001), -- emquad
- utfchar(0x2002), -- enspace
- utfchar(0x2003), -- emspace
- utfchar(0x2004), -- threeperemspace
- utfchar(0x2005), -- fourperemspace
- utfchar(0x2006), -- sixperemspace
- utfchar(0x2007), -- figurespace
- utfchar(0x2008), -- punctuationspace
- utfchar(0x2009), -- breakablethinspace
- utfchar(0x200A), -- hairspace
- utfchar(0x200B), -- zerowidthspace
- utfchar(0x202F), -- narrownobreakspace
- utfchar(0x205F), -- math thinspace
-}
-
--- now we can make:
-
-patterns.iwordtoken = patterns.wordtoken - patterns.invisibles
-patterns.iwordpattern = patterns.iwordtoken^3
-
--- require("themes/scite-context-theme")
-
--- In order to deal with some bug in additional styles (I have no cue what is
--- wrong, but additional styles get ignored and clash somehow) I just copy the
--- original lexer code ... see original for comments.
diff --git a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-context.lua b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-context.lua
index 0894e86884f..6c0293fbdef 100644
--- a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-context.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-context.lua
@@ -1,4 +1,4 @@
return {
- ["constants"]={ "zerocount", "minusone", "minustwo", "plusone", "plustwo", "plusthree", "plusfour", "plusfive", "plussix", "plusseven", "pluseight", "plusnine", "plusten", "plussixteen", "plushundred", "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard", "zeropoint", "onepoint", "halfapoint", "onebasepoint", "maxdimen", "scaledpoint", "thousandpoint", "points", "halfpoint", "zeroskip", "zeromuskip", "onemuskip", "pluscxxvii", "pluscxxviii", "pluscclv", "pluscclvi", "normalpagebox", "endoflinetoken", "outputnewlinechar", "emptytoks", "empty", "undefined", "voidbox", "emptybox", "emptyvbox", "emptyhbox", "bigskipamount", "medskipamount", "smallskipamount", "fmtname", "fmtversion", "texengine", "texenginename", "texengineversion", "luatexengine", "pdftexengine", "xetexengine", "unknownengine", "etexversion", "pdftexversion", "xetexversion", "xetexrevision", "activecatcode", "bgroup", "egroup", "endline", "conditionaltrue", "conditionalfalse", "attributeunsetvalue", "uprotationangle", "rightrotationangle", "downrotationangle", "leftrotationangle", "inicatcodes", "ctxcatcodes", "texcatcodes", "notcatcodes", "txtcatcodes", "vrbcatcodes", "prtcatcodes", "nilcatcodes", "luacatcodes", "tpacatcodes", "tpbcatcodes", "xmlcatcodes", "escapecatcode", "begingroupcatcode", "endgroupcatcode", "mathshiftcatcode", "alignmentcatcode", "endoflinecatcode", "parametercatcode", "superscriptcatcode", "subscriptcatcode", "ignorecatcode", "spacecatcode", "lettercatcode", "othercatcode", "activecatcode", "commentcatcode", "invalidcatcode", "tabasciicode", "newlineasciicode", "formfeedasciicode", "endoflineasciicode", "endoffileasciicode", "spaceasciicode", "hashasciicode", "dollarasciicode", "commentasciicode", "ampersandasciicode", "colonasciicode", "backslashasciicode", "circumflexasciicode", "underscoreasciicode", "leftbraceasciicode", "barasciicode", "rightbraceasciicode", "tildeasciicode", "delasciicode", "lessthanasciicode", "morethanasciicode", "doublecommentsignal", "atsignasciicode", "exclamationmarkasciicode", "questionmarkasciicode", "doublequoteasciicode", "singlequoteasciicode", "forwardslashasciicode", "primeasciicode", "activemathcharcode", "activetabtoken", "activeformfeedtoken", "activeendoflinetoken", "batchmodecode", "nonstopmodecode", "scrollmodecode", "errorstopmodecode", "bottomlevelgroupcode", "simplegroupcode", "hboxgroupcode", "adjustedhboxgroupcode", "vboxgroupcode", "vtopgroupcode", "aligngroupcode", "noaligngroupcode", "outputgroupcode", "mathgroupcode", "discretionarygroupcode", "insertgroupcode", "vcentergroupcode", "mathchoicegroupcode", "semisimplegroupcode", "mathshiftgroupcode", "mathleftgroupcode", "vadjustgroupcode", "charnodecode", "hlistnodecode", "vlistnodecode", "rulenodecode", "insertnodecode", "marknodecode", "adjustnodecode", "ligaturenodecode", "discretionarynodecode", "whatsitnodecode", "mathnodecode", "gluenodecode", "kernnodecode", "penaltynodecode", "unsetnodecode", "mathsnodecode", "charifcode", "catifcode", "numifcode", "dimifcode", "oddifcode", "vmodeifcode", "hmodeifcode", "mmodeifcode", "innerifcode", "voidifcode", "hboxifcode", "vboxifcode", "xifcode", "eofifcode", "trueifcode", "falseifcode", "caseifcode", "definedifcode", "csnameifcode", "fontcharifcode", "fontslantperpoint", "fontinterwordspace", "fontinterwordstretch", "fontinterwordshrink", "fontexheight", "fontemwidth", "fontextraspace", "slantperpoint", "interwordspace", "interwordstretch", "interwordshrink", "exheight", "emwidth", "extraspace", "mathsupdisplay", "mathsupnormal", "mathsupcramped", "mathsubnormal", "mathsubcombined", "mathaxisheight", "startmode", "stopmode", "startnotmode", "stopnotmode", "startmodeset", "stopmodeset", "doifmode", "doifmodeelse", "doifnotmode", "startallmodes", "stopallmodes", "startnotallmodes", "stopnotallmodes", "doifallmodes", "doifallmodeselse", "doifnotallmodes", "startenvironment", "stopenvironment", "environment", "startcomponent", "stopcomponent", "component", "startproduct", "stopproduct", "product", "startproject", "stopproject", "project", "starttext", "stoptext", "startnotext", "stopnotext", "startdocument", "stopdocument", "documentvariable", "setupdocument", "startmodule", "stopmodule", "usemodule", "usetexmodule", "useluamodule", "startTEXpage", "stopTEXpage", "enablemode", "disablemode", "preventmode", "globalenablemode", "globaldisablemode", "globalpreventmode", "pushmode", "popmode", "typescriptone", "typescripttwo", "typescriptthree", "mathsizesuffix", "mathordcode", "mathopcode", "mathbincode", "mathrelcode", "mathopencode", "mathclosecode", "mathpunctcode", "mathalphacode", "mathinnercode", "mathnothingcode", "mathlimopcode", "mathnolopcode", "mathboxcode", "mathchoicecode", "mathaccentcode", "mathradicalcode", "constantnumber", "constantnumberargument", "constantdimen", "constantdimenargument", "constantemptyargument", "continueifinputfile", "luastringsep", "!!bs", "!!es" },
- ["helpers"]={ "startsetups", "stopsetups", "startxmlsetups", "stopxmlsetups", "startluasetups", "stopluasetups", "starttexsetups", "stoptexsetups", "startrawsetups", "stoprawsetups", "startlocalsetups", "stoplocalsetups", "starttexdefinition", "stoptexdefinition", "starttexcode", "stoptexcode", "startcontextcode", "stopcontextcode", "doifsetupselse", "doifsetups", "doifnotsetups", "setup", "setups", "texsetup", "xmlsetup", "luasetup", "directsetup", "doifelsecommandhandler", "doifnotcommandhandler", "doifcommandhandler", "newmode", "setmode", "resetmode", "newsystemmode", "setsystemmode", "resetsystemmode", "pushsystemmode", "popsystemmode", "booleanmodevalue", "newcount", "newdimen", "newskip", "newmuskip", "newbox", "newtoks", "newread", "newwrite", "newmarks", "newinsert", "newattribute", "newif", "newlanguage", "newfamily", "newfam", "newhelp", "then", "begcsname", "strippedcsname", "firstargumentfalse", "firstargumenttrue", "secondargumentfalse", "secondargumenttrue", "thirdargumentfalse", "thirdargumenttrue", "fourthargumentfalse", "fourthargumenttrue", "fifthargumentfalse", "fifthsargumenttrue", "sixthargumentfalse", "sixtsargumenttrue", "doglobal", "dodoglobal", "redoglobal", "resetglobal", "donothing", "dontcomplain", "forgetall", "donetrue", "donefalse", "htdp", "unvoidbox", "hfilll", "vfilll", "mathbox", "mathlimop", "mathnolop", "mathnothing", "mathalpha", "currentcatcodetable", "defaultcatcodetable", "catcodetablename", "newcatcodetable", "startcatcodetable", "stopcatcodetable", "startextendcatcodetable", "stopextendcatcodetable", "pushcatcodetable", "popcatcodetable", "restorecatcodes", "setcatcodetable", "letcatcodecommand", "defcatcodecommand", "uedcatcodecommand", "hglue", "vglue", "hfillneg", "vfillneg", "hfilllneg", "vfilllneg", "ruledhss", "ruledhfil", "ruledhfill", "ruledhfilneg", "ruledhfillneg", "normalhfillneg", "ruledvss", "ruledvfil", "ruledvfill", "ruledvfilneg", "ruledvfillneg", "normalvfillneg", "ruledhbox", "ruledvbox", "ruledvtop", "ruledvcenter", "ruledhskip", "ruledvskip", "ruledkern", "ruledmskip", "ruledmkern", "ruledhglue", "ruledvglue", "normalhglue", "normalvglue", "ruledpenalty", "filledhboxb", "filledhboxr", "filledhboxg", "filledhboxc", "filledhboxm", "filledhboxy", "filledhboxk", "scratchcounter", "globalscratchcounter", "scratchdimen", "globalscratchdimen", "scratchskip", "globalscratchskip", "scratchmuskip", "globalscratchmuskip", "scratchtoks", "globalscratchtoks", "scratchbox", "globalscratchbox", "availablehsize", "localhsize", "setlocalhsize", "nextbox", "dowithnextbox", "dowithnextboxcs", "dowithnextboxcontent", "dowithnextboxcontentcs", "scratchwidth", "scratchheight", "scratchdepth", "scratchoffset", "scratchdistance", "scratchhsize", "scratchvsize", "scratchxoffset", "scratchyoffset", "scratchhoffset", "scratchvoffset", "scratchxposition", "scratchyposition", "scratchtopoffset", "scratchbottomoffset", "scratchleftoffset", "scratchrightoffset", "scratchcounterone", "scratchcountertwo", "scratchcounterthree", "scratchdimenone", "scratchdimentwo", "scratchdimenthree", "scratchskipone", "scratchskiptwo", "scratchskipthree", "scratchmuskipone", "scratchmuskiptwo", "scratchmuskipthree", "scratchtoksone", "scratchtokstwo", "scratchtoksthree", "scratchboxone", "scratchboxtwo", "scratchboxthree", "scratchnx", "scratchny", "scratchmx", "scratchmy", "scratchunicode", "scratchleftskip", "scratchrightskip", "scratchtopskip", "scratchbottomskip", "doif", "doifnot", "doifelse", "doifinset", "doifnotinset", "doifinsetelse", "doifnextcharelse", "doifnextoptionalelse", "doifnextbgroupelse", "doifnextparenthesiselse", "doiffastoptionalcheckelse", "doifundefinedelse", "doifdefinedelse", "doifundefined", "doifdefined", "doifelsevalue", "doifvalue", "doifnotvalue", "doifnothing", "doifsomething", "doifelsenothing", "doifsomethingelse", "doifvaluenothing", "doifvaluesomething", "doifelsevaluenothing", "doifdimensionelse", "doifnumberelse", "doifnumber", "doifnotnumber", "doifcommonelse", "doifcommon", "doifnotcommon", "doifinstring", "doifnotinstring", "doifinstringelse", "doifassignmentelse", "docheckassignment", "tracingall", "tracingnone", "loggingall", "removetoks", "appendtoks", "prependtoks", "appendtotoks", "prependtotoks", "to", "endgraf", "endpar", "everyendpar", "reseteverypar", "finishpar", "empty", "null", "space", "quad", "enspace", "obeyspaces", "obeylines", "obeyedspace", "obeyedline", "normalspace", "executeifdefined", "singleexpandafter", "doubleexpandafter", "tripleexpandafter", "dontleavehmode", "removelastspace", "removeunwantedspaces", "keepunwantedspaces", "wait", "writestatus", "define", "defineexpandable", "redefine", "setmeasure", "setemeasure", "setgmeasure", "setxmeasure", "definemeasure", "freezemeasure", "measure", "measured", "installcorenamespace", "getvalue", "getuvalue", "setvalue", "setevalue", "setgvalue", "setxvalue", "letvalue", "letgvalue", "resetvalue", "undefinevalue", "ignorevalue", "setuvalue", "setuevalue", "setugvalue", "setuxvalue", "globallet", "glet", "udef", "ugdef", "uedef", "uxdef", "checked", "unique", "getparameters", "geteparameters", "getgparameters", "getxparameters", "forgetparameters", "copyparameters", "getdummyparameters", "dummyparameter", "directdummyparameter", "setdummyparameter", "letdummyparameter", "usedummystyleandcolor", "usedummystyleparameter", "usedummycolorparameter", "processcommalist", "processcommacommand", "quitcommalist", "quitprevcommalist", "processaction", "processallactions", "processfirstactioninset", "processallactionsinset", "unexpanded", "expanded", "startexpanded", "stopexpanded", "protected", "protect", "unprotect", "firstofoneargument", "firstoftwoarguments", "secondoftwoarguments", "firstofthreearguments", "secondofthreearguments", "thirdofthreearguments", "firstoffourarguments", "secondoffourarguments", "thirdoffourarguments", "fourthoffourarguments", "firstoffivearguments", "secondoffivearguments", "thirdoffivearguments", "fourthoffivearguments", "fifthoffivearguments", "firstofsixarguments", "secondofsixarguments", "thirdofsixarguments", "fourthofsixarguments", "fifthofsixarguments", "sixthofsixarguments", "firstofoneunexpanded", "gobbleoneargument", "gobbletwoarguments", "gobblethreearguments", "gobblefourarguments", "gobblefivearguments", "gobblesixarguments", "gobblesevenarguments", "gobbleeightarguments", "gobbleninearguments", "gobbletenarguments", "gobbleoneoptional", "gobbletwooptionals", "gobblethreeoptionals", "gobblefouroptionals", "gobblefiveoptionals", "dorecurse", "doloop", "exitloop", "dostepwiserecurse", "recurselevel", "recursedepth", "dofastloopcs", "dowith", "newconstant", "setnewconstant", "newconditional", "settrue", "setfalse", "setconstant", "newmacro", "setnewmacro", "newfraction", "newsignal", "dosingleempty", "dodoubleempty", "dotripleempty", "doquadrupleempty", "doquintupleempty", "dosixtupleempty", "doseventupleempty", "dosingleargument", "dodoubleargument", "dotripleargument", "doquadrupleargument", "doquintupleargument", "dosixtupleargument", "doseventupleargument", "dosinglegroupempty", "dodoublegroupempty", "dotriplegroupempty", "doquadruplegroupempty", "doquintuplegroupempty", "permitspacesbetweengroups", "dontpermitspacesbetweengroups", "nopdfcompression", "maximumpdfcompression", "normalpdfcompression", "modulonumber", "dividenumber", "getfirstcharacter", "doiffirstcharelse", "startnointerference", "stopnointerference", "twodigits", "threedigits", "strut", "setstrut", "strutbox", "strutht", "strutdp", "strutwd", "struthtdp", "begstrut", "endstrut", "lineheight", "ordordspacing", "ordopspacing", "ordbinspacing", "ordrelspacing", "ordopenspacing", "ordclosespacing", "ordpunctspacing", "ordinnerspacing", "opordspacing", "opopspacing", "opbinspacing", "oprelspacing", "opopenspacing", "opclosespacing", "oppunctspacing", "opinnerspacing", "binordspacing", "binopspacing", "binbinspacing", "binrelspacing", "binopenspacing", "binclosespacing", "binpunctspacing", "bininnerspacing", "relordspacing", "relopspacing", "relbinspacing", "relrelspacing", "relopenspacing", "relclosespacing", "relpunctspacing", "relinnerspacing", "openordspacing", "openopspacing", "openbinspacing", "openrelspacing", "openopenspacing", "openclosespacing", "openpunctspacing", "openinnerspacing", "closeordspacing", "closeopspacing", "closebinspacing", "closerelspacing", "closeopenspacing", "closeclosespacing", "closepunctspacing", "closeinnerspacing", "punctordspacing", "punctopspacing", "punctbinspacing", "punctrelspacing", "punctopenspacing", "punctclosespacing", "punctpunctspacing", "punctinnerspacing", "innerordspacing", "inneropspacing", "innerbinspacing", "innerrelspacing", "inneropenspacing", "innerclosespacing", "innerpunctspacing", "innerinnerspacing", "normalreqno", "startimath", "stopimath", "normalstartimath", "normalstopimath", "startdmath", "stopdmath", "normalstartdmath", "normalstopdmath", "uncramped", "cramped", "triggermathstyle", "mathstylefont", "mathsmallstylefont", "mathstyleface", "mathsmallstyleface", "mathstylecommand", "mathpalette", "mathstylehbox", "mathstylevbox", "mathstylevcenter", "mathstylevcenteredhbox", "mathstylevcenteredvbox", "mathtext", "setmathsmalltextbox", "setmathtextbox", "triggerdisplaystyle", "triggertextstyle", "triggerscriptstyle", "triggerscriptscriptstyle", "triggeruncrampedstyle", "triggercrampedstyle", "triggersmallstyle", "triggeruncrampedsmallstyle", "triggercrampedsmallstyle", "triggerbigstyle", "triggeruncrampedbigstyle", "triggercrampedbigstyle", "luaexpr", "expdoifelse", "expdoif", "expdoifnot", "expdoifcommonelse", "expdoifinsetelse", "ctxdirectlua", "ctxlatelua", "ctxsprint", "ctxwrite", "ctxcommand", "ctxdirectcommand", "ctxlatecommand", "ctxreport", "ctxlua", "luacode", "lateluacode", "directluacode", "registerctxluafile", "ctxloadluafile", "luaversion", "luamajorversion", "luaminorversion", "ctxluacode", "luaconditional", "luaexpanded", "startluaparameterset", "stopluaparameterset", "luaparameterset", "definenamedlua", "obeylualines", "obeyluatokens", "startluacode", "stopluacode", "startlua", "stoplua", "carryoverpar", "Umathbotaccent" },
+ ["constants"]={ "zerocount", "minusone", "minustwo", "plusone", "plustwo", "plusthree", "plusfour", "plusfive", "plussix", "plusseven", "pluseight", "plusnine", "plusten", "plussixteen", "plushundred", "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard", "zeropoint", "onepoint", "halfapoint", "onebasepoint", "maxdimen", "scaledpoint", "thousandpoint", "points", "halfpoint", "zeroskip", "zeromuskip", "onemuskip", "pluscxxvii", "pluscxxviii", "pluscclv", "pluscclvi", "normalpagebox", "endoflinetoken", "outputnewlinechar", "emptytoks", "empty", "undefined", "voidbox", "emptybox", "emptyvbox", "emptyhbox", "bigskipamount", "medskipamount", "smallskipamount", "fmtname", "fmtversion", "texengine", "texenginename", "texengineversion", "luatexengine", "pdftexengine", "xetexengine", "unknownengine", "etexversion", "pdftexversion", "xetexversion", "xetexrevision", "activecatcode", "bgroup", "egroup", "endline", "conditionaltrue", "conditionalfalse", "attributeunsetvalue", "uprotationangle", "rightrotationangle", "downrotationangle", "leftrotationangle", "inicatcodes", "ctxcatcodes", "texcatcodes", "notcatcodes", "txtcatcodes", "vrbcatcodes", "prtcatcodes", "nilcatcodes", "luacatcodes", "tpacatcodes", "tpbcatcodes", "xmlcatcodes", "ctdcatcodes", "escapecatcode", "begingroupcatcode", "endgroupcatcode", "mathshiftcatcode", "alignmentcatcode", "endoflinecatcode", "parametercatcode", "superscriptcatcode", "subscriptcatcode", "ignorecatcode", "spacecatcode", "lettercatcode", "othercatcode", "activecatcode", "commentcatcode", "invalidcatcode", "tabasciicode", "newlineasciicode", "formfeedasciicode", "endoflineasciicode", "endoffileasciicode", "spaceasciicode", "hashasciicode", "dollarasciicode", "commentasciicode", "ampersandasciicode", "colonasciicode", "backslashasciicode", "circumflexasciicode", "underscoreasciicode", "leftbraceasciicode", "barasciicode", "rightbraceasciicode", "tildeasciicode", "delasciicode", "lessthanasciicode", "morethanasciicode", "doublecommentsignal", "atsignasciicode", "exclamationmarkasciicode", "questionmarkasciicode", "doublequoteasciicode", "singlequoteasciicode", "forwardslashasciicode", "primeasciicode", "activemathcharcode", "activetabtoken", "activeformfeedtoken", "activeendoflinetoken", "batchmodecode", "nonstopmodecode", "scrollmodecode", "errorstopmodecode", "bottomlevelgroupcode", "simplegroupcode", "hboxgroupcode", "adjustedhboxgroupcode", "vboxgroupcode", "vtopgroupcode", "aligngroupcode", "noaligngroupcode", "outputgroupcode", "mathgroupcode", "discretionarygroupcode", "insertgroupcode", "vcentergroupcode", "mathchoicegroupcode", "semisimplegroupcode", "mathshiftgroupcode", "mathleftgroupcode", "vadjustgroupcode", "charnodecode", "hlistnodecode", "vlistnodecode", "rulenodecode", "insertnodecode", "marknodecode", "adjustnodecode", "ligaturenodecode", "discretionarynodecode", "whatsitnodecode", "mathnodecode", "gluenodecode", "kernnodecode", "penaltynodecode", "unsetnodecode", "mathsnodecode", "charifcode", "catifcode", "numifcode", "dimifcode", "oddifcode", "vmodeifcode", "hmodeifcode", "mmodeifcode", "innerifcode", "voidifcode", "hboxifcode", "vboxifcode", "xifcode", "eofifcode", "trueifcode", "falseifcode", "caseifcode", "definedifcode", "csnameifcode", "fontcharifcode", "fontslantperpoint", "fontinterwordspace", "fontinterwordstretch", "fontinterwordshrink", "fontexheight", "fontemwidth", "fontextraspace", "slantperpoint", "interwordspace", "interwordstretch", "interwordshrink", "exheight", "emwidth", "extraspace", "mathsupdisplay", "mathsupnormal", "mathsupcramped", "mathsubnormal", "mathsubcombined", "mathaxisheight", "startmode", "stopmode", "startnotmode", "stopnotmode", "startmodeset", "stopmodeset", "doifmode", "doifmodeelse", "doifnotmode", "startmodeset", "stopmodeset", "startallmodes", "stopallmodes", "startnotallmodes", "stopnotallmodes", "doifallmodes", "doifallmodeselse", "doifnotallmodes", "startenvironment", "stopenvironment", "environment", "startcomponent", "stopcomponent", "component", "startproduct", "stopproduct", "product", "startproject", "stopproject", "project", "starttext", "stoptext", "startnotext", "stopnotext", "startdocument", "stopdocument", "documentvariable", "setupdocument", "startmodule", "stopmodule", "usemodule", "usetexmodule", "useluamodule", "setupmodule", "currentmoduleparameter", "moduleparameter", "startTEXpage", "stopTEXpage", "enablemode", "disablemode", "preventmode", "globalenablemode", "globaldisablemode", "globalpreventmode", "pushmode", "popmode", "typescriptone", "typescripttwo", "typescriptthree", "mathsizesuffix", "mathordcode", "mathopcode", "mathbincode", "mathrelcode", "mathopencode", "mathclosecode", "mathpunctcode", "mathalphacode", "mathinnercode", "mathnothingcode", "mathlimopcode", "mathnolopcode", "mathboxcode", "mathchoicecode", "mathaccentcode", "mathradicalcode", "constantnumber", "constantnumberargument", "constantdimen", "constantdimenargument", "constantemptyargument", "continueifinputfile", "luastringsep", "!!bs", "!!es", "lefttorightmark", "righttoleftmark", "breakablethinspace", "nobreakspace", "narrownobreakspace", "zerowidthnobreakspace", "ideographicspace", "ideographichalffillspace", "twoperemspace", "threeperemspace", "fourperemspace", "fiveperemspace", "sixperemspace", "figurespace", "punctuationspace", "hairspace", "zerowidthspace", "zerowidthnonjoiner", "zerowidthjoiner", "zwnj", "zwj" },
+ ["helpers"]={ "startsetups", "stopsetups", "startxmlsetups", "stopxmlsetups", "startluasetups", "stopluasetups", "starttexsetups", "stoptexsetups", "startrawsetups", "stoprawsetups", "startlocalsetups", "stoplocalsetups", "starttexdefinition", "stoptexdefinition", "starttexcode", "stoptexcode", "startcontextcode", "stopcontextcode", "startcontextdefinitioncode", "stopcontextdefinitioncode", "doifsetupselse", "doifsetups", "doifnotsetups", "setup", "setups", "texsetup", "xmlsetup", "luasetup", "directsetup", "doifelsecommandhandler", "doifnotcommandhandler", "doifcommandhandler", "newmode", "setmode", "resetmode", "newsystemmode", "setsystemmode", "resetsystemmode", "pushsystemmode", "popsystemmode", "booleanmodevalue", "newcount", "newdimen", "newskip", "newmuskip", "newbox", "newtoks", "newread", "newwrite", "newmarks", "newinsert", "newattribute", "newif", "newlanguage", "newfamily", "newfam", "newhelp", "then", "begcsname", "strippedcsname", "firstargumentfalse", "firstargumenttrue", "secondargumentfalse", "secondargumenttrue", "thirdargumentfalse", "thirdargumenttrue", "fourthargumentfalse", "fourthargumenttrue", "fifthargumentfalse", "fifthsargumenttrue", "sixthargumentfalse", "sixtsargumenttrue", "doglobal", "dodoglobal", "redoglobal", "resetglobal", "donothing", "dontcomplain", "forgetall", "donetrue", "donefalse", "htdp", "unvoidbox", "hfilll", "vfilll", "mathbox", "mathlimop", "mathnolop", "mathnothing", "mathalpha", "currentcatcodetable", "defaultcatcodetable", "catcodetablename", "newcatcodetable", "startcatcodetable", "stopcatcodetable", "startextendcatcodetable", "stopextendcatcodetable", "pushcatcodetable", "popcatcodetable", "restorecatcodes", "setcatcodetable", "letcatcodecommand", "defcatcodecommand", "uedcatcodecommand", "hglue", "vglue", "hfillneg", "vfillneg", "hfilllneg", "vfilllneg", "ruledhss", "ruledhfil", "ruledhfill", "ruledhfilneg", "ruledhfillneg", "normalhfillneg", "ruledvss", "ruledvfil", "ruledvfill", "ruledvfilneg", "ruledvfillneg", "normalvfillneg", "ruledhbox", "ruledvbox", "ruledvtop", "ruledvcenter", "ruledmbox", "ruledhskip", "ruledvskip", "ruledkern", "ruledmskip", "ruledmkern", "ruledhglue", "ruledvglue", "normalhglue", "normalvglue", "ruledpenalty", "filledhboxb", "filledhboxr", "filledhboxg", "filledhboxc", "filledhboxm", "filledhboxy", "filledhboxk", "scratchcounter", "globalscratchcounter", "scratchdimen", "globalscratchdimen", "scratchskip", "globalscratchskip", "scratchmuskip", "globalscratchmuskip", "scratchtoks", "globalscratchtoks", "scratchbox", "globalscratchbox", "normalbaselineskip", "normallineskip", "normallineskiplimit", "availablehsize", "localhsize", "setlocalhsize", "nextbox", "dowithnextbox", "dowithnextboxcs", "dowithnextboxcontent", "dowithnextboxcontentcs", "scratchwidth", "scratchheight", "scratchdepth", "scratchoffset", "scratchdistance", "scratchhsize", "scratchvsize", "scratchxoffset", "scratchyoffset", "scratchhoffset", "scratchvoffset", "scratchxposition", "scratchyposition", "scratchtopoffset", "scratchbottomoffset", "scratchleftoffset", "scratchrightoffset", "scratchcounterone", "scratchcountertwo", "scratchcounterthree", "scratchdimenone", "scratchdimentwo", "scratchdimenthree", "scratchskipone", "scratchskiptwo", "scratchskipthree", "scratchmuskipone", "scratchmuskiptwo", "scratchmuskipthree", "scratchtoksone", "scratchtokstwo", "scratchtoksthree", "scratchboxone", "scratchboxtwo", "scratchboxthree", "scratchnx", "scratchny", "scratchmx", "scratchmy", "scratchunicode", "scratchleftskip", "scratchrightskip", "scratchtopskip", "scratchbottomskip", "doif", "doifnot", "doifelse", "doifinset", "doifnotinset", "doifinsetelse", "doifnextcharelse", "doifnextoptionalelse", "doifnextoptionalcselse", "doiffastoptionalcheckelse", "doifnextbgroupelse", "doifnextbgroupcselse", "doifnextparenthesiselse", "doifundefinedelse", "doifdefinedelse", "doifundefined", "doifdefined", "doifelsevalue", "doifvalue", "doifnotvalue", "doifnothing", "doifsomething", "doifelsenothing", "doifsomethingelse", "doifvaluenothing", "doifvaluesomething", "doifelsevaluenothing", "doifdimensionelse", "doifnumberelse", "doifnumber", "doifnotnumber", "doifcommonelse", "doifcommon", "doifnotcommon", "doifinstring", "doifnotinstring", "doifinstringelse", "doifassignmentelse", "docheckassignment", "tracingall", "tracingnone", "loggingall", "removetoks", "appendtoks", "prependtoks", "appendtotoks", "prependtotoks", "to", "endgraf", "endpar", "everyendpar", "reseteverypar", "finishpar", "empty", "null", "space", "quad", "enspace", "obeyspaces", "obeylines", "obeyedspace", "obeyedline", "normalspace", "executeifdefined", "singleexpandafter", "doubleexpandafter", "tripleexpandafter", "dontleavehmode", "removelastspace", "removeunwantedspaces", "keepunwantedspaces", "wait", "writestatus", "define", "defineexpandable", "redefine", "setmeasure", "setemeasure", "setgmeasure", "setxmeasure", "definemeasure", "freezemeasure", "measure", "measured", "installcorenamespace", "getvalue", "getuvalue", "setvalue", "setevalue", "setgvalue", "setxvalue", "letvalue", "letgvalue", "resetvalue", "undefinevalue", "ignorevalue", "setuvalue", "setuevalue", "setugvalue", "setuxvalue", "globallet", "glet", "udef", "ugdef", "uedef", "uxdef", "checked", "unique", "getparameters", "geteparameters", "getgparameters", "getxparameters", "forgetparameters", "copyparameters", "getdummyparameters", "dummyparameter", "directdummyparameter", "setdummyparameter", "letdummyparameter", "usedummystyleandcolor", "usedummystyleparameter", "usedummycolorparameter", "processcommalist", "processcommacommand", "quitcommalist", "quitprevcommalist", "processaction", "processallactions", "processfirstactioninset", "processallactionsinset", "unexpanded", "expanded", "startexpanded", "stopexpanded", "protected", "protect", "unprotect", "firstofoneargument", "firstoftwoarguments", "secondoftwoarguments", "firstofthreearguments", "secondofthreearguments", "thirdofthreearguments", "firstoffourarguments", "secondoffourarguments", "thirdoffourarguments", "fourthoffourarguments", "firstoffivearguments", "secondoffivearguments", "thirdoffivearguments", "fourthoffivearguments", "fifthoffivearguments", "firstofsixarguments", "secondofsixarguments", "thirdofsixarguments", "fourthofsixarguments", "fifthofsixarguments", "sixthofsixarguments", "firstofoneunexpanded", "gobbleoneargument", "gobbletwoarguments", "gobblethreearguments", "gobblefourarguments", "gobblefivearguments", "gobblesixarguments", "gobblesevenarguments", "gobbleeightarguments", "gobbleninearguments", "gobbletenarguments", "gobbleoneoptional", "gobbletwooptionals", "gobblethreeoptionals", "gobblefouroptionals", "gobblefiveoptionals", "dorecurse", "doloop", "exitloop", "dostepwiserecurse", "recurselevel", "recursedepth", "dofastloopcs", "dowith", "newconstant", "setnewconstant", "setconstant", "setconstantvalue", "newconditional", "settrue", "setfalse", "settruevalue", "setfalsevalue", "newmacro", "setnewmacro", "newfraction", "newsignal", "dosingleempty", "dodoubleempty", "dotripleempty", "doquadrupleempty", "doquintupleempty", "dosixtupleempty", "doseventupleempty", "dosingleargument", "dodoubleargument", "dotripleargument", "doquadrupleargument", "doquintupleargument", "dosixtupleargument", "doseventupleargument", "dosinglegroupempty", "dodoublegroupempty", "dotriplegroupempty", "doquadruplegroupempty", "doquintuplegroupempty", "permitspacesbetweengroups", "dontpermitspacesbetweengroups", "nopdfcompression", "maximumpdfcompression", "normalpdfcompression", "modulonumber", "dividenumber", "getfirstcharacter", "doiffirstcharelse", "startnointerference", "stopnointerference", "twodigits", "threedigits", "leftorright", "strut", "setstrut", "strutbox", "strutht", "strutdp", "strutwd", "struthtdp", "begstrut", "endstrut", "lineheight", "ordordspacing", "ordopspacing", "ordbinspacing", "ordrelspacing", "ordopenspacing", "ordclosespacing", "ordpunctspacing", "ordinnerspacing", "opordspacing", "opopspacing", "opbinspacing", "oprelspacing", "opopenspacing", "opclosespacing", "oppunctspacing", "opinnerspacing", "binordspacing", "binopspacing", "binbinspacing", "binrelspacing", "binopenspacing", "binclosespacing", "binpunctspacing", "bininnerspacing", "relordspacing", "relopspacing", "relbinspacing", "relrelspacing", "relopenspacing", "relclosespacing", "relpunctspacing", "relinnerspacing", "openordspacing", "openopspacing", "openbinspacing", "openrelspacing", "openopenspacing", "openclosespacing", "openpunctspacing", "openinnerspacing", "closeordspacing", "closeopspacing", "closebinspacing", "closerelspacing", "closeopenspacing", "closeclosespacing", "closepunctspacing", "closeinnerspacing", "punctordspacing", "punctopspacing", "punctbinspacing", "punctrelspacing", "punctopenspacing", "punctclosespacing", "punctpunctspacing", "punctinnerspacing", "innerordspacing", "inneropspacing", "innerbinspacing", "innerrelspacing", "inneropenspacing", "innerclosespacing", "innerpunctspacing", "innerinnerspacing", "normalreqno", "startimath", "stopimath", "normalstartimath", "normalstopimath", "startdmath", "stopdmath", "normalstartdmath", "normalstopdmath", "uncramped", "cramped", "triggermathstyle", "mathstylefont", "mathsmallstylefont", "mathstyleface", "mathsmallstyleface", "mathstylecommand", "mathpalette", "mathstylehbox", "mathstylevbox", "mathstylevcenter", "mathstylevcenteredhbox", "mathstylevcenteredvbox", "mathtext", "setmathsmalltextbox", "setmathtextbox", "triggerdisplaystyle", "triggertextstyle", "triggerscriptstyle", "triggerscriptscriptstyle", "triggeruncrampedstyle", "triggercrampedstyle", "triggersmallstyle", "triggeruncrampedsmallstyle", "triggercrampedsmallstyle", "triggerbigstyle", "triggeruncrampedbigstyle", "triggercrampedbigstyle", "luaexpr", "expdoifelse", "expdoif", "expdoifnot", "expdoifcommonelse", "expdoifinsetelse", "ctxdirectlua", "ctxlatelua", "ctxsprint", "ctxwrite", "ctxcommand", "ctxdirectcommand", "ctxlatecommand", "ctxreport", "ctxlua", "luacode", "lateluacode", "directluacode", "registerctxluafile", "ctxloadluafile", "luaversion", "luamajorversion", "luaminorversion", "ctxluacode", "luaconditional", "luaexpanded", "startluaparameterset", "stopluaparameterset", "luaparameterset", "definenamedlua", "obeylualines", "obeyluatokens", "startluacode", "stopluacode", "startlua", "stoplua", "startctxfunction", "stopctxfunction", "ctxfunction", "startctxfunctiondefinition", "stopctxfunctiondefinition", "carryoverpar", "assumelongusagecs", "Umathbotaccent", "righttolefthbox", "lefttorighthbox", "righttoleftvbox", "lefttorightvbox", "righttoleftvtop", "lefttorightvtop", "rtlhbox", "ltrhbox", "rtlvbox", "ltrvbox", "rtlvtop", "ltrvtop", "autodirhbox", "autodirvbox", "autodirvtop", "lefttoright", "righttoleft", "synchronizelayoutdirection", "synchronizedisplaydirection", "synchronizeinlinedirection", "lesshyphens", "morehyphens", "nohyphens", "dohyphens", "Ucheckedstartdisplaymath", "Ucheckedstopdisplaymath" },
} \ No newline at end of file
diff --git a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metafun.lua b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metafun.lua
index 524a1ca494f..50b9ecec48a 100644
--- a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metafun.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metafun.lua
@@ -1,4 +1,4 @@
return {
- ["commands"]={ "sqr", "log", "ln", "exp", "inv", "pow", "pi", "radian", "tand", "cotd", "sin", "cos", "tan", "cot", "atan", "asin", "acos", "invsin", "invcos", "acosh", "asinh", "sinh", "cosh", "paired", "tripled", "unitcircle", "fulldiamond", "unitdiamond", "fullsquare", "llcircle", "lrcircle", "urcircle", "ulcircle", "tcircle", "bcircle", "lcircle", "rcircle", "lltriangle", "lrtriangle", "urtriangle", "ultriangle", "smoothed", "cornered", "superellipsed", "randomized", "squeezed", "enlonged", "shortened", "punked", "curved", "unspiked", "simplified", "blownup", "stretched", "enlarged", "leftenlarged", "topenlarged", "rightenlarged", "bottomenlarged", "crossed", "laddered", "randomshifted", "interpolated", "paralleled", "cutends", "peepholed", "llenlarged", "lrenlarged", "urenlarged", "ulenlarged", "llmoved", "lrmoved", "urmoved", "ulmoved", "rightarrow", "leftarrow", "centerarrow", "boundingbox", "innerboundingbox", "outerboundingbox", "pushboundingbox", "popboundingbox", "bottomboundary", "leftboundary", "topboundary", "rightboundary", "xsized", "ysized", "xysized", "sized", "xyscaled", "intersection_point", "intersection_found", "penpoint", "bbwidth", "bbheight", "withshade", "withlinearshading", "withcircularshading", "withfromshadecolor", "withtoshadecolor", "withshading", "shadedinto", "withcircularshade", "withlinearshade", "cmyk", "spotcolor", "multitonecolor", "namedcolor", "drawfill", "undrawfill", "inverted", "uncolored", "softened", "grayed", "greyed", "onlayer", "along", "graphictext", "loadfigure", "externalfigure", "withmask", "figure", "register", "bitmapimage", "colordecimals", "ddecimal", "dddecimal", "ddddecimal", "textext", "thetextext", "rawtextext", "textextoffset", "verbatim", "thelabel", "label", "autoalign", "transparent", "withtransparency", "property", "properties", "withproperties", "asgroup", "infont", "set_linear_vector", "linear_shade", "define_linear_shade", "define_circular_linear_shade", "define_sampled_linear_shade", "set_circular_vector", "circular_shade", "define_circular_shade", "define_circular_linear_shade", "define_sampled_circular_shade", "space", "CRLF", "grayscale", "greyscale", "withgray", "withgrey", "colorpart", "readfile", "clearxy", "unitvector", "center", "epsed", "anchored", "originpath", "infinite", "break", "xstretched", "ystretched", "snapped", "pathconnectors", "function", "constructedpath", "constructedpairs", "punkedfunction", "curvedfunction", "tightfunction", "punkedpath", "curvedpath", "tightpath", "punkedpairs", "curvedpairs", "tightpairs", "evenly", "oddly", "condition", "pushcurrentpicture", "popcurrentpicture", "arrowpath", "tensecircle", "roundedsquare", "colortype", "whitecolor", "blackcolor", "normalfill", "normaldraw", "visualizepaths", "naturalizepaths", "drawboundary", "drawwholepath", "visualizeddraw", "visualizedfill", "draworigin", "drawboundingbox", "drawpath", "drawpoint", "drawpoints", "drawcontrolpoints", "drawcontrollines", "drawpointlabels", "drawlineoptions", "drawpointoptions", "drawcontroloptions", "drawlabeloptions", "draworiginoptions", "drawboundoptions", "drawpathoptions", "resetdrawoptions", "decorated", "redecorated", "undecorated" },
+ ["commands"]={ "sqr", "log", "ln", "exp", "inv", "pow", "pi", "radian", "tand", "cotd", "sin", "cos", "tan", "cot", "atan", "asin", "acos", "invsin", "invcos", "invtan", "acosh", "asinh", "sinh", "cosh", "paired", "tripled", "unitcircle", "fulldiamond", "unitdiamond", "fullsquare", "llcircle", "lrcircle", "urcircle", "ulcircle", "tcircle", "bcircle", "lcircle", "rcircle", "lltriangle", "lrtriangle", "urtriangle", "ultriangle", "smoothed", "cornered", "superellipsed", "randomized", "squeezed", "enlonged", "shortened", "punked", "curved", "unspiked", "simplified", "blownup", "stretched", "enlarged", "leftenlarged", "topenlarged", "rightenlarged", "bottomenlarged", "crossed", "laddered", "randomshifted", "interpolated", "paralleled", "cutends", "peepholed", "llenlarged", "lrenlarged", "urenlarged", "ulenlarged", "llmoved", "lrmoved", "urmoved", "ulmoved", "rightarrow", "leftarrow", "centerarrow", "boundingbox", "innerboundingbox", "outerboundingbox", "pushboundingbox", "popboundingbox", "bottomboundary", "leftboundary", "topboundary", "rightboundary", "xsized", "ysized", "xysized", "sized", "xyscaled", "intersection_point", "intersection_found", "penpoint", "bbwidth", "bbheight", "withshade", "withlinearshading", "withcircularshading", "withfromshadecolor", "withtoshadecolor", "withshading", "shadedinto", "withcircularshade", "withlinearshade", "cmyk", "spotcolor", "multitonecolor", "namedcolor", "drawfill", "undrawfill", "inverted", "uncolored", "softened", "grayed", "greyed", "onlayer", "along", "graphictext", "loadfigure", "externalfigure", "withmask", "figure", "register", "bitmapimage", "colordecimals", "ddecimal", "dddecimal", "ddddecimal", "textext", "thetextext", "rawtextext", "textextoffset", "verbatim", "thelabel", "label", "autoalign", "transparent", "withtransparency", "property", "properties", "withproperties", "asgroup", "infont", "set_linear_vector", "linear_shade", "define_linear_shade", "define_circular_linear_shade", "define_sampled_linear_shade", "set_circular_vector", "circular_shade", "define_circular_shade", "define_circular_linear_shade", "define_sampled_circular_shade", "space", "CRLF", "grayscale", "greyscale", "withgray", "withgrey", "colorpart", "readfile", "clearxy", "unitvector", "center", "epsed", "anchored", "originpath", "infinite", "break", "xstretched", "ystretched", "snapped", "pathconnectors", "function", "constructedpath", "constructedpairs", "punkedfunction", "curvedfunction", "tightfunction", "punkedpath", "curvedpath", "tightpath", "punkedpairs", "curvedpairs", "tightpairs", "evenly", "oddly", "condition", "pushcurrentpicture", "popcurrentpicture", "arrowpath", "tensecircle", "roundedsquare", "colortype", "whitecolor", "blackcolor", "normalfill", "normaldraw", "visualizepaths", "naturalizepaths", "drawboundary", "drawwholepath", "visualizeddraw", "visualizedfill", "draworigin", "drawboundingbox", "drawpath", "drawpoint", "drawpoints", "drawcontrolpoints", "drawcontrollines", "drawpointlabels", "drawlineoptions", "drawpointoptions", "drawcontroloptions", "drawlabeloptions", "draworiginoptions", "drawboundoptions", "drawpathoptions", "resetdrawoptions", "undashed", "decorated", "redecorated", "undecorated", "passvariable", "passarrayvariable", "tostring", "format", "formatted", "startpassingvariable", "stoppassingvariable" },
["internals"]={ "nocolormodel", "greycolormodel", "graycolormodel", "rgbcolormodel", "cmykcolormodel", "shadefactor", "textextoffset", "normaltransparent", "multiplytransparent", "screentransparent", "overlaytransparent", "softlighttransparent", "hardlighttransparent", "colordodgetransparent", "colorburntransparent", "darkentransparent", "lightentransparent", "differencetransparent", "exclusiontransparent", "huetransparent", "saturationtransparent", "colortransparent", "luminositytransparent", "metapostversion", "maxdimensions" },
} \ No newline at end of file
diff --git a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metapost.lua b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metapost.lua
index df97c0a268a..766ea90da88 100644
--- a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metapost.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-metapost.lua
@@ -1,7 +1,7 @@
return {
- ["commands"]={ "beginfig", "endfig", "rotatedaround", "reflectedabout", "arrowhead", "currentpen", "currentpicture", "cuttings", "defaultfont", "extra_beginfig", "extra_endfig", "ditto", "EOF", "down", "evenly", "fullcircle", "halfcircle", "identity", "in", "left", "origin", "pensquare", "quartercircle", "right", "unitsquare", "up", "withdots", "abs", "bbox", "ceiling", "center", "cutafter", "cutbefore", "dir", "directionpoint", "div", "dotprod", "intersectionpoint", "inverse", "mod", "round", "unitvector", "whatever", "cutdraw", "draw", "drawarrow", "drawdblarrow", "fill", "filldraw", "drawdot", "loggingall", "interact", "tracingall", "tracingnone", "pickup", "undraw", "unfill", "unfilldraw", "buildcycle", "dashpattern", "decr", "dotlabel", "dotlabels", "drawoptions", "incr", "label", "labels", "max", "min", "thelabel", "z", "beginchar", "blacker", "capsule_end", "change_width", "define_blacker_pixels", "define_corrected_pixels", "define_good_x_pixels", "define_good_y_pixels", "define_horizontal_corrected_pixels", "define_pixels", "define_whole_blacker_pixels", "define_whole_pixels", "define_whole_vertical_blacker_pixels", "define_whole_vertical_pixels", "endchar", "extra_beginchar", "extra_endchar", "extra_setup", "font_coding_scheme", "clearxy", "clearit", "clearpen", "shipit", "font_extra_space", "exitunless", "relax", "hide", "gobble", "gobbled", "stop", "blankpicture", "counterclockwise", "tensepath", "takepower", "direction", "softjoin", "makelabel", "rotatedabout", "flex", "superellipse", "erase", "image", "nullpen", "savepen", "clearpen", "penpos", "penlabels", "range", "numtok", "thru", "z", "laboff", "bye", "red", "green", "blue", "cyan", "magenta", "yellow", "black", "white", "background", "graypart", "graycolor", "mm", "pt", "dd", "bp", "cm", "pc", "cc", "in" },
+ ["commands"]={ "beginfig", "endfig", "beginglyph", "endglyph", "charscale", "rotatedaround", "reflectedabout", "arrowhead", "currentpen", "currentpicture", "cuttings", "defaultfont", "extra_beginfig", "extra_endfig", "ditto", "EOF", "down", "evenly", "fullcircle", "halfcircle", "identity", "in", "left", "origin", "pensquare", "quartercircle", "right", "unitsquare", "up", "withdots", "abs", "bbox", "ceiling", "center", "cutafter", "cutbefore", "dir", "directionpoint", "div", "dotprod", "intersectionpoint", "inverse", "mod", "round", "unitvector", "whatever", "cutdraw", "draw", "drawarrow", "drawdblarrow", "fill", "filldraw", "drawdot", "loggingall", "interact", "tracingall", "tracingnone", "pickup", "undraw", "unfill", "unfilldraw", "buildcycle", "dashpattern", "decr", "dotlabel", "dotlabels", "drawoptions", "incr", "label", "labels", "max", "min", "thelabel", "z", "beginchar", "blacker", "capsule_end", "change_width", "define_blacker_pixels", "define_corrected_pixels", "define_good_x_pixels", "define_good_y_pixels", "define_horizontal_corrected_pixels", "define_pixels", "define_whole_blacker_pixels", "define_whole_pixels", "define_whole_vertical_blacker_pixels", "define_whole_vertical_pixels", "endchar", "extra_beginchar", "extra_endchar", "extra_setup", "font_coding_scheme", "clearxy", "clearit", "clearpen", "shipit", "font_extra_space", "exitunless", "relax", "hide", "gobble", "gobbled", "stop", "blankpicture", "counterclockwise", "tensepath", "takepower", "direction", "softjoin", "makelabel", "rotatedabout", "flex", "superellipse", "erase", "image", "nullpen", "savepen", "clearpen", "penpos", "penlabels", "range", "numtok", "thru", "z", "laboff", "bye", "red", "green", "blue", "cyan", "magenta", "yellow", "black", "white", "background", "graypart", "graycolor", "mm", "pt", "dd", "bp", "cm", "pc", "cc", "in", "triplet", "quadruplet" },
["internals"]={ "mitered", "rounded", "beveled", "butt", "squared", "eps", "epsilon", "infinity", "bboxmargin", "ahlength", "ahangle", "labeloffset", "dotlabeldiam", "defaultpen", "defaultscale", "join_radius", "pen_lft", "pen_rt", "pen_top", "pen_bot" },
- ["primitives"]={ "charcode", "day", "linecap", "linejoin", "miterlimit", "month", "pausing", "prologues", "showstopping", "time", "tracingcapsules", "tracingchoices", "mpprocset", "tracingcommands", "tracingequations", "tracinglostchars", "tracingmacros", "tracingonline", "tracingoutput", "tracingrestores", "tracingspecs", "tracingstats", "tracingtitles", "truecorners", "warningcheck", "year", "false", "nullpicture", "pencircle", "true", "and", "angle", "arclength", "arctime", "ASCII", "boolean", "bot", "char", "color", "cosd", "cycle", "decimal", "directiontime", "floor", "fontsize", "hex", "infont", "intersectiontimes", "known", "length", "llcorner", "lrcorner", "makepath", "makepen", "mexp", "mlog", "normaldeviate", "not", "numeric", "oct", "odd", "or", "path", "pair", "pen", "penoffset", "picture", "point", "postcontrol", "precontrol", "reverse", "rotated", "scaled", "shifted", "sind", "slanted", "sqrt", "str", "string", "subpath", "substring", "transform", "transformed", "ulcorner", "uniformdeviate", "unknown", "urcorner", "xpart", "xscaled", "xxpart", "xypart", "ypart", "yscaled", "yxpart", "yypart", "zscaled", "addto", "clip", "input", "interim", "let", "newinternal", "save", "setbounds", "shipout", "show", "showdependencies", "showtoken", "showvariable", "special", "begingroup", "endgroup", "of", "curl", "tension", "and", "controls", "interpath", "on", "off", "def", "vardef", "enddef", "expr", "suffix", "text", "primary", "secondary", "tertiary", "primarydef", "secondarydef", "tertiarydef", "randomseed", "also", "contour", "doublepath", "withcolor", "withpen", "dashed", "if", "else", "elseif", "fi", "for", "endfor", "forever", "exitif", "within", "forsuffixes", "downto", "upto", "step", "until", "charlist", "extensible", "fontdimen", "headerbyte", "kern", "ligtable", "boundarychar", "chardp", "charext", "charht", "charic", "charwd", "designsize", "fontmaking", "charexists", "cullit", "currenttransform", "gfcorners", "grayfont", "hround", "imagerules", "lowres_fix", "nodisplays", "notransforms", "openit", "displaying", "currentwindow", "screen_rows", "screen_cols", "pixels_per_inch", "cull", "display", "openwindow", "numspecial", "totalweight", "autorounding", "fillin", "proofing", "tracingpens", "xoffset", "chardx", "granularity", "smoothing", "turningcheck", "yoffset", "chardy", "hppp", "tracingedges", "vppp", "extra_beginfig", "extra_endfig", "mpxbreak", "endinput", "message", "delimiters", "turningnumber", "errmessage", "readstring", "scantokens", "end", "outer", "inner", "write", "to", "readfrom", "withprescript", "withpostscript", "top", "bot", "lft", "rt", "ulft", "urt", "llft", "lrt", "redpart", "greenpart", "bluepart", "cyanpart", "magentapart", "yellowpart", "blackpart", "greypart", "rgbcolor", "cmykcolor", "greycolor", "graycolor", "colormodel", "graypart", "dashpart", "penpart", "stroked", "filled", "textual", "clipped", "bounded", "expandafter" },
+ ["primitives"]={ "charcode", "day", "linecap", "linejoin", "miterlimit", "month", "pausing", "prologues", "showstopping", "time", "tracingcapsules", "tracingchoices", "mpprocset", "tracingcommands", "tracingequations", "tracinglostchars", "tracingmacros", "tracingonline", "tracingoutput", "tracingrestores", "tracingspecs", "tracingstats", "tracingtitles", "truecorners", "warningcheck", "year", "false", "nullpicture", "pencircle", "true", "and", "angle", "arclength", "arctime", "ASCII", "boolean", "bot", "char", "color", "cosd", "cycle", "decimal", "directiontime", "floor", "fontsize", "hex", "infont", "intersectiontimes", "known", "length", "llcorner", "lrcorner", "makepath", "makepen", "mexp", "mlog", "normaldeviate", "not", "numeric", "oct", "odd", "or", "path", "pair", "pen", "penoffset", "picture", "point", "postcontrol", "precontrol", "reverse", "rotated", "scaled", "shifted", "sind", "slanted", "sqrt", "str", "string", "subpath", "substring", "transform", "transformed", "ulcorner", "uniformdeviate", "unknown", "urcorner", "xpart", "xscaled", "xxpart", "xypart", "ypart", "yscaled", "yxpart", "yypart", "zscaled", "addto", "clip", "input", "interim", "let", "newinternal", "save", "setbounds", "shipout", "show", "showdependencies", "showtoken", "showvariable", "special", "begingroup", "endgroup", "of", "curl", "tension", "and", "controls", "interpath", "on", "off", "def", "vardef", "enddef", "expr", "suffix", "text", "primary", "secondary", "tertiary", "primarydef", "secondarydef", "tertiarydef", "randomseed", "also", "contour", "doublepath", "withcolor", "withcmykcolor", "withpen", "dashed", "if", "else", "elseif", "fi", "for", "endfor", "forever", "exitif", "within", "forsuffixes", "downto", "upto", "step", "until", "charlist", "extensible", "fontdimen", "headerbyte", "kern", "ligtable", "boundarychar", "chardp", "charext", "charht", "charic", "charwd", "designsize", "fontmaking", "charexists", "cullit", "currenttransform", "gfcorners", "grayfont", "hround", "imagerules", "lowres_fix", "nodisplays", "notransforms", "openit", "displaying", "currentwindow", "screen_rows", "screen_cols", "pixels_per_inch", "cull", "display", "openwindow", "numspecial", "totalweight", "autorounding", "fillin", "proofing", "tracingpens", "xoffset", "chardx", "granularity", "smoothing", "turningcheck", "yoffset", "chardy", "hppp", "tracingedges", "vppp", "extra_beginfig", "extra_endfig", "mpxbreak", "endinput", "message", "delimiters", "turningnumber", "errmessage", "readstring", "scantokens", "end", "outer", "inner", "write", "to", "readfrom", "withprescript", "withpostscript", "top", "bot", "lft", "rt", "ulft", "urt", "llft", "lrt", "redpart", "greenpart", "bluepart", "cyanpart", "magentapart", "yellowpart", "blackpart", "greypart", "prescriptpart", "postscriptpart", "rgbcolor", "cmykcolor", "greycolor", "graycolor", "colormodel", "graypart", "dashpart", "penpart", "stroked", "filled", "textual", "clipped", "bounded", "pathpart", "expandafter", "minute", "hour", "outputformat", "outputtemplate", "filenametemplate", "fontmapfile", "fontmapline", "fontpart", "fontsize", "glyph", "restoreclipcolor", "troffmode" },
["shortcuts"]={ "..", "...", "--", "---", "&" },
["tex"]={ "btex", "etex", "verbatimtex" },
} \ No newline at end of file
diff --git a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-tex.lua b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-tex.lua
index 7d710740c29..415b7412834 100644
--- a/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-tex.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/data/scite-context-data-tex.lua
@@ -1,9 +1,9 @@
return {
["aleph"]={ "AlephVersion", "Alephminorversion", "Alephrevision", "Alephversion", "Omegaminorversion", "Omegarevision", "Omegaversion", "boxdir", "pagebottomoffset", "pagerightoffset" },
["etex"]={ "botmarks", "clubpenalties", "currentgrouplevel", "currentgrouptype", "currentifbranch", "currentiflevel", "currentiftype", "detokenize", "dimexpr", "displaywidowpenalties", "eTeXVersion", "eTeXminorversion", "eTeXrevision", "eTeXversion", "everyeof", "firstmarks", "fontchardp", "fontcharht", "fontcharic", "fontcharwd", "glueexpr", "glueshrink", "glueshrinkorder", "gluestretch", "gluestretchorder", "gluetomu", "ifcsname", "ifdefined", "iffontchar", "interactionmode", "interlinepenalties", "lastlinefit", "lastnodetype", "marks", "muexpr", "mutoglue", "numexpr", "pagediscards", "parshapedimen", "parshapeindent", "parshapelength", "predisplaydirection", "protected", "readline", "savinghyphcodes", "savingvdiscards", "scantokens", "showgroups", "showifs", "showtokens", "splitbotmarks", "splitdiscards", "splitfirstmarks", "topmarks", "tracingassigns", "tracinggroups", "tracingifs", "tracingnesting", "tracingscantokens", "unexpanded", "unless", "widowpenalties" },
- ["luatex"]={ "Uchar", "Udelcode", "Udelcodenum", "Udelimiter", "Udelimiterover", "Udelimiterunder", "Umathaccent", "Umathaxis", "Umathbinbinspacing", "Umathbinclosespacing", "Umathbininnerspacing", "Umathbinopenspacing", "Umathbinopspacing", "Umathbinordspacing", "Umathbinpunctspacing", "Umathbinrelspacing", "Umathchar", "Umathchardef", "Umathcharnum", "Umathclosebinspacing", "Umathcloseclosespacing", "Umathcloseinnerspacing", "Umathcloseopenspacing", "Umathcloseopspacing", "Umathcloseordspacing", "Umathclosepunctspacing", "Umathcloserelspacing", "Umathcode", "Umathcodenum", "Umathconnectoroverlapmin", "Umathfractiondelsize", "Umathfractiondenomdown", "Umathfractiondenomvgap", "Umathfractionnumup", "Umathfractionnumvgap", "Umathfractionrule", "Umathinnerbinspacing", "Umathinnerclosespacing", "Umathinnerinnerspacing", "Umathinneropenspacing", "Umathinneropspacing", "Umathinnerordspacing", "Umathinnerpunctspacing", "Umathinnerrelspacing", "Umathlimitabovebgap", "Umathlimitabovekern", "Umathlimitabovevgap", "Umathlimitbelowbgap", "Umathlimitbelowkern", "Umathlimitbelowvgap", "Umathopbinspacing", "Umathopclosespacing", "Umathopenbinspacing", "Umathopenclosespacing", "Umathopeninnerspacing", "Umathopenopenspacing", "Umathopenopspacing", "Umathopenordspacing", "Umathopenpunctspacing", "Umathopenrelspacing", "Umathoperatorsize", "Umathopinnerspacing", "Umathopopenspacing", "Umathopopspacing", "Umathopordspacing", "Umathoppunctspacing", "Umathoprelspacing", "Umathordbinspacing", "Umathordclosespacing", "Umathordinnerspacing", "Umathordopenspacing", "Umathordopspacing", "Umathordordspacing", "Umathordpunctspacing", "Umathordrelspacing", "Umathoverbarkern", "Umathoverbarrule", "Umathoverbarvgap", "Umathoverdelimiterbgap", "Umathoverdelimitervgap", "Umathpunctbinspacing", "Umathpunctclosespacing", "Umathpunctinnerspacing", "Umathpunctopenspacing", "Umathpunctopspacing", "Umathpunctordspacing", "Umathpunctpunctspacing", "Umathpunctrelspacing", "Umathquad", "Umathradicaldegreeafter", "Umathradicaldegreebefore", "Umathradicaldegreeraise", "Umathradicalkern", "Umathradicalrule", "Umathradicalvgap", "Umathrelbinspacing", "Umathrelclosespacing", "Umathrelinnerspacing", "Umathrelopenspacing", "Umathrelopspacing", "Umathrelordspacing", "Umathrelpunctspacing", "Umathrelrelspacing", "Umathspaceafterscript", "Umathstackdenomdown", "Umathstacknumup", "Umathstackvgap", "Umathsubshiftdown", "Umathsubshiftdrop", "Umathsubsupshiftdown", "Umathsubsupvgap", "Umathsubtopmax", "Umathsupbottommin", "Umathsupshiftdrop", "Umathsupshiftup", "Umathsupsubbottommax", "Umathunderbarkern", "Umathunderbarrule", "Umathunderbarvgap", "Umathunderdelimiterbgap", "Umathunderdelimitervgap", "Uoverdelimiter", "Uradical", "Uroot", "Ustack", "Ustartdisplaymath", "Ustartmath", "Ustopdisplaymath", "Ustopmath", "Usubscript", "Usuperscript", "Uunderdelimiter", "alignmark", "aligntab", "attribute", "attributedef", "catcodetable", "clearmarks", "crampeddisplaystyle", "crampedscriptscriptstyle", "crampedscriptstyle", "crampedtextstyle", "fontid", "formatname", "gleaders", "ifabsdim", "ifabsnum", "ifprimitive", "initcatcodetable", "latelua", "luaescapestring", "luastartup", "luatexdatestamp", "luatexrevision", "luatexversion", "mathstyle", "nokerns", "noligs", "outputbox", "pageleftoffset", "pagetopoffset", "postexhyphenchar", "posthyphenchar", "preexhyphenchar", "prehyphenchar", "primitive", "savecatcodetable", "scantextokens", "suppressfontnotfounderror", "suppressifcsnameerror", "suppresslongerror", "suppressoutererror", "synctex" },
+ ["luatex"]={ "Uchar", "Udelcode", "Udelcodenum", "Udelimiter", "Udelimiterover", "Udelimiterunder", "Umathaccent", "Umathaxis", "Umathbinbinspacing", "Umathbinclosespacing", "Umathbininnerspacing", "Umathbinopenspacing", "Umathbinopspacing", "Umathbinordspacing", "Umathbinpunctspacing", "Umathbinrelspacing", "Umathchar", "Umathchardef", "Umathcharnum", "Umathclosebinspacing", "Umathcloseclosespacing", "Umathcloseinnerspacing", "Umathcloseopenspacing", "Umathcloseopspacing", "Umathcloseordspacing", "Umathclosepunctspacing", "Umathcloserelspacing", "Umathcode", "Umathcodenum", "Umathconnectoroverlapmin", "Umathfractiondelsize", "Umathfractiondenomdown", "Umathfractiondenomvgap", "Umathfractionnumup", "Umathfractionnumvgap", "Umathfractionrule", "Umathinnerbinspacing", "Umathinnerclosespacing", "Umathinnerinnerspacing", "Umathinneropenspacing", "Umathinneropspacing", "Umathinnerordspacing", "Umathinnerpunctspacing", "Umathinnerrelspacing", "Umathlimitabovebgap", "Umathlimitabovekern", "Umathlimitabovevgap", "Umathlimitbelowbgap", "Umathlimitbelowkern", "Umathlimitbelowvgap", "Umathopbinspacing", "Umathopclosespacing", "Umathopenbinspacing", "Umathopenclosespacing", "Umathopeninnerspacing", "Umathopenopenspacing", "Umathopenopspacing", "Umathopenordspacing", "Umathopenpunctspacing", "Umathopenrelspacing", "Umathoperatorsize", "Umathopinnerspacing", "Umathopopenspacing", "Umathopopspacing", "Umathopordspacing", "Umathoppunctspacing", "Umathoprelspacing", "Umathordbinspacing", "Umathordclosespacing", "Umathordinnerspacing", "Umathordopenspacing", "Umathordopspacing", "Umathordordspacing", "Umathordpunctspacing", "Umathordrelspacing", "Umathoverbarkern", "Umathoverbarrule", "Umathoverbarvgap", "Umathoverdelimiterbgap", "Umathoverdelimitervgap", "Umathpunctbinspacing", "Umathpunctclosespacing", "Umathpunctinnerspacing", "Umathpunctopenspacing", "Umathpunctopspacing", "Umathpunctordspacing", "Umathpunctpunctspacing", "Umathpunctrelspacing", "Umathquad", "Umathradicaldegreeafter", "Umathradicaldegreebefore", "Umathradicaldegreeraise", "Umathradicalkern", "Umathradicalrule", "Umathradicalvgap", "Umathrelbinspacing", "Umathrelclosespacing", "Umathrelinnerspacing", "Umathrelopenspacing", "Umathrelopspacing", "Umathrelordspacing", "Umathrelpunctspacing", "Umathrelrelspacing", "Umathspaceafterscript", "Umathstackdenomdown", "Umathstacknumup", "Umathstackvgap", "Umathsubshiftdown", "Umathsubshiftdrop", "Umathsubsupshiftdown", "Umathsubsupvgap", "Umathsubtopmax", "Umathsupbottommin", "Umathsupshiftdrop", "Umathsupshiftup", "Umathsupsubbottommax", "Umathunderbarkern", "Umathunderbarrule", "Umathunderbarvgap", "Umathunderdelimiterbgap", "Umathunderdelimitervgap", "Uoverdelimiter", "Uradical", "Uroot", "Ustack", "Ustartdisplaymath", "Ustartmath", "Ustopdisplaymath", "Ustopmath", "Usubscript", "Usuperscript", "Uunderdelimiter", "alignmark", "aligntab", "attribute", "attributedef", "catcodetable", "clearmarks", "crampeddisplaystyle", "crampedscriptscriptstyle", "crampedscriptstyle", "crampedtextstyle", "fontid", "formatname", "gleaders", "ifabsdim", "ifabsnum", "ifprimitive", "initcatcodetable", "latelua", "luaescapestring", "luastartup", "luatexdatestamp", "luatexrevision", "luatexversion", "luafunction", "mathstyle", "nokerns", "noligs", "outputbox", "pageleftoffset", "pagetopoffset", "postexhyphenchar", "posthyphenchar", "preexhyphenchar", "prehyphenchar", "primitive", "savecatcodetable", "scantextokens", "suppressfontnotfounderror", "suppressifcsnameerror", "suppresslongerror", "suppressoutererror", "synctex" },
["omega"]={ "OmegaVersion", "bodydir", "chardp", "charht", "charit", "charwd", "leftghost", "localbrokenpenalty", "localinterlinepenalty", "localleftbox", "localrightbox", "mathdir", "odelcode", "odelimiter", "omathaccent", "omathchar", "omathchardef", "omathcode", "oradical", "pagedir", "pageheight", "pagewidth", "pardir", "rightghost", "textdir" },
["pdftex"]={ "efcode", "expanded", "ifincsname", "ifpdfabsdim", "ifpdfabsnum", "ifpdfprimitive", "leftmarginkern", "letterspacefont", "lpcode", "pdfadjustspacing", "pdfannot", "pdfcatalog", "pdfcolorstack", "pdfcolorstackinit", "pdfcompresslevel", "pdfcopyfont", "pdfcreationdate", "pdfdecimaldigits", "pdfdest", "pdfdestmargin", "pdfdraftmode", "pdfeachlinedepth", "pdfeachlineheight", "pdfendlink", "pdfendthread", "pdffirstlineheight", "pdffontattr", "pdffontexpand", "pdffontname", "pdffontobjnum", "pdffontsize", "pdfgamma", "pdfgentounicode", "pdfglyphtounicode", "pdfhorigin", "pdfignoreddimen", "pdfimageapplygamma", "pdfimagegamma", "pdfimagehicolor", "pdfimageresolution", "pdfincludechars", "pdfinclusioncopyfonts", "pdfinclusionerrorlevel", "pdfinfo", "pdfinsertht", "pdflastannot", "pdflastlinedepth", "pdflastlink", "pdflastobj", "pdflastxform", "pdflastximage", "pdflastximagecolordepth", "pdflastximagepages", "pdflastxpos", "pdflastypos", "pdflinkmargin", "pdfliteral", "pdfmapfile", "pdfmapline", "pdfminorversion", "pdfnames", "pdfnoligatures", "pdfnormaldeviate", "pdfobj", "pdfobjcompresslevel", "pdfoptionpdfminorversion", "pdfoutline", "pdfoutput", "pdfpageattr", "pdfpagebox", "pdfpageheight", "pdfpageref", "pdfpageresources", "pdfpagesattr", "pdfpagewidth", "pdfpkmode", "pdfpkresolution", "pdfprimitive", "pdfprotrudechars", "pdfpxdimen", "pdfrandomseed", "pdfrefobj", "pdfrefxform", "pdfrefximage", "pdfreplacefont", "pdfrestore", "pdfretval", "pdfsave", "pdfsavepos", "pdfsetmatrix", "pdfsetrandomseed", "pdfstartlink", "pdfstartthread", "pdftexbanner", "pdftexrevision", "pdftexversion", "pdfthread", "pdfthreadmargin", "pdftracingfonts", "pdftrailer", "pdfuniformdeviate", "pdfuniqueresname", "pdfvorigin", "pdfxform", "pdfxformattr", "pdfxformname", "pdfxformresources", "pdfximage", "pdfximagebbox", "quitvmode", "rightmarginkern", "rpcode", "tagcode" },
- ["tex"]={ "-", "/", "AlephVersion", "Alephminorversion", "Alephrevision", "Alephversion", "OmegaVersion", "Omegaminorversion", "Omegarevision", "Omegaversion", "Udelcode", "Udelcodenum", "Udelimiter", "Udelimiterover", "Udelimiterunder", "Umathaccent", "Umathaxis", "Umathbinbinspacing", "Umathbinclosespacing", "Umathbininnerspacing", "Umathbinopenspacing", "Umathbinopspacing", "Umathbinordspacing", "Umathbinpunctspacing", "Umathbinrelspacing", "Umathchar", "Umathchardef", "Umathcharnum", "Umathclosebinspacing", "Umathcloseclosespacing", "Umathcloseinnerspacing", "Umathcloseopenspacing", "Umathcloseopspacing", "Umathcloseordspacing", "Umathclosepunctspacing", "Umathcloserelspacing", "Umathcode", "Umathcodenum", "Umathconnectoroverlapmin", "Umathfractiondelsize", "Umathfractiondenomdown", "Umathfractiondenomvgap", "Umathfractionnumup", "Umathfractionnumvgap", "Umathfractionrule", "Umathinnerbinspacing", "Umathinnerclosespacing", "Umathinnerinnerspacing", "Umathinneropenspacing", "Umathinneropspacing", "Umathinnerordspacing", "Umathinnerpunctspacing", "Umathinnerrelspacing", "Umathlimitabovebgap", "Umathlimitabovekern", "Umathlimitabovevgap", "Umathlimitbelowbgap", "Umathlimitbelowkern", "Umathlimitbelowvgap", "Umathopbinspacing", "Umathopclosespacing", "Umathopenbinspacing", "Umathopenclosespacing", "Umathopeninnerspacing", "Umathopenopenspacing", "Umathopenopspacing", "Umathopenordspacing", "Umathopenpunctspacing", "Umathopenrelspacing", "Umathoperatorsize", "Umathopinnerspacing", "Umathopopenspacing", "Umathopopspacing", "Umathopordspacing", "Umathoppunctspacing", "Umathoprelspacing", "Umathordbinspacing", "Umathordclosespacing", "Umathordinnerspacing", "Umathordopenspacing", "Umathordopspacing", "Umathordordspacing", "Umathordpunctspacing", "Umathordrelspacing", "Umathoverbarkern", "Umathoverbarrule", "Umathoverbarvgap", "Umathoverdelimiterbgap", "Umathoverdelimitervgap", "Umathpunctbinspacing", "Umathpunctclosespacing", "Umathpunctinnerspacing", "Umathpunctopenspacing", "Umathpunctopspacing", "Umathpunctordspacing", "Umathpunctpunctspacing", "Umathpunctrelspacing", "Umathquad", "Umathradicaldegreeafter", "Umathradicaldegreebefore", "Umathradicaldegreeraise", "Umathradicalkern", "Umathradicalrule", "Umathradicalvgap", "Umathrelbinspacing", "Umathrelclosespacing", "Umathrelinnerspacing", "Umathrelopenspacing", "Umathrelopspacing", "Umathrelordspacing", "Umathrelpunctspacing", "Umathrelrelspacing", "Umathspaceafterscript", "Umathstackdenomdown", "Umathstacknumup", "Umathstackvgap", "Umathsubshiftdown", "Umathsubshiftdrop", "Umathsubsupshiftdown", "Umathsubsupvgap", "Umathsubtopmax", "Umathsupbottommin", "Umathsupshiftdrop", "Umathsupshiftup", "Umathsupsubbottommax", "Umathunderbarkern", "Umathunderbarrule", "Umathunderbarvgap", "Umathunderdelimiterbgap", "Umathunderdelimitervgap", "Uoverdelimiter", "Uradical", "Uroot", "Ustack", "Ustartdisplaymath", "Ustartmath", "Ustopdisplaymath", "Ustopmath", "Usubscript", "Usuperscript", "Uunderdelimiter", "above", "abovedisplayshortskip", "abovedisplayskip", "abovewithdelims", "accent", "adjdemerits", "advance", "afterassignment", "aftergroup", "alignmark", "aligntab", "atop", "atopwithdelims", "attribute", "attributedef", "badness", "baselineskip", "batchmode", "begingroup", "belowdisplayshortskip", "belowdisplayskip", "binoppenalty", "bodydir", "botmark", "botmarks", "box", "boxdir", "boxmaxdepth", "brokenpenalty", "catcode", "catcodetable", "char", "chardef", "chardp", "charht", "charit", "charwd", "cleaders", "clearmarks", "closein", "closeout", "clubpenalties", "clubpenalty", "copy", "count", "countdef", "cr", "crampeddisplaystyle", "crampedscriptscriptstyle", "crampedscriptstyle", "crampedtextstyle", "crcr", "csname", "currentgrouplevel", "currentgrouptype", "currentifbranch", "currentiflevel", "currentiftype", "day", "deadcycles", "def", "defaulthyphenchar", "defaultskewchar", "delcode", "delimiter", "delimiterfactor", "delimitershortfall", "detokenize", "dimen", "dimendef", "dimexpr", "directlua", "discretionary", "displayindent", "displaylimits", "displaystyle", "displaywidowpenalties", "displaywidowpenalty", "displaywidth", "divide", "doublehyphendemerits", "dp", "dump", "eTeXVersion", "eTeXminorversion", "eTeXrevision", "eTeXversion", "edef", "efcode", "else", "emergencystretch", "end", "endcsname", "endgroup", "endinput", "endlinechar", "eqno", "errhelp", "errmessage", "errorcontextlines", "errorstopmode", "escapechar", "everycr", "everydisplay", "everyeof", "everyhbox", "everyjob", "everymath", "everypar", "everyvbox", "exhyphenchar", "exhyphenpenalty", "expandafter", "expanded", "fam", "fi", "finalhyphendemerits", "firstmark", "firstmarks", "floatingpenalty", "font", "fontchardp", "fontcharht", "fontcharic", "fontcharwd", "fontdimen", "fontid", "fontname", "formatname", "futurelet", "gdef", "gleaders", "global", "globaldefs", "glueexpr", "glueshrink", "glueshrinkorder", "gluestretch", "gluestretchorder", "gluetomu", "halign", "hangafter", "hangindent", "hbadness", "hbox", "hfil", "hfill", "hfilneg", "hfuzz", "hoffset", "holdinginserts", "hrule", "hsize", "hskip", "hss", "ht", "hyphenation", "hyphenchar", "hyphenpenalty", "if", "ifabsdim", "ifabsnum", "ifcase", "ifcat", "ifcsname", "ifdefined", "ifdim", "ifeof", "iffalse", "iffontchar", "ifhbox", "ifhmode", "ifincsname", "ifinner", "ifmmode", "ifnum", "ifodd", "ifpdfabsdim", "ifpdfabsnum", "ifpdfprimitive", "ifprimitive", "iftrue", "ifvbox", "ifvmode", "ifvoid", "ifx", "ignorespaces", "immediate", "indent", "initcatcodetable", "input", "inputlineno", "insert", "insertpenalties", "interactionmode", "interlinepenalties", "interlinepenalty", "jobname", "kern", "language", "lastbox", "lastkern", "lastlinefit", "lastnodetype", "lastpenalty", "lastskip", "latelua", "lccode", "leaders", "left", "leftghost", "lefthyphenmin", "leftmarginkern", "leftskip", "leqno", "let", "letterspacefont", "limits", "linepenalty", "lineskip", "lineskiplimit", "localbrokenpenalty", "localinterlinepenalty", "localleftbox", "localrightbox", "long", "looseness", "lower", "lowercase", "lpcode", "luaescapestring", "luastartup", "luatexdatestamp", "luatexrevision", "luatexversion", "mag", "mark", "marks", "mathaccent", "mathbin", "mathchar", "mathchardef", "mathchoice", "mathclose", "mathcode", "mathdir", "mathinner", "mathop", "mathopen", "mathord", "mathpunct", "mathrel", "mathstyle", "mathsurround", "maxdeadcycles", "maxdepth", "meaning", "medmuskip", "message", "middle", "mkern", "month", "moveleft", "moveright", "mskip", "muexpr", "multiply", "muskip", "muskipdef", "mutoglue", "newlinechar", "noalign", "noboundary", "noexpand", "noindent", "nokerns", "noligs", "nolimits", "nolocaldirs", "nolocalwhatsits", "nonscript", "nonstopmode", "nulldelimiterspace", "nullfont", "number", "numexpr", "odelcode", "odelimiter", "omathaccent", "omathchar", "omathchardef", "omathcode", "omit", "openin", "openout", "or", "oradical", "outer", "output", "outputbox", "outputpenalty", "over", "overfullrule", "overline", "overwithdelims", "pagebottomoffset", "pagedepth", "pagedir", "pagediscards", "pagefilllstretch", "pagefillstretch", "pagefilstretch", "pagegoal", "pageheight", "pageleftoffset", "pagerightoffset", "pageshrink", "pagestretch", "pagetopoffset", "pagetotal", "pagewidth", "par", "pardir", "parfillskip", "parindent", "parshape", "parshapedimen", "parshapeindent", "parshapelength", "parskip", "patterns", "pausing", "pdfadjustspacing", "pdfannot", "pdfcatalog", "pdfcolorstack", "pdfcolorstackinit", "pdfcompresslevel", "pdfcopyfont", "pdfcreationdate", "pdfdecimaldigits", "pdfdest", "pdfdestmargin", "pdfdraftmode", "pdfeachlinedepth", "pdfeachlineheight", "pdfendlink", "pdfendthread", "pdffirstlineheight", "pdffontattr", "pdffontexpand", "pdffontname", "pdffontobjnum", "pdffontsize", "pdfgamma", "pdfgentounicode", "pdfglyphtounicode", "pdfhorigin", "pdfignoreddimen", "pdfimageapplygamma", "pdfimagegamma", "pdfimagehicolor", "pdfimageresolution", "pdfincludechars", "pdfinclusioncopyfonts", "pdfinclusionerrorlevel", "pdfinfo", "pdfinsertht", "pdflastannot", "pdflastlinedepth", "pdflastlink", "pdflastobj", "pdflastxform", "pdflastximage", "pdflastximagecolordepth", "pdflastximagepages", "pdflastxpos", "pdflastypos", "pdflinkmargin", "pdfliteral", "pdfmapfile", "pdfmapline", "pdfminorversion", "pdfnames", "pdfnoligatures", "pdfnormaldeviate", "pdfobj", "pdfobjcompresslevel", "pdfoptionpdfminorversion", "pdfoutline", "pdfoutput", "pdfpageattr", "pdfpagebox", "pdfpageheight", "pdfpageref", "pdfpageresources", "pdfpagesattr", "pdfpagewidth", "pdfpkmode", "pdfpkresolution", "pdfprimitive", "pdfprotrudechars", "pdfpxdimen", "pdfrandomseed", "pdfrefobj", "pdfrefxform", "pdfrefximage", "pdfreplacefont", "pdfrestore", "pdfretval", "pdfsave", "pdfsavepos", "pdfsetmatrix", "pdfsetrandomseed", "pdfstartlink", "pdfstartthread", "pdftexbanner", "pdftexrevision", "pdftexversion", "pdfthread", "pdfthreadmargin", "pdftracingfonts", "pdftrailer", "pdfuniformdeviate", "pdfuniqueresname", "pdfvorigin", "pdfxform", "pdfxformattr", "pdfxformname", "pdfxformresources", "pdfximage", "pdfximagebbox", "penalty", "postdisplaypenalty", "postexhyphenchar", "posthyphenchar", "predisplaydirection", "predisplaypenalty", "predisplaysize", "preexhyphenchar", "prehyphenchar", "pretolerance", "prevdepth", "prevgraf", "primitive", "protected", "quitvmode", "radical", "raise", "read", "readline", "relax", "relpenalty", "right", "rightghost", "righthyphenmin", "rightmarginkern", "rightskip", "romannumeral", "rpcode", "savecatcodetable", "savinghyphcodes", "savingvdiscards", "scantextokens", "scantokens", "scriptfont", "scriptscriptfont", "scriptscriptstyle", "scriptspace", "scriptstyle", "scrollmode", "setbox", "setlanguage", "sfcode", "shipout", "show", "showbox", "showboxbreadth", "showboxdepth", "showgroups", "showifs", "showlists", "showthe", "showtokens", "skewchar", "skip", "skipdef", "spacefactor", "spaceskip", "span", "special", "splitbotmark", "splitbotmarks", "splitdiscards", "splitfirstmark", "splitfirstmarks", "splitmaxdepth", "splittopskip", "string", "suppressfontnotfounderror", "suppressifcsnameerror", "suppresslongerror", "suppressoutererror", "synctex", "tabskip", "tagcode", "textdir", "textfont", "textstyle", "the", "thickmuskip", "thinmuskip", "time", "toks", "toksdef", "tolerance", "topmark", "topmarks", "topskip", "tracingassigns", "tracingcommands", "tracinggroups", "tracingifs", "tracinglostchars", "tracingmacros", "tracingnesting", "tracingonline", "tracingoutput", "tracingpages", "tracingparagraphs", "tracingrestores", "tracingscantokens", "tracingstats", "uccode", "uchyph", "underline", "unexpanded", "unhbox", "unhcopy", "unkern", "unless", "unpenalty", "unskip", "unvbox", "unvcopy", "uppercase", "vadjust", "valign", "vbadness", "vbox", "vcenter", "vfil", "vfill", "vfilneg", "vfuzz", "voffset", "vrule", "vsize", "vskip", "vsplit", "vss", "vtop", "wd", "widowpenalties", "widowpenalty", "write", "xdef", "xleaders", "xspaceskip", "year" },
+ ["tex"]={ "-", "/", "AlephVersion", "Alephminorversion", "Alephrevision", "Alephversion", "OmegaVersion", "Omegaminorversion", "Omegarevision", "Omegaversion", "Udelcode", "Udelcodenum", "Udelimiter", "Udelimiterover", "Udelimiterunder", "Umathaccent", "Umathaxis", "Umathbinbinspacing", "Umathbinclosespacing", "Umathbininnerspacing", "Umathbinopenspacing", "Umathbinopspacing", "Umathbinordspacing", "Umathbinpunctspacing", "Umathbinrelspacing", "Umathchar", "Umathchardef", "Umathcharnum", "Umathclosebinspacing", "Umathcloseclosespacing", "Umathcloseinnerspacing", "Umathcloseopenspacing", "Umathcloseopspacing", "Umathcloseordspacing", "Umathclosepunctspacing", "Umathcloserelspacing", "Umathcode", "Umathcodenum", "Umathconnectoroverlapmin", "Umathfractiondelsize", "Umathfractiondenomdown", "Umathfractiondenomvgap", "Umathfractionnumup", "Umathfractionnumvgap", "Umathfractionrule", "Umathinnerbinspacing", "Umathinnerclosespacing", "Umathinnerinnerspacing", "Umathinneropenspacing", "Umathinneropspacing", "Umathinnerordspacing", "Umathinnerpunctspacing", "Umathinnerrelspacing", "Umathlimitabovebgap", "Umathlimitabovekern", "Umathlimitabovevgap", "Umathlimitbelowbgap", "Umathlimitbelowkern", "Umathlimitbelowvgap", "Umathopbinspacing", "Umathopclosespacing", "Umathopenbinspacing", "Umathopenclosespacing", "Umathopeninnerspacing", "Umathopenopenspacing", "Umathopenopspacing", "Umathopenordspacing", "Umathopenpunctspacing", "Umathopenrelspacing", "Umathoperatorsize", "Umathopinnerspacing", "Umathopopenspacing", "Umathopopspacing", "Umathopordspacing", "Umathoppunctspacing", "Umathoprelspacing", "Umathordbinspacing", "Umathordclosespacing", "Umathordinnerspacing", "Umathordopenspacing", "Umathordopspacing", "Umathordordspacing", "Umathordpunctspacing", "Umathordrelspacing", "Umathoverbarkern", "Umathoverbarrule", "Umathoverbarvgap", "Umathoverdelimiterbgap", "Umathoverdelimitervgap", "Umathpunctbinspacing", "Umathpunctclosespacing", "Umathpunctinnerspacing", "Umathpunctopenspacing", "Umathpunctopspacing", "Umathpunctordspacing", "Umathpunctpunctspacing", "Umathpunctrelspacing", "Umathquad", "Umathradicaldegreeafter", "Umathradicaldegreebefore", "Umathradicaldegreeraise", "Umathradicalkern", "Umathradicalrule", "Umathradicalvgap", "Umathrelbinspacing", "Umathrelclosespacing", "Umathrelinnerspacing", "Umathrelopenspacing", "Umathrelopspacing", "Umathrelordspacing", "Umathrelpunctspacing", "Umathrelrelspacing", "Umathspaceafterscript", "Umathstackdenomdown", "Umathstacknumup", "Umathstackvgap", "Umathsubshiftdown", "Umathsubshiftdrop", "Umathsubsupshiftdown", "Umathsubsupvgap", "Umathsubtopmax", "Umathsupbottommin", "Umathsupshiftdrop", "Umathsupshiftup", "Umathsupsubbottommax", "Umathunderbarkern", "Umathunderbarrule", "Umathunderbarvgap", "Umathunderdelimiterbgap", "Umathunderdelimitervgap", "Uoverdelimiter", "Uradical", "Uroot", "Ustack", "Ustartdisplaymath", "Ustartmath", "Ustopdisplaymath", "Ustopmath", "Usubscript", "Usuperscript", "Uunderdelimiter", "above", "abovedisplayshortskip", "abovedisplayskip", "abovewithdelims", "accent", "adjdemerits", "advance", "afterassignment", "aftergroup", "alignmark", "aligntab", "atop", "atopwithdelims", "attribute", "attributedef", "badness", "baselineskip", "batchmode", "begingroup", "belowdisplayshortskip", "belowdisplayskip", "binoppenalty", "bodydir", "botmark", "botmarks", "box", "boxdir", "boxmaxdepth", "brokenpenalty", "catcode", "catcodetable", "char", "chardef", "cleaders", "clearmarks", "closein", "closeout", "clubpenalties", "clubpenalty", "copy", "count", "countdef", "cr", "crampeddisplaystyle", "crampedscriptscriptstyle", "crampedscriptstyle", "crampedtextstyle", "crcr", "csname", "currentgrouplevel", "currentgrouptype", "currentifbranch", "currentiflevel", "currentiftype", "day", "deadcycles", "def", "defaulthyphenchar", "defaultskewchar", "delcode", "delimiter", "delimiterfactor", "delimitershortfall", "detokenize", "dimen", "dimendef", "dimexpr", "directlua", "discretionary", "displayindent", "displaylimits", "displaystyle", "displaywidowpenalties", "displaywidowpenalty", "displaywidth", "divide", "doublehyphendemerits", "dp", "dump", "eTeXVersion", "eTeXminorversion", "eTeXrevision", "eTeXversion", "edef", "efcode", "else", "emergencystretch", "end", "endcsname", "endgroup", "endinput", "endlinechar", "eqno", "errhelp", "errmessage", "errorcontextlines", "errorstopmode", "escapechar", "everycr", "everydisplay", "everyeof", "everyhbox", "everyjob", "everymath", "everypar", "everyvbox", "exhyphenchar", "exhyphenpenalty", "expandafter", "expanded", "fam", "fi", "finalhyphendemerits", "firstmark", "firstmarks", "floatingpenalty", "font", "fontchardp", "fontcharht", "fontcharic", "fontcharwd", "fontdimen", "fontid", "fontname", "formatname", "futurelet", "gdef", "gleaders", "global", "globaldefs", "glueexpr", "glueshrink", "glueshrinkorder", "gluestretch", "gluestretchorder", "gluetomu", "halign", "hangafter", "hangindent", "hbadness", "hbox", "hfil", "hfill", "hfilneg", "hfuzz", "hoffset", "holdinginserts", "hrule", "hsize", "hskip", "hss", "ht", "hyphenation", "hyphenchar", "hyphenpenalty", "if", "ifabsdim", "ifabsnum", "ifcase", "ifcat", "ifcsname", "ifdefined", "ifdim", "ifeof", "iffalse", "iffontchar", "ifhbox", "ifhmode", "ifincsname", "ifinner", "ifmmode", "ifnum", "ifodd", "ifpdfabsdim", "ifpdfabsnum", "ifpdfprimitive", "ifprimitive", "iftrue", "ifvbox", "ifvmode", "ifvoid", "ifx", "ignorespaces", "immediate", "indent", "initcatcodetable", "input", "inputlineno", "insert", "insertpenalties", "interactionmode", "interlinepenalties", "interlinepenalty", "jobname", "kern", "language", "lastbox", "lastkern", "lastlinefit", "lastnodetype", "lastpenalty", "lastskip", "latelua", "lccode", "leaders", "left", "leftghost", "lefthyphenmin", "leftmarginkern", "leftskip", "leqno", "let", "letterspacefont", "limits", "linepenalty", "lineskip", "lineskiplimit", "localbrokenpenalty", "localinterlinepenalty", "localleftbox", "localrightbox", "long", "looseness", "lower", "lowercase", "lpcode", "luaescapestring", "luastartup", "luatexdatestamp", "luatexrevision", "luatexversion", "mag", "mark", "marks", "mathaccent", "mathbin", "mathchar", "mathchardef", "mathchoice", "mathclose", "mathcode", "mathdir", "mathinner", "mathop", "mathopen", "mathord", "mathpunct", "mathrel", "mathstyle", "mathsurround", "maxdeadcycles", "maxdepth", "meaning", "medmuskip", "message", "middle", "mkern", "month", "moveleft", "moveright", "mskip", "muexpr", "multiply", "muskip", "muskipdef", "mutoglue", "newlinechar", "noalign", "noboundary", "noexpand", "noindent", "nokerns", "noligs", "nolimits", "nolocaldirs", "nolocalwhatsits", "nonscript", "nonstopmode", "nulldelimiterspace", "nullfont", "number", "numexpr", "odelcode", "odelimiter", "omathaccent", "omathchar", "omathchardef", "omathcode", "omit", "openin", "openout", "or", "oradical", "outer", "output", "outputbox", "outputpenalty", "over", "overfullrule", "overline", "overwithdelims", "pagebottomoffset", "pagedepth", "pagedir", "pagediscards", "pagefilllstretch", "pagefillstretch", "pagefilstretch", "pagegoal", "pageheight", "pageleftoffset", "pagerightoffset", "pageshrink", "pagestretch", "pagetopoffset", "pagetotal", "pagewidth", "par", "pardir", "parfillskip", "parindent", "parshape", "parshapedimen", "parshapeindent", "parshapelength", "parskip", "patterns", "pausing", "pdfadjustspacing", "pdfannot", "pdfcatalog", "pdfcolorstack", "pdfcolorstackinit", "pdfcompresslevel", "pdfcopyfont", "pdfcreationdate", "pdfdecimaldigits", "pdfdest", "pdfdestmargin", "pdfdraftmode", "pdfeachlinedepth", "pdfeachlineheight", "pdfendlink", "pdfendthread", "pdffirstlineheight", "pdffontattr", "pdffontexpand", "pdffontname", "pdffontobjnum", "pdffontsize", "pdfgamma", "pdfgentounicode", "pdfglyphtounicode", "pdfhorigin", "pdfignoreddimen", "pdfimageapplygamma", "pdfimagegamma", "pdfimagehicolor", "pdfimageresolution", "pdfincludechars", "pdfinclusioncopyfonts", "pdfinclusionerrorlevel", "pdfinfo", "pdfinsertht", "pdflastannot", "pdflastlinedepth", "pdflastlink", "pdflastobj", "pdflastxform", "pdflastximage", "pdflastximagecolordepth", "pdflastximagepages", "pdflastxpos", "pdflastypos", "pdflinkmargin", "pdfliteral", "pdfmapfile", "pdfmapline", "pdfminorversion", "pdfnames", "pdfnoligatures", "pdfnormaldeviate", "pdfobj", "pdfobjcompresslevel", "pdfoptionpdfminorversion", "pdfoutline", "pdfoutput", "pdfpageattr", "pdfpagebox", "pdfpageheight", "pdfpageref", "pdfpageresources", "pdfpagesattr", "pdfpagewidth", "pdfpkmode", "pdfpkresolution", "pdfprimitive", "pdfprotrudechars", "pdfpxdimen", "pdfrandomseed", "pdfrefobj", "pdfrefxform", "pdfrefximage", "pdfreplacefont", "pdfrestore", "pdfretval", "pdfsave", "pdfsavepos", "pdfsetmatrix", "pdfsetrandomseed", "pdfstartlink", "pdfstartthread", "pdftexbanner", "pdftexrevision", "pdftexversion", "pdfthread", "pdfthreadmargin", "pdftracingfonts", "pdftrailer", "pdfuniformdeviate", "pdfuniqueresname", "pdfvorigin", "pdfxform", "pdfxformattr", "pdfxformname", "pdfxformresources", "pdfximage", "pdfximagebbox", "penalty", "postdisplaypenalty", "postexhyphenchar", "posthyphenchar", "predisplaydirection", "predisplaypenalty", "predisplaysize", "preexhyphenchar", "prehyphenchar", "pretolerance", "prevdepth", "prevgraf", "primitive", "protected", "quitvmode", "radical", "raise", "read", "readline", "relax", "relpenalty", "right", "rightghost", "righthyphenmin", "rightmarginkern", "rightskip", "romannumeral", "rpcode", "savecatcodetable", "savinghyphcodes", "savingvdiscards", "scantextokens", "scantokens", "scriptfont", "scriptscriptfont", "scriptscriptstyle", "scriptspace", "scriptstyle", "scrollmode", "setbox", "setlanguage", "sfcode", "shipout", "show", "showbox", "showboxbreadth", "showboxdepth", "showgroups", "showifs", "showlists", "showthe", "showtokens", "skewchar", "skip", "skipdef", "spacefactor", "spaceskip", "span", "special", "splitbotmark", "splitbotmarks", "splitdiscards", "splitfirstmark", "splitfirstmarks", "splitmaxdepth", "splittopskip", "string", "suppressfontnotfounderror", "suppressifcsnameerror", "suppresslongerror", "suppressoutererror", "synctex", "tabskip", "tagcode", "textdir", "textfont", "textstyle", "the", "thickmuskip", "thinmuskip", "time", "toks", "toksdef", "tolerance", "topmark", "topmarks", "topskip", "tracingassigns", "tracingcommands", "tracinggroups", "tracingifs", "tracinglostchars", "tracingmacros", "tracingnesting", "tracingonline", "tracingoutput", "tracingpages", "tracingparagraphs", "tracingrestores", "tracingscantokens", "tracingstats", "uccode", "uchyph", "underline", "unexpanded", "unhbox", "unhcopy", "unkern", "unless", "unpenalty", "unskip", "unvbox", "unvcopy", "uppercase", "vadjust", "valign", "vbadness", "vbox", "vcenter", "vfil", "vfill", "vfilneg", "vfuzz", "voffset", "vrule", "vsize", "vskip", "vsplit", "vss", "vtop", "wd", "widowpenalties", "widowpenalty", "write", "xdef", "xleaders", "xspaceskip", "year" },
["xetex"]={ "XeTeXversion" },
} \ No newline at end of file
diff --git a/Master/texmf-dist/context/data/scite/lexers/lexer.lua b/Master/texmf-dist/context/data/scite/lexers/lexer.lua
new file mode 100644
index 00000000000..9582f6a76b6
--- /dev/null
+++ b/Master/texmf-dist/context/data/scite/lexers/lexer.lua
@@ -0,0 +1,3 @@
+-- this works ok:
+
+return require("scite-context-lexer")
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-bibtex.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-bibtex.lua
new file mode 100644
index 00000000000..88b070e5ee2
--- /dev/null
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-bibtex.lua
@@ -0,0 +1,176 @@
+local info = {
+ version = 1.002,
+ comment = "scintilla lpeg lexer for bibtex",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files",
+}
+
+local global, string, table, lpeg = _G, string, table, lpeg
+local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
+local type = type
+
+local lexer = require("lexer")
+local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+local exact_match = lexer.exact_match
+
+local bibtexlexer = lexer.new("xml","scite-context-lexer-xml")
+local whitespace = bibtexlexer.whitespace
+
+ local escape, left, right = P("\\"), P('{'), P('}')
+
+ patterns.balanced = P {
+ [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0,
+ [2] = left * V(1) * right
+ }
+
+-- taken from bibl-bib.lua
+
+local anything = patterns.anything
+local percent = P("%")
+local start = P("@")
+local comma = P(",")
+local hash = P("#")
+local escape = P("\\")
+local single = P("'")
+local double = P('"')
+local left = P('{')
+local right = P('}')
+local lineending = S("\n\r")
+local space = S(" \t\n\r\f")
+local spaces = space^1
+local equal = P("=")
+
+local keyword = (R("az","AZ","09") + S("@_:-"))^1
+local s_quoted = ((escape*single) + spaces^1 + (1-single))^0
+local d_quoted = ((escape*double) + spaces^1 + (1-double))^0
+local balanced = patterns.balanced
+
+local t_spacing = token(whitespace, space^1)
+local t_optionalws = token("default", space^1)^0
+
+local t_equal = token("operator",equal)
+local t_left = token("grouping",left)
+local t_right = token("grouping",right)
+local t_comma = token("operator",comma)
+local t_hash = token("operator",hash)
+
+local t_s_value = token("operator",single)
+ * token("text",s_quoted)
+ * token("operator",single)
+local t_d_value = token("operator",double)
+ * token("text",d_quoted)
+ * token("operator",double)
+local t_b_value = token("operator",left)
+ * token("text",balanced)
+ * token("operator",right)
+local t_r_value = token("text",keyword)
+
+local t_keyword = token("keyword",keyword)
+local t_key = token("command",keyword)
+local t_label = token("warning",keyword)
+
+local t_somevalue = t_s_value + t_d_value + t_b_value + t_r_value
+local t_value = t_somevalue
+ * ((t_optionalws * t_hash * t_optionalws) * t_somevalue)^0
+
+local t_assignment = t_optionalws
+ * t_key
+ * t_optionalws
+ * t_equal
+ * t_optionalws
+ * t_value
+
+local t_shortcut = t_keyword
+ * t_optionalws
+ * t_left
+ * t_optionalws
+ * (t_assignment * t_comma^0)^0
+ * t_optionalws
+ * t_right
+
+local t_definition = t_keyword
+ * t_optionalws
+ * t_left
+ * t_optionalws
+ * t_label
+ * t_optionalws
+ * t_comma
+ * (t_assignment * t_comma^0)^0
+ * t_optionalws
+ * t_right
+
+local t_comment = t_keyword
+ * t_optionalws
+ * t_left
+ * token("text",(1-t_right)^0)
+ * t_optionalws
+ * t_right
+
+local t_forget = token("comment",percent^1 * (1-lineending)^0)
+
+local t_rest = token("default",anything)
+
+-- this kind of lexing seems impossible as the size of the buffer passed to the lexer is not
+-- large enough .. but we can cheat and use this:
+--
+-- function OnOpen(filename) editor:Colourise(1,editor.TextLength) end -- or is it 0?
+
+bibtexlexer._rules = {
+ { "whitespace", t_spacing },
+ { "forget", t_forget },
+ { "shortcut", t_shortcut },
+ { "definition", t_definition },
+ { "comment", t_comment },
+ { "rest", t_rest },
+}
+
+-- local t_assignment = t_key
+-- * t_optionalws
+-- * t_equal
+-- * t_optionalws
+-- * t_value
+--
+-- local t_shortcut = t_keyword
+-- * t_optionalws
+-- * t_left
+--
+-- local t_definition = t_keyword
+-- * t_optionalws
+-- * t_left
+-- * t_optionalws
+-- * t_label
+-- * t_optionalws
+-- * t_comma
+--
+-- bibtexlexer._rules = {
+-- { "whitespace", t_spacing },
+-- { "assignment", t_assignment },
+-- { "definition", t_definition },
+-- { "shortcut", t_shortcut },
+-- { "right", t_right },
+-- { "comma", t_comma },
+-- { "forget", t_forget },
+-- { "comment", t_comment },
+-- { "rest", t_rest },
+-- }
+
+bibtexlexer._tokenstyles = context.styleset
+
+bibtexlexer._foldpattern = P("{") + P("}")
+
+bibtexlexer._foldsymbols = {
+ _patterns = {
+ "{",
+ "}",
+ },
+ ["grouping"] = {
+ ["{"] = 1,
+ ["}"] = -1,
+ },
+}
+
+return bibtexlexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cld.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cld.lua
index 1e30c18a287..3442a195c1b 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cld.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cld.lua
@@ -6,13 +6,14 @@ local info = {
license = "see context related readme files",
}
-local lexer = lexer
+local lexer = require("lexer")
+local context = lexer.context
+local patterns = context.patterns
-local cldlexer = { _NAME = "cld", _FILENAME = "scite-context-lexer-cld" }
-local whitespace = lexer.WHITESPACE -- maybe we need to fix this
-local context = lexer.context
+local cldlexer = lexer.new("cld","scite-context-lexer-cld")
+local lualexer = lexer.load("scite-context-lexer-lua")
-local lualexer = lexer.load('scite-context-lexer-lua')
+-- can probably be done nicer now, a bit of a hack
cldlexer._rules = lualexer._rules_cld
cldlexer._tokenstyles = lualexer._tokenstyles
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp-web.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp-web.lua
new file mode 100644
index 00000000000..daa9221bad0
--- /dev/null
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp-web.lua
@@ -0,0 +1,23 @@
+local info = {
+ version = 1.002,
+ comment = "scintilla lpeg lexer for cpp web",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files",
+}
+
+local lexer = require("lexer")
+local context = lexer.context
+local patterns = context.patterns
+
+local cppweblexer = lexer.new("cpp-web","scite-context-lexer-cpp")
+local cpplexer = lexer.load("scite-context-lexer-cpp")
+
+-- can probably be done nicer now, a bit of a hack
+
+cppweblexer._rules = cpplexer._rules_web
+cppweblexer._tokenstyles = cpplexer._tokenstyles
+cppweblexer._foldsymbols = cpplexer._foldsymbols
+cppweblexer._directives = cpplexer._directives
+
+return cppweblexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp.lua
new file mode 100644
index 00000000000..31180e6a574
--- /dev/null
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-cpp.lua
@@ -0,0 +1,188 @@
+local info = {
+ version = 1.002,
+ comment = "scintilla lpeg lexer for cpp",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files",
+}
+
+-- looks liks the original cpp lexer but web ready (so nothing special here yet)
+
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+
+local lexer = require("lexer")
+local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+local exact_match = lexer.exact_match
+
+local cpplexer = lexer.new("cpp","scite-context-lexer-cpp")
+local whitespace = cpplexer.whitespace
+
+local keywords = { -- copied from cpp.lua
+ -- c
+ "asm", "auto", "break", "case", "const", "continue", "default", "do", "else",
+ "extern", "false", "for", "goto", "if", "inline", "register", "return",
+ "sizeof", "static", "switch", "true", "typedef", "volatile", "while",
+ "restrict",
+ -- hm
+ "_Bool", "_Complex", "_Pragma", "_Imaginary",
+ -- c++.
+ "catch", "class", "const_cast", "delete", "dynamic_cast", "explicit",
+ "export", "friend", "mutable", "namespace", "new", "operator", "private",
+ "protected", "public", "signals", "slots", "reinterpret_cast",
+ "static_assert", "static_cast", "template", "this", "throw", "try", "typeid",
+ "typename", "using", "virtual"
+}
+
+local datatypes = { -- copied from cpp.lua
+ "bool", "char", "double", "enum", "float", "int", "long", "short", "signed",
+ "struct", "union", "unsigned", "void"
+}
+
+local macros = { -- copied from cpp.lua
+ "define", "elif", "else", "endif", "error", "if", "ifdef", "ifndef", "import",
+ "include", "line", "pragma", "undef", "using", "warning"
+}
+
+local space = patterns.space -- S(" \n\r\t\f\v")
+local any = patterns.any
+local restofline = patterns.restofline
+local startofline = patterns.startofline
+
+local squote = P("'")
+local dquote = P('"')
+local period = P(".")
+local escaped = P("\\") * P(1)
+local slashes = P("//")
+local begincomment = P("/*")
+local endcomment = P("*/")
+local percent = P("%")
+
+local hexadecimal = patterns.hexadecimal
+local decimal = patterns.decimal
+local float = patterns.float
+local integer = P("-")^-1 * (hexadecimal + decimal) -- also in patterns ?
+
+local spacing = token(whitespace, space^1)
+local rest = token("default", any)
+
+local shortcomment = token("comment", slashes * restofline^0)
+local longcomment = token("comment", begincomment * (1-endcomment)^0 * endcomment^-1)
+
+local shortstring = token("quote", dquote) -- can be shared
+ * token("string", (escaped + (1-dquote))^0)
+ * token("quote", dquote)
+ + token("quote", squote)
+ * token("string", (escaped + (1-squote))^0)
+ * token("quote", squote)
+
+local number = token("number", float + integer)
+
+local validword = R("AZ","az","__") * R("AZ","az","__","09")^0
+local identifier = token("default",validword)
+
+local operator = token("special", S("+-*/%^!=<>;:{}[]().&|?~"))
+
+----- optionalspace = spacing^0
+
+local p_keywords = exact_match(keywords )
+local p_datatypes = exact_match(datatypes)
+local p_macros = exact_match(macros)
+
+local keyword = token("keyword", p_keywords)
+local datatype = token("keyword", p_datatypes)
+local identifier = token("default", validword)
+
+local macro = token("data", #P("#") * startofline * P("#") * S("\t ")^0 * p_macros)
+
+cpplexer._rules = {
+ { "whitespace", spacing },
+ { "keyword", keyword },
+ { "type", datatype },
+ { "identifier", identifier },
+ { "string", shortstring },
+ { "longcomment", longcomment },
+ { "shortcomment", shortcomment },
+ { "number", number },
+ { "macro", macro },
+ { "operator", operator },
+ { "rest", rest },
+}
+
+local web = lexer.loadluafile("scite-context-lexer-web-snippets")
+
+if web then
+
+ lexer.inform("supporting web snippets in cpp lexer")
+
+ cpplexer._rules_web = {
+ { "whitespace", spacing },
+ { "keyword", keyword },
+ { "type", datatype },
+ { "identifier", identifier },
+ { "string", shortstring },
+ { "longcomment", longcomment },
+ { "shortcomment", shortcomment },
+ { "web", web.pattern },
+ { "number", number },
+ { "macro", macro },
+ { "operator", operator },
+ { "rest", rest },
+ }
+
+else
+
+ lexer.report("not supporting web snippets in cpp lexer")
+
+ cpplexer._rules_web = {
+ { "whitespace", spacing },
+ { "keyword", keyword },
+ { "type", datatype },
+ { "identifier", identifier },
+ { "string", shortstring },
+ { "longcomment", longcomment },
+ { "shortcomment", shortcomment },
+ { "number", number },
+ { "macro", macro },
+ { "operator", operator },
+ { "rest", rest },
+ }
+
+end
+
+cpplexer._tokenstyles = context.styleset
+
+cpplexer._foldpattern = P("/*") + P("*/") + S("{}") -- separate entry else interference
+
+cpplexer._foldsymbols = {
+ _patterns = {
+ "[{}]",
+ "/%*",
+ "%*/",
+ },
+ -- ["data"] = { -- macro
+ -- ["region"] = 1,
+ -- ["endregion"] = -1,
+ -- ["if"] = 1,
+ -- ["ifdef"] = 1,
+ -- ["ifndef"] = 1,
+ -- ["endif"] = -1,
+ -- },
+ ["special"] = { -- operator
+ ["{"] = 1,
+ ["}"] = -1,
+ },
+ ["comment"] = {
+ ["/*"] = 1,
+ ["*/"] = -1,
+ }
+}
+
+-- -- by indentation:
+
+cpplexer._foldpatterns = nil
+cpplexer._foldsymbols = nil
+
+return cpplexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua
index fdec301bef2..855adbe4ea2 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua
@@ -1,20 +1,21 @@
local info = {
version = 1.002,
- comment = "scintilla lpeg lexer for lua",
+ comment = "scintilla lpeg lexer for lua longstrings",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files",
}
-local lexer = lexer
-local token = lexer.token
-local P = lpeg.P
-
-local stringlexer = { _NAME = "lua-longstring", _FILENAME = "scite-context-lexer-lua-longstring" }
-local whitespace = lexer.WHITESPACE
+local lexer = require("lexer") -- require("scite-context-lexer")
local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+
+local stringlexer = lexer.new("lua-longstring","scite-context-lexer-lua-longstring")
+local whitespace = stringlexer.whitespace
-local space = lexer.space
+local space = patterns.space
local nospace = 1 - space
local p_spaces = token(whitespace, space ^1)
@@ -25,6 +26,6 @@ stringlexer._rules = {
{ "string", p_string },
}
-stringlexer._tokenstyles = lexer.context.styleset
+stringlexer._tokenstyles = context.styleset
return stringlexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua.lua
index 9672e110e4f..c44d586bab5 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-lua.lua
@@ -6,56 +6,68 @@ local info = {
license = "see context related readme files",
}
--- todo: _G.print (keep _G colored)
-
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
+-- beware: all multiline is messy, so even if it's no lexer, it should be an embedded lexer
+-- we probably could use a local whitespace variant but this is cleaner
-local lexer = lexer
-local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing
-local P, R, S, C, Cg, Cb, Cs, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpeg.Cb, lpeg.Cs, lpeg.Cmt
+local P, R, S, C, Cmt, Cp = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cmt, lpeg.Cp
local match, find = string.match, string.find
local setmetatable = setmetatable
--- beware: all multiline is messy, so even if it's no lexer, it should be an embedded lexer
--- we probably could use a local whitespace variant but this is cleaner
-
-local lualexer = { _NAME = "lua", _FILENAME = "scite-context-lexer-lua" }
-local whitespace = lexer.WHITESPACE
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+local exact_match = lexer.exact_match
+local just_match = lexer.just_match
+
+local lualexer = lexer.new("lua","scite-context-lexer-lua")
+local whitespace = lualexer.whitespace
local stringlexer = lexer.load("scite-context-lexer-lua-longstring")
-local directives = { } -- communication channel
+local directives = { } -- communication channel
-- this will be extended
+-- we could combine some in a hash that returns the class that then makes the token
+-- this can save time on large files
+
local keywords = {
- 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', -- 'goto',
- 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true',
- 'until', 'while',
+ "and", "break", "do", "else", "elseif", "end", "false", "for", "function", -- "goto",
+ "if", "in", "local", "nil", "not", "or", "repeat", "return", "then", "true",
+ "until", "while",
}
local functions = {
- 'assert', 'collectgarbage', 'dofile', 'error', 'getmetatable',
- 'ipairs', 'load', 'loadfile', 'module', 'next', 'pairs',
- 'pcall', 'print', 'rawequal', 'rawget', 'rawset', 'require',
- 'setmetatable', 'tonumber', 'tostring', 'type', 'unpack', 'xpcall', 'select',
+ "assert", "collectgarbage", "dofile", "error", "getmetatable",
+ "ipairs", "load", "loadfile", "module", "next", "pairs",
+ "pcall", "print", "rawequal", "rawget", "rawset", "require",
+ "setmetatable", "tonumber", "tostring", "type", "unpack", "xpcall", "select",
"string", "table", "coroutine", "debug", "file", "io", "lpeg", "math", "os", "package", "bit32",
}
local constants = {
- '_G', '_VERSION', '_M', '...', '_ENV',
+ "_G", "_VERSION", "_M", "...", "_ENV",
-- here too
- '__add', '__call', '__concat', '__div', '__eq', '__gc', '__index',
- '__le', '__lt', '__metatable', '__mode', '__mul', '__newindex',
- '__pow', '__sub', '__tostring', '__unm',
+ "__add", "__call", "__concat", "__div", "__idiv", "__eq", "__gc", "__index",
+ "__le", "__lt", "__metatable", "__mode", "__mul", "__newindex",
+ "__pow", "__sub", "__tostring", "__unm", "__len",
+ "__pairs", "__ipairs",
+ "NaN",
}
+-- local tokenmappings = { }
+--
+-- for i=1,#keywords do tokenmappings[keywords [i]] = "keyword" }
+-- for i=1,#functions do tokenmappings[functions[i]] = "function" }
+-- for i=1,#constants do tokenmappings[constants[i]] = "constant" }
+
local internals = { -- __
- 'add', 'call', 'concat', 'div', 'eq', 'gc', 'index',
- 'le', 'lt', 'metatable', 'mode', 'mul', 'newindex',
- 'pow', 'sub', 'tostring', 'unm',
+ "add", "call", "concat", "div", "eq", "gc", "index",
+ "le", "lt", "metatable", "mode", "mul", "newindex",
+ "pow", "sub", "tostring", "unm", "len",
}
local depricated = {
@@ -65,7 +77,9 @@ local depricated = {
}
local csnames = { -- todo: option
+ "commands",
"context",
+ "ctx",
"metafun",
"metapost",
}
@@ -79,14 +93,14 @@ local longonestart = P("[[")
local longonestop = P("]]")
local longonestring = (1-longonestop)^0
-local longtwostart = P('[') * Cmt(equals,setlevel) * P('[')
-local longtwostop = P(']') * equals * P(']')
+local longtwostart = P("[") * Cmt(equals,setlevel) * P("[")
+local longtwostop = P("]") * equals * P("]")
local sentinels = { } setmetatable(sentinels, { __index = function(t,k) local v = "]" .. k .. "]" t[k] = v return v end })
local longtwostring = P(function(input,index)
if level then
- -- local sentinel = ']' .. level .. ']'
+ -- local sentinel = "]" .. level .. "]"
local sentinel = sentinels[level]
local _, stop = find(input,sentinel,index,true)
return stop and stop + 1 - #sentinel or #input + 1
@@ -97,32 +111,33 @@ end)
local longtwostring_end = P(function(input,index)
if level then
- -- local sentinel = ']' .. level .. ']'
+ -- local sentinel = "]" .. level .. "]"
local sentinel = sentinels[level]
local _, stop = find(input,sentinel,index,true)
return stop and stop + 1 or #input + 1
end
end)
-local longcomment = Cmt(#('[[' + ('[' * C(equals) * '[')), function(input,index,level)
- -- local sentinel = ']' .. level .. ']'
+local longcomment = Cmt(#("[[" + ("[" * C(equals) * "[")), function(input,index,level)
+ -- local sentinel = "]" .. level .. "]"
local sentinel = sentinels[level]
local _, stop = find(input,sentinel,index,true)
return stop and stop + 1 or #input + 1
end)
-local space = lexer.space -- S(" \n\r\t\f\v")
-local any = lexer.any
+local space = patterns.space -- S(" \n\r\t\f\v")
+local any = patterns.any
+local eol = patterns.eol
local squote = P("'")
local dquote = P('"')
local escaped = P("\\") * P(1)
-local dashes = P('--')
+local dashes = P("--")
local spacing = token(whitespace, space^1)
local rest = token("default", any)
-local shortcomment = token("comment", dashes * lexer.nonnewline^0)
+local shortcomment = token("comment", dashes * (1-eol)^0)
local longcomment = token("comment", dashes * longcomment)
-- fails on very long string with \ at end of lines (needs embedded lexer)
@@ -147,18 +162,23 @@ local string = shortstring
lexer.embed_lexer(lualexer, stringlexer, token("quote",longtwostart), token("string",longtwostring_body) * token("quote",longtwostring_end))
-local integer = P("-")^-1 * (lexer.hex_num + lexer.dec_num)
-local number = token("number", lexer.float + integer)
+local integer = P("-")^-1 * (patterns.hexadecimal + patterns.decimal)
+local number = token("number", patterns.float + integer)
-- officially 127-255 are ok but not utf so useless
-local validword = R("AZ","az","__") * R("AZ","az","__","09")^0
+----- validword = R("AZ","az","__") * R("AZ","az","__","09")^0
+
+local utf8character = P(1) * R("\128\191")^1
+local validword = (R("AZ","az","__") + utf8character) * (R("AZ","az","__","09") + utf8character)^0
+local validsuffix = (R("AZ","az") + utf8character) * (R("AZ","az","__","09") + utf8character)^0
local identifier = token("default",validword)
----- operator = token("special", P('..') + P('~=') + S('+-*/%^#=<>;:,.{}[]()')) -- maybe split off {}[]()
----- operator = token("special", S('+-*/%^#=<>;:,{}[]()') + P('..') + P('.') + P('~=') ) -- maybe split off {}[]()
-local operator = token("special", S('+-*/%^#=<>;:,{}[]().') + P('~=') ) -- no ^1 because of nested lexers
+----- operator = token("special", S('+-*/%^#=<>;:,{}[]().') + P('~=') ) -- no ^1 because of nested lexers
+local operator = token("special", S('+-*/%^#=<>;:,{}[]().|~')) -- no ^1 because of nested lexers
local structure = token("special", S('{}[]()'))
@@ -177,8 +197,7 @@ local p_functions = exact_match(functions)
local p_constants = exact_match(constants)
local p_internals = P("__")
* exact_match(internals)
-local p_csnames = exact_match(csnames)
-
+local p_csnames = just_match(csnames)
local keyword = token("keyword", p_keywords)
local builtin = token("plain", p_functions)
local constant = token("data", p_constants)
@@ -186,8 +205,10 @@ local internal = token("data", p_internals)
local csname = token("user", p_csnames)
* (
optionalspace * hasargument
- + ( optionalspace * token("special", S(".:")) * optionalspace * token("user", validword) )^1
+ + ( optionalspace * token("special", S(".:")) * optionalspace * token("user", validword ) )^1
+ + token("user", P("_") * validsuffix)
)
+
local identifier = token("default", validword)
* ( optionalspace * token("special", S(".:")) * optionalspace * (
token("warning", p_keywords) +
@@ -195,22 +216,33 @@ local identifier = token("default", validword)
token("default", validword )
) )^0
+-- local t = { } for k, v in next, tokenmappings do t[#t+1] = k end t = table.concat(t)
+-- -- local experimental = (S(t)^1) / function(s) return tokenmappings[s] end * Cp()
+--
+-- local experimental = Cmt(S(t)^1, function(_,i,s)
+-- local t = tokenmappings[s]
+-- if t then
+-- return true, t, i
+-- end
+-- end)
+
lualexer._rules = {
- { 'whitespace', spacing },
- { 'keyword', keyword },
- -- { 'structure', structure },
- { 'function', builtin },
- { 'csname', csname },
- { 'constant', constant },
- { 'goto', gotokeyword },
- { 'identifier', identifier },
- { 'string', string },
- { 'number', number },
- { 'longcomment', longcomment },
- { 'shortcomment', shortcomment },
- { 'label', gotolabel },
- { 'operator', operator },
- { 'rest', rest },
+ { "whitespace", spacing },
+ { "keyword", keyword }, -- can be combined
+ -- { "structure", structure },
+ { "function", builtin }, -- can be combined
+ { "constant", constant }, -- can be combined
+ -- { "experimental", experimental }, -- works but better split
+ { "csname", csname },
+ { "goto", gotokeyword },
+ { "identifier", identifier },
+ { "string", string },
+ { "number", number },
+ { "longcomment", longcomment },
+ { "shortcomment", shortcomment },
+ { "label", gotolabel },
+ { "operator", operator },
+ { "rest", rest },
}
-- -- experiment
@@ -245,18 +277,18 @@ lualexer._rules = {
-- }
--
-- lualexer._rules = {
--- { 'whitespace', spacing },
--- { 'whatever', whatever },
--- { 'csname', csname },
--- { 'goto', gotokeyword },
--- { 'identifier', identifier },
--- { 'string', string },
--- { 'number', number },
--- { 'longcomment', longcomment },
--- { 'shortcomment', shortcomment },
--- { 'label', gotolabel },
--- { 'operator', operator },
--- { 'rest', rest },
+-- { "whitespace", spacing },
+-- { "whatever", whatever },
+-- { "csname", csname },
+-- { "goto", gotokeyword },
+-- { "identifier", identifier },
+-- { "string", string },
+-- { "number", number },
+-- { "longcomment", longcomment },
+-- { "shortcomment", shortcomment },
+-- { "label", gotolabel },
+-- { "operator", operator },
+-- { "rest", rest },
-- }
lualexer._tokenstyles = context.styleset
@@ -268,26 +300,26 @@ lualexer._foldpattern = (P("end") + P("if") + P("do") + P("function") + P("repea
lualexer._foldsymbols = {
_patterns = {
- '[a-z][a-z]+',
- '[{}%[%]]',
+ "[a-z][a-z]+",
+ "[{}%[%]]",
},
- ['keyword'] = { -- challenge: if=0 then=1 else=-1 elseif=-1
- ['if'] = 1, -- if .. [then|else] .. end
- ['do'] = 1, -- [while] do .. end
- ['function'] = 1, -- function .. end
- ['repeat'] = 1, -- repeat .. until
- ['until'] = -1,
- ['end'] = -1,
+ ["keyword"] = { -- challenge: if=0 then=1 else=-1 elseif=-1
+ ["if"] = 1, -- if .. [then|else] .. end
+ ["do"] = 1, -- [while] do .. end
+ ["function"] = 1, -- function .. end
+ ["repeat"] = 1, -- repeat .. until
+ ["until"] = -1,
+ ["end"] = -1,
},
- ['comment'] = {
- ['['] = 1, [']'] = -1,
+ ["comment"] = {
+ ["["] = 1, ["]"] = -1,
},
- -- ['quote'] = { -- confusing
- -- ['['] = 1, [']'] = -1,
+ -- ["quote"] = { -- confusing
+ -- ["["] = 1, ["]"] = -1,
-- },
- ['special'] = {
- -- ['('] = 1, [')'] = -1,
- ['{'] = 1, ['}'] = -1,
+ ["special"] = {
+ -- ["("] = 1, [")"] = -1,
+ ["{"] = 1, ["}"] = -1,
},
}
@@ -295,9 +327,9 @@ lualexer._foldsymbols = {
local cstoken = R("az","AZ","\127\255") + S("@!?_")
local texcsname = P("\\") * cstoken^1
-local commentline = P('%') * (1-S("\n\r"))^0
+local commentline = P("%") * (1-S("\n\r"))^0
-local texcomment = token('comment', Cmt(commentline, function() return directives.cld_inline end))
+local texcomment = token("comment", Cmt(commentline, function() return directives.cld_inline end))
local longthreestart = P("\\!!bs")
local longthreestop = P("\\!!es")
@@ -307,7 +339,7 @@ local texstring = token("quote", longthreestart)
* token("string", longthreestring)
* token("quote", longthreestop)
--- local texcommand = token("user", texcsname)
+----- texcommand = token("user", texcsname)
local texcommand = token("warning", texcsname)
-- local texstring = token("quote", longthreestart)
@@ -320,22 +352,22 @@ local texcommand = token("warning", texcsname)
lualexer._directives = directives
lualexer._rules_cld = {
- { 'whitespace', spacing },
- { 'texstring', texstring },
- { 'texcomment', texcomment },
- { 'texcommand', texcommand },
- -- { 'structure', structure },
- { 'keyword', keyword },
- { 'function', builtin },
- { 'csname', csname },
- { 'constant', constant },
- { 'identifier', identifier },
- { 'string', string },
- { 'longcomment', longcomment },
- { 'shortcomment', shortcomment }, -- should not be used inline so best signal it as comment (otherwise complex state till end of inline)
- { 'number', number },
- { 'operator', operator },
- { 'rest', rest },
+ { "whitespace", spacing },
+ { "texstring", texstring },
+ { "texcomment", texcomment },
+ { "texcommand", texcommand },
+ -- { "structure", structure },
+ { "keyword", keyword },
+ { "function", builtin },
+ { "csname", csname },
+ { "constant", constant },
+ { "identifier", identifier },
+ { "string", string },
+ { "longcomment", longcomment },
+ { "shortcomment", shortcomment }, -- should not be used inline so best signal it as comment (otherwise complex state till end of inline)
+ { "number", number },
+ { "operator", operator },
+ { "rest", rest },
}
return lualexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-mps.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-mps.lua
index 96c5e9c3cb9..b87ea83cb53 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-mps.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-mps.lua
@@ -6,17 +6,19 @@ local info = {
license = "see context related readme files",
}
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
-
-local lexer = lexer
local global, string, table, lpeg = _G, string, table, lpeg
-local token, exact_match = lexer.token, lexer.exact_match
-local P, R, S, V, C, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt
+local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
local type = type
-local metafunlexer = { _NAME = "mps", _FILENAME = "scite-context-lexer-mps" }
-local whitespace = lexer.WHITESPACE
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+local exact_match = lexer.exact_match
+
+local metafunlexer = lexer.new("mps","scite-context-lexer-mps")
+local whitespace = metafunlexer.whitespace
local metapostprimitives = { }
local metapostinternals = { }
@@ -65,79 +67,97 @@ do
end
-local space = lexer.space -- S(" \n\r\t\f\v")
-local any = lexer.any
+local space = patterns.space -- S(" \n\r\t\f\v")
+local any = patterns.any
local dquote = P('"')
-local cstoken = R("az","AZ") + P("_")
-local mptoken = R("az","AZ")
+local cstoken = patterns.idtoken
+local mptoken = patterns.alpha
local leftbrace = P("{")
local rightbrace = P("}")
-local number = context.patterns.real
+local number = patterns.real
local cstokentex = R("az","AZ","\127\255") + S("@!?_")
-- we could collapse as in tex
local spacing = token(whitespace, space^1)
-local rest = token('default', any)
-local comment = token('comment', P('%') * (1-S("\n\r"))^0)
-local internal = token('reserved', exact_match(mergedshortcuts,false))
-local shortcut = token('data', exact_match(mergedinternals))
-local helper = token('command', exact_match(metafuncommands))
-local plain = token('plain', exact_match(metapostcommands))
-local quoted = token('quote', dquote)
- * token('string', P(1-dquote)^0)
- * token('quote', dquote)
-local texstuff = token('quote', P("btex ") + P("verbatimtex "))
- * token('string', P(1-P(" etex"))^0)
- * token('quote', P(" etex"))
-local primitive = token('primitive', exact_match(metapostprimitives))
-local identifier = token('default', cstoken^1)
-local number = token('number', number)
-local grouping = token('grouping', S("()[]{}")) -- can be an option
-local special = token('special', S("#()[]{}<>=:\"")) -- or else := <> etc split
-local texlike = token('warning', P("\\") * cstokentex^1)
-local extra = token('extra', S("`~%^&_-+*/\'|\\"))
+local rest = token("default", any)
+local comment = token("comment", P("%") * (1-S("\n\r"))^0)
+local internal = token("reserved", exact_match(mergedshortcuts,false))
+local shortcut = token("data", exact_match(mergedinternals))
+local helper = token("command", exact_match(metafuncommands))
+local plain = token("plain", exact_match(metapostcommands))
+local quoted = token("quote", dquote)
+ * token("string", P(1-dquote)^0)
+ * token("quote", dquote)
+local texstuff = token("quote", P("btex ") + P("verbatimtex "))
+ * token("string", P(1-P(" etex"))^0)
+ * token("quote", P(" etex"))
+local primitive = token("primitive", exact_match(metapostprimitives))
+local identifier = token("default", cstoken^1)
+local number = token("number", number)
+local grouping = token("grouping", S("()[]{}")) -- can be an option
+local special = token("special", S("#()[]{}<>=:\"")) -- or else := <> etc split
+local texlike = token("warning", P("\\") * cstokentex^1)
+local extra = token("extra", P("+-+") + P("++") + S("`~%^&_-+*/\'|\\"))
local nested = P { leftbrace * (V(1) + (1-rightbrace))^0 * rightbrace }
-local texlike = token('embedded', P("\\") * (P("MP") + P("mp")) * mptoken^1)
+local texlike = token("embedded", P("\\") * (P("MP") + P("mp")) * mptoken^1)
* spacing^0
- * token('grouping', leftbrace)
- * token('rest', (nested + (1-rightbrace))^0 )
- * token('grouping', rightbrace)
- + token('warning', P("\\") * cstokentex^1)
+ * token("grouping", leftbrace)
+ * token("default", (nested + (1-rightbrace))^0 )
+ * token("grouping", rightbrace)
+ + token("warning", P("\\") * cstokentex^1)
+
+-- lua: we assume: lua ( "lua code" )
+
+local cldlexer = lexer.load("scite-context-lexer-cld","mps-cld")
+
+local startlua = P("lua") * space^0 * P('(') * space^0 * P('"')
+local stoplua = P('"') * space^0 * P(')')
+
+local startluacode = token("embedded", startlua)
+local stopluacode = #stoplua * token("embedded", stoplua)
+
+lexer.embed_lexer(metafunlexer, cldlexer, startluacode, stopluacode)
metafunlexer._rules = {
- { 'whitespace', spacing },
- { 'comment', comment },
- { 'internal', internal },
- { 'shortcut', shortcut },
- { 'helper', helper },
- { 'plain', plain },
- { 'primitive', primitive },
- { 'texstuff', texstuff },
- { 'identifier', identifier },
- { 'number', number },
- { 'quoted', quoted },
- -- { 'grouping', grouping }, -- can be an option
- { 'special', special },
- { 'texlike', texlike },
- { 'extra', extra },
- { 'rest', rest },
+ { "whitespace", spacing },
+ { "comment", comment },
+ { "internal", internal },
+ { "shortcut", shortcut },
+ { "helper", helper },
+ { "plain", plain },
+ { "primitive", primitive },
+ { "texstuff", texstuff },
+ { "identifier", identifier },
+ { "number", number },
+ { "quoted", quoted },
+ -- { "grouping", grouping }, -- can be an option
+ { "special", special },
+ { "texlike", texlike },
+ { "extra", extra },
+ { "rest", rest },
}
metafunlexer._tokenstyles = context.styleset
-metafunlexer._foldpattern = R("az")^2 -- separate entry else interference
+metafunlexer._foldpattern = patterns.lower^2 -- separate entry else interference
metafunlexer._foldsymbols = {
_patterns = {
- '[a-z][a-z]+',
+ "[a-z][a-z]+",
},
- ["primitive"] = {
+ ["plain"] = {
["beginfig"] = 1,
["endfig"] = -1,
+ ["beginglyph"] = 1,
+ ["endglyph"] = -1,
+ -- ["begingraph"] = 1,
+ -- ["endgraph"] = -1,
+ },
+ ["primitive"] = {
["def"] = 1,
["vardef"] = 1,
["primarydef"] = 1,
@@ -152,4 +172,6 @@ metafunlexer._foldsymbols = {
}
}
+-- if inspect then inspect(metafunlexer) end
+
return metafunlexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-object.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-object.lua
index 6d0b6d8da8e..1fb95838a9c 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-object.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-object.lua
@@ -1,24 +1,29 @@
local info = {
version = 1.002,
- comment = "scintilla lpeg lexer for pdf",
+ comment = "scintilla lpeg lexer for pdf objects",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files",
}
-local lexer = lexer
-local token = lexer.token
+-- no longer used: nesting lexers with whitespace in start/stop is unreliable
+
local P, R, S, C, V = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.V
-local pdfobjectlexer = { _NAME = "pdf-object", _FILENAME = "scite-context-lexer-pdf-object" }
-local whitespace = lexer.WHITESPACE -- triggers states
+local lexer = require("lexer")
local context = lexer.context
local patterns = context.patterns
-local space = lexer.space
-local somespace = space^1
+local token = lexer.token
+
+local pdfobjectlexer = lexer.new("pdfobj","scite-context-lexer-pdf-object")
+local whitespace = pdfobjectlexer.whitespace
-local newline = S("\n\r")
+local space = patterns.space
+local spacing = patterns.spacing
+local nospacing = patterns.nospacing
+local anything = patterns.anything
+local newline = patterns.eol
local real = patterns.real
local cardinal = patterns.cardinal
@@ -27,7 +32,6 @@ local rparent = P(")")
local langle = P("<")
local rangle = P(">")
local escape = P("\\")
-local anything = P(1)
local unicodetrigger = P("feff")
local nametoken = 1 - space - S("<>/[]()")
@@ -35,48 +39,50 @@ local name = P("/") * nametoken^1
local p_string = P { ( escape * anything + lparent * V(1) * rparent + (1 - rparent) )^0 }
-local t_spacing = token(whitespace, space^1)
-local t_spaces = token(whitespace, space^1)^0
+local t_spacing = token(whitespace, spacing)
+local t_spaces = token(whitespace, spacing)^0
+local t_rest = token("default", nospacing) -- anything
local p_stream = P("stream")
local p_endstream = P("endstream")
------ p_obj = P("obj")
+local p_obj = P("obj")
local p_endobj = P("endobj")
local p_reference = P("R")
local p_objectnumber = patterns.cardinal
-local p_comment = P('%') * (1-S("\n\r"))^0
+local p_comment = P("%") * (1-S("\n\r"))^0
-local string = token("quote", lparent)
+local t_string = token("quote", lparent)
* token("string", p_string)
* token("quote", rparent)
-local unicode = token("quote", langle)
+local t_unicode = token("quote", langle)
* token("plain", unicodetrigger)
* token("string", (1-rangle)^1)
* token("quote", rangle)
-local whatsit = token("quote", langle)
+local t_whatsit = token("quote", langle)
* token("string", (1-rangle)^1)
* token("quote", rangle)
-local keyword = token("command", name)
-local constant = token("constant", name)
-local number = token('number', real)
--- local reference = token("number", cardinal)
--- * t_spacing
--- * token("number", cardinal)
-local reserved = token("number", P("true") + P("false") + P("NULL"))
-local reference = token("warning", cardinal)
+local t_keyword = token("command", name)
+local t_constant = token("constant", name)
+local t_number = token("number", real)
+-- t_reference = token("number", cardinal)
+-- * t_spacing
+-- * token("number", cardinal)
+local t_reserved = token("number", P("true") + P("false") + P("NULL"))
+local t_reference = token("warning", cardinal)
* t_spacing
- * token("warning", cardinal)
+ * token("warning", cardinal)
* t_spacing
* token("keyword", p_reference)
+
local t_comment = token("comment", p_comment)
--- t_openobject = token("number", p_objectnumber)
--- * t_spacing
--- * token("number", p_objectnumber)
--- * t_spacing
--- * token("keyword", p_obj)
-local t_closeobject = token("keyword", p_endobj)
+local t_openobject = token("warning", p_objectnumber * spacing)
+-- * t_spacing
+ * token("warning", p_objectnumber * spacing)
+-- * t_spacing
+ * token("keyword", p_obj)
+local t_closeobject = token("keyword", p_endobj)
local t_opendictionary = token("grouping", P("<<"))
local t_closedictionary = token("grouping", P(">>"))
@@ -84,32 +90,45 @@ local t_closedictionary = token("grouping", P(">>"))
local t_openarray = token("grouping", P("["))
local t_closearray = token("grouping", P("]"))
+-- todo: comment
+
local t_stream = token("keyword", p_stream)
--- * token("default", newline * (1-newline*p_endstream*newline)^1 * newline)
- * token("default", (1 - p_endstream)^1)
+-- * token("default", newline * (1-newline*p_endstream*newline)^1 * newline)
+-- * token("text", (1 - p_endstream)^1)
+ * (token("text", (1 - p_endstream-spacing)^1) + t_spacing)^1
* token("keyword", p_endstream)
local t_dictionary = { "dictionary",
- dictionary = t_opendictionary * (t_spaces * keyword * t_spaces * V("whatever"))^0 * t_spaces * t_closedictionary,
+ dictionary = t_opendictionary * (t_spaces * t_keyword * t_spaces * V("whatever"))^0 * t_spaces * t_closedictionary,
array = t_openarray * (t_spaces * V("whatever"))^0 * t_spaces * t_closearray,
- whatever = V("dictionary") + V("array") + constant + reference + string + unicode + number + whatsit,
+ whatever = V("dictionary") + V("array") + t_constant + t_reference + t_string + t_unicode + t_number + t_reserved + t_whatsit,
}
+----- t_object = { "object", -- weird that we need to catch the end here (probably otherwise an invalid lpeg)
+----- object = t_spaces * (V("dictionary") * t_spaces * t_stream^-1 + V("array") + V("number") + t_spaces) * t_spaces * t_closeobject,
+----- dictionary = t_opendictionary * (t_spaces * t_keyword * t_spaces * V("whatever"))^0 * t_spaces * t_closedictionary,
+----- array = t_openarray * (t_spaces * V("whatever"))^0 * t_spaces * t_closearray,
+----- whatever = V("dictionary") + V("array") + t_constant + t_reference + t_string + t_unicode + t_number + t_reserved + t_whatsit,
+----- number = t_number,
+----- }
+
local t_object = { "object", -- weird that we need to catch the end here (probably otherwise an invalid lpeg)
- object = t_spaces * (V("dictionary") * t_spaces * t_stream^-1 + V("array") + V("number") + t_spaces) * t_spaces * t_closeobject,
- dictionary = t_opendictionary * (t_spaces * keyword * t_spaces * V("whatever"))^0 * t_spaces * t_closedictionary,
- array = t_openarray * (t_spaces * V("whatever"))^0 * t_spaces * t_closearray,
- number = number,
- whatever = V("dictionary") + V("array") + constant + reference + string + unicode + number + reserved + whatsit,
+ dictionary = t_dictionary.dictionary,
+ array = t_dictionary.array,
+ whatever = t_dictionary.whatever,
+ object = t_openobject^-1 * t_spaces * (V("dictionary") * t_spaces * t_stream^-1 + V("array") + V("number") + t_spaces) * t_spaces * t_closeobject,
+ number = t_number,
}
pdfobjectlexer._shared = {
- dictionary = t_dictionary,
+ dictionary = t_dictionary,
+ object = t_object,
+ stream = t_stream,
}
pdfobjectlexer._rules = {
- { 'whitespace', t_spacing },
- { 'object', t_object },
+ { "whitespace", t_spacing }, -- in fact, here we don't want whitespace as it's top level lexer work
+ { "object", t_object },
}
pdfobjectlexer._tokenstyles = context.styleset
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua
index f205e913053..7097c41a68d 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua
@@ -6,44 +6,36 @@ local info = {
license = "see context related readme files",
}
-local lexer = lexer
-local token = lexer.token
-local P, R = lpeg.P, lpeg.R
+-- no longer used: nesting lexers with whitespace in start/stop is unreliable
--- xref
--- cardinal cardinal [character]
--- ..
--- %%EOF | startxref | trailer
+local P, R = lpeg.P, lpeg.R
-local pdfxreflexer = { _NAME = "pdf-xref", _FILENAME = "scite-context-lexer-pdf-xref" }
-local whitespace = lexer.WHITESPACE -- triggers states
+local lexer = require("lexer")
local context = lexer.context
local patterns = context.patterns
-local pdfobjectlexer = lexer.load("scite-context-lexer-pdf-object")
+local token = lexer.token
+
+local pdfxreflexer = lexer.new("pdfxref","scite-context-lexer-pdf-xref")
+local whitespace = pdfxreflexer.whitespace
local spacing = patterns.spacing
+local cardinal = patterns.cardinal
+local alpha = patterns.alpha
local t_spacing = token(whitespace, spacing)
-local p_trailer = P("trailer")
-
-local t_number = token("number", R("09")^1)
- * t_spacing
- * token("number", R("09")^1)
- * t_spacing
- * (token("keyword", R("az","AZ")) * t_spacing)^-1
-
-local t_xref = t_number^1
+local p_xref = P("xref")
+local t_xref = token("keyword",p_xref)
+ * token("number", cardinal * spacing * cardinal * spacing)
--- local t_xref = token("default", (1-p_trailer)^1)
--- * token("keyword", p_trailer)
--- * t_spacing
--- * pdfobjectlexer._shared.dictionary
+local t_number = token("number", cardinal * spacing * cardinal * spacing)
+ * token("keyword", alpha)
pdfxreflexer._rules = {
- { 'whitespace', t_spacing },
- { 'xref', t_xref },
+ { "whitespace", t_spacing },
+ { "xref", t_xref },
+ { "number", t_number },
}
pdfxreflexer._tokenstyles = context.styleset
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf.lua
index 685fdb16e71..f8e4e73804e 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-pdf.lua
@@ -6,75 +6,199 @@ local info = {
license = "see context related readme files",
}
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
+-- pdf is normally startic .. i.e. not edited so we don't really
+-- need embedded lexers.
-local lexer = lexer
-local token = lexer.token
-local P, R, S = lpeg.P, lpeg.R, lpeg.S
-
-local pdflexer = { _NAME = "pdf", _FILENAME = "scite-context-lexer-pdf" }
-local whitespace = lexer.WHITESPACE -- triggers states
-
-local pdfobjectlexer = lexer.load("scite-context-lexer-pdf-object")
-local pdfxreflexer = lexer.load("scite-context-lexer-pdf-xref")
+local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
+local lexer = require("lexer")
local context = lexer.context
local patterns = context.patterns
+local token = lexer.token
+
+local pdflexer = lexer.new("pdf","scite-context-lexer-pdf")
+local whitespace = pdflexer.whitespace
+
+----- pdfobjectlexer = lexer.load("scite-context-lexer-pdf-object")
+----- pdfxreflexer = lexer.load("scite-context-lexer-pdf-xref")
+
+local anything = patterns.anything
local space = patterns.space
local spacing = patterns.spacing
local nospacing = patterns.nospacing
local anything = patterns.anything
local restofline = patterns.restofline
-local t_spacing = token(whitespace, spacing)
-local t_rest = token("default", nospacing) -- anything
+local t_whitespace = token(whitespace, spacing)
+local t_spacing = token("default", spacing)
+----- t_rest = token("default", nospacing)
+local t_rest = token("default", anything)
+
+local p_comment = P("%") * restofline
+local t_comment = token("comment", p_comment)
+
+-- whatever
+
+local space = patterns.space
+local spacing = patterns.spacing
+local nospacing = patterns.nospacing
+local anything = patterns.anything
+local newline = patterns.eol
+local real = patterns.real
+local cardinal = patterns.cardinal
+local alpha = patterns.alpha
+
+local lparent = P("(")
+local rparent = P(")")
+local langle = P("<")
+local rangle = P(">")
+local escape = P("\\")
+local unicodetrigger = P("feff")
+local nametoken = 1 - space - S("<>/[]()")
+local name = P("/") * nametoken^1
+
+local p_string = P { ( escape * anything + lparent * V(1) * rparent + (1 - rparent) )^0 }
+
+local t_spacing = token("default", spacing)
+local t_spaces = token("default", spacing)^0
+local t_rest = token("default", nospacing) -- anything
+
+local p_stream = P("stream")
+local p_endstream = P("endstream")
local p_obj = P("obj")
local p_endobj = P("endobj")
-local p_xref = P("xref")
-local p_startxref = P("startxref")
-local p_eof = P("%%EOF")
-local p_trailer = P("trailer")
+local p_reference = P("R")
local p_objectnumber = patterns.cardinal
-local p_comment = P('%') * restofline
-
-local t_comment = token("comment", p_comment)
-local t_openobject = token("warning", p_objectnumber)
+local p_comment = P("%") * (1-S("\n\r"))^0
+
+local t_string = token("quote", lparent)
+ * token("string", p_string)
+ * token("quote", rparent)
+local t_unicode = token("quote", langle)
+ * token("plain", unicodetrigger)
+ * token("string", (1-rangle)^1)
+ * token("quote", rangle)
+local t_whatsit = token("quote", langle)
+ * token("string", (1-rangle)^1)
+ * token("quote", rangle)
+local t_keyword = token("command", name)
+local t_constant = token("constant", name)
+local t_number = token("number", real)
+-- t_reference = token("number", cardinal)
+-- * t_spacing
+-- * token("number", cardinal)
+local t_reserved = token("number", P("true") + P("false") + P("NULL"))
+-- t_reference = token("warning", cardinal * spacing * cardinal * spacing)
+-- * token("keyword", p_reference)
+local t_reference = token("warning", cardinal)
* t_spacing
- * token("warning", p_objectnumber)
+ * token("warning", cardinal)
* t_spacing
- * token("keyword", p_obj)
- * t_spacing^0
-local t_closeobject = token("keyword", p_endobj)
+ * token("keyword", p_reference)
--- We could do clever xref parsing but why should we (i.e. we should check for
--- the xref body. As a pdf file is not edited, we could do without a nested
--- lexer anyway.
+local t_comment = token("comment", p_comment)
+
+local t_openobject = token("warning", p_objectnumber)
+ * t_spacing
+ * token("warning", p_objectnumber)
+ * t_spacing
+ * token("keyword", p_obj)
+-- t_openobject = token("warning", p_objectnumber * spacing)
+-- * token("warning", p_objectnumber * spacing)
+-- * token("keyword", p_obj)
+local t_closeobject = token("keyword", p_endobj)
+
+local t_opendictionary = token("grouping", P("<<"))
+local t_closedictionary = token("grouping", P(">>"))
+
+local t_openarray = token("grouping", P("["))
+local t_closearray = token("grouping", P("]"))
+
+local t_stream = token("keyword", p_stream)
+ * token("text", (1 - p_endstream)^1)
+ * token("keyword", p_endstream)
+
+local t_dictionary = { "dictionary",
+ dictionary = t_opendictionary * (t_spaces * t_keyword * t_spaces * V("whatever"))^0 * t_spaces * t_closedictionary,
+ array = t_openarray * (t_spaces * V("whatever"))^0 * t_spaces * t_closearray,
+ whatever = V("dictionary") + V("array") + t_constant + t_reference + t_string + t_unicode + t_number + t_reserved + t_whatsit,
+ }
+
+local t_object = { "object", -- weird that we need to catch the end here (probably otherwise an invalid lpeg)
+ dictionary = t_dictionary.dictionary,
+ array = t_dictionary.array,
+ whatever = t_dictionary.whatever,
+ object = t_openobject * t_spaces * (V("dictionary")^-1 * t_spaces * t_stream^-1 + V("array") + V("number") + t_spaces) * t_spaces * t_closeobject,
+ number = t_number,
+ }
+
+-- objects ... sometimes NUL characters play havoc ... and in xref we have
+-- issues with embedded lexers that have spaces in the start and stop
+-- conditions and this cannot be handled well either ... so, an imperfect
+-- solution ... but anyway, there is not that much that can end up in
+-- the root of the tree see we're sort of safe
+local p_trailer = P("trailer")
local t_trailer = token("keyword", p_trailer)
* t_spacing
- * pdfobjectlexer._shared.dictionary
+ * t_dictionary
+-- t_trailer = token("keyword", p_trailer * spacing)
+-- * t_dictionary
-local t_openxref = token("plain", p_xref)
-local t_closexref = token("plain", p_startxref)
- + token("comment", p_eof)
- + t_trailer
-local t_startxref = token("plain", p_startxref)
+local p_startxref = P("startxref")
+local t_startxref = token("keyword", p_startxref)
* t_spacing
- * token("number", R("09")^1)
+ * token("number", cardinal)
+-- t_startxref = token("keyword", p_startxref * spacing)
+-- * token("number", cardinal)
-lexer.embed_lexer(pdflexer, pdfobjectlexer, t_openobject, t_closeobject)
-lexer.embed_lexer(pdflexer, pdfxreflexer, t_openxref, t_closexref)
+local p_xref = P("xref")
+local t_xref = token("keyword",p_xref)
+ * t_spacing
+ * token("number", cardinal)
+ * t_spacing
+ * token("number", cardinal)
+ * spacing
+-- t_xref = token("keyword",p_xref)
+-- * token("number", spacing * cardinal * spacing * cardinal * spacing)
+
+local t_number = token("number", cardinal)
+ * t_spacing
+ * token("number", cardinal)
+ * t_spacing
+ * token("keyword", S("fn"))
+-- t_number = token("number", cardinal * spacing * cardinal * spacing)
+-- * token("keyword", S("fn"))
pdflexer._rules = {
- { 'whitespace', t_spacing },
- { 'comment', t_comment },
- { 'xref', t_startxref },
- { 'rest', t_rest },
+ { "whitespace", t_whitespace },
+ { "object", t_object },
+ { "comment", t_comment },
+ { "trailer", t_trailer },
+ { "startxref", t_startxref },
+ { "xref", t_xref },
+ { "number", t_number },
+ { "rest", t_rest },
}
pdflexer._tokenstyles = context.styleset
+-- lexer.inspect(pdflexer)
+
+-- collapser: obj endobj stream endstream
+
+pdflexer._foldpattern = p_obj + p_endobj + p_stream + p_endstream
+
+pdflexer._foldsymbols = {
+ ["keyword"] = {
+ ["obj"] = 1,
+ ["endobj"] = -1,
+ ["stream"] = 1,
+ ["endstream"] = -1,
+ },
+}
+
return pdflexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex-web.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex-web.lua
new file mode 100644
index 00000000000..5d8859c2649
--- /dev/null
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex-web.lua
@@ -0,0 +1,23 @@
+local info = {
+ version = 1.002,
+ comment = "scintilla lpeg lexer for tex web",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files",
+}
+
+local lexer = require("lexer")
+local context = lexer.context
+local patterns = context.patterns
+
+local texweblexer = lexer.new("tex-web","scite-context-lexer-tex")
+local texlexer = lexer.load("scite-context-lexer-tex")
+
+-- can probably be done nicer now, a bit of a hack
+
+texweblexer._rules = texlexer._rules_web
+texweblexer._tokenstyles = texlexer._tokenstyles
+texweblexer._foldsymbols = texlexer._foldsymbols
+texweblexer._directives = texlexer._directives
+
+return texweblexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex.lua
index cbe6d261d66..d67be2cd84a 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-tex.lua
@@ -24,33 +24,26 @@ local info = {
-- local interface = props["keywordclass.macros.context.en"]
-- local interface = lexer.get_property("keywordclass.macros.context.en","")
- -- it seems that whitespace triggers the lexer when embedding happens, but this
- -- is quite fragile due to duplicate styles .. lexer.WHITESPACE is a number
- -- (initially) ... _NAME vs filename (but we don't want to overwrite files)
-
- -- this lexer does not care about other macro packages (one can of course add a fake
- -- interface but it's not on the agenda)
-
]]--
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
-
-local lexer = lexer
local global, string, table, lpeg = _G, string, table, lpeg
-local token, exact_match = lexer.token, lexer.exact_match
local P, R, S, V, C, Cmt, Cp, Cc, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt, lpeg.Cp, lpeg.Cc, lpeg.Ct
local type, next = type, next
local find, match, lower, upper = string.find, string.match, string.lower, string.upper
--- module(...)
-
-local contextlexer = { _NAME = "tex", _FILENAME = "scite-context-lexer-tex" }
-local whitespace = lexer.WHITESPACE
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
+local inform = context.inform
-local cldlexer = lexer.load('scite-context-lexer-cld')
------ cldlexer = lexer.load('scite-context-lexer-lua')
-local mpslexer = lexer.load('scite-context-lexer-mps')
+local token = lexer.token
+local exact_match = lexer.exact_match
+
+local contextlexer = lexer.new("tex","scite-context-lexer-tex")
+local whitespace = contextlexer.whitespace
+
+local cldlexer = lexer.load("scite-context-lexer-cld")
+local mpslexer = lexer.load("scite-context-lexer-mps")
local commands = { en = { } }
local primitives = { }
@@ -64,7 +57,9 @@ do -- todo: only once, store in global
local definitions = context.loaddefinitions("scite-context-data-interfaces")
if definitions then
+ local list = { }
for interface, list in next, definitions do
+ list[#list+1] = interface
local c = { }
for i=1,#list do
c[list[i]] = true
@@ -79,6 +74,7 @@ do -- todo: only once, store in global
end
commands[interface] = c
end
+ inform("context user interfaces '%s' supported",table.concat(list," "))
end
local definitions = context.loaddefinitions("scite-context-data-context")
@@ -146,13 +142,16 @@ local validminimum = 3
-- % language=uk
-local knownpreamble = Cmt(#P("% "), function(input,i,_) -- todo : utfbomb
+-- fails (empty loop message) ... latest lpeg issue?
+
+local knownpreamble = Cmt(P("% "), function(input,i,_) -- todo : utfbomb, was #P("% ")
if i < 10 then
validwords, validminimum = false, 3
- local s, e, word = find(input,'^(.+)[\n\r]',i) -- combine with match
+ local s, e, word = find(input,"^(.+)[\n\r]",i) -- combine with match
if word then
local interface = match(word,"interface=([a-z]+)")
- if interface then
+ if interface and #interface == 2 then
+ inform("enabling context user interface '%s'",interface)
currentcommands = commands[interface] or commands.en or { }
end
local language = match(word,"language=([a-z]+)")
@@ -170,7 +169,7 @@ end)
-- local helpers_hash = { } for i=1,#helpers do helpers_hash [helpers [i]] = true end
-- local primitives_hash = { } for i=1,#primitives do primitives_hash[primitives[i]] = true end
--- local specialword = Ct( P('\\') * Cmt( C(cstoken^1), function(input,i,s)
+-- local specialword = Ct( P("\\") * Cmt( C(cstoken^1), function(input,i,s)
-- if currentcommands[s] then
-- return true, "command", i
-- elseif constants_hash[s] then
@@ -184,7 +183,7 @@ end)
-- end
-- end) )
--- local specialword = P('\\') * Cmt( C(cstoken^1), function(input,i,s)
+-- local specialword = P("\\") * Cmt( C(cstoken^1), function(input,i,s)
-- if currentcommands[s] then
-- return true, { "command", i }
-- elseif constants_hash[s] then
@@ -202,11 +201,11 @@ end)
-- 10pt
-local commentline = P('%') * (1-S("\n\r"))^0
+local commentline = P("%") * (1-S("\n\r"))^0
local endline = S("\n\r")^1
-local space = lexer.space -- S(" \n\r\t\f\v")
-local any = lexer.any
+local space = patterns.space -- S(" \n\r\t\f\v")
+local any = patterns.any
local backslash = P("\\")
local hspace = S(" \t")
@@ -219,7 +218,7 @@ local p_command = backslash * knowncommand
local p_constant = backslash * exact_match(constants)
local p_helper = backslash * exact_match(helpers)
local p_primitive = backslash * exact_match(primitives)
-local p_ifprimitive = P('\\if') * cstoken^1
+local p_ifprimitive = P("\\if") * cstoken^1
local p_csname = backslash * (cstoken^1 + P(1))
local p_grouping = S("{$}")
local p_special = S("#()[]<>=\"")
@@ -299,24 +298,24 @@ local p_invisible = invisibles^1
local spacing = token(whitespace, p_spacing )
-local rest = token('default', p_rest )
-local preamble = token('preamble', p_preamble )
-local comment = token('comment', p_comment )
-local command = token('command', p_command )
-local constant = token('data', p_constant )
-local helper = token('plain', p_helper )
-local primitive = token('primitive', p_primitive )
-local ifprimitive = token('primitive', p_ifprimitive)
-local reserved = token('reserved', p_reserved )
-local csname = token('user', p_csname )
-local grouping = token('grouping', p_grouping )
-local number = token('number', p_number )
- * token('constant', p_unit )
-local special = token('special', p_special )
-local reserved = token('reserved', p_reserved ) -- reserved internal preproc
-local extra = token('extra', p_extra )
-local invisible = token('invisible', p_invisible )
-local text = token('default', p_text )
+local rest = token("default", p_rest )
+local preamble = token("preamble", p_preamble )
+local comment = token("comment", p_comment )
+local command = token("command", p_command )
+local constant = token("data", p_constant )
+local helper = token("plain", p_helper )
+local primitive = token("primitive", p_primitive )
+local ifprimitive = token("primitive", p_ifprimitive)
+local reserved = token("reserved", p_reserved )
+local csname = token("user", p_csname )
+local grouping = token("grouping", p_grouping )
+local number = token("number", p_number )
+ * token("constant", p_unit )
+local special = token("special", p_special )
+local reserved = token("reserved", p_reserved ) -- reserved internal preproc
+local extra = token("extra", p_extra )
+local invisible = token("invisible", p_invisible )
+local text = token("default", p_text )
local word = p_word
----- startluacode = token("grouping", P("\\startluacode"))
@@ -390,18 +389,21 @@ contextlexer._reset_parser = function()
end
local luaenvironment = P("lua") * (P("setups") + P("code") + P(true))
+ + P("ctxfunction") * (P("definition") + P(true))
local inlinelua = P("\\") * (
- P("ctx") * ( P("lua") + P("command") + P("late") * (P("lua") + P("command")) )
- + P("cld") * ( P("command") + P("context") )
+ P("ctx") * (P("lua") + P("command") + P("late") * (P("lua") + P("command")) + P("function"))
+ + P("cld") * (P("command") + P("context"))
+ P("luaexpr")
+ (P("direct") + P("late")) * P("lua")
)
local startlua = P("\\start") * Cmt(luaenvironment,startdisplaylua)
+ + P("<?lua") * Cmt(P(true),startdisplaylua)
+ inlinelua * space^0 * ( Cmt(P("{"),startinlinelua) )
local stoplua = P("\\stop") * Cmt(luaenvironment,stopdisplaylua)
+ + P("?>") * Cmt(P(true),stopdisplaylua)
+ Cmt(P("{"),stopinlinelua_b)
+ Cmt(P("}"),stopinlinelua_e)
@@ -413,7 +415,7 @@ local metafuncall = ( P("reusable") + P("usable") + P("unique") + P("
+ P("MPpositiongraphic")
local metafunenvironment = metafuncall -- ( P("use") + P("reusable") + P("unique") ) * ("MPgraphic")
- + P("MP") * ( P("code")+ P("page") + P("inclusions") + P("initializations") + P("definitions") + P("extensions") + P("graphic") )
+ + P("MP") * ( P("code")+ P("page") + P("inclusions") + P("initializations") + P("definitions") + P("extensions") + P("graphic") + P("calculation") )
local startmetafun = P("\\start") * metafunenvironment
local stopmetafun = P("\\stop") * metafunenvironment -- todo match start
@@ -432,9 +434,6 @@ local callers = token("embedded", P("\\") * metafuncall) * metafu
lexer.embed_lexer(contextlexer, cldlexer, startluacode, stopluacode)
lexer.embed_lexer(contextlexer, mpslexer, startmetafuncode, stopmetafuncode)
--- Watch the text grabber, after all, we're talking mostly of text (beware,
--- no punctuation here as it can be special. We might go for utf here.
-
contextlexer._rules = {
{ "whitespace", spacing },
{ "preamble", preamble },
@@ -458,11 +457,61 @@ contextlexer._rules = {
{ "rest", rest },
}
-contextlexer._tokenstyles = context.styleset
--- contextlexer._tokenstyles = context.stylesetcopy() -- experiment
+-- Watch the text grabber, after all, we're talking mostly of text (beware,
+-- no punctuation here as it can be special). We might go for utf here.
+
+local web = lexer.loadluafile("scite-context-lexer-web-snippets")
+
+if web then
+
+ lexer.inform("supporting web snippets in tex lexer")
+
+ contextlexer._rules_web = {
+ { "whitespace", spacing },
+ { "text", text }, -- non words
+ { "comment", comment },
+ { "constant", constant },
+ { "callers", callers },
+ { "helper", helper },
+ { "command", command },
+ { "primitive", primitive },
+ { "ifprimitive", ifprimitive },
+ { "reserved", reserved },
+ { "csname", csname },
+ { "grouping", grouping },
+ { "special", special },
+ { "extra", extra },
+ { "invisible", invisible },
+ { "web", web.pattern },
+ { "rest", rest },
+ }
+
+else
+
+ lexer.report("not supporting web snippets in tex lexer")
+
+ contextlexer._rules_web = {
+ { "whitespace", spacing },
+ { "text", text }, -- non words
+ { "comment", comment },
+ { "constant", constant },
+ { "callers", callers },
+ { "helper", helper },
+ { "command", command },
+ { "primitive", primitive },
+ { "ifprimitive", ifprimitive },
+ { "reserved", reserved },
+ { "csname", csname },
+ { "grouping", grouping },
+ { "special", special },
+ { "extra", extra },
+ { "invisible", invisible },
+ { "rest", rest },
+ }
--- contextlexer._tokenstyles[#contextlexer._tokenstyles + 1] = { cldlexer._NAME..'_whitespace', lexer.style_whitespace }
--- contextlexer._tokenstyles[#contextlexer._tokenstyles + 1] = { mpslexer._NAME..'_whitespace', lexer.style_whitespace }
+end
+
+contextlexer._tokenstyles = context.styleset
local environment = {
["\\start"] = 1, ["\\stop"] = -1,
@@ -493,4 +542,6 @@ contextlexer._foldsymbols = { -- these need to be style references
["grouping"] = group,
}
+-- context.inspect(contextlexer)
+
return contextlexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-txt.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-txt.lua
index fe062fb94e5..43eec2c3522 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-txt.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-txt.lua
@@ -6,22 +6,23 @@ local info = {
license = "see context related readme files",
}
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
-
-local lexer = lexer
-local token = lexer.token
-local P, S, Cmt, Cp, Ct = lpeg.P, lpeg.S, lpeg.Cmt, lpeg.Cp, lpeg.Ct
+local P, S, Cmt, Cp = lpeg.P, lpeg.S, lpeg.Cmt, lpeg.Cp
local find, match = string.find, string.match
-local textlexer = { _NAME = "txt", _FILENAME = "scite-context-lexer-txt" }
-local whitespace = lexer.WHITESPACE
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+
+local textlexer = lexer.new("txt","scite-context-lexer-txt")
+local whitespace = textlexer.whitespace
-local space = lexer.space
-local any = lexer.any
+local space = patterns.space
+local any = patterns.any
+local wordtoken = patterns.wordtoken
+local wordpattern = patterns.wordpattern
-local wordtoken = context.patterns.wordtoken
-local wordpattern = context.patterns.wordpattern
local checkedword = context.checkedword
local styleofword = context.styleofword
local setwordlist = context.setwordlist
@@ -36,10 +37,10 @@ local validminimum = 3
-- [#!-%] language=uk
-local p_preamble = Cmt(#(S("#!-%") * P(" ")), function(input,i,_) -- todo: utf bomb
+local p_preamble = Cmt((S("#!-%") * P(" ")), function(input,i,_) -- todo: utf bomb no longer #
if i == 1 then -- < 10 then
validwords, validminimum = false, 3
- local s, e, line = find(input,'^[#!%-%%](.+)[\n\r]',i)
+ local s, e, line = find(input,"^[#!%-%%](.+)[\n\r]",i)
if line then
local language = match(line,"language=([a-z]+)")
if language then
@@ -54,7 +55,6 @@ local t_preamble =
token("preamble", p_preamble)
local t_word =
--- Ct( wordpattern / function(s) return styleofword(validwords,validminimum,s) end * Cp() ) -- the function can be inlined
wordpattern / function(s) return styleofword(validwords,validminimum,s) end * Cp() -- the function can be inlined
local t_text =
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web-snippets.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web-snippets.lua
new file mode 100644
index 00000000000..196a545bc0e
--- /dev/null
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web-snippets.lua
@@ -0,0 +1,133 @@
+local info = {
+ version = 1.002,
+ comment = "scintilla lpeg lexer for web snippets",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files",
+}
+
+local P, R, S, C, Cg, Cb, Cs, Cmt, lpegmatch = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpeg.Cb, lpeg.Cs, lpeg.Cmt, lpeg.match
+
+local lexer = require("lexer")
+local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+
+local websnippets = { }
+
+local space = patterns.space -- S(" \n\r\t\f\v")
+local any = patterns.any
+local restofline = patterns.restofline
+local startofline = patterns.startofline
+
+local squote = P("'")
+local dquote = P('"')
+local period = P(".")
+
+local t_whitespace = token(whitespace, space^1)
+local t_spacing = token("default", space^1)
+local t_rest = token("default", any)
+
+-- the web subset
+
+local p_beginofweb = P("@")
+local p_endofweb = P("@>")
+
+-- @, @/ @| @# @+ @; @[ @]
+
+local p_directive_1 = p_beginofweb * S(",/|#+;[]")
+local t_directive_1 = token("label",p_directive_1)
+
+-- @.text @>(monospaced)
+-- @:text @>(macro driven)
+-- @= verbose@>
+-- @! underlined @>
+-- @t text @> (hbox)
+-- @q ignored @>
+
+local p_typeset = p_beginofweb * S(".:=!tq")
+local t_typeset = token("label",p_typeset) * token("warning",(1-p_endofweb)^1) * token("label",p_endofweb)
+
+-- @^index@>
+
+local p_index = p_beginofweb * P("^")
+local t_index = token("label",p_index) * token("function",(1-p_endofweb)^1) * token("label",p_endofweb)
+
+-- @f text renderclass
+
+local p_render = p_beginofweb * S("f")
+local t_render = token("label",p_render) * t_spacing * token("warning",(1-space)^1) * t_spacing * token("label",(1-space)^1)
+
+-- @s idem
+-- @p idem
+-- @& strip (spaces before)
+-- @h
+
+local p_directive_2 = p_beginofweb * S("sp&h")
+local t_directive_2 = token("label",p_directive_2)
+
+-- @< ... @> [=|+=|]
+-- @(foo@>
+
+local p_reference = p_beginofweb * S("<(")
+local t_reference = token("label",p_reference) * token("function",(1-p_endofweb)^1) * token("label",p_endofweb * (P("+=") + P("="))^-1)
+
+-- @'char' (ascii code)
+
+local p_character = p_beginofweb * S("'")
+local t_character = token("label",p_character) * token("reserved",(1-squote)^1) * token("label",squote)
+
+-- @l nonascii
+
+local p_nonascii = p_beginofweb * S("l")
+local t_nonascii = token("label",p_nonascii) * t_spacing * token("reserved",(1-space)^1)
+
+-- @x @y @z changefile
+-- @i webfile
+
+local p_filename = p_beginofweb * S("xyzi")
+local t_filename = token("label",p_filename) * t_spacing * token("reserved",(1-space)^1)
+
+-- @@ escape
+
+local p_escape = p_beginofweb * p_beginofweb
+local t_escape = token("text",p_escape)
+
+-- structure
+
+-- @* title.
+
+-- local p_section = p_beginofweb * P("*")^1
+-- local t_section = token("label",p_section) * t_spacing * token("function",(1-period)^1) * token("label",period)
+
+-- @ explanation
+
+-- local p_explanation = p_beginofweb
+-- local t_explanation = token("label",p_explanation) * t_spacing^1
+
+-- @d macro
+
+-- local p_macro = p_beginofweb * P("d")
+-- local t_macro = token("label",p_macro)
+
+-- @c code
+
+-- local p_code = p_beginofweb * P("c")
+-- local t_code = token("label",p_code)
+
+websnippets.pattern = P (
+ t_typeset
+ + t_index
+ + t_render
+ + t_reference
+ + t_filename
+ + t_directive_1
+ + t_directive_2
+ + t_character
+ + t_nonascii
+ + t_escape
+)
+
+
+return websnippets
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web.lua
index f59a3205dd8..86ae76644d0 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-web.lua
@@ -1,155 +1,67 @@
local info = {
- version = 1.002,
- comment = "scintilla lpeg lexer for w",
+ version = 1.003,
+ comment = "scintilla lpeg lexer for web",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files",
}
--- this will be extended
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
-
-local lexer = lexer
-local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing
-local P, R, S, C, Cg, Cb, Cs, Cmt, lpegmatch = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpeg.Cb, lpeg.Cs, lpeg.Cmt, lpeg.match
-local setmetatable = setmetatable
-
-local weblexer = { _NAME = "web", _FILENAME = "scite-context-lexer-web" }
-local whitespace = lexer.WHITESPACE
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
-local keywords = { -- copied from cpp.lua
- -- c
- 'asm', 'auto', 'break', 'case', 'const', 'continue', 'default', 'do', 'else',
- 'extern', 'false', 'for', 'goto', 'if', 'inline', 'register', 'return',
- 'sizeof', 'static', 'switch', 'true', 'typedef', 'volatile', 'while',
- 'restrict',
- -- hm
- '_Bool', '_Complex', '_Pragma', '_Imaginary',
- -- c++.
- 'catch', 'class', 'const_cast', 'delete', 'dynamic_cast', 'explicit',
- 'export', 'friend', 'mutable', 'namespace', 'new', 'operator', 'private',
- 'protected', 'public', 'signals', 'slots', 'reinterpret_cast',
- 'static_assert', 'static_cast', 'template', 'this', 'throw', 'try', 'typeid',
- 'typename', 'using', 'virtual'
-}
-
-local datatypes = { -- copied from cpp.lua
- 'bool', 'char', 'double', 'enum', 'float', 'int', 'long', 'short', 'signed',
- 'struct', 'union', 'unsigned', 'void'
-}
-
-local macros = { -- copied from cpp.lua
- 'define', 'elif', 'else', 'endif', 'error', 'if', 'ifdef', 'ifndef', 'import',
- 'include', 'line', 'pragma', 'undef', 'using', 'warning'
-}
-
-local space = lexer.space -- S(" \n\r\t\f\v")
-local any = lexer.any
-local patterns = context.patterns
-local restofline = patterns.restofline
-local startofline = patterns.startofline
-
-local squote = P("'")
-local dquote = P('"')
-local escaped = P("\\") * P(1)
-local slashes = P('//')
-local begincomment = P("/*")
-local endcomment = P("*/")
-local percent = P("%")
-
-local spacing = token(whitespace, space^1)
-local rest = token("default", any)
-
-local shortcomment = token("comment", slashes * restofline^0)
-local longcomment = token("comment", begincomment * (1-endcomment)^0 * endcomment^-1)
-local texcomment = token("comment", percent * restofline^0)
-
-local shortstring = token("quote", dquote) -- can be shared
- * token("string", (escaped + (1-dquote))^0)
- * token("quote", dquote)
- + token("quote", squote)
- * token("string", (escaped + (1-squote))^0)
- * token("quote", squote)
+local token = lexer.token
+local exact_match = lexer.exact_match
-local integer = P("-")^-1 * (lexer.hex_num + lexer.dec_num)
-local number = token("number", lexer.float + integer)
+local weblexer = lexer.new("web","scite-context-lexer-web")
+local whitespace = weblexer.whitespace
-local validword = R("AZ","az","__") * R("AZ","az","__","09")^0
+local space = patterns.space -- S(" \n\r\t\f\v")
+local any = patterns.any
+local restofline = patterns.restofline
+local startofline = patterns.startofline
-local identifier = token("default",validword)
+local period = P(".")
+local percent = P("%")
-local operator = token("special", S('+-*/%^!=<>;:{}[]().&|?~'))
+local spacing = token(whitespace, space^1)
+local rest = token("default", any)
------ optionalspace = spacing^0
+local eop = P("@>")
+local eos = eop * P("+")^-1 * P("=")
-local p_keywords = exact_match(keywords )
-local p_datatypes = exact_match(datatypes)
-local p_macros = exact_match(macros)
+-- we can put some of the next in the web-snippets file
+-- is f okay here?
-local keyword = token("keyword", p_keywords)
-local datatype = token("keyword", p_datatypes)
-local identifier = token("default", validword)
+local texcomment = token("comment", percent * restofline^0)
-local macro = token("data", #P('#') * startofline * P('#') * S('\t ')^0 * p_macros)
+local texpart = token("label",P("@")) * #spacing
+ + token("label",P("@") * P("*")^1) * token("function",(1-period)^1) * token("label",period)
+local midpart = token("label",P("@d")) * #spacing
+ + token("label",P("@f")) * #spacing
+local cpppart = token("label",P("@c")) * #spacing
+ + token("label",P("@p")) * #spacing
+ + token("label",P("@") * S("<(")) * token("function",(1-eop)^1) * token("label",eos)
-local beginweb = P("@")
-local endweb = P("@c")
+local anypart = P("@") * ( P("*")^1 + S("dfcp") + space^1 + S("<(") * (1-eop)^1 * eos )
+local limbo = 1 - anypart - percent
-local webcomment = token("comment", #beginweb * startofline * beginweb * (1-endweb)^0 * endweb)
+local texlexer = lexer.load("scite-context-lexer-tex-web")
+local cpplexer = lexer.load("scite-context-lexer-cpp-web")
-local texlexer = lexer.load('scite-context-lexer-tex')
+lexer.embed_lexer(weblexer, texlexer, texpart + limbo, #anypart)
+lexer.embed_lexer(weblexer, cpplexer, cpppart + midpart, #anypart)
-lexer.embed_lexer(weblexer, texlexer, #beginweb * startofline * token("comment",beginweb), token("comment",endweb))
+local texcomment = token("comment", percent * restofline^0)
weblexer._rules = {
- { 'whitespace', spacing },
- { 'keyword', keyword },
- { 'type', datatype },
- { 'identifier', identifier },
- { 'string', shortstring },
- -- { 'webcomment', webcomment },
- { 'texcomment', texcomment },
- { 'longcomment', longcomment },
- { 'shortcomment', shortcomment },
- { 'number', number },
- { 'macro', macro },
- { 'operator', operator },
- { 'rest', rest },
+ { "whitespace", spacing },
+ { "texcomment", texcomment }, -- else issues with first tex section
+ { "rest", rest },
}
weblexer._tokenstyles = context.styleset
-weblexer._foldpattern = P("/*") + P("*/") + S("{}") -- separate entry else interference
-
-weblexer._foldsymbols = {
- _patterns = {
- '[{}]',
- '/%*',
- '%*/',
- },
- -- ["data"] = { -- macro
- -- ['region'] = 1,
- -- ['endregion'] = -1,
- -- ['if'] = 1,
- -- ['ifdef'] = 1,
- -- ['ifndef'] = 1,
- -- ['endif'] = -1,
- -- },
- ["special"] = { -- operator
- ['{'] = 1,
- ['}'] = -1,
- },
- ["comment"] = {
- ['/*'] = 1,
- ['*/'] = -1,
- }
-}
-
--- -- by indentation:
---
-weblexer._foldpatterns = nil
-weblexer._foldsymbols = nil
-
return weblexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua
index 97253e14035..e6276da0d37 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua
@@ -6,23 +6,26 @@ local info = {
license = "see context related readme files",
}
-local lexer = lexer
-local token = lexer.token
local P = lpeg.P
-local xmlcdatalexer = { _NAME = "xml-cdata", _FILENAME = "scite-context-lexer-xml-cdata" }
-local whitespace = lexer.WHITESPACE -- triggers states
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
-local space = lexer.space
+local token = lexer.token
+
+local xmlcdatalexer = lexer.new("xml-cdata","scite-context-lexer-xml-cdata")
+local whitespace = xmlcdatalexer.whitespace
+
+local space = patterns.space
local nospace = 1 - space - P("]]>")
-local p_spaces = token(whitespace, space ^1)
-local p_cdata = token("comment", nospace^1)
+local t_spaces = token(whitespace, space ^1)
+local t_cdata = token("comment", nospace^1)
xmlcdatalexer._rules = {
- { "whitespace", p_spaces },
- { "cdata", p_cdata },
+ { "whitespace", t_spaces },
+ { "cdata", t_cdata },
}
xmlcdatalexer._tokenstyles = context.styleset
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-comment.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-comment.lua
index 104310f9417..b5b3fefe07b 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-comment.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-comment.lua
@@ -6,37 +6,28 @@ local info = {
license = "see context related readme files",
}
-local lexer = lexer
-local token = lexer.token
local P = lpeg.P
-local xmlcommentlexer = { _NAME = "xml-comment", _FILENAME = "scite-context-lexer-xml-comment" }
-local whitespace = lexer.WHITESPACE
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
-local space = lexer.space
-local nospace = 1 - space - P("-->")
+local token = lexer.token
-local p_spaces = token(whitespace, space ^1)
-local p_comment = token("comment", nospace^1)
+local xmlcommentlexer = lexer.new("xml-comment","scite-context-lexer-xml-comment")
+local whitespace = xmlcommentlexer.whitespace
+
+local space = patterns.space
+local nospace = 1 - space - P("-->")
+
+local t_spaces = token(whitespace, space ^1)
+local t_comment = token("comment", nospace^1)
xmlcommentlexer._rules = {
- { "whitespace", p_spaces },
- { "comment", p_comment },
+ { "whitespace", t_spaces },
+ { "comment", t_comment },
}
xmlcommentlexer._tokenstyles = context.styleset
-xmlcommentlexer._foldpattern = P("<!--") + P("-->")
-
-xmlcommentlexer._foldsymbols = {
- _patterns = {
- "<%!%-%-", "%-%->", -- comments
- },
- ["comment"] = {
- ["<!--"] = 1,
- ["-->" ] = -1,
- }
-}
-
return xmlcommentlexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-script.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-script.lua
index fd1aae7f70c..bbb938dc588 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-script.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml-script.lua
@@ -1,28 +1,31 @@
local info = {
version = 1.002,
- comment = "scintilla lpeg lexer for xml cdata",
+ comment = "scintilla lpeg lexer for xml script",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files",
}
-local lexer = lexer
-local token = lexer.token
local P = lpeg.P
-local xmlscriptlexer = { _NAME = "xml-script", _FILENAME = "scite-context-lexer-xml-script" }
-local whitespace = lexer.WHITESPACE -- triggers states
-local context = lexer.context
+local lexer = require("lexer")
+local context = lexer.context
+local patterns = context.patterns
-local space = lexer.space
-local nospace = 1 - space - (P("</") * P("script") + P("SCRIPT")) * P(">")
+local token = lexer.token
-local p_spaces = token(whitespace, space ^1)
-local p_cdata = token("default", nospace^1)
+local xmlscriptlexer = lexer.new("xml-script","scite-context-lexer-xml-script")
+local whitespace = xmlscriptlexer.whitespace
+
+local space = patterns.space
+local nospace = 1 - space - (P("</") * P("script") + P("SCRIPT")) * P(">")
+
+local t_spaces = token(whitespace, space ^1)
+local t_script = token("default", nospace^1)
xmlscriptlexer._rules = {
- { "whitespace", p_spaces },
- { "script", p_cdata },
+ { "whitespace", t_spaces },
+ { "script", t_script },
}
xmlscriptlexer._tokenstyles = context.styleset
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml.lua
index 241e2259123..77c89b1d677 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer-xml.lua
@@ -12,26 +12,28 @@ local info = {
-- todo: parse entities in attributes
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
-
-local lexer = lexer
local global, string, table, lpeg = _G, string, table, lpeg
-local token, exact_match = lexer.token, lexer.exact_match
-local P, R, S, V, C, Cmt, Ct, Cp = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt, lpeg.Ct, lpeg.Cp
+local P, R, S, C, Cmt, Cp = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cmt, lpeg.Cp
local type = type
local match, find = string.match, string.find
-local xmllexer = { _NAME = "xml", _FILENAME = "scite-context-lexer-xml" }
-local whitespace = lexer.WHITESPACE -- triggers states
+local lexer = require("lexer")
local context = lexer.context
+local patterns = context.patterns
+
+local token = lexer.token
+local exact_match = lexer.exact_match
-local xmlcommentlexer = lexer.load("scite-context-lexer-xml-comment") -- indirect (some issue with the lexer framework)
-local xmlcdatalexer = lexer.load("scite-context-lexer-xml-cdata") -- indirect (some issue with the lexer framework)
-local xmlscriptlexer = lexer.load("scite-context-lexer-xml-script") -- indirect (some issue with the lexer framework)
-local lualexer = lexer.load("scite-context-lexer-lua") --
+local xmllexer = lexer.new("xml","scite-context-lexer-xml")
+local whitespace = xmllexer.whitespace
-local space = lexer.space -- S(" \t\n\r\v\f")
-local any = lexer.any -- P(1)
+local xmlcommentlexer = lexer.load("scite-context-lexer-xml-comment")
+local xmlcdatalexer = lexer.load("scite-context-lexer-xml-cdata")
+local xmlscriptlexer = lexer.load("scite-context-lexer-xml-script")
+local lualexer = lexer.load("scite-context-lexer-lua")
+
+local space = patterns.space
+local any = patterns.any
local dquote = P('"')
local squote = P("'")
@@ -40,7 +42,7 @@ local semicolon = P(";")
local equal = P("=")
local ampersand = P("&")
-local name = (R("az","AZ","09") + S('_-.'))^1
+local name = (R("az","AZ","09") + S("_-."))^1
local openbegin = P("<")
local openend = P("</")
local closebegin = P("/>") + P(">")
@@ -84,12 +86,12 @@ local validminimum = 3
--
-- <?context-directive editor language us ?>
-local p_preamble = Cmt(#P("<?xml "), function(input,i,_) -- todo: utf bomb
+local t_preamble = Cmt(P("<?xml "), function(input,i,_) -- todo: utf bomb, no longer #
if i < 200 then
validwords, validminimum = false, 3
local language = match(input,"^<%?xml[^>]*%?>%s*<%?context%-directive%s+editor%s+language%s+(..)%s+%?>")
-- if not language then
- -- language = match(input,'^<%?xml[^>]*language=[\"\'](..)[\"\'][^>]*%?>',i)
+ -- language = match(input,"^<%?xml[^>]*language=[\"\'](..)[\"\'][^>]*%?>",i)
-- end
if language then
validwords, validminimum = setwordlist(language)
@@ -98,24 +100,23 @@ local p_preamble = Cmt(#P("<?xml "), function(input,i,_) -- todo: utf bomb
return false
end)
-local p_word =
+local t_word =
-- Ct( iwordpattern / function(s) return styleofword(validwords,validminimum,s) end * Cp() ) -- the function can be inlined
iwordpattern / function(s) return styleofword(validwords,validminimum,s) end * Cp() -- the function can be inlined
-local p_rest =
+local t_rest =
token("default", any)
-local p_text =
+local t_text =
token("default", (1-S("<>&")-space)^1)
-local p_spacing =
+local t_spacing =
token(whitespace, space^1)
--- token("whitespace", space^1)
-local p_optionalwhitespace =
- p_spacing^0
+local t_optionalwhitespace =
+ token("default", space^1)^0
-local p_localspacing =
+local t_localspacing =
token("default", space^1)
-- Because we want a differently colored open and close we need an embedded lexer (whitespace
@@ -123,22 +124,22 @@ local p_localspacing =
-- Even using different style keys is not robust as they can be shared. I'll fix the main
-- lexer code.
-local p_sstring =
+local t_sstring =
token("quote",dquote)
* token("string",(1-dquote)^0) -- different from context
* token("quote",dquote)
-local p_dstring =
+local t_dstring =
token("quote",squote)
* token("string",(1-squote)^0) -- different from context
* token("quote",squote)
--- local p_comment =
+-- local t_comment =
-- token("command",opencomment)
-- * token("comment",(1-closecomment)^0) -- different from context
-- * token("command",closecomment)
--- local p_cdata =
+-- local t_cdata =
-- token("command",opencdata)
-- * token("comment",(1-closecdata)^0) -- different from context
-- * token("command",closecdata)
@@ -156,74 +157,74 @@ local p_dstring =
-- <!ENTITY xxxx PUBLIC "yyyy" >
-- <!ENTITY xxxx "yyyy" >
-local p_docstr = p_dstring + p_sstring
+local t_docstr = t_dstring + t_sstring
-local p_docent = token("command",P("<!ENTITY"))
- * p_optionalwhitespace
+local t_docent = token("command",P("<!ENTITY"))
+ * t_optionalwhitespace
* token("keyword",name)
- * p_optionalwhitespace
+ * t_optionalwhitespace
* (
(
token("constant",P("SYSTEM"))
- * p_optionalwhitespace
- * p_docstr
- * p_optionalwhitespace
+ * t_optionalwhitespace
+ * t_docstr
+ * t_optionalwhitespace
* token("constant",P("NDATA"))
- * p_optionalwhitespace
+ * t_optionalwhitespace
* token("keyword",name)
) + (
token("constant",P("PUBLIC"))
- * p_optionalwhitespace
- * p_docstr
+ * t_optionalwhitespace
+ * t_docstr
) + (
- p_docstr
+ t_docstr
)
)
- * p_optionalwhitespace
+ * t_optionalwhitespace
* token("command",P(">"))
-local p_docele = token("command",P("<!ELEMENT"))
- * p_optionalwhitespace
+local t_docele = token("command",P("<!ELEMENT"))
+ * t_optionalwhitespace
* token("keyword",name)
- * p_optionalwhitespace
+ * t_optionalwhitespace
* token("command",P("("))
* (
- p_spacing
+ t_localspacing
+ token("constant",P("#CDATA") + P("#PCDATA") + P("ANY"))
+ token("text",P(","))
+ token("comment",(1-S(",)"))^1)
)^1
* token("command",P(")"))
- * p_optionalwhitespace
+ * t_optionalwhitespace
* token("command",P(">"))
-local p_docset = token("command",P("["))
- * p_optionalwhitespace
- * ((p_optionalwhitespace * (p_docent + p_docele))^1 + token("comment",(1-P("]"))^0))
- * p_optionalwhitespace
+local t_docset = token("command",P("["))
+ * t_optionalwhitespace
+ * ((t_optionalwhitespace * (t_docent + t_docele))^1 + token("comment",(1-P("]"))^0))
+ * t_optionalwhitespace
* token("command",P("]"))
-local p_doctype = token("command",P("<!DOCTYPE"))
- * p_optionalwhitespace
+local t_doctype = token("command",P("<!DOCTYPE"))
+ * t_optionalwhitespace
* token("keyword",name)
- * p_optionalwhitespace
+ * t_optionalwhitespace
* (
(
token("constant",P("PUBLIC"))
- * p_optionalwhitespace
- * p_docstr
- * p_optionalwhitespace
- * p_docstr
- * p_optionalwhitespace
+ * t_optionalwhitespace
+ * t_docstr
+ * t_optionalwhitespace
+ * t_docstr
+ * t_optionalwhitespace
) + (
token("constant",P("SYSTEM"))
- * p_optionalwhitespace
- * p_docstr
- * p_optionalwhitespace
+ * t_optionalwhitespace
+ * t_docstr
+ * t_optionalwhitespace
)
)^-1
- * p_docset^-1
- * p_optionalwhitespace
+ * t_docset^-1
+ * t_optionalwhitespace
* token("command",P(">"))
lexer.embed_lexer(xmllexer, lualexer, token("command", openlua), token("command", closelua))
@@ -231,7 +232,7 @@ lexer.embed_lexer(xmllexer, xmlcommentlexer, token("command", opencomment), toke
lexer.embed_lexer(xmllexer, xmlcdatalexer, token("command", opencdata), token("command", closecdata))
lexer.embed_lexer(xmllexer, xmlscriptlexer, token("command", openscript), token("command", closescript))
--- local p_name =
+-- local t_name =
-- token("plain",name)
-- * (
-- token("default",colon)
@@ -239,11 +240,11 @@ lexer.embed_lexer(xmllexer, xmlscriptlexer, token("command", openscript), toke
-- )
-- + token("keyword",name)
-local p_name = -- more robust
+local t_name = -- more robust
token("plain",name * colon)^-1
* token("keyword",name)
--- local p_key =
+-- local t_key =
-- token("plain",name)
-- * (
-- token("default",colon)
@@ -251,81 +252,82 @@ local p_name = -- more robust
-- )
-- + token("constant",name)
-local p_key =
+local t_key =
token("plain",name * colon)^-1
* token("constant",name)
-local p_attributes = (
- p_optionalwhitespace
- * p_key
- * p_optionalwhitespace
+local t_attributes = (
+ t_optionalwhitespace
+ * t_key
+ * t_optionalwhitespace
* token("plain",equal)
- * p_optionalwhitespace
- * (p_dstring + p_sstring)
- * p_optionalwhitespace
+ * t_optionalwhitespace
+ * (t_dstring + t_sstring)
+ * t_optionalwhitespace
)^0
-local p_open =
+local t_open =
token("keyword",openbegin)
* (
- p_name
- * p_optionalwhitespace
- * p_attributes
+ t_name
+ * t_optionalwhitespace
+ * t_attributes
* token("keyword",closebegin)
+
token("error",(1-closebegin)^1)
)
-local p_close =
+local t_close =
token("keyword",openend)
* (
- p_name
- * p_optionalwhitespace
+ t_name
+ * t_optionalwhitespace
* token("keyword",closeend)
+
token("error",(1-closeend)^1)
)
-local p_entity =
+local t_entity =
token("constant",entity)
-local p_instruction =
+local t_instruction =
token("command",openinstruction * P("xml"))
- * p_optionalwhitespace
- * p_attributes
- * p_optionalwhitespace
+ * t_optionalwhitespace
+ * t_attributes
+ * t_optionalwhitespace
* token("command",closeinstruction)
+ token("command",openinstruction * name)
* token("default",(1-closeinstruction)^1)
* token("command",closeinstruction)
-local p_invisible =
+local t_invisible =
token("invisible",invisibles^1)
--- local p_preamble =
--- token('preamble', p_preamble )
+-- local t_preamble =
+-- token("preamble", t_preamble )
xmllexer._rules = {
- { "whitespace", p_spacing },
- { "preamble", p_preamble },
- { "word", p_word },
- -- { "text", p_text },
- -- { "comment", p_comment },
- -- { "cdata", p_cdata },
- { "doctype", p_doctype },
- { "instruction", p_instruction },
- { "close", p_close },
- { "open", p_open },
- { "entity", p_entity },
- { "invisible", p_invisible },
- { "rest", p_rest },
+ { "whitespace", t_spacing },
+ { "preamble", t_preamble },
+ { "word", t_word },
+ -- { "text", t_text },
+ -- { "comment", t_comment },
+ -- { "cdata", t_cdata },
+ { "doctype", t_doctype },
+ { "instruction", t_instruction },
+ { "close", t_close },
+ { "open", t_open },
+ { "entity", t_entity },
+ { "invisible", t_invisible },
+ { "rest", t_rest },
}
xmllexer._tokenstyles = context.styleset
xmllexer._foldpattern = P("</") + P("<") + P("/>") -- separate entry else interference
++ P("<!--") + P("-->")
-xmllexer._foldsymbols = { -- somehow doesn't work yet
+xmllexer._foldsymbols = {
_patterns = {
"</",
"/>",
@@ -336,6 +338,13 @@ xmllexer._foldsymbols = { -- somehow doesn't work yet
["/>"] = -1,
["<"] = 1,
},
+ ["command"] = {
+ ["</"] = -1,
+ ["/>"] = -1,
+ ["<!--"] = 1,
+ ["-->"] = -1,
+ ["<"] = 1,
+ },
}
return xmllexer
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua
index 816d9583bca..6335af91151 100644
--- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua
@@ -1,5 +1,5 @@
local info = {
- version = 1.324,
+ version = 1.400,
comment = "basics for scintilla lpeg lexer for context/metafun",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
@@ -8,162 +8,667 @@ local info = {
}
--- todo: move all code here
--- todo: explore adapted dll ... properties + init
--- todo: play with hotspot and other properties
+-- todo: hook into context resolver etc
+-- todo: only old api in lexers, rest in context subnamespace
+-- todo: make sure we can run in one state .. copies or shared?
+-- todo: auto-nesting
+
+local log = false
+local trace = false
+local detail = false
+local show = false -- nice for tracing (also for later)
+local collapse = false -- can save some 15% (maybe easier on scintilla)
+local inspect = false -- can save some 15% (maybe easier on scintilla)
+
+-- local log = true
+-- local trace = true
--- The fold and lex functions are copied and patched from original code by Mitchell (see
--- lexer.lua). All errors are mine. The ability to use lpeg is a real nice adition and a
--- brilliant move. The code is a byproduct of the (mainly Lua based) textadept (still a
--- rapidly moving target) that unfortunately misses a realtime output pane. On the other
--- hand, SciTE is somewhat crippled by the fact that we cannot pop in our own (language
--- dependent) lexer into the output pane (somehow the errorlist lexer is hard coded into
--- the editor). Hopefully that will change some day.
---
--- Starting with SciTE version 3.20 there is an issue with coloring. As we still lack
--- a connection with scite itself (properties as well as printing to the log pane) we
--- cannot trace this (on windows). As far as I can see, there are no fundamental
--- changes in lexer.lua or LexLPeg.cxx so it must be in scintilla itself. So for the
--- moment I stick to 3.10. Indicators are: no lexing of 'next' and 'goto <label>' in the
--- Lua lexer and no brace highlighting either. Interesting is that it does work ok in
--- the cld lexer (so the Lua code is okay). Also the fact that char-def.lua lexes fast
--- is a signal that the lexer quits somewhere halfway.
---
--- After checking 3.24 and adapting to the new lexer tables things are okay again. So,
--- this version assumes 3.24 or higher. In 3.24 we have a different token result, i.e. no
--- longer a { tag, pattern } but just two return values. I didn't check other changes but
--- will do that when I run into issues. I had optimized these small tables by hashing which
--- was more efficient but this is no longer needed.
---
--- In 3.3.1 another major change took place: some helper constants (maybe they're no
--- longer constants) and functions were moved into the lexer modules namespace but the
--- functions are assigned to the Lua module afterward so we cannot alias them beforehand.
---
--- I've considered making a whole copy and patch the other functions too as we need
--- an extra nesting model. However, I don't want to maintain too much. An unfortunate
--- change in 3.03 is that no longer a script can be specified. This means that instead
--- of loading the extensions via the properties file, we now need to load them in our
--- own lexers, unless of course we replace lexer.lua completely (which adds another
--- installation issue).
---
--- Another change has been that _LEXERHOME is no longer available. It looks like more and
--- more functionality gets dropped so maybe at some point we need to ship our own dll/so
--- files. For instance, I'd like to have access to the current filename and other scite
--- properties. For instance, we could cache some info with each file, if only we had
--- knowledge of what file we're dealing with.
---
--- For huge files folding can be pretty slow and I do have some large ones that I keep
--- open all the time. Loading is normally no ussue, unless one has remembered the status
--- and the cursor is at the last line of a 200K line file. Optimizing the fold function
--- brought down loading of char-def.lua from 14 sec => 8 sec. Replacing the word_match
--- function and optimizing the lex function gained another 2+ seconds. A 6 second load
--- is quite ok for me. The changed lexer table structure (no subtables) brings loading
--- down to a few seconds.
---
--- When the lexer path is copied to the textadept lexer path, and the theme definition to
--- theme path (as lexer.lua), the lexer works there as well. When I have time and motive
--- I will make a proper setup file to tune the look and feel a bit and associate suffixes
--- with the context lexer. The textadept editor has a nice style tracing option but lacks
--- the tabs for selecting files that scite has. It also has no integrated run that pipes
--- to the log pane (I wonder if it could borrow code from the console2 project). Interesting
--- is that the jit version of textadept crashes on lexing large files (and does not feel
--- faster either).
---
--- Function load(lexer_name) starts with _M.WHITESPACE = lexer_name..'_whitespace' which
--- means that we need to have it frozen at the moment we load another lexer. Because spacing
--- is used to revert to a parent lexer we need to make sure that we load children as late
--- as possible in order not to get the wrong whitespace trigger. This took me quite a while
--- to figure out (not being that familiar with the internals). The lex and fold functions
--- have been optimized. It is a pitty that there is no proper print available. Another thing
--- needed is a default style in ourown theme style definition, as otherwise we get wrong
--- nested lexers, especially if they are larger than a view. This is the hardest part of
+-- GET GOING
+--
+-- You need to copy this file over lexer.lua. In principle other lexers could
+-- work too but not now. Maybe some day. All patterns will move into the patterns
+-- name space. I might do the same with styles. If you run an older version of
+-- SciTE you can take one of the archives. Pre 3.41 versions can just be copied
+-- to the right path, as there we still use part of the normal lexer.
+--
+-- REMARK
+--
+-- We started using lpeg lexing as soon as it came available. Because we had
+-- rather demanding files an dalso wanted to use nested lexers, we ended up with
+-- our own variant (more robust and faster). As a consequence successive versions
+-- had to be adapted to changes in the (still unstable) api. In addition to
+-- lexing we also have spell checking and such.
+--
+-- STATUS
+--
+-- todo: maybe use a special stripped version of the dll (stable api)
+-- todo: play with hotspot and other properties
+-- wish: access to all scite properties and in fact integrate in scite
+-- todo: add proper tracing and so .. not too hard as we can run on mtxrun
+-- todo: get rid of these lexers.STYLE_XX and lexers.XX (hide such details)
+--
+-- HISTORY
+--
+-- The fold and lex functions are copied and patched from original code by Mitchell
+-- (see lexer.lua). All errors are mine. The ability to use lpeg is a real nice
+-- adition and a brilliant move. The code is a byproduct of the (mainly Lua based)
+-- textadept (still a rapidly moving target) that unfortunately misses a realtime
+-- output pane. On the other hand, SciTE is somewhat crippled by the fact that we
+-- cannot pop in our own (language dependent) lexer into the output pane (somehow
+-- the errorlist lexer is hard coded into the editor). Hopefully that will change
+-- some day.
+--
+-- Starting with SciTE version 3.20 there is an issue with coloring. As we still
+-- lack a connection with SciTE itself (properties as well as printing to the log
+-- pane) and we cannot trace this (on windows). As far as I can see, there are no
+-- fundamental changes in lexer.lua or LexLPeg.cxx so it must be in Scintilla
+-- itself. So for the moment I stick to 3.10. Indicators are: no lexing of 'next'
+-- and 'goto <label>' in the Lua lexer and no brace highlighting either. Interesting
+-- is that it does work ok in the cld lexer (so the Lua code is okay). Also the fact
+-- that char-def.lua lexes fast is a signal that the lexer quits somewhere halfway.
+-- Maybe there are some hard coded limitations on the amount of styles and/or length
+-- if names.
+--
+-- After checking 3.24 and adapting to the new lexer tables things are okay again.
+-- So, this version assumes 3.24 or higher. In 3.24 we have a different token
+-- result, i.e. no longer a { tag, pattern } but just two return values. I didn't
+-- check other changes but will do that when I run into issues. I had optimized
+-- these small tables by hashing which was more efficient but this is no longer
+-- needed. For the moment we keep some of that code around as I don't know what
+-- happens in future versions.
+--
+-- In 3.31 another major change took place: some helper constants (maybe they're no
+-- longer constants) and functions were moved into the lexer modules namespace but
+-- the functions are assigned to the Lua module afterward so we cannot alias them
+-- beforehand. We're probably getting close to a stable interface now. I've
+-- considered making a whole copy and patch the other functions too as we need an
+-- extra nesting model. However, I don't want to maintain too much. An unfortunate
+-- change in 3.03 is that no longer a script can be specified. This means that
+-- instead of loading the extensions via the properties file, we now need to load
+-- them in our own lexers, unless of course we replace lexer.lua completely (which
+-- adds another installation issue).
+--
+-- Another change has been that _LEXERHOME is no longer available. It looks like
+-- more and more functionality gets dropped so maybe at some point we need to ship
+-- our own dll/so files. For instance, I'd like to have access to the current
+-- filename and other scite properties. For instance, we could cache some info with
+-- each file, if only we had knowledge of what file we're dealing with.
+--
+-- For huge files folding can be pretty slow and I do have some large ones that I
+-- keep open all the time. Loading is normally no ussue, unless one has remembered
+-- the status and the cursor is at the last line of a 200K line file. Optimizing the
+-- fold function brought down loading of char-def.lua from 14 sec => 8 sec.
+-- Replacing the word_match function and optimizing the lex function gained another
+-- 2+ seconds. A 6 second load is quite ok for me. The changed lexer table structure
+-- (no subtables) brings loading down to a few seconds.
+--
+-- When the lexer path is copied to the textadept lexer path, and the theme
+-- definition to theme path (as lexer.lua), the lexer works there as well. When I
+-- have time and motive I will make a proper setup file to tune the look and feel a
+-- bit and associate suffixes with the context lexer. The textadept editor has a
+-- nice style tracing option but lacks the tabs for selecting files that scite has.
+-- It also has no integrated run that pipes to the log pane. Interesting is that the
+-- jit version of textadept crashes on lexing large files (and does not feel faster
+-- either; maybe a side effect of known limitations).
+--
+-- Function load(lexer_name) starts with _lexers.WHITESPACE = lexer_name ..
+-- '_whitespace' which means that we need to have it frozen at the moment we load
+-- another lexer. Because spacing is used to revert to a parent lexer we need to
+-- make sure that we load children as late as possible in order not to get the wrong
+-- whitespace trigger. This took me quite a while to figure out (not being that
+-- familiar with the internals). The lex and fold functions have been optimized. It
+-- is a pitty that there is no proper print available. Another thing needed is a
+-- default style in our own theme style definition, as otherwise we get wrong nested
+-- lexers, especially if they are larger than a view. This is the hardest part of
-- getting things right.
--
--- Eventually it might be safer to copy the other methods from lexer.lua here as well so
--- that we have no dependencies, apart from the c library (for which at some point the api
--- will be stable I hope).
+-- It's a pitty that there is no scintillua library for the OSX version of scite.
+-- Even better would be to have the scintillua library as integral part of scite as
+-- that way I could use OSX alongside windows and linux (depending on needs). Also
+-- nice would be to have a proper interface to scite then because currently the
+-- lexer is rather isolated and the lua version does not provide all standard
+-- libraries. It would also be good to have lpeg support in the regular scite lua
+-- extension (currently you need to pick it up from someplace else).
+--
+-- With 3.41 the interface changed again so it gets time to look into the C++ code
+-- and consider compiling and patching myself. Loading is more complicated not as
+-- the lexer gets loaded automatically so we have little control over extending the
+-- code now. After a few days trying all kind of solutions I decided to follow a
+-- different approach: drop in a complete replacement. This of course means that I
+-- need to keep track of even more changes (which for sure will happen) but at least
+-- I get rid of interferences. The api (lexing and configuration) is simply too
+-- unstable across versions. Maybe in a few years things have stabelized. (Or maybe
+-- it's not really expected that one writes lexers at all.) A side effect is that I
+-- now no longer will use shipped lexers but just the built-in ones. Not that it
+-- matters much as the context lexers cover what I need (and I can always write
+-- more).
+--
+-- In fact, the transition to 3.41 was triggered by an unfateful update of Ubuntu
+-- which left me with an incompatible SciTE and lexer library and updating was not
+-- possible due to the lack of 64 bit libraries. We'll see what the future brings.
+--
+-- Promissing is that the library now can use another Lua instance so maybe some day
+-- it will get properly in SciTE and we can use more clever scripting.
+--
+-- In some lexers we use embedded ones even if we could do it directly, The reason is
+-- that when the end token is edited (e.g. -->), backtracking to the space before the
+-- begin token (e.g. <!--) results in applying the surrounding whitespace which in
+-- turn means that when the end token is edited right, backtracking doesn't go back.
+-- One solution (in the dll) would be to backtrack several space categories. After all,
+-- lexing is quite fast (applying the result is much slower).
--
--- It's a pitty that there is no scintillua library for the OSX version of scite. Even
--- better would be to have the scintillua library as integral part of scite as that way I
--- could use OSX alongside windows and linux (depending on needs). Also nice would be to
--- have a proper interface to scite then because currently the lexer is rather isolated and the
--- lua version does not provide all standard libraries. It would also be good to have lpeg
--- support in the regular scite lua extension (currently you need to pick it up from someplace
--- else).
+-- For some reason the first blob of text tends to go wrong (pdf and web). It would be
+-- nice to have 'whole doc' initial lexing. Quite fishy as it makes it impossible to
+-- lex the first part well (for already opened documents) because only a partial
+-- text is passed.
+--
+-- So, maybe I should just write this from scratch (assuming more generic usage)
+-- because after all, the dll expects just tables, based on a string. I can then also
+-- do some more aggressive resource sharing (needed when used generic).
+--
+-- I think that nested lexers are still bugged (esp over longer ranges). It never was
+-- robust or maybe it's simply not meant for too complex cases. The 3.24 version was
+-- probably the best so far. The fact that styles bleed between lexers even if their
+-- states are isolated is an issue. Another issus is that zero characters in the
+-- text passed to the lexer can mess things up (pdf files have them in streams).
+--
+-- For more complex 'languages', like web or xml, we need to make sure that we use
+-- e.g. 'default' for spacing that makes up some construct. Ok, we then still have a
+-- backtracking issue but less.
+--
+-- TODO
+--
+-- I can make an export to context, but first I'll redo the code that makes the grammar,
+-- as we only seem to need
+--
+-- lexer._TOKENSTYLES : table
+-- lexer._CHILDREN : flag
+-- lexer._EXTRASTYLES : table
+-- lexer._GRAMMAR : flag
+--
+-- lexers.load : function
+-- lexers.lex : function
+--
+-- So, if we drop compatibility with other lex definitions, we can make things simpler.
-local lpeg = require 'lpeg'
+-- TRACING
+--
+-- The advantage is that we now can check more easily with regular Lua. We can also
+-- use wine and print to the console (somehow stdout is intercepted there.) So, I've
+-- added a bit of tracing. Interesting is to notice that each document gets its own
+-- instance which has advantages but also means that when we are spellchecking we
+-- reload the word lists each time. (In the past I assumed a shared instance and took
+-- some precautions.)
+
+local lpeg = require("lpeg")
-local R, P, S, C, V, Cp, Cs, Ct, Cmt, Cc, Cf, Cg, Carg = lpeg.R, lpeg.P, lpeg.S, lpeg.C, lpeg.V, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Carg
-local lpegmatch = lpeg.match
-local find, gmatch, match, lower, upper, gsub = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub
-local concat = table.concat
local global = _G
-local type, next, setmetatable, rawset = type, next, setmetatable, rawset
+local find, gmatch, match, lower, upper, gsub, sub, format = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub, string.sub, string.format
+local concat = table.concat
+local type, next, setmetatable, rawset, tonumber, tostring = type, next, setmetatable, rawset, tonumber, tostring
+local R, P, S, V, C, Cp, Cs, Ct, Cmt, Cc, Cf, Cg, Carg = lpeg.R, lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Carg
+local lpegmatch = lpeg.match
+
+local nesting = 0
+
+local function report(fmt,str,...)
+ if log then
+ if str then
+ fmt = format(fmt,str,...)
+ end
+ print(format("scite lpeg lexer > %s > %s",nesting == 0 and "-" or nesting,fmt))
+ end
+end
--- less confusing as we also use lexer for the current lexer and local _M = lexer is just ugly
+local function inform(...)
+ if log and trace then
+ report(...)
+ end
+end
-local lexers = lexer
+inform("loading context lexer module (global table: %s)",tostring(global))
--- these helpers are set afterwards so we delay their initialization ... there is no need to alias each time again
+if not package.searchpath then
-local get_style_at, get_indent_amount, get_property, get_fold_level, FOLD_BASE, FOLD_HEADER, FOLD_BLANK, initialize
+ -- Unfortunately the io library is only available when we end up
+ -- in this branch of code.
+
+ inform("using adapted function 'package.searchpath' (if used at all)")
+
+ function package.searchpath(name,path)
+ local tried = { }
+ for part in gmatch(path,"[^;]+") do
+ local filename = gsub(part,"%?",name)
+ local f = io.open(filename,"r")
+ if f then
+ inform("file found on path: %s",filename)
+ f:close()
+ return filename
+ end
+ tried[#tried + 1] = format("no file '%s'",filename)
+ end
+ -- added: local path .. for testing
+ local f = io.open(filename,"r")
+ if f then
+ inform("file found on current path: %s",filename)
+ f:close()
+ return filename
+ end
+ --
+ tried[#tried + 1] = format("no file '%s'",filename)
+ return nil, concat(tried,"\n")
+ end
+
+end
+
+local lexers = { }
+local context = { }
+lexers.context = context
+
+local patterns = { }
+context.patterns = patterns -- todo: lexers.patterns
+
+context.report = report
+context.inform = inform
+
+lexers.LEXERPATH = package.path -- can be multiple paths separated by ;
+lexers.LEXERPATH = "./?.lua" -- good enough, will be set anyway (was
+
+if resolvers then
+ -- todo: set LEXERPATH
+ -- todo: set report
+end
+
+local usedlexers = { }
+local parent_lexer = nil
+
+-- The problem with styles is that there is some nasty interaction with scintilla
+-- and each version of lexer dll/so has a different issue. So, from now on we will
+-- just add them here. There is also a limit on some 30 styles. Maybe I should
+-- hash them in order to reuse.
+
+-- todo: work with proper hashes and analyze what styles are really used by a
+-- lexer
+
+local default = {
+ "nothing", "whitespace", "comment", "string", "number", "keyword",
+ "identifier", "operator", "error", "preprocessor", "constant", "variable",
+ "function", "type", "label", "embedded",
+ "quote", "special", "extra", "reserved", "okay", "warning",
+ "command", "internal", "preamble", "grouping", "primitive", "plain",
+ "user",
+ -- not used (yet) .. we cross the 32 boundary so had to patch the initializer, see (1)
+ "char", "class", "data", "definition", "invisible", "regex",
+ "standout", "tag",
+ "text",
+}
+
+local predefined = {
+ "default", "linenumber", "bracelight", "bracebad", "controlchar",
+ "indentguide", "calltip"
+}
+
+-- Bah ... ugly ... nicer would be a proper hash .. we now have properties
+-- as well as STYLE_* and some connection between them ... why .. ok, we
+-- could delay things but who cares. Anyway, at this moment the properties
+-- are still unknown.
+
+local function preparestyles(list)
+ local reverse = { }
+ for i=1,#list do
+ local k = list[i]
+ local K = upper(k)
+ local s = "style." .. k
+ lexers[K] = k -- is this used
+ lexers["STYLE_"..K] = "$(" .. k .. ")"
+ reverse[k] = true
+ end
+ return reverse
+end
+
+local defaultstyles = preparestyles(default)
+local predefinedstyles = preparestyles(predefined)
+
+-- These helpers are set afterwards so we delay their initialization ... there
+-- is no need to alias each time again and this way we can more easily adapt
+-- to updates.
+
+-- These keep changing (values, functions, tables ...) so we nee to check these
+-- with each update. Some of them are set in the loader (the require 'lexer' is
+-- in fact not a real one as the lexer code is loaded in the dll). It's also not
+-- getting more efficient.
+
+-- FOLD_BASE = lexers.FOLD_BASE or SC_FOLDLEVELBASE
+-- FOLD_HEADER = lexers.FOLD_HEADER or SC_FOLDLEVELHEADERFLAG
+-- FOLD_BLANK = lexers.FOLD_BLANK or SC_FOLDLEVELWHITEFLAG
+-- get_style_at = lexers.get_style_at or GetStyleAt
+-- get_indent_amount = lexers.get_indent_amount or GetIndentAmount
+-- get_property = lexers.get_property or GetProperty
+-- get_fold_level = lexers.get_fold_level or GetFoldLevel
+
+-- It needs checking: do we have access to all properties now? I'll clean
+-- this up anyway as I want a simple clean and stable model.
+
+-- This is somewhat messy. The lexer dll provides some virtual fields:
+--
+-- + property
+-- + property_int
+-- + style_at
+-- + fold_level
+-- + indent_amount
+--
+-- but for some reasons not:
+--
+-- + property_expanded
+--
+-- As a consequence we need to define it here because otherwise the
+-- lexer will crash. The fuzzy thing is that we don't have to define
+-- the property and property_int tables but we do have to define the
+-- expanded beforehand. The folding properties are no longer interfaced
+-- so the interface to scite is now rather weak (only a few hard coded
+-- properties).
+
+local FOLD_BASE = 0
+local FOLD_HEADER = 0
+local FOLD_BLANK = 0
+
+local style_at = { }
+local indent_amount = { }
+local fold_level = { }
+
+local function check_main_properties()
+ if not lexers.property then
+ lexers.property = { }
+ end
+ if not lexers.property_int then
+ lexers.property_int = setmetatable({ }, {
+ __index = function(t,k)
+ -- why the tostring .. it relies on lua casting to a number when
+ -- doing a comparison
+ return tonumber(lexers.property[k]) or 0 -- tostring removed
+ end,
+ __newindex = function(t,k,v)
+ report("properties are read-only, '%s' is not changed",k)
+ end,
+ })
+ end
+end
+
+lexers.property_expanded = setmetatable({ }, {
+ __index = function(t,k)
+ -- better be safe for future changes .. what if at some point this is
+ -- made consistent in the dll ... we need to keep an eye on that
+ local property = lexers.property
+ if not property then
+ check_main_properties()
+ end
+ --
+ return gsub(property[k],"[$%%]%b()", function(k)
+ return t[sub(k,3,-2)]
+ end)
+ end,
+ __newindex = function(t,k,v)
+ report("properties are read-only, '%s' is not changed",k)
+ end,
+})
-initialize = function()
- FOLD_BASE = lexers.FOLD_BASE or SC_FOLDLEVELBASE
- FOLD_HEADER = lexers.FOLD_HEADER or SC_FOLDLEVELHEADERFLAG
- FOLD_BLANK = lexers.FOLD_BLANK or SC_FOLDLEVELWHITEFLAG
- get_style_at = lexers.get_style_at or GetStyleAt
- get_indent_amount = lexers.get_indent_amount or GetIndentAmount
- get_property = lexers.get_property or GetProperty
- get_fold_level = lexers.get_fold_level or GetFoldLevel
+-- A downward compatible feature but obsolete:
+
+-- local function get_property(tag,default)
+-- return lexers.property_int[tag] or lexers.property[tag] or default
+-- end
+
+-- We still want our own properties (as it keeps changing so better play
+-- safe from now on):
+
+local function check_properties(lexer)
+ if lexer.properties then
+ return lexer
+ end
+ check_main_properties()
+ -- we use a proxy
+ local mainproperties = lexers.property
+ local properties = { }
+ local expanded = setmetatable({ }, {
+ __index = function(t,k)
+ return gsub(properties[k] or mainproperties[k],"[$%%]%b()", function(k)
+ return t[sub(k,3,-2)]
+ end)
+ end,
+ })
+ lexer.properties = setmetatable(properties, {
+ __index = mainproperties,
+ __call = function(t,k,default) -- expands
+ local v = expanded[k]
+ local t = type(default)
+ if t == "number" then
+ return tonumber(v) or default
+ elseif t == "boolean" then
+ return v == nil and default or v
+ else
+ return v or default
+ end
+ end,
+ })
+ return lexer
+end
+
+-- do
+-- lexers.property = { foo = 123, red = "R" }
+-- local a = check_properties({}) print("a.foo",a.properties.foo)
+-- a.properties.foo = "bar" print("a.foo",a.properties.foo)
+-- a.properties.foo = "bar:$(red)" print("a.foo",a.properties.foo) print("a.foo",a.properties("foo"))
+-- end
+
+local function set(value,default)
+ if value == 0 or value == false or value == "0" then
+ return false
+ elseif value == 1 or value == true or value == "1" then
+ return true
+ else
+ return default
+ end
+end
+
+local function check_context_properties()
+ local property = lexers.property -- let's hope that this stays
+ log = set(property["lexer.context.log"], log)
+ trace = set(property["lexer.context.trace"], trace)
+ detail = set(property["lexer.context.detail"], detail)
+ show = set(property["lexer.context.show"], show)
+ collapse = set(property["lexer.context.collapse"],collapse)
+ inspect = set(property["lexer.context.inspect"], inspect)
+end
+
+function context.registerproperties(p) -- global
+ check_main_properties()
+ local property = lexers.property -- let's hope that this stays
+ for k, v in next, p do
+ property[k] = v
+ end
+ check_context_properties()
+end
+
+context.properties = setmetatable({ }, {
+ __index = lexers.property,
+ __newindex = function(t,k,v)
+ check_main_properties()
+ lexers.property[k] = v
+ check_context_properties()
+ end,
+})
+
+-- We want locals to we set them delayed. Once.
+
+local function initialize()
+ FOLD_BASE = lexers.FOLD_BASE
+ FOLD_HEADER = lexers.FOLD_HEADER
+ FOLD_BLANK = lexers.FOLD_BLANK
+ --
+ style_at = lexers.style_at -- table
+ indent_amount = lexers.indent_amount -- table
+ fold_level = lexers.fold_level -- table
+ --
+ check_main_properties()
--
initialize = nil
end
--- we create our own extra namespace for extensions and helpers
+-- Style handler.
+--
+-- The property table will be set later (after loading) by the library. The
+-- styleset is not needed any more as we predefine all styles as defaults
+-- anyway (too bug sensitive otherwise).
+
+local function toproperty(specification)
+ local serialized = { }
+ for key, value in next, specification do
+ if value == true then
+ serialized[#serialized+1] = key
+ elseif type(value) == "table" then
+ serialized[#serialized+1] = key .. ":" .. "#" .. value[1] .. value[2] .. value[3]
+ else
+ serialized[#serialized+1] = key .. ":" .. tostring(value)
+ end
+ end
+ return concat(serialized,",")
+end
-lexers.context = lexers.context or { }
-local context = lexers.context
+local function tostyles(styles)
+ local styleset = { }
+ local property = lexers.property or { }
+ for k, v in next, styles do
+ v = toproperty(v)
+ styleset[k] = v
+ property["style."..k] = v
+ end
+ return styleset
+end
-context.patterns = context.patterns or { }
-local patterns = context.patterns
+context.toproperty = toproperty
+context.tostyles = tostyles
-lexers._CONTEXTEXTENSIONS = true
+local function sortedkeys(hash)
+ local t, n = { }, 0
+ for k, v in next, hash do
+ t[#t+1] = k
+ local l = #tostring(k)
+ if l > n then
+ n = l
+ end
+ end
+ table.sort(t)
+ return t, n
+end
+
+-- If we had one instance/state of Lua as well as all regular libraries
+-- preloaded we could use the context base libraries. So, let's go poor-
+-- mans solution now.
+
+function context.registerstyles(styles)
+ local styleset = tostyles(styles)
+ context.styles = styles
+ context.styleset = styleset
+ if trace then
+ if detail then
+ local t, n = sortedkeys(styleset)
+ local template = " %-" .. n .. "s : %s"
+ report("initializing styleset:")
+ for i=1,#t do
+ local k = t[i]
+ report(template,k,styleset[k])
+ end
+ else
+ report("initializing styleset")
+ end
+ end
+end
+
+-- Some spell checking related stuff. Unfortunately we cannot use a path set
+-- by property. This will get a hook for resolvers.
local locations = {
- -- lexers.context.path,
- "data", -- optional data directory
- "..", -- regular scite directory
+ "context/lexers", -- context lexers
+ "context/lexers/data", -- context lexers
+ "../lexers", -- original lexers
+ "../lexers/data", -- original lexers
+ ".", -- whatever
+ "./data", -- whatever
}
local function collect(name)
--- local definitions = loadfile(name .. ".luc") or loadfile(name .. ".lua")
- local okay, definitions = pcall(function () return require(name) end)
- if okay then
- if type(definitions) == "function" then
- definitions = definitions()
+ local root = gsub(lexers.LEXERPATH or ".","/.-lua$","") .. "/" -- this is a horrible hack
+ -- report("module '%s' locating '%s'",tostring(lexers),name)
+ for i=1,#locations do
+ local fullname = root .. locations[i] .. "/" .. name .. ".lua" -- so we can also check for .luc
+ if trace then
+ report("attempt to locate '%s'",fullname)
end
- if type(definitions) == "table" then
- return definitions
+ local okay, result = pcall(function () return dofile(fullname) end)
+ if okay then
+ return result, fullname
+ end
+ end
+end
+
+function context.loadluafile(name)
+ local data, fullname = collect(name)
+ if data then
+ if trace then
+ report("lua file '%s' has been loaded",fullname)
end
+ return data, fullname
end
+ report("unable to load lua file '%s'",name)
end
+-- in fact we could share more as we probably process the data but then we need
+-- to have a more advanced helper
+
+local cache = { }
+
function context.loaddefinitions(name)
- for i=1,#locations do
- local data = collect(locations[i] .. "/" .. name)
- if data then
- return data
+ local data = cache[name]
+ if data then
+ if trace then
+ report("reusing definitions '%s'",name)
end
+ return data
+ elseif trace and data == false then
+ report("definitions '%s' were not found",name)
end
+ local data, fullname = collect(name)
+ if not data then
+ report("unable to load definition file '%s'",name)
+ data = false
+ elseif trace then
+ report("definition file '%s' has been loaded",fullname)
+ if detail then
+ local t, n = sortedkeys(data)
+ local template = " %-" .. n .. "s : %s"
+ for i=1,#t do
+ local k = t[i]
+ local v = data[k]
+ if type(v) ~= "table" then
+ report(template,k,tostring(v))
+ elseif #v > 0 then
+ report(template,k,#v)
+ else
+ -- no need to show hash
+ end
+ end
+ end
+ end
+ cache[name] = data
+ return type(data) == "table" and data
end
function context.word_match(words,word_chars,case_insensitive)
- local chars = '%w_' -- maybe just "" when word_chars
+ local chars = "%w_" -- maybe just "" when word_chars
if word_chars then
- chars = '^([' .. chars .. gsub(word_chars,'([%^%]%-])', '%%%1') ..']+)'
+ chars = "^([" .. chars .. gsub(word_chars,"([%^%]%-])", "%%%1") .."]+)"
else
- chars = '^([' .. chars ..']+)'
+ chars = "^([" .. chars .."]+)"
end
if case_insensitive then
local word_list = { }
@@ -186,39 +691,108 @@ function context.word_match(words,word_chars,case_insensitive)
end
end
-local idtoken = R("az","AZ","\127\255","__")
-local digit = R("09")
-local sign = S("+-")
-local period = P(".")
-local space = S(" \n\r\t\f\v")
-
-patterns.idtoken = idtoken
-
-patterns.digit = digit
-patterns.sign = sign
-patterns.period = period
+-- Patterns are grouped in a separate namespace but the regular lexers expect
+-- shortcuts to be present in the lexers library. Maybe I'll incorporate some
+-- of l-lpeg later.
+
+do
+
+ local anything = P(1)
+ local idtoken = R("az","AZ","\127\255","__")
+ local digit = R("09")
+ local sign = S("+-")
+ local period = P(".")
+ local octdigit = R("07")
+ local hexdigit = R("09","AF","af")
+ local lower = R("az")
+ local upper = R("AZ")
+ local alpha = upper + lower
+ local space = S(" \n\r\t\f\v")
+ local eol = S("\r\n")
+ local backslash = P("\\")
+ local decimal = digit^1
+ local octal = P("0")
+ * octdigit^1
+ local hexadecimal = P("0") * S("xX")
+ * (hexdigit^0 * period * hexdigit^1 + hexdigit^1 * period * hexdigit^0 + hexdigit^1)
+ * (S("pP") * sign^-1 * hexdigit^1)^-1 -- *
+
+ patterns.idtoken = idtoken
+ patterns.digit = digit
+ patterns.sign = sign
+ patterns.period = period
+ patterns.octdigit = octdigit
+ patterns.hexdigit = hexdigit
+ patterns.ascii = R("\000\127") -- useless
+ patterns.extend = R("\000\255") -- useless
+ patterns.control = R("\000\031")
+ patterns.lower = lower
+ patterns.upper = upper
+ patterns.alpha = alpha
+ patterns.decimal = decimal
+ patterns.octal = octal
+ patterns.hexadecimal = hexadecimal
+ patterns.float = sign^-1
+ * (digit^0 * period * digit^1 + digit^1 * period * digit^0 + digit^1)
+ * S("eE") * sign^-1 * digit^1 -- *
+ patterns.cardinal = decimal
+
+ patterns.signeddecimal = sign^-1 * decimal
+ patterns.signedoctal = sign^-1 * octal
+ patterns.signedhexadecimal = sign^-1 * hexadecimal
+ patterns.integer = sign^-1 * (hexadecimal + octal + decimal)
+ patterns.real =
+ sign^-1 * ( -- at most one
+ digit^1 * period * digit^0 -- 10.0 10.
+ + digit^0 * period * digit^1 -- 0.10 .10
+ + digit^1 -- 10
+ )
+
+ patterns.anything = anything
+ patterns.any = anything
+ patterns.restofline = (1-eol)^1
+ patterns.space = space
+ patterns.spacing = space^1
+ patterns.nospacing = (1-space)^1
+ patterns.eol = eol
+ patterns.newline = P("\r\n") + eol
+
+ local endof = S("\n\r\f")
+
+ patterns.startofline = P(function(input,index)
+ return (index == 1 or lpegmatch(endof,input,index-1)) and index
+ end)
+
+ -- These are the expected ones for other lexers. Maybe all in own namespace
+ -- and provide compatibility layer. or should I just remove them?
+
+ lexers.any = anything
+ lexers.ascii = ascii
+ lexers.extend = extend
+ lexers.alpha = alpha
+ lexers.digit = digit
+ lexers.alnum = alnum
+ lexers.lower = lower
+ lexers.upper = upper
+ lexers.xdigit = hexdigit
+ lexers.cntrl = control
+ lexers.graph = R("!~")
+ lexers.print = R(" ~")
+ lexers.punct = R("!/", ":@", "[\'", "{~")
+ lexers.space = space
+ lexers.newline = S("\r\n\f")^1
+ lexers.nonnewline = 1 - lexers.newline
+ lexers.nonnewline_esc = 1 - (lexers.newline + '\\') + backslash * anything
+ lexers.dec_num = decimal
+ lexers.oct_num = octal
+ lexers.hex_num = hexadecimal
+ lexers.integer = integer
+ lexers.float = float
+ lexers.word = (alpha + "_") * (alpha + digit + "_")^0 -- weird, why digits
-patterns.cardinal = digit^1
-patterns.integer = sign^-1 * digit^1
-
-patterns.real =
- sign^-1 * ( -- at most one
- digit^1 * period * digit^0 -- 10.0 10.
- + digit^0 * period * digit^1 -- 0.10 .10
- + digit^1 -- 10
- )
-
-patterns.restofline = (1-S("\n\r"))^1
-patterns.space = space
-patterns.spacing = space^1
-patterns.nospacing = (1-space)^1
-patterns.anything = P(1)
-
-local endof = S("\n\r\f")
+end
-patterns.startofline = P(function(input,index)
- return (index == 1 or lpegmatch(endof,input,index-1)) and index
-end)
+-- end of patterns
function context.exact_match(words,word_chars,case_insensitive)
local characters = concat(words)
@@ -227,7 +801,7 @@ function context.exact_match(words,word_chars,case_insensitive)
word_chars = ""
end
if type(word_chars) == "string" then
- pattern = S(characters) + idtoken
+ pattern = S(characters) + patterns.idtoken
if case_insensitive then
pattern = pattern + S(upper(characters)) + S(lower(characters))
end
@@ -239,16 +813,28 @@ function context.exact_match(words,word_chars,case_insensitive)
end
if case_insensitive then
local list = { }
- for i=1,#words do
- list[lower(words[i])] = true
+ if #words == 0 then
+ for k, v in next, words do
+ list[lower(k)] = v
+ end
+ else
+ for i=1,#words do
+ list[lower(words[i])] = true
+ end
end
return Cmt(pattern^1, function(_,i,s)
return list[lower(s)] -- and i or nil
end)
else
local list = { }
- for i=1,#words do
- list[words[i]] = true
+ if #words == 0 then
+ for k, v in next, words do
+ list[k] = v
+ end
+ else
+ for i=1,#words do
+ list[words[i]] = true
+ end
end
return Cmt(pattern^1, function(_,i,s)
return list[s] -- and i or nil
@@ -256,6 +842,14 @@ function context.exact_match(words,word_chars,case_insensitive)
end
end
+function context.just_match(words)
+ local p = P(words[1])
+ for i=2,#words do
+ p = p + P(words[i])
+ end
+ return p
+end
+
-- spell checking (we can only load lua files)
--
-- return {
@@ -278,6 +872,7 @@ function context.setwordlist(tag,limit) -- returns hash (lowercase keys and orig
if not list then
list = context.loaddefinitions("spell-" .. tag)
if not list or type(list) ~= "table" then
+ report("invalid spell checking list for '%s'",tag)
list = { words = false, min = 3 }
else
list.words = list.words or false
@@ -285,6 +880,9 @@ function context.setwordlist(tag,limit) -- returns hash (lowercase keys and orig
end
lists[tag] = list
end
+ if trace then
+ report("enabling spell checking for '%s' with minimum '%s'",tag,list.min)
+ end
return list.words, list.min
end
@@ -349,7 +947,7 @@ setmetatable(h_table, { __index = function(t,level) local v = { level, FOLD_HEAD
setmetatable(b_table, { __index = function(t,level) local v = { level, FOLD_BLANK } t[level] = v return v end })
setmetatable(n_table, { __index = function(t,level) local v = { level } t[level] = v return v end })
-local newline = P("\r\n") + S("\r\n")
+local newline = patterns.newline
local p_yes = Cp() * Cs((1-newline)^1) * newline^-1
local p_nop = newline
@@ -367,7 +965,7 @@ local function fold_by_parsing(text,start_pos,start_line,start_level,lexer)
if fold_pattern then
-- if no functions are found then we could have a faster one
fold_pattern = Cp() * C(fold_pattern) / function(s,match)
- local symbols = fold_symbols[get_style_at(start_pos + s)]
+ local symbols = fold_symbols[style_at[start_pos + s]]
if symbols then
local l = symbols[match]
if l then
@@ -397,13 +995,13 @@ local function fold_by_parsing(text,start_pos,start_line,start_level,lexer)
local fold_symbols_patterns = fold_symbols._patterns
local action_y = function(pos,line)
for j = 1, #fold_symbols_patterns do
- for s, match in gmatch(line,fold_symbols_patterns[j]) do -- '()('..patterns[i]..')'
- local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)]
+ for s, match in gmatch(line,fold_symbols_patterns[j]) do -- "()(" .. patterns[i] .. ")"
+ local symbols = fold_symbols[style_at[start_pos + pos + s - 1]]
local l = symbols and symbols[match]
local t = type(l)
- if t == 'number' then
+ if t == "number" then
current_level = current_level + l
- elseif t == 'function' then
+ elseif t == "function" then
current_level = current_level + l(text, pos, line, s, match)
end
end
@@ -451,7 +1049,7 @@ end
local folds, current_line, prev_level
local function action_y()
- local current_level = FOLD_BASE + get_indent_amount(current_line)
+ local current_level = FOLD_BASE + indent_amount[current_line]
if current_level > prev_level then -- next level
local i = current_line - 1
local f
@@ -485,7 +1083,7 @@ local function action_n()
current_line = current_line + 1
end
-local pattern = ( S("\t ")^0 * ( (1-S("\n\r"))^1 / action_y + P(true) / action_n) * newline )^0
+local pattern = ( S("\t ")^0 * ( (1-patterns.eol)^1 / action_y + P(true) / action_n) * newline )^0
local function fold_by_indentation(text,start_pos,start_line,start_level)
-- initialize
@@ -521,14 +1119,13 @@ local threshold_by_parsing = 512 * 1024 -- we don't know the filesize yet
local threshold_by_indentation = 512 * 1024 -- we don't know the filesize yet
local threshold_by_line = 512 * 1024 -- we don't know the filesize yet
-function context.fold(text,start_pos,start_line,start_level) -- hm, we had size thresholds .. where did they go
- if text == '' then
+function context.fold(lexer,text,start_pos,start_line,start_level) -- hm, we had size thresholds .. where did they go
+ if text == "" then
return { }
end
if initialize then
initialize()
end
- local lexer = global._LEXER
local fold_by_lexer = lexer._fold
local fold_by_symbols = lexer._foldsymbols
local filesize = 0 -- we don't know that
@@ -536,15 +1133,15 @@ function context.fold(text,start_pos,start_line,start_level) -- hm, we had size
if filesize <= threshold_by_lexer then
return fold_by_lexer(text,start_pos,start_line,start_level,lexer)
end
- elseif fold_by_symbols then -- and get_property('fold.by.parsing',1) > 0 then
+ elseif fold_by_symbols then -- and lexer.properties("fold.by.parsing",1) > 0 then
if filesize <= threshold_by_parsing then
return fold_by_parsing(text,start_pos,start_line,start_level,lexer)
end
- elseif get_property('fold.by.indentation',1) > 0 then
+ elseif lexer.properties("fold.by.indentation",1) > 0 then
if filesize <= threshold_by_indentation then
return fold_by_indentation(text,start_pos,start_line,start_level,lexer)
end
- elseif get_property('fold.by.line',1) > 0 then
+ elseif lexer.properties("fold.by.line",1) > 0 then
if filesize <= threshold_by_line then
return fold_by_line(text,start_pos,start_line,start_level,lexer)
end
@@ -554,7 +1151,7 @@ end
-- The following code is mostly unchanged:
-local function add_rule(lexer,id,rule)
+local function add_rule(lexer,id,rule) -- unchanged
if not lexer._RULES then
lexer._RULES = { }
lexer._RULEORDER = { }
@@ -563,31 +1160,101 @@ local function add_rule(lexer,id,rule)
lexer._RULEORDER[#lexer._RULEORDER + 1] = id
end
-local function add_style(lexer,token_name,style)
- local len = lexer._STYLES.len
- if len == 32 then
- len = len + 8
+-- I finally figured out that adding more styles was an issue because of several
+-- reasons:
+--
+-- + in old versions there was a limit in the amount, so we overran the built-in
+-- hard coded scintilla range
+-- + then, the add_style function didn't check for already known ones, so again
+-- we had an overrun (with some magic that could be avoided)
+-- + then, when I messed with a new default set I realized that there is no check
+-- in initializing _TOKENSTYLES (here the inspect function helps)
+-- + of course it was mostly a side effect of passing all the used styles to the
+-- _tokenstyles instead of only the not-default ones but such a thing should not
+-- matter (read: intercepted)
+--
+-- This finally removed a head-ache and was revealed by lots of tracing, which I
+-- should have built in way earlier.
+
+local function add_style(lexer,token_name,style) -- changed a bit around 3.41
+ -- We don't add styles that are already defined as this can overflow the
+ -- amount possible (in old versions of scintilla).
+ if defaultstyles[token_name] then
+ if trace and detail then
+ report("default style '%s' is ignored as extra style",token_name)
+ end
+ return
+ elseif predefinedstyles[token_name] then
+ if trace and detail then
+ report("predefined style '%s' is ignored as extra style",token_name)
+ end
+ return
+ else
+ if trace and detail then
+ report("adding extra style '%s' as '%s'",token_name,style)
+ end
+ end
+ -- This is unchanged. We skip the dangerous zone.
+ local num_styles = lexer._numstyles
+ if num_styles == 32 then
+ num_styles = num_styles + 8
end
- if len >= 128 then
- print('Too many styles defined (128 MAX)')
+ if num_styles >= 255 then
+ report("there can't be more than %s styles",255)
end
- lexer._TOKENS[token_name] = len
- lexer._STYLES[len] = style
- lexer._STYLES.len = len + 1
+ lexer._TOKENSTYLES[token_name] = num_styles
+ lexer._EXTRASTYLES[token_name] = style
+ lexer._numstyles = num_styles + 1
end
-local function join_tokens(lexer)
- local patterns = lexer._RULES
- local order = lexer._RULEORDER
- local token_rule = patterns[order[1]]
- for i=2,#order do
- token_rule = token_rule + patterns[order[i]]
+local function check_styles(lexer)
+ -- Here we also use a check for the dangerous zone. That way we can have a
+ -- larger default set. The original code just assumes that #default is less
+ -- than the dangerous zone's start.
+ local numstyles = 0
+ local tokenstyles = { }
+ for i=1, #default do
+ if numstyles == 32 then
+ numstyles = numstyles + 8
+ end
+ tokenstyles[default[i]] = numstyles
+ numstyles = numstyles + 1
+ end
+ -- Unchanged.
+ for i=1, #predefined do
+ tokenstyles[predefined[i]] = i + 31
end
- lexer._TOKENRULE = token_rule
- return token_rule
+ lexer._TOKENSTYLES = tokenstyles
+ lexer._numstyles = numstyles
+ lexer._EXTRASTYLES = { }
+ return lexer
end
-local function add_lexer(grammar, lexer, token_rule)
+-- At some point an 'any' append showed up in the original code ...
+-- but I see no need to catch that case ... beter fix the specification.
+--
+-- hm, why are many joined twice
+
+local function join_tokens(lexer) -- slightly different from the original (no 'any' append)
+ local patterns = lexer._RULES
+ local order = lexer._RULEORDER
+ -- report("lexer: %s, tokens: %s",lexer._NAME,table.concat(order," + "))
+ if patterns and order then
+ local token_rule = patterns[order[1]] -- normally whitespace
+ for i=2,#order do
+ token_rule = token_rule + patterns[order[i]]
+ end
+ if lexer._TYPE ~= "context" then
+ token_rule = token_rule + lexers.token(lexers.DEFAULT, patterns.any)
+ end
+ lexer._TOKENRULE = token_rule
+ return token_rule
+ else
+ return P(1)
+ end
+end
+
+local function add_lexer(grammar, lexer) -- mostly the same as the original
local token_rule = join_tokens(lexer)
local lexer_name = lexer._NAME
local children = lexer._CHILDREN
@@ -598,20 +1265,24 @@ local function add_lexer(grammar, lexer, token_rule)
end
local child_name = child._NAME
local rules = child._EMBEDDEDRULES[lexer_name]
- local rules_token_rule = grammar['__'..child_name] or rules.token_rule
- grammar[child_name] = (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1 * V(lexer_name)
- local embedded_child = '_' .. child_name
- grammar[embedded_child] = rules.start_rule * (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1
+ local rules_token_rule = grammar["__" .. child_name] or rules.token_rule
+ local pattern = (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1
+ grammar[child_name] = pattern * V(lexer_name)
+ local embedded_child = "_" .. child_name
+ grammar[embedded_child] = rules.start_rule * pattern
token_rule = V(embedded_child) + token_rule
end
- grammar['__' .. lexer_name] = token_rule
+ if trace then
+ report("adding lexer '%s' with %s children",lexer_name,#children)
+ end
+ grammar["__" .. lexer_name] = token_rule
grammar[lexer_name] = token_rule^0
end
-local function build_grammar(lexer, initial_rule)
+local function build_grammar(lexer,initial_rule) -- same as the original
local children = lexer._CHILDREN
+ local lexer_name = lexer._NAME
if children then
- local lexer_name = lexer._NAME
if not initial_rule then
initial_rule = lexer_name
end
@@ -619,17 +1290,90 @@ local function build_grammar(lexer, initial_rule)
add_lexer(grammar, lexer)
lexer._INITIALRULE = initial_rule
lexer._GRAMMAR = Ct(P(grammar))
+ if trace then
+ report("building grammar for '%s' with whitespace '%s'and %s children",lexer_name,lexer.whitespace or "?",#children)
+ end
else
lexer._GRAMMAR = Ct(join_tokens(lexer)^0)
+ if trace then
+ report("building grammar for '%s' with whitespace '%s'",lexer_name,lexer.whitespace or "?")
+ end
end
end
--- so far. We need these local functions in the next one.
+-- So far. We need these local functions in the next one.
local lineparsers = { }
-function context.lex(text,init_style)
- local lexer = global._LEXER
+local maxmatched = 100
+
+local function collapsed(t)
+ local lasttoken = nil
+ local lastindex = nil
+ for i=1,#t,2 do
+ local token = t[i]
+ local position = t[i+1]
+ if token == lasttoken then
+ t[lastindex] = position
+ elseif lastindex then
+ lastindex = lastindex + 1
+ t[lastindex] = token
+ lastindex = lastindex + 1
+ t[lastindex] = position
+ lasttoken = token
+ else
+ lastindex = i+1
+ lasttoken = token
+ end
+ end
+ for i=#t,lastindex+1,-1 do
+ t[i] = nil
+ end
+ return t
+end
+
+local function matched(lexer,grammar,text)
+ -- text = string.gsub(text,"\z","!")
+ local t = lpegmatch(grammar,text)
+ if trace then
+ if show then
+ report("output of lexer: %s (max %s entries)",lexer._NAME,maxmatched)
+ local s = lexer._TOKENSTYLES
+ local p = 1
+ for i=1,2*maxmatched,2 do
+ local n = i + 1
+ local ti = t[i]
+ local tn = t[n]
+ if ti then
+ local txt = sub(text,p,tn-1)
+ if txt then
+ txt = gsub(txt,"[%s]"," ")
+ else
+ txt = "!no text!"
+ end
+ report("%4i : %s > %s (%s) (%s)",n/2,ti,tn,s[ti] or "!unset!",txt)
+ p = tn
+ else
+ break
+ end
+ end
+ end
+ report("lexer results: %s, length: %s, ranges: %s",lexer._NAME,#text,#t/2)
+ if collapse then
+ t = collapsed(t)
+ report("lexer collapsed: %s, length: %s, ranges: %s",lexer._NAME,#text,#t/2)
+ end
+ elseif collapse then
+ t = collapsed(t)
+ end
+ return t
+end
+
+-- Todo: make nice generic lexer (extra argument with start/stop commands) for
+-- context itself.
+
+function context.lex(lexer,text,init_style)
+ -- local lexer = global._LEXER
local grammar = lexer._GRAMMAR
if initialize then
initialize()
@@ -656,7 +1400,7 @@ function context.lex(text,init_style)
offset = offset + length
if noftokens > 0 and tokens[noftokens] ~= offset then
noftokens = noftokens + 1
- tokens[noftokens] = 'default'
+ tokens[noftokens] = "default"
noftokens = noftokens + 1
tokens[noftokens] = offset + 1
end
@@ -666,10 +1410,7 @@ function context.lex(text,init_style)
end
lpegmatch(lineparser,text)
return tokens
-
elseif lexer._CHILDREN then
- -- as we cannot print, tracing is not possible ... this might change as we can as well
- -- generate them all in one go (sharing as much as possible)
local hash = lexer._HASH -- hm, was _hash
if not hash then
hash = { }
@@ -678,12 +1419,13 @@ function context.lex(text,init_style)
grammar = hash[init_style]
if grammar then
lexer._GRAMMAR = grammar
+ -- lexer._GRAMMAR = lexer._GRAMMAR or grammar
else
- for style, style_num in next, lexer._TOKENS do
+ for style, style_num in next, lexer._TOKENSTYLES do
if style_num == init_style then
-- the name of the lexers is filtered from the whitespace
- -- specification
- local lexer_name = match(style,'^(.+)_whitespace') or lexer._NAME
+ -- specification .. weird code, should be a reverse hash
+ local lexer_name = match(style,"^(.+)_whitespace") or lexer._NAME
if lexer._INITIALRULE ~= lexer_name then
grammar = hash[lexer_name]
if not grammar then
@@ -698,147 +1440,579 @@ function context.lex(text,init_style)
grammar = grammar or lexer._GRAMMAR
hash[init_style] = grammar
end
- return lpegmatch(grammar,text)
+ if trace then
+ report("lexing '%s' with initial style '%s' and %s children",lexer._NAME,#lexer._CHILDREN or 0,init_style)
+ end
+ return matched(lexer,grammar,text)
else
- return lpegmatch(grammar,text)
+ if trace then
+ report("lexing '%s' with initial style '%s'",lexer._NAME,init_style)
+ end
+ return matched(lexer,grammar,text)
end
end
--- todo: keywords: one lookup and multiple matches
-
--- function context.token(name, patt)
--- return Ct(patt * Cc(name) * Cp())
--- end
---
--- -- hm, changed in 3.24 .. no longer a table
+-- hm, changed in 3.24 .. no longer small table but one table:
function context.token(name, patt)
return patt * Cc(name) * Cp()
end
+-- The next ones were mostly unchanged (till now), we moved it here when 3.41
+-- became close to impossible to combine with cq. overload and a merge was
+-- the only solution. It makes later updates more painful but the update to
+-- 3.41 was already a bit of a nightmare anyway.
+
+-- Loading lexers is rather interwoven with what the dll/so sets and
+-- it changes over time. So, we need to keep an eye on changes. One
+-- problem that we always faced were the limitations in length of
+-- lexer names (as they get app/prepended occasionally to strings with
+-- a hard coded limit). So, we always used alternative names and now need
+-- to make sure this doesn't clash. As I no longer intend to use shipped
+-- lexers I could strip away some of the code in the future, but keeping
+-- it as reference makes sense.
+
+-- I spend quite some time figuring out why 3.41 didn't work or crashed which
+-- is hard when no stdout is available and when the io library is absent. In
+-- the end of of the problems was in the _NAME setting. We set _NAME
+-- to e.g. 'tex' but load from a file with a longer name, which we do
+-- as we don't want to clash with existing files, we end up in
+-- lexers not being found.
+
+local whitespaces = { }
+
+local function push_whitespace(name)
+ table.insert(whitespaces,lexers.WHITESPACE or "whitespace")
+ lexers.WHITESPACE = name .. "_whitespace"
+end
+
+local function pop_whitespace()
+ lexers.WHITESPACE = table.remove(whitespaces) or "whitespace"
+end
+
+local function check_whitespace(lexer,name)
+ if lexer then
+ lexer.whitespace = (name or lexer.name or lexer._NAME) .. "_whitespace"
+ end
+end
+
+function context.new(name,filename)
+ local lexer = {
+ _TYPE = "context",
+ --
+ _NAME = name, -- used for token building
+ _FILENAME = filename, -- for diagnostic purposed
+ --
+ name = name,
+ filename = filename,
+ }
+ if trace then
+ report("initializing lexer tagged '%s' from file '%s'",name,filename or name)
+ end
+ check_whitespace(lexer)
+ check_styles(lexer)
+ check_properties(lexer)
+ return lexer
+end
+
+local function nolexer(name)
+ local lexer = {
+ _TYPE = "unset",
+ _NAME = name,
+ -- _rules = { },
+ }
+ check_styles(lexer)
+ check_whitespace(lexer)
+ check_properties(lexer)
+ return lexer
+end
+
+local function load_lexer(name,namespace)
+ if trace then
+ report("loading lexer file '%s'",name)
+ end
+ push_whitespace(namespace or name) -- for traditional lexers .. no alt_name yet
+ local lexer, fullname = context.loadluafile(name)
+ pop_whitespace()
+ if not lexer then
+ report("invalid lexer file '%s'",name)
+ elseif trace then
+ report("lexer file '%s' has been loaded",fullname)
+ end
+ if type(lexer) ~= "table" then
+ if trace then
+ report("lexer file '%s' gets a dummy lexer",name)
+ end
+ return nolexer(name)
+ end
+ if lexer._TYPE ~= "context" then
+ lexer._TYPE = "native"
+ check_styles(lexer)
+ check_whitespace(lexer,namespace or name)
+ check_properties(lexer)
+ end
+ if not lexer._NAME then
+ lexer._NAME = name -- so: filename
+ end
+ if name ~= namespace then
+ lexer._NAME = namespace
+ end
+ return lexer
+end
+
+-- tracing ...
+
+local function inspect_lexer(lexer,level)
+ -- If we had the regular libs available I could use the usual
+ -- helpers.
+ local parent = lexer._lexer
+ lexer._lexer = nil -- prevent endless recursion
+ local name = lexer._NAME
+ local function showstyles_1(tag,styles)
+ local numbers = { }
+ for k, v in next, styles do
+ numbers[v] = k
+ end
+ -- sort by number and make number hash too
+ local keys = sortedkeys(numbers)
+ for i=1,#keys do
+ local k = keys[i]
+ local v = numbers[k]
+ report("[%s %s] %s %s = %s",level,name,tag,k,v)
+ end
+ end
+ local function showstyles_2(tag,styles)
+ local keys = sortedkeys(styles)
+ for i=1,#keys do
+ local k = keys[i]
+ local v = styles[k]
+ report("[%s %s] %s %s = %s",level,name,tag,k,v)
+ end
+ end
+ local keys = sortedkeys(lexer)
+ for i=1,#keys do
+ local k = keys[i]
+ local v = lexer[k]
+ report("[%s %s] root key : %s = %s",level,name,k,tostring(v))
+ end
+ showstyles_1("token style",lexer._TOKENSTYLES)
+ showstyles_2("extra style",lexer._EXTRASTYLES)
+ local children = lexer._CHILDREN
+ if children then
+ for i=1,#children do
+ inspect_lexer(children[i],level+1)
+ end
+ end
+ lexer._lexer = parent
+end
+
+function context.inspect(lexer)
+ inspect_lexer(lexer,0)
+end
+
+-- An optional second argument has been introduced so that one can embed a lexer
+-- more than once ... maybe something to look into (as not it's done by remembering
+-- the start sequence ... quite okay but maybe suboptimal ... anyway, never change
+-- a working solution).
+
+-- namespace can be automatic: if parent then use name of parent (chain)
+
+function context.loadlexer(filename,namespace)
+ nesting = nesting + 1
+ if not namespace then
+ namespace = filename
+ end
+ local lexer = usedlexers[namespace] -- we load by filename but the internal name can be short
+ if lexer then
+ if trace then
+ report("reusing lexer '%s'",namespace)
+ end
+ nesting = nesting - 1
+ return lexer
+ elseif trace then
+ report("loading lexer '%s'",namespace)
+ end
+ --
+ if initialize then
+ initialize()
+ end
+ --
+ parent_lexer = nil
+ --
+ lexer = load_lexer(filename,namespace) or nolexer(filename,namespace)
+ usedlexers[filename] = lexer
+ --
+ if not lexer._rules and not lexer._lexer then
+ lexer._lexer = parent_lexer
+ end
+ --
+ if lexer._lexer then
+ local _l = lexer._lexer
+ local _r = lexer._rules
+ local _s = lexer._tokenstyles
+ if not _l._tokenstyles then
+ _l._tokenstyles = { }
+ end
+ if _r then
+ local rules = _l._rules
+ local name = lexer.name
+ for i=1,#_r do
+ local rule = _r[i]
+ rules[#rules + 1] = {
+ name .. "_" .. rule[1],
+ rule[2],
+ }
+ end
+ end
+ if _s then
+ local tokenstyles = _l._tokenstyles
+ for token, style in next, _s do
+ tokenstyles[token] = style
+ end
+ end
+ lexer = _l
+ end
+ --
+ local _r = lexer._rules
+ if _r then
+ local _s = lexer._tokenstyles
+ if _s then
+ for token, style in next, _s do
+ add_style(lexer, token, style)
+ end
+ end
+ for i=1,#_r do
+ local rule = _r[i]
+ add_rule(lexer, rule[1], rule[2])
+ end
+ build_grammar(lexer)
+ end
+ --
+ add_style(lexer, lexer.whitespace, lexers.STYLE_WHITESPACE)
+ --
+ local foldsymbols = lexer._foldsymbols
+ if foldsymbols then
+ local patterns = foldsymbols._patterns
+ if patterns then
+ for i = 1, #patterns do
+ patterns[i] = "()(" .. patterns[i] .. ")"
+ end
+ end
+ end
+ --
+ lexer.lex = lexers.lex
+ lexer.fold = lexers.fold
+ --
+ nesting = nesting - 1
+ --
+ if inspect then
+ context.inspect(lexer)
+ end
+ --
+ return lexer
+end
+
+function context.embed_lexer(parent, child, start_rule, end_rule) -- mostly the same as the original
+ local embeddedrules = child._EMBEDDEDRULES
+ if not embeddedrules then
+ embeddedrules = { }
+ child._EMBEDDEDRULES = embeddedrules
+ end
+ if not child._RULES then
+ local rules = child._rules
+ if not rules then
+ report("child lexer '%s' has no rules",child._NAME or "unknown")
+ rules = { }
+ child._rules = rules
+ end
+ for i=1,#rules do
+ local rule = rules[i]
+ add_rule(child, rule[1], rule[2])
+ end
+ end
+ embeddedrules[parent._NAME] = {
+ ["start_rule"] = start_rule,
+ ["token_rule"] = join_tokens(child),
+ ["end_rule"] = end_rule
+ }
+ local children = parent._CHILDREN
+ if not children then
+ children = { }
+ parent._CHILDREN = children
+ end
+ children[#children + 1] = child
+ local tokenstyles = parent._tokenstyles
+ if not tokenstyles then
+ tokenstyles = { }
+ parent._tokenstyles = tokenstyles
+ end
+ local childname = child._NAME
+ local whitespace = childname .. "_whitespace"
+ tokenstyles[whitespace] = lexers.STYLE_WHITESPACE -- all these STYLE_THINGS will go .. just a proper hash
+ if trace then
+ report("using whitespace '%s' as trigger for '%s' with property '%s'",whitespace,childname,lexers.STYLE_WHITESPACE)
+ end
+ local childstyles = child._tokenstyles
+ if childstyles then
+ for token, style in next, childstyles do
+ tokenstyles[token] = style
+ end
+ end
+ child._lexer = parent
+ parent_lexer = parent
+end
+
+-- we now move the adapted code to the lexers namespace
+
+lexers.new = context.new
+lexers.load = context.loadlexer
+------.loadlexer = context.loadlexer
+lexers.loadluafile = context.loadluafile
+lexers.embed_lexer = context.embed_lexer
lexers.fold = context.fold
lexers.lex = context.lex
lexers.token = context.token
+lexers.word_match = context.word_match
lexers.exact_match = context.exact_match
+lexers.just_match = context.just_match
+lexers.inspect = context.inspect
+lexers.report = context.report
+lexers.inform = context.inform
-- helper .. alas ... the lexer's lua instance is rather crippled .. not even
-- math is part of it
-local floor = math and math.floor
-local char = string.char
+do
+
+ local floor = math and math.floor
+ local char = string.char
+
+ if not floor then
+
+ floor = function(n)
+ return tonumber(format("%d",n))
+ end
+
+ math = math or { }
+
+ math.floor = floor
+
+ end
+
+ local function utfchar(n)
+ if n < 0x80 then
+ return char(n)
+ elseif n < 0x800 then
+ return char(
+ 0xC0 + floor(n/0x40),
+ 0x80 + (n % 0x40)
+ )
+ elseif n < 0x10000 then
+ return char(
+ 0xE0 + floor(n/0x1000),
+ 0x80 + (floor(n/0x40) % 0x40),
+ 0x80 + (n % 0x40)
+ )
+ elseif n < 0x40000 then
+ return char(
+ 0xF0 + floor(n/0x40000),
+ 0x80 + floor(n/0x1000),
+ 0x80 + (floor(n/0x40) % 0x40),
+ 0x80 + (n % 0x40)
+ )
+ else
+ -- return char(
+ -- 0xF1 + floor(n/0x1000000),
+ -- 0x80 + floor(n/0x40000),
+ -- 0x80 + floor(n/0x1000),
+ -- 0x80 + (floor(n/0x40) % 0x40),
+ -- 0x80 + (n % 0x40)
+ -- )
+ return "?"
+ end
+ end
+
+ context.utfchar = utfchar
-if not floor then
+ -- a helper from l-lpeg:
- floor = function(n)
- return tonumber(string.format("%d",n))
+ local function make(t)
+ local p
+ for k, v in next, t do
+ if not p then
+ if next(v) then
+ p = P(k) * make(v)
+ else
+ p = P(k)
+ end
+ else
+ if next(v) then
+ p = p + P(k) * make(v)
+ else
+ p = p + P(k)
+ end
+ end
+ end
+ return p
end
- math = math or { }
+ function lpeg.utfchartabletopattern(list)
+ local tree = { }
+ for i=1,#list do
+ local t = tree
+ for c in gmatch(list[i],".") do
+ if not t[c] then
+ t[c] = { }
+ end
+ t = t[c]
+ end
+ end
+ return make(tree)
+ end
- math.floor = floor
+ patterns.invisibles = lpeg.utfchartabletopattern {
+ utfchar(0x00A0), -- nbsp
+ utfchar(0x2000), -- enquad
+ utfchar(0x2001), -- emquad
+ utfchar(0x2002), -- enspace
+ utfchar(0x2003), -- emspace
+ utfchar(0x2004), -- threeperemspace
+ utfchar(0x2005), -- fourperemspace
+ utfchar(0x2006), -- sixperemspace
+ utfchar(0x2007), -- figurespace
+ utfchar(0x2008), -- punctuationspace
+ utfchar(0x2009), -- breakablethinspace
+ utfchar(0x200A), -- hairspace
+ utfchar(0x200B), -- zerowidthspace
+ utfchar(0x202F), -- narrownobreakspace
+ utfchar(0x205F), -- math thinspace
+ }
+
+ -- now we can make:
+
+ patterns.iwordtoken = patterns.wordtoken - patterns.invisibles
+ patterns.iwordpattern = patterns.iwordtoken^3
end
-local function utfchar(n)
- if n < 0x80 then
- return char(n)
- elseif n < 0x800 then
- return char(
- 0xC0 + floor(n/0x40),
- 0x80 + (n % 0x40)
- )
- elseif n < 0x10000 then
- return char(
- 0xE0 + floor(n/0x1000),
- 0x80 + (floor(n/0x40) % 0x40),
- 0x80 + (n % 0x40)
- )
- elseif n < 0x40000 then
- return char(
- 0xF0 + floor(n/0x40000),
- 0x80 + floor(n/0x1000),
- 0x80 + (floor(n/0x40) % 0x40),
- 0x80 + (n % 0x40)
- )
+-- The following helpers are not used, partyally replace by other mechanism and
+-- when needed I'll first optimize them. I only made them somewhat more readable.
+
+function lexers.delimited_range(chars, single_line, no_escape, balanced) -- unchanged
+ local s = sub(chars,1,1)
+ local e = #chars == 2 and sub(chars,2,2) or s
+ local range
+ local b = balanced and s or ""
+ local n = single_line and "\n" or ""
+ if no_escape then
+ local invalid = S(e .. n .. b)
+ range = patterns.any - invalid
+ else
+ local invalid = S(e .. n .. b) + patterns.backslash
+ range = patterns.any - invalid + patterns.backslash * patterns.any
+ end
+ if balanced and s ~= e then
+ return P {
+ s * (range + V(1))^0 * e
+ }
else
- -- return char(
- -- 0xF1 + floor(n/0x1000000),
- -- 0x80 + floor(n/0x40000),
- -- 0x80 + floor(n/0x1000),
- -- 0x80 + (floor(n/0x40) % 0x40),
- -- 0x80 + (n % 0x40)
- -- )
- return "?"
+ return s * range^0 * P(e)^-1
end
end
-context.utfchar = utfchar
+function lexers.starts_line(patt) -- unchanged
+ return P ( function(input, index)
+ if index == 1 then
+ return index
+ end
+ local char = sub(input,index - 1,index - 1)
+ if char == "\n" or char == "\r" or char == "\f" then
+ return index
+ end
+ end ) * patt
+end
--- a helper from l-lpeg:
+function lexers.last_char_includes(s) -- unchanged
+ s = "[" .. gsub(s,"[-%%%[]", "%%%1") .. "]"
+ return P ( function(input, index)
+ if index == 1 then
+ return index
+ end
+ local i = index
+ while match(sub(input,i - 1,i - 1),"[ \t\r\n\f]") do
+ i = i - 1
+ end
+ if match(sub(input,i - 1,i - 1),s) then
+ return index
+ end
+ end)
+end
-local gmatch = string.gmatch
+function lexers.nested_pair(start_chars, end_chars) -- unchanged
+ local s = start_chars
+ local e = P(end_chars)^-1
+ return P {
+ s * (patterns.any - s - end_chars + V(1))^0 * e
+ }
+end
-local function make(t)
- local p
- for k, v in next, t do
- if not p then
- if next(v) then
- p = P(k) * make(v)
- else
- p = P(k)
+local function prev_line_is_comment(prefix, text, pos, line, s) -- unchanged
+ local start = find(line,"%S")
+ if start < s and not find(line,prefix,start,true) then
+ return false
+ end
+ local p = pos - 1
+ if sub(text,p,p) == "\n" then
+ p = p - 1
+ if sub(text,p,p) == "\r" then
+ p = p - 1
+ end
+ if sub(text,p,p) ~= "\n" then
+ while p > 1 and sub(text,p - 1,p - 1) ~= "\n"
+ do p = p - 1
end
- else
- if next(v) then
- p = p + P(k) * make(v)
- else
- p = p + P(k)
+ while find(sub(text,p,p),"^[\t ]$") do
+ p = p + 1
end
+ return sub(text,p,p + #prefix - 1) == prefix
end
end
- return p
+ return false
end
-function lpeg.utfchartabletopattern(list)
- local tree = { }
- for i=1,#list do
- local t = tree
- for c in gmatch(list[i],".") do
- if not t[c] then
- t[c] = { }
- end
- t = t[c]
- end
- end
- return make(tree)
-end
-
-patterns.invisibles = lpeg.utfchartabletopattern {
- utfchar(0x00A0), -- nbsp
- utfchar(0x2000), -- enquad
- utfchar(0x2001), -- emquad
- utfchar(0x2002), -- enspace
- utfchar(0x2003), -- emspace
- utfchar(0x2004), -- threeperemspace
- utfchar(0x2005), -- fourperemspace
- utfchar(0x2006), -- sixperemspace
- utfchar(0x2007), -- figurespace
- utfchar(0x2008), -- punctuationspace
- utfchar(0x2009), -- breakablethinspace
- utfchar(0x200A), -- hairspace
- utfchar(0x200B), -- zerowidthspace
- utfchar(0x202F), -- narrownobreakspace
- utfchar(0x205F), -- math thinspace
-}
-
--- now we can make:
-
-patterns.iwordtoken = patterns.wordtoken - patterns.invisibles
-patterns.iwordpattern = patterns.iwordtoken^3
+local function next_line_is_comment(prefix, text, pos, line, s)
+ local p = find(text,"\n",pos + s)
+ if p then
+ p = p + 1
+ while find(sub(text,p,p),"^[\t ]$") do
+ p = p + 1
+ end
+ return sub(text,p,p + #prefix - 1) == prefix
+ end
+ return false
+end
--- require("themes/scite-context-theme")
+function lexers.fold_line_comments(prefix)
+ local property_int = lexers.property_int
+ return function(text, pos, line, s)
+ if property_int["fold.line.comments"] == 0 then
+ return 0
+ end
+ if s > 1 and match(line,"^%s*()") < s then
+ return 0
+ end
+ local prev_line_comment = prev_line_is_comment(prefix, text, pos, line, s)
+ local next_line_comment = next_line_is_comment(prefix, text, pos, line, s)
+ if not prev_line_comment and next_line_comment then
+ return 1
+ end
+ if prev_line_comment and not next_line_comment then
+ return -1
+ end
+ return 0
+ end
+end
--- In order to deal with some bug in additional styles (I have no cue what is
--- wrong, but additional styles get ignored and clash somehow) I just copy the
--- original lexer code ... see original for comments.
+-- done
return lexers
diff --git a/Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme-keep.lua b/Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme-keep.lua
deleted file mode 100644
index 7f9423d9a30..00000000000
--- a/Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme-keep.lua
+++ /dev/null
@@ -1,233 +0,0 @@
-local info = {
- version = 1.002,
- comment = "theme for scintilla lpeg lexer for context/metafun",
- author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
- copyright = "PRAGMA ADE / ConTeXt Development Team",
- license = "see context related readme files",
-}
-
--- context_path = string.split(os.resultof("mtxrun --find-file context.mkiv"))[1] or ""
--- global.trace("OEPS") -- how do we get access to the regular lua extensions
-
--- The regular styles set the main lexer styles table but we avoid that in order not
--- to end up with updating issues. We just use another table.
-
--- if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
-
-local context_path = "t:/sources" -- c:/data/tex-context/tex/texmf-context/tex/base
-local font_name = 'Dejavu Sans Mono'
-local font_size = 14
-
-if not WIN32 then
- font_name = '!' .. font_name
-end
-
-local color = lexer.color
-local style = lexer.style
-
-lexer.context = lexer.context or { }
-local context = lexer.context
-
-context.path = context_path
-
-colors = {
- red = color('7F', '00', '00'),
- green = color('00', '7F', '00'),
- blue = color('00', '00', '7F'),
- cyan = color('00', '7F', '7F'),
- magenta = color('7F', '00', '7F'),
- yellow = color('7F', '7F', '00'),
- orange = color('B0', '7F', '00'),
- --
- white = color('FF', 'FF', 'FF'),
- light = color('CF', 'CF', 'CF'),
- grey = color('80', '80', '80'),
- dark = color('4F', '4F', '4F'),
- black = color('00', '00', '00'),
- --
- selection = color('F7', 'F7', 'F7'),
- logpanel = color('E7', 'E7', 'E7'),
- textpanel = color('CF', 'CF', 'CF'),
- linepanel = color('A7', 'A7', 'A7'),
- tippanel = color('44', '44', '44'),
- --
- right = color('00', '00', 'FF'),
- wrong = color('FF', '00', '00'),
-}
-
-colors.teal = colors.cyan
-colors.purple = colors.magenta
-
--- to be set:
---
--- style_nothing
--- style_class
--- style_comment
--- style_constant
--- style_definition
--- style_error
--- style_function
--- style_keyword
--- style_number
--- style_operator
--- style_string
--- style_preproc
--- style_tag
--- style_type
--- style_variable
--- style_embedded
--- style_label
--- style_regex
--- style_identifier
---
--- style_line_number
--- style_bracelight
--- style_bracebad
--- style_controlchar
--- style_indentguide
--- style_calltip
-
-style_default = style {
- font = font_name,
- size = font_size,
- fore = colors.black,
- back = colors.textpanel,
-}
-
-style_nothing = style {
- -- empty
-}
-
-style_number = style { fore = colors.cyan }
-style_comment = style { fore = colors.yellow }
-style_string = style { fore = colors.magenta }
-style_keyword = style { fore = colors.blue, bold = true }
-
-style_quote = style { fore = colors.blue, bold = true }
-style_special = style { fore = colors.blue }
-style_extra = style { fore = colors.yellow }
-
-style_embedded = style { fore = colors.black, bold = true }
-
-style_char = style { fore = colors.magenta }
-style_reserved = style { fore = colors.magenta, bold = true }
-style_class = style { fore = colors.black, bold = true }
-style_constant = style { fore = colors.cyan, bold = true }
-style_definition = style { fore = colors.black, bold = true }
-style_okay = style { fore = colors.dark }
-style_error = style { fore = colors.red }
-style_warning = style { fore = colors.orange }
-style_invisible = style { back = colors.orange }
-style_function = style { fore = colors.black, bold = true }
-style_operator = style { fore = colors.blue }
-style_preproc = style { fore = colors.yellow, bold = true }
-style_tag = style { fore = colors.cyan }
-style_type = style { fore = colors.blue }
-style_variable = style { fore = colors.black }
-style_identifier = style_nothing
-
-style_standout = style { fore = colors.orange, bold = true }
-
-style_line_number = style { back = colors.linepanel }
-style_bracelight = style_standout
-style_bracebad = style_standout
-style_indentguide = style { fore = colors.linepanel, back = colors.white }
-style_calltip = style { fore = colors.white, back = colors.tippanel }
-style_controlchar = style_nothing
-
-style_label = style { fore = colors.red, bold = true } -- style { fore = colors.cyan, bold = true }
-style_regex = style_string
-
-style_command = style { fore = colors.green, bold = true }
-
--- only bold seems to work
-
-lexer.style_nothing = style_nothing
-lexer.style_class = style_class
-lexer.style_comment = style_comment
-lexer.style_constant = style_constant
-lexer.style_definition = style_definition
-lexer.style_error = style_error
-lexer.style_function = style_function
-lexer.style_keyword = style_keyword
-lexer.style_number = style_number
-lexer.style_operator = style_operator
-lexer.style_string = style_string
-lexer.style_preproc = style_preproc
-lexer.style_tag = style_tag
-lexer.style_type = style_type
-lexer.style_variable = style_variable
-lexer.style_embedded = style_embedded
-lexer.style_label = style_label
-lexer.style_regex = style_regex
-lexer.style_identifier = style_nothing
-
-local styles = { -- as we have globals we could do with less
-
- -- ["whitespace"] = style_whitespace, -- not to be set!
-
-["default"] = style_nothing,
-["number"] = style_number,
-["comment"] = style_comment,
-["keyword"] = style_keyword,
-["string"] = style_string,
-["preproc"] = style_preproc,
-
- ["reserved"] = style_reserved,
- ["internal"] = style_standout,
-
- ["command"] = style_command,
- ["preamble"] = style_comment,
- ["embedded"] = style_embedded,
- ["grouping"] = style { fore = colors.red },
-["label"] = style_label,
- ["primitive"] = style_keyword,
- ["plain"] = style { fore = colors.dark, bold = true },
- ["user"] = style { fore = colors.green },
- ["data"] = style_constant,
- ["special"] = style_special,
- ["extra"] = style_extra,
- ["quote"] = style_quote,
-
- ["okay"] = style_okay,
- ["warning"] = style_warning,
- ["invisible"] = style_invisible,
-["error"] = style_error,
-
-}
-
--- Old method (still available):
-
-local styleset = { }
-
-for k, v in next, styles do
- styleset[#styleset+1] = { k, v }
-end
-
-context.styles = styles
-context.styleset = styleset
-
--- We need to be sparse due to some limitation (and the number of built in styles
--- growing).
-
--- function context.newstyleset(list)
--- local t = { }
--- if list then
--- for i=1,#list do
--- t[list[i]] = true
--- end
--- end
--- return t
--- end
-
--- function context.usestyle(set,name)
--- set[name] = true
--- return name
--- end
-
--- function context.usestyleset(set)
--- local t = { }
--- for k, _ in next, set do
--- t[#t+1] = { k, styles[k] or styles.default }
--- end
--- end
diff --git a/Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme.lua b/Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme.lua
index e32fe9dee15..b0c63fe3941 100644
--- a/Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme.lua
+++ b/Master/texmf-dist/context/data/scite/lexers/themes/scite-context-theme.lua
@@ -7,217 +7,144 @@ local info = {
}
-- context_path = string.split(os.resultof("mtxrun --find-file context.mkiv"))[1] or ""
--- global.trace("OEPS") -- how do we get access to the regular lua extensions
--- The regular styles set the main lexer styles table but we avoid that in order not
--- to end up with updating issues. We just use another table.
-
-if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end
-
-local context_path = "t:/sources" -- c:/data/tex-context/tex/texmf-context/tex/base
-local font_name = 'Dejavu Sans Mono'
-local font_size = 14
-
-if not WIN32 then
- font_name = '!' .. font_name
-end
-
-local color = lexer.color
-local style = lexer.style
+-- What used to be proper Lua definitions are in 3.42 SciTE properties although
+-- integration is still somewhat half. Also, the indexed style specification is
+-- now a hash (which indeed makes more sense). However, the question is: am I
+-- going to rewrite the style bit? It anyway makes more sense to keep this file
+-- somewhat neutral as we no longer need to be compatible. However, we cannot be
+-- sure of helpers being present yet when this file is loaded, so we are somewhat
+-- crippled. On the other hand, I don't see other schemes being used with the
+-- context lexers.
+
+-- The next kludge is no longer needed which is good!
+--
+-- if GTK then -- WIN32 GTK OSX CURSES
+-- font_name = '!' .. font_name
+-- end
-lexer.context = lexer.context or { }
-local context = lexer.context
+-- I need to play with these, some work ok:
+--
+-- eolfilled noteolfilled
+-- characterset:u|l
+-- visible notvisible
+-- changeable notchangeable (this way we can protect styles, e.g. preamble?)
+-- hotspot nothotspot
-context.path = context_path
+local font_name = 'Dejavu Sans Mono'
+local font_size = '14'
local colors = {
- red = color('7F', '00', '00'),
- green = color('00', '7F', '00'),
- blue = color('00', '00', '7F'),
- cyan = color('00', '7F', '7F'),
- magenta = color('7F', '00', '7F'),
- yellow = color('7F', '7F', '00'),
- orange = color('B0', '7F', '00'),
+ red = { '7F', '00', '00' },
+ green = { '00', '7F', '00' },
+ blue = { '00', '00', '7F' },
+ cyan = { '00', '7F', '7F' },
+ magenta = { '7F', '00', '7F' },
+ yellow = { '7F', '7F', '00' },
+ orange = { 'B0', '7F', '00' },
--
- white = color('FF', 'FF', 'FF'),
- light = color('CF', 'CF', 'CF'),
- grey = color('80', '80', '80'),
- dark = color('4F', '4F', '4F'),
- black = color('00', '00', '00'),
+ white = { 'FF', 'FF', 'FF' },
+ light = { 'CF', 'CF', 'CF' },
+ grey = { '80', '80', '80' },
+ dark = { '4F', '4F', '4F' },
+ black = { '00', '00', '00' },
--
- selection = color('F7', 'F7', 'F7'),
- logpanel = color('E7', 'E7', 'E7'),
- textpanel = color('CF', 'CF', 'CF'),
- linepanel = color('A7', 'A7', 'A7'),
- tippanel = color('44', '44', '44'),
+ selection = { 'F7', 'F7', 'F7' },
+ logpanel = { 'E7', 'E7', 'E7' },
+ textpanel = { 'CF', 'CF', 'CF' },
+ linepanel = { 'A7', 'A7', 'A7' },
+ tippanel = { '44', '44', '44' },
--
- right = color('00', '00', 'FF'),
- wrong = color('FF', '00', '00'),
+ right = { '00', '00', 'FF' },
+ wrong = { 'FF', '00', '00' },
}
-colors.teal = colors.cyan
-colors.purple = colors.magenta
-
-lexer.colors = colors
-
--- defaults:
-
-local style_nothing = style { }
------ style_whitespace = style { }
-local style_comment = style { fore = colors.yellow }
-local style_string = style { fore = colors.magenta }
-local style_number = style { fore = colors.cyan }
-local style_keyword = style { fore = colors.blue, bold = true }
-local style_identifier = style_nothing
-local style_operator = style { fore = colors.blue }
-local style_error = style { fore = colors.red }
-local style_preproc = style { fore = colors.yellow, bold = true }
-local style_constant = style { fore = colors.cyan, bold = true }
-local style_variable = style { fore = colors.black }
-local style_function = style { fore = colors.black, bold = true }
-local style_class = style { fore = colors.black, bold = true }
-local style_type = style { fore = colors.blue }
-local style_label = style { fore = colors.red, bold = true }
-local style_regex = style { fore = colors.magenta }
-
--- reserved:
-
-local style_default = style { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel }
-local style_text = style { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel }
-local style_line_number = style { back = colors.linepanel }
-local style_bracelight = style { fore = colors.orange, bold = true }
-local style_bracebad = style { fore = colors.orange, bold = true }
-local style_indentguide = style { fore = colors.linepanel, back = colors.white }
-local style_calltip = style { fore = colors.white, back = colors.tippanel }
-local style_controlchar = style_nothing
-
--- extras:
-
-local style_quote = style { fore = colors.blue, bold = true }
-local style_special = style { fore = colors.blue }
-local style_extra = style { fore = colors.yellow }
-local style_embedded = style { fore = colors.black, bold = true }
------ style_char = style { fore = colors.magenta }
-local style_reserved = style { fore = colors.magenta, bold = true }
-local style_definition = style { fore = colors.black, bold = true }
-local style_okay = style { fore = colors.dark }
-local style_warning = style { fore = colors.orange }
-local style_invisible = style { back = colors.orange }
-local style_tag = style { fore = colors.cyan }
------ style_standout = style { fore = colors.orange, bold = true }
-local style_command = style { fore = colors.green, bold = true }
-local style_internal = style { fore = colors.orange, bold = true }
-
-local style_preamble = style { fore = colors.yellow }
-local style_grouping = style { fore = colors.red }
-local style_primitive = style { fore = colors.blue, bold = true }
-local style_plain = style { fore = colors.dark, bold = true }
-local style_user = style { fore = colors.green }
-local style_data = style { fore = colors.cyan, bold = true }
-
-
--- used by the generic lexer:
-
-lexer.style_nothing = style_nothing -- 0
------.whitespace = style_whitespace -- 1
-lexer.style_comment = style_comment -- 2
-lexer.style_string = style_string -- 3
-lexer.style_number = style_number -- 4
-lexer.style_keyword = style_keyword -- 5
-lexer.style_identifier = style_nothing -- 6
-lexer.style_operator = style_operator -- 7
-lexer.style_error = style_error -- 8
-lexer.style_preproc = style_preproc -- 9
-lexer.style_constant = style_constant -- 10
-lexer.style_variable = style_variable -- 11
-lexer.style_function = style_function -- 12
-lexer.style_class = style_class -- 13
-lexer.style_type = style_type -- 14
-lexer.style_label = style_label -- 15
-lexer.style_regex = style_regexp -- 16
-
-lexer.style_default = style_default -- 32
-lexer.style_line_number = style_line_number -- 33
-lexer.style_bracelight = style_bracelight -- 34
-lexer.style_bracebad = style_bracebad -- 35
-lexer.style_indentguide = style_indentguide -- 36
-lexer.style_calltip = style_calltip -- 37
-lexer.style_controlchar = style_controlchar -- 38
-
-local styles = { -- as we have globals we could do with less
-
- -- ["whitespace"] = style_whitespace, -- not to be set!
- ["default"] = style_nothing, -- else no good backtracking to start-of-child
- -- ["number"] = style_number,
- -- ["comment"] = style_comment,
- -- ["keyword"] = style_keyword,
- -- ["string"] = style_string,
- -- ["preproc"] = style_preproc,
- -- ["error"] = style_error,
- -- ["label"] = style_label,
-
- ["invisible"] = style_invisible,
- ["quote"] = style_quote,
- ["special"] = style_special,
- ["extra"] = style_extra,
- ["embedded"] = style_embedded,
- -- ["char"] = style_char,
- ["reserved"] = style_reserved,
- -- ["definition"] = style_definition,
- ["okay"] = style_okay,
- ["warning"] = style_warning,
- -- ["standout"] = style_standout,
- ["command"] = style_command,
- ["internal"] = style_internal,
- ["preamble"] = style_preamble,
- ["grouping"] = style_grouping,
- ["primitive"] = style_primitive,
- ["plain"] = style_plain,
- ["user"] = style_user,
- ["data"] = style_data,
-
- ["text"] = style_text, -- style_default
+local styles = {
+
+ ["whitespace"] = { },
+ ["default"] = { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel },
+ ["default"] = { font = font_name, size = font_size, fore = colors.black },
+ ["number"] = { fore = colors.cyan },
+ ["comment"] = { fore = colors.yellow },
+ ["keyword"] = { fore = colors.blue, bold = true },
+ ["string"] = { fore = colors.magenta },
+ -- ["preproc"] = { fore = colors.yellow, bold = true },
+ ["error"] = { fore = colors.red },
+ ["label"] = { fore = colors.red, bold = true },
+
+ ["nothing"] = { },
+ ["class"] = { fore = colors.black, bold = true },
+ ["function"] = { fore = colors.black, bold = true },
+ ["constant"] = { fore = colors.cyan, bold = true },
+ ["operator"] = { fore = colors.blue },
+ ["regex"] = { fore = colors.magenta },
+ ["preprocessor"] = { fore = colors.yellow, bold = true },
+ ["tag"] = { fore = colors.cyan },
+ ["type"] = { fore = colors.blue },
+ ["variable"] = { fore = colors.black },
+ ["identifier"] = { },
+
+ ["linenumber"] = { back = colors.linepanel },
+ ["bracelight"] = { fore = colors.orange, bold = true },
+ ["bracebad"] = { fore = colors.orange, bold = true },
+ ["controlchar"] = { },
+ ["indentguide"] = { fore = colors.linepanel, back = colors.white },
+ ["calltip"] = { fore = colors.white, back = colors.tippanel },
+
+ ["invisible"] = { back = colors.orange },
+ ["quote"] = { fore = colors.blue, bold = true },
+ ["special"] = { fore = colors.blue },
+ ["extra"] = { fore = colors.yellow },
+ ["embedded"] = { fore = colors.black, bold = true },
+ ["char"] = { fore = colors.magenta },
+ ["reserved"] = { fore = colors.magenta, bold = true },
+ ["definition"] = { fore = colors.black, bold = true },
+ ["okay"] = { fore = colors.dark },
+ ["warning"] = { fore = colors.orange },
+ ["standout"] = { fore = colors.orange, bold = true },
+ ["command"] = { fore = colors.green, bold = true },
+ ["internal"] = { fore = colors.orange, bold = true },
+ ["preamble"] = { fore = colors.yellow },
+ ["grouping"] = { fore = colors.red },
+ ["primitive"] = { fore = colors.blue, bold = true },
+ ["plain"] = { fore = colors.dark, bold = true },
+ ["user"] = { fore = colors.green },
+ ["data"] = { fore = colors.cyan, bold = true },
+
+ -- equal to default:
+
+ ["text"] = { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel },
+ ["text"] = { font = font_name, size = font_size, fore = colors.black },
}
-local styleset = { }
-
-for k, v in next, styles do
- styleset[#styleset+1] = { k, v }
-end
+local properties = {
+ ["fold.by.parsing"] = 1,
+ ["fold.by.indentation"] = 0,
+ ["fold.by.line"] = 0,
+ ["fold.line.comments"] = 0,
+ --
+ ["lexer.context.log"] = 1, -- log errors and warnings
+ ["lexer.context.trace"] = 0, -- show loading, initializations etc
+ ["lexer.context.detail"] = 0, -- show more detail when tracing
+ ["lexer.context.show"] = 0, -- show result of lexing
+ ["lexer.context.collapse"] = 0, -- make lexing results somewhat more efficient
+ ["lexer.context.inspect"] = 0, -- show some info about lexer (styles and so)
+ --
+-- ["lexer.context.log"] = 1, -- log errors and warnings
+-- ["lexer.context.trace"] = 1, -- show loading, initializations etc
+}
-context.styles = styles
-context.styleset = styleset
+local lexer = lexer or require("lexer")
+local context = lexer.context
-function context.stylesetcopy()
- local t = { }
- for i=1,#styleset do
- t[i] = styleset[i]
+if context then
+ context.inform("loading context (style) properties")
+ if context.registerstyles then
+ context.registerstyles(styles)
+ end
+ if context.registerproperties then
+ context.registerproperties(properties)
end
- t[#t+1] = { "whitespace", style_nothing }
- return t
end
--- We can be sparse if needed:
-
--- function context.newstyleset(list)
--- local t = { }
--- if list then
--- for i=1,#list do
--- t[list[i]] = true
--- end
--- end
--- return t
--- end
-
--- function context.usestyle(set,name)
--- set[name] = true
--- return name
--- end
-
--- function context.usestyleset(set)
--- local t = { }
--- for k, _ in next, set do
--- t[#t+1] = { k, styles[k] or styles.default }
--- end
--- end