diff options
Diffstat (limited to 'Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua')
-rw-r--r-- | Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua | 416 |
1 files changed, 80 insertions, 336 deletions
diff --git a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua index 7883177b41e..816d9583bca 100644 --- a/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua +++ b/Master/texmf-dist/context/data/scite/lexers/scite-context-lexer.lua @@ -10,9 +10,15 @@ local info = { -- todo: move all code here -- todo: explore adapted dll ... properties + init +-- todo: play with hotspot and other properties -- The fold and lex functions are copied and patched from original code by Mitchell (see --- lexer.lua). All errors are mine. +-- lexer.lua). All errors are mine. The ability to use lpeg is a real nice adition and a +-- brilliant move. The code is a byproduct of the (mainly Lua based) textadept (still a +-- rapidly moving target) that unfortunately misses a realtime output pane. On the other +-- hand, SciTE is somewhat crippled by the fact that we cannot pop in our own (language +-- dependent) lexer into the output pane (somehow the errorlist lexer is hard coded into +-- the editor). Hopefully that will change some day. -- -- Starting with SciTE version 3.20 there is an issue with coloring. As we still lack -- a connection with scite itself (properties as well as printing to the log pane) we @@ -26,7 +32,12 @@ local info = { -- After checking 3.24 and adapting to the new lexer tables things are okay again. So, -- this version assumes 3.24 or higher. In 3.24 we have a different token result, i.e. no -- longer a { tag, pattern } but just two return values. I didn't check other changes but --- will do that when I run into issues. +-- will do that when I run into issues. I had optimized these small tables by hashing which +-- was more efficient but this is no longer needed. +-- +-- In 3.3.1 another major change took place: some helper constants (maybe they're no +-- longer constants) and functions were moved into the lexer modules namespace but the +-- functions are assigned to the Lua module afterward so we cannot alias them beforehand. -- -- I've considered making a whole copy and patch the other functions too as we need -- an extra nesting model. However, I don't want to maintain too much. An unfortunate @@ -89,24 +100,38 @@ local concat = table.concat local global = _G local type, next, setmetatable, rawset = type, next, setmetatable, rawset -if lexer then - -- in recent c++ code the lexername and loading is hard coded -elseif _LEXERHOME then - dofile(_LEXERHOME .. '/lexer.lua') -- pre 3.03 situation -else - dofile('lexer.lua') -- whatever +-- less confusing as we also use lexer for the current lexer and local _M = lexer is just ugly + +local lexers = lexer + +-- these helpers are set afterwards so we delay their initialization ... there is no need to alias each time again + +local get_style_at, get_indent_amount, get_property, get_fold_level, FOLD_BASE, FOLD_HEADER, FOLD_BLANK, initialize + +initialize = function() + FOLD_BASE = lexers.FOLD_BASE or SC_FOLDLEVELBASE + FOLD_HEADER = lexers.FOLD_HEADER or SC_FOLDLEVELHEADERFLAG + FOLD_BLANK = lexers.FOLD_BLANK or SC_FOLDLEVELWHITEFLAG + get_style_at = lexers.get_style_at or GetStyleAt + get_indent_amount = lexers.get_indent_amount or GetIndentAmount + get_property = lexers.get_property or GetProperty + get_fold_level = lexers.get_fold_level or GetFoldLevel + -- + initialize = nil end -lexer.context = lexer.context or { } -local context = lexer.context +-- we create our own extra namespace for extensions and helpers + +lexers.context = lexers.context or { } +local context = lexers.context context.patterns = context.patterns or { } local patterns = context.patterns -lexer._CONTEXTEXTENSIONS = true +lexers._CONTEXTEXTENSIONS = true local locations = { - -- lexer.context.path, + -- lexers.context.path, "data", -- optional data directory "..", -- regular scite directory } @@ -133,8 +158,6 @@ function context.loaddefinitions(name) end end --- maybe more efficient: - function context.word_match(words,word_chars,case_insensitive) local chars = '%w_' -- maybe just "" when word_chars if word_chars then @@ -268,36 +291,9 @@ end patterns.wordtoken = R("az","AZ","\127\255") patterns.wordpattern = patterns.wordtoken^3 -- todo: if limit and #s < limit then --- -- pre 3.24: --- --- function context.checkedword(validwords,validminimum,s,i) -- ,limit --- if not validwords then -- or #s < validminimum then --- return true, { "text", i } -- { "default", i } --- else --- -- keys are lower --- local word = validwords[s] --- if word == s then --- return true, { "okay", i } -- exact match --- elseif word then --- return true, { "warning", i } -- case issue --- else --- local word = validwords[lower(s)] --- if word == s then --- return true, { "okay", i } -- exact match --- elseif word then --- return true, { "warning", i } -- case issue --- elseif upper(s) == s then --- return true, { "warning", i } -- probably a logo or acronym --- else --- return true, { "error", i } --- end --- end --- end --- end - function context.checkedword(validwords,validminimum,s,i) -- ,limit if not validwords then -- or #s < validminimum then - return true, "text", i -- { "default", i } + return true, "text", i -- true, "default", i else -- keys are lower local word = validwords[s] @@ -347,100 +343,12 @@ end -- overloaded functions -local FOLD_BASE = SC_FOLDLEVELBASE -local FOLD_HEADER = SC_FOLDLEVELHEADERFLAG -local FOLD_BLANK = SC_FOLDLEVELWHITEFLAG - -local get_style_at = GetStyleAt -local get_property = GetProperty -local get_indent_amount = GetIndentAmount - -local h_table, b_table, n_table = { }, { }, { } +local h_table, b_table, n_table = { }, { }, { } -- from the time small tables were used (optimization) setmetatable(h_table, { __index = function(t,level) local v = { level, FOLD_HEADER } t[level] = v return v end }) setmetatable(b_table, { __index = function(t,level) local v = { level, FOLD_BLANK } t[level] = v return v end }) setmetatable(n_table, { __index = function(t,level) local v = { level } t[level] = v return v end }) --- -- todo: move the local functions outside (see below) .. old variant < 3.24 --- --- local newline = P("\r\n") + S("\r\n") --- local p_yes = Cp() * Cs((1-newline)^1) * newline^-1 --- local p_nop = newline --- --- local function fold_by_parsing(text,start_pos,start_line,start_level,lexer) --- local foldsymbols = lexer._foldsymbols --- if not foldsymbols then --- return { } --- end --- local patterns = foldsymbols._patterns --- if not patterns then --- return { } --- end --- local nofpatterns = #patterns --- if nofpatterns == 0 then --- return { } --- end --- local folds = { } --- local line_num = start_line --- local prev_level = start_level --- local current_level = prev_level --- local validmatches = foldsymbols._validmatches --- if not validmatches then --- validmatches = { } --- for symbol, matches in next, foldsymbols do -- whatever = { start = 1, stop = -1 } --- if not find(symbol,"^_") then -- brrr --- for s, _ in next, matches do --- validmatches[s] = true --- end --- end --- end --- foldsymbols._validmatches = validmatches --- end --- -- of course we could instead build a nice lpeg checker .. something for --- -- a rainy day with a stack of new cd's at hand --- local function action_y(pos,line) --- for i=1,nofpatterns do --- for s, m in gmatch(line,patterns[i]) do --- if validmatches[m] then --- local symbols = foldsymbols[get_style_at(start_pos + pos + s - 1)] --- if symbols then --- local action = symbols[m] --- if action then --- if type(action) == 'number' then -- we could store this in validmatches if there was only one symbol category --- current_level = current_level + action --- else --- current_level = current_level + action(text,pos,line,s,m) --- end --- if current_level < FOLD_BASE then --- current_level = FOLD_BASE --- end --- end --- end --- end --- end --- end --- if current_level > prev_level then --- folds[line_num] = h_table[prev_level] -- { prev_level, FOLD_HEADER } --- else --- folds[line_num] = n_table[prev_level] -- { prev_level } --- end --- prev_level = current_level --- line_num = line_num + 1 --- end --- local function action_n() --- folds[line_num] = b_table[prev_level] -- { prev_level, FOLD_BLANK } --- line_num = line_num + 1 --- end --- if lexer._reset_parser then --- lexer._reset_parser() --- end --- local lpegpattern = (p_yes/action_y + p_nop/action_n)^0 -- not too efficient but indirect function calls are neither but --- lpegmatch(lpegpattern,text) -- keys are not pressed that fast ... large files are slow anyway --- return folds --- end - --- The 3.24 variant; no longer subtable optimization is needed: - local newline = P("\r\n") + S("\r\n") local p_yes = Cp() * Cs((1-newline)^1) * newline^-1 local p_nop = newline @@ -458,38 +366,6 @@ local function fold_by_parsing(text,start_pos,start_line,start_level,lexer) -- if fold_pattern then -- if no functions are found then we could have a faster one - - -- fold_pattern = Cp() * C(fold_pattern) * Carg(1) / function(s,match,pos) - -- local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)] - -- local l = symbols and symbols[match] - -- if l then - -- local t = type(l) - -- if t == 'number' then - -- current_level = current_level + l - -- elseif t == 'function' then - -- current_level = current_level + l(text, pos, line, s, match) - -- end - -- end - -- end - -- fold_pattern = (fold_pattern + P(1))^0 - -- local action_y = function(pos,line) - -- lpegmatch(fold_pattern,line,1,pos) - -- folds[line_num] = prev_level - -- if current_level > prev_level then - -- folds[line_num] = prev_level + FOLD_HEADER - -- end - -- if current_level < FOLD_BASE then - -- current_level = FOLD_BASE - -- end - -- prev_level = current_level - -- line_num = line_num + 1 - -- end - -- local action_n = function() - -- folds[line_num] = prev_level + FOLD_BLANK - -- line_num = line_num + 1 - -- end - -- pattern = (p_yes/action_y + p_nop/action_n)^0 - fold_pattern = Cp() * C(fold_pattern) / function(s,match) local symbols = fold_symbols[get_style_at(start_pos + s)] if symbols then @@ -562,99 +438,16 @@ local function fold_by_parsing(text,start_pos,start_line,start_level,lexer) prev_level = _start_level_ current_level = prev_level lpegmatch(pattern,text) --- return folds -local t = folds -folds = nil -return t -- so folds can be collected + -- make folds collectable + local t = folds + folds = nil + return t end folders[lexer] = folder end return folder(text,start_pos,start_line,start_level,lexer) end --- local function fold_by_indentation(text,start_pos,start_line,start_level) --- local folds = { } --- local current_line = start_line --- local prev_level = start_level --- for line in gmatch(text,'[\t ]*(.-)\r?\n') do --- if line ~= "" then --- local current_level = FOLD_BASE + get_indent_amount(current_line) --- if current_level > prev_level then -- next level --- local i = current_line - 1 --- while true do --- local f = folds[i] --- if f and f[2] == FOLD_BLANK then --- i = i - 1 --- else --- break --- end --- end --- local f = folds[i] --- if f then --- f[2] = FOLD_HEADER --- end -- low indent --- folds[current_line] = n_table[current_level] -- { current_level } -- high indent --- elseif current_level < prev_level then -- prev level --- local f = folds[current_line - 1] --- if f then --- f[1] = prev_level -- high indent --- end --- folds[current_line] = n_table[current_level] -- { current_level } -- low indent --- else -- same level --- folds[current_line] = n_table[prev_level] -- { prev_level } --- end --- prev_level = current_level --- else --- folds[current_line] = b_table[prev_level] -- { prev_level, FOLD_BLANK } --- end --- current_line = current_line + 1 --- end --- return folds --- end - --- local function fold_by_indentation(text,start_pos,start_line,start_level) --- local folds = { } --- local current_line = start_line --- local prev_level = start_level --- for line in gmatch(text,'[\t ]*(.-)\r?\n') do --- if line ~= '' then --- local current_level = FOLD_BASE + get_indent_amount(current_line) --- if current_level > prev_level then -- next level --- local i = current_line - 1 --- local f --- while true do --- f = folds[i] --- if not f then --- break --- elseif f[2] == FOLD_BLANK then --- i = i - 1 --- else --- f[2] = FOLD_HEADER -- low indent --- break --- end --- end --- folds[current_line] = { current_level } -- high indent --- elseif current_level < prev_level then -- prev level --- local f = folds[current_line - 1] --- if f then --- f[1] = prev_level -- high indent --- end --- folds[current_line] = { current_level } -- low indent --- else -- same level --- folds[current_line] = { prev_level } --- end --- prev_level = current_level --- else --- folds[current_line] = { prev_level, FOLD_BLANK } --- end --- current_line = current_line + 1 --- end --- for line, level in next, folds do --- folds[line] = level[1] + (level[2] or 0) --- end --- return folds --- end - local folds, current_line, prev_level local function action_y() @@ -707,18 +500,17 @@ local function fold_by_indentation(text,start_pos,start_line,start_level) for line, level in next, folds do folds[line] = level[1] + (level[2] or 0) end - -- done --- return folds -local t = folds -folds = nil -return t -- so folds can be collected + -- done, make folds collectable + local t = folds + folds = nil + return t end local function fold_by_line(text,start_pos,start_line,start_level) local folds = { } -- can also be lpeg'd for _ in gmatch(text,".-\r?\n") do - folds[start_line] = n_table[start_level] -- { start_level } + folds[start_line] = n_table[start_level] -- { start_level } -- stile tables ? needs checking start_line = start_line + 1 end return folds @@ -733,10 +525,13 @@ function context.fold(text,start_pos,start_line,start_level) -- hm, we had size if text == '' then return { } end - local lexer = global._LEXER - local fold_by_lexer = lexer._fold + if initialize then + initialize() + end + local lexer = global._LEXER + local fold_by_lexer = lexer._fold local fold_by_symbols = lexer._foldsymbols - local filesize = 0 -- we don't know that + local filesize = 0 -- we don't know that if fold_by_lexer then if filesize <= threshold_by_lexer then return fold_by_lexer(text,start_pos,start_line,start_level,lexer) @@ -759,16 +554,16 @@ end -- The following code is mostly unchanged: -local function add_rule(lexer, id, rule) +local function add_rule(lexer,id,rule) if not lexer._RULES then - lexer._RULES = {} - lexer._RULEORDER = {} + lexer._RULES = { } + lexer._RULEORDER = { } end lexer._RULES[id] = rule lexer._RULEORDER[#lexer._RULEORDER + 1] = id end -local function add_style(lexer, token_name, style) +local function add_style(lexer,token_name,style) local len = lexer._STYLES.len if len == 32 then len = len + 8 @@ -777,39 +572,40 @@ local function add_style(lexer, token_name, style) print('Too many styles defined (128 MAX)') end lexer._TOKENS[token_name] = len - lexer._STYLES[len] = style - lexer._STYLES.len = len + 1 + lexer._STYLES[len] = style + lexer._STYLES.len = len + 1 end local function join_tokens(lexer) - local patterns, order = lexer._RULES, lexer._RULEORDER + local patterns = lexer._RULES + local order = lexer._RULEORDER local token_rule = patterns[order[1]] for i=2,#order do token_rule = token_rule + patterns[order[i]] end lexer._TOKENRULE = token_rule - return lexer._TOKENRULE + return token_rule end local function add_lexer(grammar, lexer, token_rule) local token_rule = join_tokens(lexer) local lexer_name = lexer._NAME - local children = lexer._CHILDREN + local children = lexer._CHILDREN for i=1,#children do local child = children[i] if child._CHILDREN then add_lexer(grammar, child) end - local child_name = child._NAME - local rules = child._EMBEDDEDRULES[lexer_name] - local rules_token_rule = grammar['__'..child_name] or rules.token_rule - grammar[child_name] = (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1 * V(lexer_name) - local embedded_child = '_' .. child_name + local child_name = child._NAME + local rules = child._EMBEDDEDRULES[lexer_name] + local rules_token_rule = grammar['__'..child_name] or rules.token_rule + grammar[child_name] = (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1 * V(lexer_name) + local embedded_child = '_' .. child_name grammar[embedded_child] = rules.start_rule * (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1 - token_rule = V(embedded_child) + token_rule + token_rule = V(embedded_child) + token_rule end grammar['__' .. lexer_name] = token_rule - grammar[lexer_name] = token_rule^0 + grammar[lexer_name] = token_rule^0 end local function build_grammar(lexer, initial_rule) @@ -829,58 +625,21 @@ local function build_grammar(lexer, initial_rule) end -- so far. We need these local functions in the next one. --- --- Before 3.24 we had tokens[..] = { category, position }, now it's a two values. local lineparsers = { } function context.lex(text,init_style) local lexer = global._LEXER local grammar = lexer._GRAMMAR + if initialize then + initialize() + end if not grammar then return { } elseif lexer._LEXBYLINE then -- we could keep token local tokens = { } local offset = 0 local noftokens = 0 - -- -- pre 3.24 - -- - -- for line in gmatch(text,'[^\r\n]*\r?\n?') do -- could be an lpeg - -- local line_tokens = lpegmatch(grammar,line) - -- if line_tokens then - -- for i=1,#line_tokens do - -- local token = line_tokens[i] - -- token[2] = token[2] + offset - -- noftokens = noftokens + 1 - -- tokens[noftokens] = token - -- end - -- end - -- offset = offset + #line - -- if noftokens > 0 and tokens[noftokens][2] ~= offset then - -- noftokens = noftokens + 1 - -- tokens[noftokens] = { 'default', offset + 1 } - -- end - -- end - - -- for line in gmatch(text,'[^\r\n]*\r?\n?') do - -- local line_tokens = lpegmatch(grammar,line) - -- if line_tokens then - -- for i=1,#line_tokens,2 do - -- noftokens = noftokens + 1 - -- tokens[noftokens] = line_tokens[i] - -- noftokens = noftokens + 1 - -- tokens[noftokens] = line_tokens[i + 1] + offset - -- end - -- end - -- offset = offset + #line - -- if noftokens > 0 and tokens[noftokens] ~= offset then - -- noftokens = noftokens + 1 - -- tokens[noftokens] = 'default' - -- noftokens = noftokens + 1 - -- tokens[noftokens] = offset + 1 - -- end - -- end - local lineparser = lineparsers[lexer] if not lineparser then -- probably a cmt is more efficient lineparser = C((1-newline)^0 * newline) / function(line) @@ -957,10 +716,10 @@ function context.token(name, patt) return patt * Cc(name) * Cp() end -lexer.fold = context.fold -lexer.lex = context.lex -lexer.token = context.token -lexer.exact_match = context.exact_match +lexers.fold = context.fold +lexers.lex = context.lex +lexers.token = context.token +lexers.exact_match = context.exact_match -- helper .. alas ... the lexer's lua instance is rather crippled .. not even -- math is part of it @@ -1053,23 +812,6 @@ function lpeg.utfchartabletopattern(list) return make(tree) end --- patterns.invisibles = --- P(utfchar(0x00A0)) -- nbsp --- + P(utfchar(0x2000)) -- enquad --- + P(utfchar(0x2001)) -- emquad --- + P(utfchar(0x2002)) -- enspace --- + P(utfchar(0x2003)) -- emspace --- + P(utfchar(0x2004)) -- threeperemspace --- + P(utfchar(0x2005)) -- fourperemspace --- + P(utfchar(0x2006)) -- sixperemspace --- + P(utfchar(0x2007)) -- figurespace --- + P(utfchar(0x2008)) -- punctuationspace --- + P(utfchar(0x2009)) -- breakablethinspace --- + P(utfchar(0x200A)) -- hairspace --- + P(utfchar(0x200B)) -- zerowidthspace --- + P(utfchar(0x202F)) -- narrownobreakspace --- + P(utfchar(0x205F)) -- math thinspace - patterns.invisibles = lpeg.utfchartabletopattern { utfchar(0x00A0), -- nbsp utfchar(0x2000), -- enquad @@ -1098,3 +840,5 @@ patterns.iwordpattern = patterns.iwordtoken^3 -- In order to deal with some bug in additional styles (I have no cue what is -- wrong, but additional styles get ignored and clash somehow) I just copy the -- original lexer code ... see original for comments. + +return lexers |