expltools (5dec24)

git-svn-id: svn://tug.org/texlive/trunk@73049 c570f23f-e606-0410-a88d-b1316a301751

5 files changed, 613 insertions, 0 deletions

diff --git a/Master/texmf-dist/scripts/expltools/explcheck-cli.lua b/Master/texmf-dist/scripts/expltools/explcheck-cli.lua
new file mode 100755
index 00000000000..69e5e1a1cee
--- /dev/null
+++ b/Master/texmf-dist/scripts/expltools/explcheck-cli.lua
@@ -0,0 +1,113 @@
+-- A command-line interface for the static analyzer explcheck.
+
+local new_issues = require("explcheck-issues")
+local format = require("explcheck-format")
+
+local preprocessing = require("explcheck-preprocessing")
+-- local lexical_analysis = require("explcheck-lexical-analysis")
+-- local syntactic_analysis = require("explcheck-syntactic-analysis")
+-- local semantic_analysis = require("explcheck-semantic-analysis")
+-- local pseudo_flow_analysis = require("explcheck-pseudo-flow-analysis")
+
+-- Deduplicate pathnames.
+local function deduplicate_pathnames(pathnames)
+  local deduplicated_pathnames = {}
+  local seen_pathnames = {}
+  for _, pathname in ipairs(pathnames) do
+    if seen_pathnames[pathname] ~= nil then
+      goto continue
+    end
+    seen_pathnames[pathname] = true
+    table.insert(deduplicated_pathnames, pathname)
+    ::continue::
+  end
+  return deduplicated_pathnames
+end
+
+-- Process all input files.
+local function main(pathnames, warnings_are_errors, max_line_length)
+  local num_warnings = 0
+  local num_errors = 0
+
+  print("Checking " .. #pathnames .. " files")
+
+  for pathname_number, pathname in ipairs(pathnames) do
+
+    -- Load an input file.
+    local file = assert(io.open(pathname, "r"), "Could not open " .. pathname .. " for reading")
+    local content = assert(file:read("*a"))
+    assert(file:close())
+    local issues = new_issues()
+
+    -- Run all processing steps.
+    local line_starting_byte_numbers, _ = preprocessing(issues, content, max_line_length)
+    if #issues.errors > 0 then
+      goto continue
+    end
+    -- lexical_analysis(issues)
+    -- syntactic_analysis(issues)
+    -- semantic_analysis(issues)
+    -- pseudo_flow_analysis(issues)
+
+    -- Print warnings and errors.
+    ::continue::
+    num_warnings = num_warnings + #issues.warnings
+    num_errors = num_errors + #issues.errors
+    format.print_results(pathname, issues, line_starting_byte_numbers, pathname_number == #pathnames)
+  end
+
+  -- Print a summary.
+  format.print_summary(#pathnames, num_warnings, num_errors)
+
+  if(num_errors > 0) then
+    return 1
+  elseif(warnings_are_errors and num_warnings > 0) then
+    return 2
+  else
+    return 0
+  end
+end
+
+local function print_usage()
+  print("Usage: " .. arg[0] .. " [OPTIONS] FILENAMES\n")
+  print("Run static analysis on expl3 files.\n")
+  print("Options:")
+  print("\t--max-line-length=N\tThe maximum line length before the warning S103 (Line too long) is produced.")
+  print("\t--warnings-are-errors\tProduce a non-zero exit code if any warnings are produced by the analysis.")
+end
+
+if #arg == 0 then
+  print_usage()
+  os.exit(1)
+else
+  -- Collect arguments.
+  local pathnames = {}
+  local warnings_are_errors = false
+  local only_pathnames_from_now_on = false
+  local max_line_length = nil
+  for _, argument in ipairs(arg) do
+    if only_pathnames_from_now_on then
+      table.insert(pathnames, argument)
+    elseif argument == "--" then
+      only_pathnames_from_now_on = true
+    elseif argument == "--help" then
+      print_usage()
+      os.exit(0)
+    elseif argument == "--warnings-are-errors" then
+      warnings_are_errors = true
+    elseif argument:sub(1, 18) == "--max-line-length=" then
+      max_line_length = tonumber(argument:sub(19))
+    elseif argument:sub(1, 2) == "--" then
+      -- An unknown argument
+      print_usage()
+      os.exit(1)
+    else
+      table.insert(pathnames, argument)
+    end
+  end
+  pathnames = deduplicate_pathnames(pathnames)
+
+  -- Run the analysis.
+  local exit_code = main(pathnames, warnings_are_errors, max_line_length)
+  os.exit(exit_code)
+end
diff --git a/Master/texmf-dist/scripts/expltools/explcheck-format.lua b/Master/texmf-dist/scripts/expltools/explcheck-format.lua
new file mode 100755
index 00000000000..5b64ab51b90
--- /dev/null
+++ b/Master/texmf-dist/scripts/expltools/explcheck-format.lua
@@ -0,0 +1,233 @@
+-- Formatting for the command-line interface of the static analyzer explcheck.
+
+-- Transform a singular into plural if the count is zero or greater than two.
+local function pluralize(singular, count)
+  if count == 1 then
+    return singular
+  else
+    return singular .. "s"
+  end
+end
+
+-- Shorten a pathname, so that it does not exceed maximum length.
+local function format_pathname(pathname, max_length)
+  -- First, replace path segments with `/.../`, keeping other segments.
+  local first_iteration = true
+  while #pathname > max_length do
+    local pattern
+    if first_iteration then
+      pattern = "([^/]*)/[^/]*/(.*)"
+    else
+      pattern = "([^/]*)/%.%.%./[^/]*/(.*)"
+    end
+    local prefix_start, _, prefix, suffix = pathname:find(pattern)
+    if prefix_start == nil or prefix_start > 1 then
+      break
+    end
+    pathname = prefix .. "/.../" .. suffix
+    first_iteration = false
+  end
+  -- If this isn't enough, remove the initial path segment and prefix the filename with `...`.
+  if #pathname > max_length then
+    local pattern
+    if first_iteration then
+      pattern = "([^/]*/?)(.*)"
+    else
+      pattern = "([^/]*/%.%.%./)(.*)"
+    end
+    local prefix_start, _, _, suffix = pathname:find(pattern)
+    if prefix_start == 1 then
+      pathname = ".../" .. suffix
+      if #pathname > max_length then
+        pathname = "..." .. suffix:sub(-(max_length - #("...")))
+      end
+    end
+  end
+  return pathname
+end
+
+-- Colorize a string using ASCII color codes.
+local function colorize(text, ...)
+  local buffer = {}
+  for _, color_code in ipairs({...}) do
+    table.insert(buffer, "\27[")
+    table.insert(buffer, tostring(color_code))
+    table.insert(buffer, "m")
+  end
+  table.insert(buffer, text)
+  table.insert(buffer, "\27[0m")
+  return table.concat(buffer, "")
+end
+
+-- Remove ASCII color codes from a string.
+local function decolorize(text)
+  return text:gsub("\27%[[0-9]+m", "")
+end
+
+-- Convert a byte number in a file to a line and column number in a file.
+local function convert_byte_to_line_and_column(line_starting_byte_numbers, byte_number)
+  local line_number = 0
+  for _, line_starting_byte_number in ipairs(line_starting_byte_numbers) do
+    if line_starting_byte_number > byte_number then
+      break
+    end
+    line_number = line_number + 1
+  end
+  assert(line_number > 0)
+  local line_starting_byte_number = line_starting_byte_numbers[line_number]
+  assert(line_starting_byte_number <= byte_number)
+  local column_number = byte_number - line_starting_byte_number + 1
+  return line_number, column_number
+end
+
+-- Print the results of analyzing a file.
+local function print_results(pathname, issues, line_starting_byte_numbers, is_last_file)
+  -- Display an overview.
+  local all_issues = {}
+  local status
+  if(#issues.errors > 0) then
+    status = (
+      colorize(
+        (
+          tostring(#issues.errors)
+          .. " "
+          .. pluralize("error", #issues.errors)
+        ), 1, 31
+      )
+    )
+    table.insert(all_issues, issues.errors)
+    if(#issues.warnings > 0) then
+      status = (
+        status
+        .. ", "
+        .. colorize(
+          (
+            tostring(#issues.warnings)
+            .. " "
+            .. pluralize("warning", #issues.warnings)
+          ), 1, 33
+        )
+      )
+      table.insert(all_issues, issues.warnings)
+    end
+  elseif(#issues.warnings > 0) then
+    status = colorize(
+      (
+        tostring(#issues.warnings)
+        .. " "
+        .. pluralize("warning", #issues.warnings)
+      ), 1, 33
+    )
+    table.insert(all_issues, issues.warnings)
+  else
+    status = colorize("OK", 1, 32)
+  end
+
+  local max_overview_length = 72
+  local prefix = "Checking "
+  local formatted_pathname = format_pathname(
+    pathname,
+    math.max(
+      (
+        max_overview_length
+        - #prefix
+        - #(" ")
+        - #decolorize(status)
+      ), 1
+    )
+  )
+  local overview = (
+    prefix
+    .. formatted_pathname
+    .. (" "):rep(
+      math.max(
+        (
+          max_overview_length
+          - #prefix
+          - #decolorize(status)
+          - #formatted_pathname
+        ), 1
+      )
+    )
+    .. status
+  )
+  io.write("\n" .. overview)
+
+  -- Display the errors, followed by warnings.
+  if #all_issues > 0 then
+    for _, warnings_or_errors in ipairs(all_issues) do
+      print()
+      -- Display the warnings/errors.
+      for _, issue in ipairs(issues.sort(warnings_or_errors)) do
+        local code = issue[1]
+        local message = issue[2]
+        local range = issue[3]
+        local position = ":"
+        if range ~= nil then
+          local line_number, column_number = convert_byte_to_line_and_column(line_starting_byte_numbers, range[1])
+          position = position .. tostring(line_number) .. ":" .. tostring(column_number) .. ":"
+        end
+        local max_line_length = 88
+        local reserved_position_length = 10
+        local reserved_suffix_length = 30
+        local label_indent = (" "):rep(4)
+        local suffix = code:upper() .. " " .. message
+        formatted_pathname = format_pathname(
+          pathname,
+          math.max(
+            (
+              max_line_length
+              - #label_indent
+              - reserved_position_length
+              - #(" ")
+              - math.max(#suffix, reserved_suffix_length)
+            ), 1
+          )
+        )
+        local line = (
+          label_indent
+          .. formatted_pathname
+          .. position
+          .. (" "):rep(
+            math.max(
+              (
+                max_line_length
+                - #label_indent
+                - #formatted_pathname
+                - #decolorize(position)
+                - math.max(#suffix, reserved_suffix_length)
+              ), 1
+            )
+          )
+          .. suffix
+          .. (" "):rep(math.max(reserved_suffix_length - #suffix, 0))
+        )
+        io.write("\n" .. line)
+      end
+    end
+    if(not is_last_file) then
+      print()
+    end
+  end
+end
+
+-- Print the summary results of analyzing multiple files.
+local function print_summary(num_pathnames, num_warnings, num_errors)
+  io.write("\n\nTotal: ")
+
+  local errors_message = tostring(num_errors) .. " " .. pluralize("error", num_errors)
+  errors_message = colorize(errors_message, 1, (num_errors > 0 and 31) or 32)
+  io.write(errors_message .. ", ")
+
+  local warnings_message = tostring(num_warnings) .. " " .. pluralize("warning", num_warnings)
+  warnings_message = colorize(warnings_message, 1, (num_warnings > 0 and 33) or 32)
+  io.write(warnings_message .. " in ")
+
+  print(tostring(num_pathnames) .. " " .. pluralize("file", num_pathnames))
+end
+
+return {
+  convert_byte_to_line_and_column = convert_byte_to_line_and_column,
+  print_results = print_results,
+  print_summary = print_summary,
+}
diff --git a/Master/texmf-dist/scripts/expltools/explcheck-issues.lua b/Master/texmf-dist/scripts/expltools/explcheck-issues.lua
new file mode 100755
index 00000000000..9f7c1be75b6
--- /dev/null
+++ b/Master/texmf-dist/scripts/expltools/explcheck-issues.lua
@@ -0,0 +1,81 @@
+-- A registry of warnings and errors identified by different processing steps.
+
+local Issues = {}
+
+function Issues.new(cls)
+  local new_object = {}
+  setmetatable(new_object, cls)
+  cls.__index = cls
+  new_object.errors = {}
+  new_object.warnings = {}
+  new_object.ignored_issues = {}
+  return new_object
+end
+
+-- Convert an issue identifier to either a table of warnings or a table of errors.
+function Issues:_get_issue_table(identifier)
+  local prefix = identifier:sub(1, 1)
+  if prefix == "s" or prefix == "w" then
+    return self.warnings
+  elseif prefix == "t" or prefix == "e" then
+    return self.errors
+  else
+    assert(false, 'Identifier "' .. identifier .. '" has an unknown prefix "' .. prefix .. '"')
+  end
+end
+
+-- Add an issue to the table of issues.
+function Issues:add(identifier, message, range_start, range_end)
+  if self.ignored_issues[identifier] then
+    return
+  end
+  local issue_table = self:_get_issue_table(identifier)
+  local range
+  if range_start == nil then
+    range = nil
+  else
+    range = {range_start, range_end}
+  end
+  table.insert(issue_table, {identifier, message, range})
+end
+
+-- Prevent an issue from being present in the table of issues.
+function Issues:ignore(identifier)
+  -- Remove the issue if it has already been added.
+  local issue_table = self:_get_issue_table(identifier)
+  local updated_issues = {}
+  for _, issue in ipairs(issue_table) do
+    if issue[1] ~= identifier then
+      table.insert(updated_issues, issue)
+    end
+  end
+  for issue_index, issue in ipairs(updated_issues) do
+    issue_table[issue_index] = issue
+  end
+  for issue_index = #updated_issues + 1, #issue_table, 1 do
+    issue_table[issue_index] = nil
+  end
+  -- Prevent the issue from being added later.
+  self.ignored_issues[identifier] = true
+end
+
+-- Sort the warnings/errors using location as the primary key.
+function Issues.sort(warnings_and_errors)
+  local sorted_warnings_and_errors = {}
+  for _, issue in ipairs(warnings_and_errors) do
+    local code = issue[1]
+    local message = issue[2]
+    local range = issue[3]
+    table.insert(sorted_warnings_and_errors, {code, message, range})
+  end
+  table.sort(sorted_warnings_and_errors, function(a, b)
+    local a_code, b_code = a[1], b[1]
+    local a_range, b_range = (a[3] and a[3][1]) or 0, (b[3] and b[3][1]) or 0
+    return a_range < b_range or (a_range == b_range and a_code < b_code)
+  end)
+  return sorted_warnings_and_errors
+end
+
+return function()
+  return Issues:new()
+end
diff --git a/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua b/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua
new file mode 100755
index 00000000000..8ab1cdc3d06
--- /dev/null
+++ b/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua
@@ -0,0 +1,183 @@
+-- The preprocessing step of static analysis determines which parts of the input files contain expl3 code.
+
+local lpeg = require("lpeg")
+local Cp, P, R, S, V = lpeg.Cp, lpeg.P, lpeg.R, lpeg.S, lpeg.V
+
+-- Define base parsers.
+---- Generic
+local any = P(1)
+local eof = -any
+
+---- Tokens
+local lbrace = P("{")
+local rbrace = P("}")
+local percent_sign = P("%")
+local backslash = P([[\]])
+local letter = R("AZ","az")
+local underscore = P("_")
+local colon = P(":")
+
+---- Spacing
+local newline = (
+  P("\n")
+  + P("\r\n")
+  + P("\r")
+)
+local linechar = any - newline
+local spacechar = S("\t ")
+local optional_spaces = spacechar^0
+local optional_spaces_and_newline_after_cs = (
+  optional_spaces
+  * (
+    newline
+    * optional_spaces
+  )^-1
+)
+
+-- Define intermediate parsers.
+---- Parts of TeX syntax
+local comment = (
+  percent_sign
+  * linechar^0
+  * newline
+  * optional_spaces
+)
+local argument = (
+  lbrace
+  * (
+    comment
+    + (any - rbrace)
+  )^0
+  * rbrace
+)
+local expl3like_control_sequence = (
+  backslash
+  * (letter - underscore - colon)^1
+  * (underscore + colon)
+  * letter^1
+)
+
+---- Standard delimiters
+local provides = (
+  P([[\ProvidesExpl]])
+  * (
+      P("Package")
+      + P("Class")
+      + P("File")
+    )
+  * optional_spaces_and_newline_after_cs
+  * comment^0
+  * argument
+  * comment^0
+  * argument
+  * comment^0
+  * argument
+  * comment^0
+  * argument
+)
+local expl_syntax_on = P([[\ExplSyntaxOn]])
+local expl_syntax_off = P([[\ExplSyntaxOff]])
+
+local function preprocessing(issues, content, max_line_length)
+  if max_line_length == nil then
+    max_line_length = 80
+  end
+
+  -- Determine the bytes where lines begin.
+  local line_starting_byte_numbers = {}
+  local function record_line(line_start)
+    table.insert(line_starting_byte_numbers, line_start)
+  end
+  local function line_too_long(range_start, range_end)
+    issues:add('s103', 'line too long', range_start, range_end + 1)
+  end
+  local line_numbers_grammar = (
+    Cp() / record_line
+    * (
+      (
+        (
+          Cp() * linechar^(max_line_length + 1) * Cp() / line_too_long
+          + linechar^0
+        )
+        * newline
+        * Cp()
+      ) / record_line
+    )^0
+  )
+  lpeg.match(line_numbers_grammar, content)
+  -- Determine which parts of the input files contain expl3 code.
+  local expl_ranges = {}
+  local function capture_range(range_start, range_end)
+    table.insert(expl_ranges, {range_start, range_end + 1})
+  end
+  local function unexpected_pattern(pattern, code, message, test)
+    return Cp() * pattern * Cp() / function(range_start, range_end)
+      if test == nil or test() then
+        issues:add(code, message, range_start, range_end + 1)
+      end
+    end
+  end
+  local num_provides = 0
+  local analysis_grammar = P{
+    "Root";
+    Root = (
+      (
+        V"NonExplPart"
+        * V"ExplPart" / capture_range
+      )^0
+      * V"NonExplPart"
+    ),
+    NonExplPart = (
+      (
+        unexpected_pattern(
+          V"Closer",
+          "w101",
+          "unexpected delimiters"
+        )
+        + unexpected_pattern(
+            expl3like_control_sequence,
+            "e102",
+            "expl3 control sequences in non-expl3 parts"
+          )
+        + (any - V"Opener")
+      )^0
+    ),
+    ExplPart = (
+      V"Opener"
+      * Cp()
+      * (
+          unexpected_pattern(
+            V"Opener",
+            "w101",
+            "unexpected delimiters"
+          )
+          + (any - V"Closer")
+        )^0
+      * Cp()
+      * (V"Closer" + eof)
+    ),
+    Opener = (
+      expl_syntax_on
+      + unexpected_pattern(
+        provides,
+        "e104",
+        [[multiple delimiters `\ProvidesExpl*` in a single file]],
+        function()
+          num_provides = num_provides + 1
+          return num_provides > 1
+        end
+      )
+    ),
+    Closer = expl_syntax_off,
+  }
+  lpeg.match(analysis_grammar, content)
+  -- If no parts were detected, assume that the whole input file is in expl3.
+  if(#expl_ranges == 0 and #content > 0) then
+    table.insert(expl_ranges, {0, #content})
+    issues:add('w100', 'no standard delimiters')
+    issues:ignore('e102')
+  end
+  return line_starting_byte_numbers, expl_ranges
+end
+
+return preprocessing
diff --git a/Master/texmf-dist/scripts/expltools/explcheck.lua b/Master/texmf-dist/scripts/expltools/explcheck.lua
new file mode 100755
index 00000000000..f2b1cfaa6d7
--- /dev/null
+++ b/Master/texmf-dist/scripts/expltools/explcheck.lua
@@ -0,0 +1,3 @@
+#!/usr/bin/env texlua
+-- A command-line interface for the static analyzer explcheck.
+require("explcheck-cli")