diff options
| author | Karl Berry <karl@freefriends.org> | 2024-12-05 21:37:35 +0000 |
|---|---|---|
| committer | Karl Berry <karl@freefriends.org> | 2024-12-05 21:37:35 +0000 |
| commit | ea85773b9fb11d3ee3449c6e31582678967bd0a4 (patch) | |
| tree | 75efc3e95de0acfcdadebc10b8c44a801068f815 /Master | |
| parent | 3f8dd2e8b435c75014a9b7a7256258dfc09f815b (diff) | |
expltools (5dec24)
git-svn-id: svn://tug.org/texlive/trunk@73049 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master')
55 files changed, 2948 insertions, 3 deletions
diff --git a/Master/bin/aarch64-linux/explcheck b/Master/bin/aarch64-linux/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/aarch64-linux/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/amd64-freebsd/explcheck b/Master/bin/amd64-freebsd/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/amd64-freebsd/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/amd64-netbsd/explcheck b/Master/bin/amd64-netbsd/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/amd64-netbsd/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/armhf-linux/explcheck b/Master/bin/armhf-linux/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/armhf-linux/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/i386-freebsd/explcheck b/Master/bin/i386-freebsd/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/i386-freebsd/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/i386-linux/explcheck b/Master/bin/i386-linux/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/i386-linux/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/i386-netbsd/explcheck b/Master/bin/i386-netbsd/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/i386-netbsd/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/i386-solaris/explcheck b/Master/bin/i386-solaris/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/i386-solaris/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/universal-darwin/explcheck b/Master/bin/universal-darwin/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/universal-darwin/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/windows/explcheck.exe b/Master/bin/windows/explcheck.exe Binary files differnew file mode 100755 index 00000000000..3332231b08c --- /dev/null +++ b/Master/bin/windows/explcheck.exe diff --git a/Master/bin/x86_64-cygwin/explcheck b/Master/bin/x86_64-cygwin/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/x86_64-cygwin/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/x86_64-darwinlegacy/explcheck b/Master/bin/x86_64-darwinlegacy/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/x86_64-darwinlegacy/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/x86_64-linux/explcheck b/Master/bin/x86_64-linux/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/x86_64-linux/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/x86_64-linuxmusl/explcheck b/Master/bin/x86_64-linuxmusl/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/x86_64-linuxmusl/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/bin/x86_64-solaris/explcheck b/Master/bin/x86_64-solaris/explcheck new file mode 120000 index 00000000000..c2893e6af1c --- /dev/null +++ b/Master/bin/x86_64-solaris/explcheck @@ -0,0 +1 @@ +../../texmf-dist/scripts/expltools/explcheck.lua
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/CHANGES.md b/Master/texmf-dist/doc/support/expltools/CHANGES.md new file mode 100644 index 00000000000..7391dae728b --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/CHANGES.md @@ -0,0 +1,5 @@ +# Changes + +## 2024-12-04 + +- The first release of the project that implements the preprocessing step. diff --git a/Master/texmf-dist/doc/support/expltools/README.md b/Master/texmf-dist/doc/support/expltools/README.md new file mode 100644 index 00000000000..04b44adc5b6 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/README.md @@ -0,0 +1,41 @@ +# Expltools: Development tools for expl3 programmers + +This repository contains the code and documentation of an expl3 static analysis tool `explcheck` outlined in the following devlog posts: + +1. [Introduction][1] +2. [Requirements][2] +3. [Related work][3] +4. [Design][4] + +On September 6, 2024, the tool has been [accepted for funding][5] by [the TeX Development Fund][6]. +The full text of the project proposal, which summarizes devlog posts 1–4 is available [here][7]. + +In the future, this repository may also contain the code of other useful development tools for expl3 programmers, such as a command-line utility similar to `grep` that will ignore whitespaces and newlines as well as other tools. + + [1]: https://witiko.github.io/Expl3-Linter-1/ + [2]: https://witiko.github.io/Expl3-Linter-2/ + [3]: https://witiko.github.io/Expl3-Linter-3/ + [4]: https://witiko.github.io/Expl3-Linter-4/ + [5]: https://tug.org/tc/devfund/grants.html + [6]: https://tug.org/tc/devfund/application.html + [7]: https://tug.org/tc/devfund/documents/2024-09-expltools.pdf + +## Usage + +You can use the tool from command-line as follows: + +``` +$ explcheck [options] [filenames] +``` + +## Notes to distributors + +The file `explcheck.lua` should be installed in the TDS directory `scripts/expltools/explcheck`. Furthermore, it should be made executable, and either symlinked to system directories as `expltools` on Unix or have a wrapper `expltools.exe` installed on Windows. + +## Authors + +- Vít Starý Novotný (<mailto:witiko@mail.muni.cz>) + +## License + +This material is dual-licensed under GNU GPL 2.0 or later and LPPL 1.3c or later. diff --git a/Master/texmf-dist/doc/support/expltools/e102.tex b/Master/texmf-dist/doc/support/expltools/e102.tex new file mode 100644 index 00000000000..0085df8b174 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/e102.tex @@ -0,0 +1,12 @@ +\ProvidesExplFile{example.tex}{2024-04-09}{1.0.0}{An example file} +\tl_new:N + \g_example_tl +\tl_gset:Nn + \g_example_tl + { Hello,~ } +\tl_gput_right:Nn + \g_example_tl + { world! } +\ExplSyntaxOff +\tl_use:N % error on this line + \g_example_tl % error on this line diff --git a/Master/texmf-dist/doc/support/expltools/e104.tex b/Master/texmf-dist/doc/support/expltools/e104.tex new file mode 100644 index 00000000000..24f441d729b --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/e104.tex @@ -0,0 +1,5 @@ +\ProvidesExplPackage + {example.sty}{2024-04-09}{1.0.0}{An example package} +\ExplSyntaxOff +\ProvidesExplClass % error on this line + {example.cls}{2024-04-09}{1.0.0}{An example class} diff --git a/Master/texmf-dist/doc/support/expltools/latexmkrc b/Master/texmf-dist/doc/support/expltools/latexmkrc new file mode 100644 index 00000000000..28c69b9594e --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/latexmkrc @@ -0,0 +1,9 @@ +# Custom latexmk configuration +## Enable shell escape +set_tex_cmds('--shell-escape %O %S'); + +## Output PDF by default and use LuaLaTeX +$pdf_mode = 4; + +## Treat warnings as errors +$warnings_as_errors = 1; diff --git a/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools.sty b/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools.sty new file mode 100644 index 00000000000..b7503826bfe --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools.sty @@ -0,0 +1,9 @@ +\NeedsTeXFormat{LaTeX2e} +\ProvidesExplPackage + {markdownthemewitiko_expltools}% + {2024-12-04}% + {0.0.2}% + {Snippets for typesetting the documentation of expltools} +\markdownSetupSnippet + { metadata } + { raw_attribute } diff --git a/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools_explcheck_project-proposal.sty b/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools_explcheck_project-proposal.sty new file mode 100644 index 00000000000..ab78ceba196 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools_explcheck_project-proposal.sty @@ -0,0 +1,19 @@ +\NeedsTeXFormat{LaTeX2e} +\ProvidesExplPackage + {markdownthemewitiko_expltools_explcheck_project-proposal}% + {2024-09-06}% + {0.0.1}% + {Snippets for typesetting the documentation of the project proposal for the % + expl3 analysis tool} +\markdownSetupSnippet + { body } + { + raw_attribute, + code = { + \tableofcontents + \clearpage + }, + renderers = { + link = { \href { #3 } { #1 } } + } + } diff --git a/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools_explcheck_warnings-and-errors.sty b/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools_explcheck_warnings-and-errors.sty new file mode 100644 index 00000000000..aa07a0f3d76 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/markdownthemewitiko_expltools_explcheck_warnings-and-errors.sty @@ -0,0 +1,211 @@ +\NeedsTeXFormat{LaTeX2e} +\ProvidesExplPackage + {markdownthemewitiko_expltools_explcheck_warnings-and-errors}% + {2024-11-29}% + {0.0.3}% + {Snippets for typesetting the documentation of the warnings and errors for % + the expl3 analysis tool} +\int_new:N + \g__expltools_explcheck_subsection_int +\tl_new:N + \g__expltools_explcheck_subsection_tl +\tl_new:N + \l__expltools_explcheck_current_label_tl +\tl_new:N + \l__expltools_explcheck_current_identifier_tl +\prop_new:N + \g__expltools_explcheck_labels_prop +\cs_new:Nn + \__expltools_explcheck_increment_counters: + { + \int_gincr:N + \g__expltools_explcheck_subsection_int + \int_compare:nTF + { \g__expltools_explcheck_subsection_int < 10 } + { + \tl_gset:Nn + \g__expltools_explcheck_subsection_tl + { 0 } + \tl_gput_right:NV + \g__expltools_explcheck_subsection_tl + \g__expltools_explcheck_subsection_int + } + { + \tl_gset:NV + \g__expltools_explcheck_subsection_tl + \g__expltools_explcheck_subsection_int + } + \tl_if_empty:NF + \l__expltools_explcheck_current_identifier_tl + { + \prop_gput:NVV + \g__expltools_explcheck_labels_prop + \l__expltools_explcheck_current_identifier_tl + \l__expltools_explcheck_current_label_tl + \exp_args:NV + \label + \l__expltools_explcheck_current_identifier_tl + } + \exp_args:NV + \index + \l__expltools_explcheck_current_label_tl + } +\markdownSetupSnippet + { body } + { + citations, + content_blocks, + definition_lists, + fenced_code, + notes, + header_attributes, + html, + raw_attribute, + relative_references, + renderers = { + underscore = { + \discretionary + { _ \textnormal { - } } + { } + { _ } + }, + link = { + \tl_set:Nn + \l_tmpa_tl + { #3 } + \tl_set:Nx + \l_tmpb_tl + { + \tl_range:Nnn + \l_tmpa_tl + { 1 } + { 1 } + } + \str_if_eq:NNT + \l_tmpb_tl + \c_hash_str + { + \tl_set:Nx + \l_tmpb_tl + { + \tl_range:Nnn + \l_tmpa_tl + { 2 } + { -1 } + } + \tl_set:Nn + \l_tmpa_tl + { \hyperref [ } + \tl_put_right:NV + \l_tmpa_tl + \l_tmpb_tl + \tl_put_right:Nn + \l_tmpa_tl + { ] } + \prop_get:NVN + \g__expltools_explcheck_labels_prop + \l_tmpb_tl + \l_tmpb_tl + \tl_put_right:Nx + \l_tmpa_tl + { { \l_tmpb_tl } } + \tl_use:N + \l_tmpa_tl + } + }, + headingOne = { + \section { #1 } + \int_gset:Nn + \g__expltools_explcheck_subsection_int + { 0 } + \tl_gset:Nn + \g__expltools_explcheck_subsection_tl + { 00 } + }, + headerAttributeContextBegin = { + \group_begin: + \markdownSetup + { + renderers = { + attributeIdentifier = { + \tl_set:Nn + \l__expltools_explcheck_current_identifier_tl + { ##1 } + }, + attributeKeyValue = { + \str_if_eq:nnT + { ##1 } + { label } + { + \tl_set:Nx + \l_tmpa_tl + { \str_uppercase:n { ##2 } } + \str_if_eq:VVF + \l_tmpa_tl + \l__expltools_explcheck_current_label_tl + { + \msg_error:nnVV + { expltools / explcheck } + { unexpected-label } + \l_tmpa_tl + \l__expltools_explcheck_current_label_tl + } + } + }, + attributeClassName = { + \tl_set:Nx + \l__expltools_explcheck_current_label_tl + { + \str_uppercase:n { ##1 } + \thesection + \g__expltools_explcheck_subsection_tl + } + \markdownSetup + { + renderers = { + headingTwo = { + \subsection + { + ####1~[ \l__expltools_explcheck_current_label_tl ] + } + \__expltools_explcheck_increment_counters: + }, + headingThree = { + \subsubsection + { + ####1~[ \l__expltools_explcheck_current_label_tl ] + } + \__expltools_explcheck_increment_counters: + }, + }, + } + } + }, + } + }, + headerAttributeContextEnd = { + \group_end: + }, + contentBlock = { + \str_case:nnF + { #1 } + { + { tex } + { + \markdownRendererInputFencedCode + { #3 } + { tex } + { tex } + } + } + { + \markdownInput + { #3 } + } + }, + }, + } +\msg_new:nnn + { expltools / explcheck } + { unexpected-label } + { Expected~section~label~"#1",~but~"#2"~was~generated } diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal-01-introduction.md b/Master/texmf-dist/doc/support/expltools/project-proposal-01-introduction.md new file mode 100644 index 00000000000..c59fae55315 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal-01-introduction.md @@ -0,0 +1,28 @@ +# Introduction + +In 2021, I used [the expl3 programming language][7] for the first time in my life. I had already been eyeing expl3 for some time and, when it came to defining a `\LaTeX`{=tex}-specific interface for processing YAML metadata in [version 2.11.0][1] of [the Markdown package for `\TeX`{=tex}][2], I took the plunge. + +After two and a half years, approximately 3.5k out of the 5k lines of `\TeX`{=tex} code in [version 3.5.0][3] of the Markdown package are written in expl3. I also developed several consumer products with it, and I have written [three][4] [journal][5] [articles][6] for my local `\TeX`{=tex} users group about it. Needless to say, expl3 has been a blast for me! + +In the Markdown package, each change is reviewed by a number of automated static analysis tools (so-called *linters*), which look for programming errors in the code. While these tools don't catch all programming errors, they have proven extremely useful in catching the typos that inevitably start trickling in after 2AM. + +Since the Markdown package contains code in different programming languages, we use many different linters such as [`shellcheck`][8] for shell scripts, [`luacheck`][9] for Lua, and [`flake8`][10] and [`pytype`][11] for Python. However, since no linters for expl3 exist, typos are often only caught by regression tests, human reviewers, and sometimes even by our users after a release. Nobody is happy about this. + +Earlier this year, I realized that, unlike `\TeX`{=tex}, expl3 has the following two properties that seem to make it well-suited to static analysis: + +1. Simple uniform syntax: (Almost) all operations are expressed as function calls. This Lisp-like quality makes is easy to convert well-behaved expl3 programs that only use high-level interfaces into abstract syntax trees. This is a prerequisite for accurate static analysis. +2. Explicit type and scope: Variables and constants are separate from functions. Each variable is either local or global. Variables and constants are explicitly typed. This information makes it easy to detect common programming errors related to the incorrect use of variables. + +For the longest time, I wanted to try my hand at building a linter from the ground up. Therefore, I decided to kill two birds with one stone and improve the tooling for expl3 while learning something new along the way by building a linter for expl3. + + [1]: https://github.com/Witiko/markdown/releases/tag/2.11.0 + [2]: https://ctan.org/pkg/markdown + [3]: https://github.com/Witiko/markdown/releases/tag/3.5.0 + [4]: http://dx.doi.org/10.5300/2022-1-4/35 + [5]: http://dx.doi.org/10.5300/2023-1-2/3 + [6]: http://dx.doi.org/10.5300/2023-3-4/153 + [7]: http://mirrors.ctan.org/macros/latex/required/l3kernel/expl3.pdf + [8]: https://www.shellcheck.net/ + [9]: https://github.com/mpeterv/luacheck + [10]: https://pypi.org/project/flake8/ + [11]: https://pypi.org/project/pytype/
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal-02-requirements.md b/Master/texmf-dist/doc/support/expltools/project-proposal-02-requirements.md new file mode 100644 index 00000000000..95626c0a874 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal-02-requirements.md @@ -0,0 +1,106 @@ +# Requirements + +In this section, I outline the requirements for the linter. These will form the basis of the design and the implementation. + +## Functional requirements + +The linter should accept a list of input expl3 files. Then, the linter should process each input file and print out issues it has identified with the file. + +Initially, the linter should recognize at least the following types of issues: + +- Style: + - Overly long lines + - Missing stylistic white-spaces + - Malformed names of functions, variables, constants, quarks, and scan marks +- Functions: + - Multiply defined functions and function variants + - Calling undefined functions and function variants + - Calling [deprecated and removed][2] functions + - Unknown argument specifiers + - Unexpected function call arguments + - Unused private functions and function variants +- Variables: + - Multiply declared variables and constants + - Using undefined variables and constants + - Using variables of incompatible types + - Using deprecated and removed variables and constants + - Setting constants and undeclared variables + - Unused variables and constants + - Locally setting global variables and vice versa + +## Non-functional requirements + +### Issues + +The linter should make distinction between two types of issues: warnings and errors. As a rule of thumb, whereas warnings are suggestions about best practices, errors will likely result in runtime errors. + +Here are three examples of warnings: + +- Missing stylistic white-spaces around curly braces +- Using deprecated functions and variables +- Unused variable or constant + +Here are three examples of errors: + +- Using an undefined message +- Calling a function with a `V`-type argument with a variable or constant that does not support `V`-type expansion +- Multiply declared variable or constant + +The overriding design goal for the initial releases of the linter should be the simplicity of implementation and robustness to unexpected input. For all issues, the linter should prefer [precision over recall][1] and only print them out when it is reasonably certain that it has understood the code, even at the expense of potentially missing some issues. + +Each issue should be assigned a unique identifier. Using these identifiers, issues can be disabled globally using a config file, for individual input files from the command-line, and for sections of code or individual lines of code using `\TeX`{=tex} comments. + +### Architecture + +To make the linter easy to use in continuous integration pipelines, it should be written in Lua 5.3 using just the standard Lua library. One possible exception is checking whether functions, variables, and other symbols from the input files are expl3 build-ins. This may require using the `texlua` interpreter and a minimal `\TeX`{=tex} distribution that includes the `\LaTeX`{=tex}3 kernel, at least initially. + +The linter should process input files in a series of discrete steps, which should be represented as Lua modules. Users should be able to import the modules into their Lua code and use them independently on the rest of the linter. + +Each step should process the input received from the previous step, identify any issues with the input, and transform the input to an output format appropriate for the next step. The default command-line script for the linter should execute all steps and print out issues from all steps. Users should be able to easily adapt the default script in the following ways: + +1. Change how the linter discovers input files. +2. Change or replace processing steps or insert additional steps. +3. Change how the linter reacts to issues with the input files. + +The linter should integrate easily with text editors. Therefore, the linter should either directly support the [language server protocol (LSP)][6] or be designed in a way that makes it easy to write an LSP wrapper for it. + +### Validation + +As a part of the test-driven development paradigm, all issues identified by a processing step should have at least one associated test in the code repository of the linter. All tests should be executed periodically during the development of the linter. + +As a part of the dogfooding paradigm, the linter should be used in the continuous integration pipeline of [the Markdown Package for `\TeX`{=tex}][3] since the initial releases of the linter in order to collect early user feedback. Other early adopters are also welcome to try the initial releases of the linter and report issues to its code repository. + +At some point, a larger-scale validation should be conducted as an experimental part of a TUGboat article that will introduce the linter to the wider `\TeX`{=tex} community. In this validation, all expl3 packages from current and historical `\TeX`{=tex} Live distributions should be processed with the linter. The results should be evaluated both quantitatively and qualitatively. While the quantitative evaluation should focus mainly on trends in how expl3 is used in packages, the qualitative evaluation should explore the shortcomings of the linter and ideas for future improvements. + +### License terms + +The linter should be [free software][8] and dual-licensed under [the GNU General Public License (GNU GPL) 2.0][12] or later and [the `\LaTeX`{=tex} Project Public License (LPPL) 1.3c][13] or later. + +The option to use GNU GPL 2.0 or later is motivated by the fact that GNU GPL 2.0 and 3.0 are [mutually incompatible][14]. Supporting both GNU GPL 2.0 and 3.0 extends the number of free open-source projects that will be able to alter and redistribute the linter. + +The option to use LPPL 1.3c is motivated by the fact that it imposes very few licensing restrictions on `\TeX`{=tex} users. Furthermore, it also preserves the integrity of `\TeX`{=tex} distributions by enforcing its naming and maintenance clauses, which ensure ongoing project stewardship and prevent confusion between modified and official versions. + +Admittedly, GNU GPL and LPPL may seem like an unusual combination, since GNU GPL is a copyleft license whereas LPPL is a permissive license. However, there are strategic benefits to offering both. + +We would offer LPPL as the primary license for derivative works within the `\TeX`{=tex} ecosystem. One downside of using LPPL is that it could potentially allow bad actors to create proprietary derivative works without contributing back to the original project. However, this trade-off helps maintain the `\TeX`{=tex} ecosystem's consistency and reliability. Incidentally, there is an element of trust in the `\TeX`{=tex} user community to voluntarily contribute improvements back, even though the license itself does not mandate it. + +We would offer GNU GPL as an alternative license for derivative works outside the `\TeX`{=tex} ecosystem. The key benefit of including GNU GPL is that it enables the code to be integrated into free open-source projects, especially those with licenses that are incompatible with LPPL's naming requirements. This opens the door for broader collaboration with the free software community. + +Notably, GNU GPL creates a one-way licensing situation: Once a derivative work is licensed under GNU GPL, it cannot be legally re-licensed under a less restrictive license like LPPL. As a result, we wouldn't be able to incorporate changes made to GNU GPL-licensed works back into the original project under LPPL without also creating two forks of the project licensed under GNU GPL 2.0 and GNU GPL 3.0, respectively. While this might seem like a downside, I view it as an important counterbalance to the potential for proprietary derivative works under LPPL. + +In summary, this dual-licensing approach allows us to maintain the integrity of the `\TeX`{=tex} ecosystem while making the project more accessible to the broader free open-source community. It provides flexibility for different use cases, though we will need to carefully manage contributions to ensure compliance with all licenses. + + [1]: https://developers.google.com/machine-learning/crash-course/classification/precision-and-recall + [2]: https://github.com/latex3/latex3/blob/main/l3kernel/doc/l3obsolete.txt + [3]: https://github.com/witiko/markdown + [4]: /Expl3-Linter-1 + [5]: /Expl3-Linter-2.5 + [6]: https://microsoft.github.io/language-server-protocol/ + [7]: https://www.gnu.org/licenses/lgpl-3.0.en.html + [8]: https://www.gnu.org/philosophy/free-sw.html + [9]: https://www.gnu.org/licenses/gpl-3.0.html + [10]: https://www.gnu.org/licenses/license-list.html#GPLCompatibleLicenses + [11]: /Expl3-Linter-3 + [12]: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html + [13]: https://www.latex-project.org/lppl/lppl-1-3c/ + [14]: https://www.gnu.org/licenses/rms-why-gplv3.en.html
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal-03-related-work.md b/Master/texmf-dist/doc/support/expltools/project-proposal-03-related-work.md new file mode 100644 index 00000000000..198275d71eb --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal-03-related-work.md @@ -0,0 +1,64 @@ +# Related work + +In this section, I review the related work in the analysis of `\TeX`{=tex} programs and documents. This related work should be considered in the design of the linter and reused whenever it is appropriate and compatible with the license of the linter. + +## Unravel + +[The unravel package][11] by Bruno Le Floch analyses of expl3 programs as well as `\TeX`{=tex} programs and documents in general. The package was suggested to me as related work by Joseph Wright in personal correspondence. + +Unlike a linter, which performs _static_ analysis by leafing through the code and makes suggestions, unravel is a _debugger_ that is used for _dynamic_ analysis. It allows the user to step through the execution of code while providing extra information about the state of `\TeX`{=tex}. Unravel is written in expl3 and emulates `\TeX`{=tex} primitives using expl3 functions. It has been released under the `\LaTeX`{=tex} Project Public License (LPPL) 1.3c. + +While both linters and debuggers are valuable in producing bug-free software, linters prevent bugs by proactively pointing out potential bugs without any user interaction, whereas debuggers are typically used interactively to determine the cause of a bug after it has already manifested. + +## Chktex, chklref, cmdtrack, lacheck, match_parens, nag, and tex2tok + +The Comprehensive `\TeX`{=tex} Archive Network (CTAN) lists related software projects on the topics of [debuging support][12] and [`\LaTeX`{=tex} quality][13], some of which I list in this section. + +[The chktex package][14] by Jens T. Berger Thielemann is a linter for the static analysis of `\LaTeX`{=tex} documents. It has been written in ANSI C and released under the GNU GPL 2.0 license. The types of issues with the input files and how they are reported to the user can be configured to some extent from the command-line and using configuration files to a larger extent. Chktex is extensible and, in addition to the configuration of existing issues, it allows the definition of new types of issues using regular expressions. + +[The lacheck package][17] by Kresten Krab Thorup is a linter for the static analysis of `\LaTeX`{=tex} documents. Similarly to chktex, lacheck has been written in ANSI C and released under the GNU GPL 1.0 license. Unlike chktex, lacheck cannot be configured either from the command-line or using configuration files. + +[The chklref package][15] by Jérôme Lelong is a linter for the static analysis of `\LaTeX`{=tex} documents. It has been written in Perl and released under the GNU GPL 3.0 license. Unlike chktex, chklref focuses just on the detection of unused labels, which often accumulate over the lifetime of a `\LaTeX`{=tex} document. + +[The match_parens package][18] by Wybo Dekker is a linter for the static analysis of expl3 programs as well as `\TeX`{=tex} programs and documents in general. It has been written in Ruby and released under the GNU GPL 1.0 license. Unlike chktex, match_parens focuses just on the detection of mismatched paired punctuation, such as parentheses, braces, brackets, and quotation marks. As such, it can also be used for the static analysis of natural text as well as programs and documents in programming and markup languages that use paired punctuation in its syntax. + +[The cmdtrack package][16] by Michael John Downes is a debugger for the dynamic analysis of `\LaTeX`{=tex} documents. It has been written in `\LaTeX`{=tex} and released under the LPPL 1.0 license. It detects unused user-defined commands, which also often accumulate over the lifetime of a `\LaTeX`{=tex} document, and mentions them in the `.log` file produced during the compilation of a `\LaTeX`{=tex} document. + +[The nag package][19] by Ulrich Michael Schwarz is a debugger for the dynamic analysis of `\LaTeX`{=tex} documents. Similarly to cmdtrack, nag has also been written in `\LaTeX`{=tex} and released under the LPPL 1.0 license. It detects the use of obsolete `\LaTeX`{=tex} commands, document classes, and packages and mentions them in the `.log` file produced during the compilation of a `\LaTeX`{=tex} document. + +[The tex2tok package][20] by Jonathan Fine is a debugger for the dynamic analysis of expl3 programs as well as `\TeX`{=tex} programs and documents in general. It has been written in `\TeX`{=tex} and released under the GNU GPL 2.0 license. It executes a `\TeX`{=tex} file and produces a new `.tok` file with a list of `\TeX`{=tex} tokens in the file. Compared to static analysis, the dynamic analysis ensures correct category codes. However, it requires the execution of the `\TeX`{=tex} file, which may take long or never complete in the presence of bugs in the code. + +## Luacheck and flake8 + +[Luacheck][21] by Peter Melnichenko and [flake8][22] by Tarek Ziade are linters for the static analysis of Lua and Python programs, respectively. They have been written in Lua and Python, respectively, and released under the MIT license. Both tools are widely used and should inform the design of my linter in terms of architecture, configuration, and extensibility. + +Similar to chktex, the types of issues with the input files and how they are reported to the user can be configured from the command-line and using configuration files. Additionally, the reporting can also be enabled or disabled in the code of the analyzed program using inline comments. + +Unlike luacheck, which is not extensible at the time of writing and only allows the configuration of existing issues, flake8 supports Python extensions that can add support for new types of issues. + +## TeXLab and digestif + +[TeXLab][23] by Eric and Patrick Förscher and [digestif][24] by Augusto Stoffel are [language servers][6] for the static analysis of `\TeX`{=tex} programs and documents. They have been written in Rust and Lua, respectively, and released under the GNU GPL 3.0 license. The language servers were suggested to me as related work by Michal Hoftich at TUG 2024. + +Whereas `\TeX`{=tex}Lab focuses on `\LaTeX`{=tex} documents, digestif also supports other formats such as `\Hologo{ConTeXt}`{=tex} and GNU Texinfo. Neither `\TeX`{=tex}Lab nor digestif support expl3 code at the time of writing. + +In terms of the programming language, license, and scope, digestif seems like the most related work to my linter. However, its GNU GPL 3.0 license is incompatible with the dual license of the linter, which prohibits code reuse. + + [1]: /Expl3-Linter-2 + [2]: /Expl3-Linter-2#license-terms + [5]: /Expl3-Linter-2.5 + [6]: https://microsoft.github.io/language-server-protocol/ + [11]: https://ctan.org/pkg/unravel + [12]: https://ctan.org/topic/debug-supp + [13]: https://ctan.org/topic/latex-qual + [14]: https://ctan.org/pkg/chktex + [15]: https://ctan.org/pkg/chklref + [16]: https://ctan.org/pkg/cmdtrack + [17]: https://ctan.org/pkg/lacheck + [18]: https://ctan.org/pkg/match_parens + [19]: https://ctan.org/pkg/nag + [20]: https://ctan.org/pkg/tex2tok + [21]: https://github.com/mpeterv/luacheck + [22]: https://github.com/pycqa/flake8 + [23]: https://ctan.org/pkg/texlab + [24]: https://ctan.org/pkg/digestif
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal-04-design.md b/Master/texmf-dist/doc/support/expltools/project-proposal-04-design.md new file mode 100644 index 00000000000..b80b0eade70 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal-04-design.md @@ -0,0 +1,75 @@ +# Design + +In this section, I outline the design of the linter and I create a code +repository for the linter. + +## Processing steps + +As outlined in the requirements, the linter will process input files in a +series of discrete steps, each represented by a single Lua module. + +Here are the individual processing steps that should be supported by the linter: + +1. Preprocessing: Determine which parts of the input files contain expl3 code. +2. Lexical analysis: Convert expl3 parts of the input files into `\TeX`{=tex} tokens. +3. Syntactic analysis: Convert `\TeX`{=tex} tokens into a tree of function calls. +4. Semantic analysis: Determine the meaning of the different function calls. +5. Flow analysis: Determine additional emergent properties of the code. + +## Warnings and errors + +As also outlined in the requirements, each processing step should identify +issues with the output and produce either a warning or an error. Furthermore, +the requirements list 16 types of issues that should be recognized by the linter +at a minimum. Lastly, the requirements require that, as a part of the +test-driven development paradigm, all issues identified by a processing step +should have at least one associated test in the code repository of the linter. + +In [a document titled "Warnings and errors for the expl3 analysis tool"][6], +I compiled a list of 66 warnings and errors that should be recognized by the +initial version of the linter. For each issue, there is also an example of +expl3 code with and without the issue. These examples can be directly converted +to tests and used during the development of the corresponding processing steps. + +## Limitations + +Due to the dynamic nature of `\TeX`{=tex}, initial versions of the linter will make some +naïve assumption and simplification during the analysis, such as: + +- Assume default expl3 [catcodes][8] everywhere. +- Ignore non-expl3 and third-party code. +- Do not analyze expansion and key–value calls. + +As a result, the initial version of the linter may not have a sufficient +understanding of expl3 code to support proper flow analysis. Instead, the +initial version of the linter may need to use pseudo-flow-analysis that would +check for simple cases of the warnings and errors from flow analysis. Future +versions of the linter should improve their code understanding to the point +where proper flow analysis can be performed. + +The warnings and errors in this document do not cover the complete expl3 +language. The limitations currently include the areas outlined in a section +of [the document with warnings and errors][6] titled "Caveats". Future versions +of the linter should improve the coverage. + +## Code repository + +I created a repository [`witiko/expltools`][3] titled "Development tools for +expl3 programmers" at GitHub. As outlined in the requirements, I dual-license the code under [GNU GPL 2.0][10] or later and [LPPL 1.3c][11] or later. + +Furthermore, I also [registered][7] the expl3 prefix `expltools`, so that it +can be used in the documentation for the linter, in other supporting expl3 code +used in the linter, and also possibly in development tools for expl3 +programmers other than the linter. + + [1]: /Expl3-Linter-2 + [2]: /Expl3-Linter-3 + [3]: https://github.com/Witiko/expltools + [4]: https://github.com/astoff/digestif/blob/7962d25/digestif/Parser.lua + [5]: https://ctan.org/pkg/digestif + [6]: https://github.com/Witiko/expltools/releases/download/2024-09-06/warnings-and-errors.pdf + [7]: https://github.com/latex3/latex3/pull/1556 + [8]: https://en.wikibooks.org/wiki/TeX/catcode + [9]: /Expl3-Linter-2#license-terms + [10]: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html + [11]: https://www.latex-project.org/lppl/lppl-1-3c/
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal-05-benefits-of-grant-funding.md b/Master/texmf-dist/doc/support/expltools/project-proposal-05-benefits-of-grant-funding.md new file mode 100644 index 00000000000..4ae89f495cf --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal-05-benefits-of-grant-funding.md @@ -0,0 +1,7 @@ +# Benefits of grant funding + +Securing this grant will significantly enhance my ability to dedicate focused and uninterrupted time to this project, enabling me to allocate at least two full weeks of work over the next 12 months. This concentrated effort will be far more productive than the fragmented hours I currently manage to find after a long day's work, ensuring that I can make substantial progress. + +Additionally, the grant will serve as a meaningful endorsement of the project's value, reflecting the community's interest and support. This recognition will not only reinforce the importance of the work but also help attract other contributors who share a commitment to advancing the project. + +Finally, the visibility that comes with receiving this grant will elevate the project's profile, making it more prominent within the TeX community and beyond. This increased visibility is crucial for attracting further interest, feedback, and potential collaborations, all of which are vital for the project's long-term success.
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal.cls b/Master/texmf-dist/doc/support/expltools/project-proposal.cls new file mode 100644 index 00000000000..9e4156d4391 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal.cls @@ -0,0 +1,16 @@ +\NeedsTeXFormat{LaTeX2e} +\ProvidesClass% + {project-proposal}% + [2024-09-06 0.0.1 Document class for typesetting the project proposal for % + the expl3 analysis tool] +\LoadClass{article} +\RequirePackage{lua-widow-control} +\RequirePackage{markdown} +\markdownSetup { + import = { + witiko/expltools = metadata, + witiko/expltools/explcheck/project-proposal = body, + } +} +\RequirePackage{hologo} +\RequirePackage{hyperref} diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal.pdf b/Master/texmf-dist/doc/support/expltools/project-proposal.pdf Binary files differnew file mode 100644 index 00000000000..a90ba61363b --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal.pdf diff --git a/Master/texmf-dist/doc/support/expltools/project-proposal.tex b/Master/texmf-dist/doc/support/expltools/project-proposal.tex new file mode 100644 index 00000000000..6161fe1105c --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/project-proposal.tex @@ -0,0 +1,14 @@ +\documentclass{project-proposal} +\begin{yaml}[snippet=metadata] +title: 'Project proposal: expl3 static analyzer' +author: Vít Starý Novotný +date: 2024-09-06 +\end{yaml} +\begin{document} +\markdownSetup{snippet=body} +\markdownInput{project-proposal-01-introduction.md} +\markdownInput{project-proposal-02-requirements.md} +\markdownInput{project-proposal-03-related-work.md} +\markdownInput{project-proposal-04-design.md} +\markdownInput{project-proposal-05-benefits-of-grant-funding.md} +\end{document} diff --git a/Master/texmf-dist/doc/support/expltools/s103.tex b/Master/texmf-dist/doc/support/expltools/s103.tex new file mode 100644 index 00000000000..898d1160da7 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/s103.tex @@ -0,0 +1,2 @@ +This line is not very long, because it is 80 characters long, not 81 characters. +This line is very long, because it is 81 characters long. % warning on this line diff --git a/Master/texmf-dist/doc/support/expltools/w100.tex b/Master/texmf-dist/doc/support/expltools/w100.tex new file mode 100644 index 00000000000..227c228efef --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/w100.tex @@ -0,0 +1,11 @@ +% file-wide warning +\tl_new:N + \g_example_tl +\tl_gset:Nn + \g_example_tl + { Hello,~ } +\tl_gput_right:Nn + \g_example_tl + { world! } +\tl_use:N + \g_example_tl diff --git a/Master/texmf-dist/doc/support/expltools/w101.tex b/Master/texmf-dist/doc/support/expltools/w101.tex new file mode 100644 index 00000000000..0af25ae1aab --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/w101.tex @@ -0,0 +1,14 @@ +\input expl3-generic +\ExplSyntaxOff % warning on this line +\ExplSyntaxOn +\tl_new:N + \g_example_tl +\tl_gset:Nn + \g_example_tl + { Hello,~ } +\ExplSyntaxOn % warning on this line +\tl_gput_right:Nn + \g_example_tl + { world! } +\tl_use:N + \g_example_tl diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors-00-introduction.md b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-00-introduction.md new file mode 100644 index 00000000000..d28b738b806 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-00-introduction.md @@ -0,0 +1,43 @@ +# Introduction + +In this document, I list the warnings and errors for the different processing steps of the expl3 linter [@starynovotny2024static3]: + +Preprocessing + +: Determine which parts of the input files contain expl3 code. + +Lexical analysis + +: Convert expl3 parts of the input files into `\TeX`{=tex} tokens. + +Syntactic analysis + +: Convert `\TeX`{=tex} tokens into a tree of function calls. + +Semantic analysis + +: Determine the meaning of the different function calls. + +Flow analysis + +: Determine additional emergent properties of the code. + +For each warning and error, I specify a unique identifier that can be used to disable the warning or error, a description of the condition for the warning or error, and a code example that demonstrates the condition and serves as a test case for the linter. + +Warnings and errors have different types that decides the prefix of their idenfitiers: + +- Warnings: + + `S` + : Style warnings + + `W` + : Other warnings + +- Errors: + + `T` + : Type errors + + `E` + : Other errors
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors-01-preprocessing.md b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-01-preprocessing.md new file mode 100644 index 00000000000..a88efa28c7a --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-01-preprocessing.md @@ -0,0 +1,29 @@ +# Preprocessing +In the preprocessing step, the expl3 analysis tool determines which parts of the input files contain expl3 code. Inline `\TeX`{=tex} comments that disable warnings and errors are also analyzed in this step. + +## No standard delimiters {.w label=w100} +An input file contains no delimiters such as `\ExplSyntaxOn`, `\ExplSyntaxOff`, `\ProvidesExplPackage`, `\ProvidesExplClass`, and `\ProvidesExplFile` [@latexteam2024interfaces, Section 2.1]. The analysis tool should assume that the whole input file is in expl3. + + /w100.tex + +## Unexpected delimiters {.w label=w101} +An input file contains extraneous `\ExplSyntaxOn` delimiters [@latexteam2024interfaces, Section 2.1] in expl3 parts or extraneous `\ExplSyntaxOff` delimiters in non-expl3 parts. + + /w101.tex + +## Expl3 control sequences in non-expl3 parts {.e label=e102} +An input file contains what looks like expl3 control sequences [@latexteam2024interfaces, Section 1.1] in non-expl3 parts. + + /e102.tex + +## Line too long {.s label=s103} +Some lines in expl3 parts are longer than 80 characters [@latexteam2024style, Section 2]. + + /s103.tex + +The maximum line length can be configured using the command-line option `--max-line-length` or with the parameter `max_line_length` of the function returned by the Lua module `explcheck-preprocessing`. + +## Multiple delimiters `\ProvidesExpl*` in a single file {.e label=e104} +An input file contains multiple delimiters `\ProvidesExplPackage`, `\ProvidesExplClass`, and `\ProvidesExplFile`. + + /e104.tex diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors-02-lexical-analysis.md b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-02-lexical-analysis.md new file mode 100644 index 00000000000..9dc0b564cd7 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-02-lexical-analysis.md @@ -0,0 +1,157 @@ +# Lexical analysis +In the lexical analysis step, the expl3 analysis tool converts the expl3 parts of the input files into a list of `\TeX`{=tex} tokens. + +## “Weird” and “Do not use” argument specifiers {.w} +Some control sequence tokens correspond to functions with `w` (weird) or `D` (do not use) argument specifiers. + +``` tex +\cs_new:Npn + \show_until_if:w % warning on this line + #1 \if:w % warning on this line + { \tl_show:n {#1} } +\show_until_if:w % warning on this line + \tex_if:D % warning on this line + \if_charcode:w % warning on this line + \if:w % warning on this line +``` + +The above example has been taken from @latexteam2024interfaces [Chapter 24]. + +## Unknown argument specifiers {.e} +Some control sequence tokens correspond to functions with unknown argument specifiers. [@latexteam2024interfaces, Section 1.1] + +``` tex +\cs_new:Nn + \example:bar % error on this line + { foo } + { bar } + { baz } +``` + +## Deprecated control sequences {.w} +Some control sequence tokens correspond to deprecated expl3 control sequences from `l3obsolete.txt` [@josephwright2024obsolete]. +<!-- vi "$(texdoc -Il l3obsolete.txt | sed -r 's/\s*[0-9]+\s*//')" --> + +``` tex +\str_lower_case:n % warning on this line + { FOO BAR } +``` + +## Removed control sequences {.e} +Some control sequence tokens correspond to removed expl3 control sequences from `l3obsolete.txt` [@josephwright2024obsolete]. +<!-- vi "$(texdoc -Il l3obsolete.txt | sed -r 's/\s*[0-9]+\s*//')" --> + +``` tex +\msg_log:n % error on this line + { + Foo~bar~ + \c_one_thousand % error on this line + } +``` + +## Missing stylistic whitespaces {.s} +Some control sequences and curly braces are not surrounded by whitespaces [@latexteam2024programming, Section 6] [@latexteam2024style, Section 3]. + +``` tex +\cs_new:Npn \foo_bar:Nn #1#2 +{ + \cs_if_exist:NTF#1 % warning on this line + { \__foo_bar:n {#2} } + { \__foo_bar:nn{#2}{literal} } % warning on this line +} +``` + +## Malformed function name {.s} +Some function have names that are not in the format `\texttt{\textbackslash\meta{module}\_\meta{description}:\meta{arg-spec}}`{=tex} [@latexteam2024programming, Section 3.2]. + +``` tex +\cs_new:Nn + \description: % warning on this line + { foo } +``` + +``` tex +\cs_new:Nn + \module__description: % warning on this line + { foo } +``` + +``` tex +\cs_new:Nn + \_module_description: % warning on this line + { foo } +``` + +``` tex +\cs_new:Nn + \__module_description: + { foo } +``` + +## Malformed variable or constant name {.s} +Some expl3 variables and constants have names that are not in the format `\texttt{\textbackslash\meta{scope}\_\meta{module}\_\meta{description}\_\meta{type}}`{=tex} [@latexteam2024programming, Section 3.2], where the `\meta{module}`{=tex} part is optional. + +``` tex +\tl_new:Nn + \g_description_box % warning on this line +\tl_new:Nn + \l__description_box % warning on this line +\tl_const:Nn + \c_description % warning on this line + { foo } +``` + +``` tex +\tl_new:Nn + \g_module_description_box +\tl_new:Nn + \l_module_description_box +\tl_const:Nn + \c__module_description_box + { foo } +``` + +An exception is made for scratch variables [@latexteam2024interfaces, Section 1.1.1]: + +``` tex +\tl_use:N + \l_tmpa_tl +\int_use:N + \l_tmpb_int +\str_use:N + \l_tmpa_str +``` + +## Malformed quark or scan mark name {.s} +Some expl3 quarks and scan marks have names that do not start with `\q_` and `\s_`, respectively [@latexteam2024programming, Chapter 19]. + +``` tex +\quark_new:N + \foo_bar % error on this line +``` + +``` tex +\quark_new:N + \q_foo_bar +``` + +``` tex +\scan_new:N + \foo_bar % error on this line +``` + +``` tex +\scan_new:N + \s_foo_bar +``` + +## Too many closing braces {.e} +An expl3 part of the input file contains too many closing braces. + +``` tex +\tl_new:N + \g_example_tl +\tl_gset:Nn + \g_example_tl + { Hello,~ } } % error on this line +``` diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors-03-syntactic-analysis.md b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-03-syntactic-analysis.md new file mode 100644 index 00000000000..18a956167cf --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-03-syntactic-analysis.md @@ -0,0 +1,26 @@ +# Syntactic analysis +In the syntactic analysis step, the expl3 analysis tool converts the list of `\TeX`{=tex} tokens into a tree of function calls. + +## Unexpected function call argument {.e} +A function is called with an unexpected argument. Partial applications are detected by analysing closing braces (`}`) and do not produce an error. + +``` tex +\cs_new:Nn + \example_foo:n + { foo~#1 } +\cs_new:Nn + \example_bar: + { \example_foo:n } +\cs_new:Nn + \example_baz: + { + \example_bar: + { bar } + } +``` + +``` tex +\cs_new:Nn + { unexpected } % error on this line + \l_tmpa_tl % error on this line +``` diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors-04-semantic-analysis.md b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-04-semantic-analysis.md new file mode 100644 index 00000000000..781386b8ba8 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-04-semantic-analysis.md @@ -0,0 +1,510 @@ +# Semantic analysis +In the semantic analysis step, the expl3 analysis tool determines the meaning of the different function calls. + +## Functions and conditional functions + +### Expanding an unexpandable variable or constant {.t} +A function with a `V`-type argument is called with a variable or constant that does not support `V`-type expansion [@latexteam2024interfaces, Section 1.1]. + +``` tex +\cs_new:Nn + \module_foo:n + { #1 } +\cs_generate_variant:Nn + \module_foo:n + { V } +\module_foo:V % error on this line + \c_false_bool +``` + +### Unused function {.w #unused-function} +A private function or conditional function is defined but unused. + +``` tex +\cs_new:Nn % warning on this line + \__module_foo: + { bar } +``` + +``` tex +\prg_new_conditional:Nnn % warning on this line + \__module_foo: + { p, T, F, TF } + { \prg_return_true: } +``` + +### Unused function variant {.w #unused-function-variant} +A private function or conditional function variant is defined but unused. + +``` tex +\cs_new:Nn + \__module_foo:n + { bar~#1 } +\cs_generate_variant:Nn % warning on this line + \__module_foo:n + { V } +\__module_foo:n + { baz } +``` + +``` tex +\prg_new_conditional:Nnn + \__module_foo:n + { p, T, F, TF } + { \prg_return_true: } +\prg_generate_conditional_variant:Nnn % warning on this line + \__module_foo:n + { TF } + { V } +\__module_foo:nTF + { foo } + { bar } + { baz } +``` + +### Function variant of incompatible type {.t} +A function variant is generated from an incompatible argument type [@latexteam2024interfaces, Section 5.2, documentation of function `\cs_generate_variant:Nn`]. + +``` tex +\cs_new:Nn + \module_foo:Nn + { bar } +\cs_generate_variant:Nn % error on this line + \module_foo:Nn + { nn } +\cs_generate_variant:Nn % error on this line + \module_foo:Nn + { NN } +``` + +### Protected predicate function {.e} +A protected predicate function is defined. + +``` tex +\prg_new_protected_conditional:Nnn + \module_foo: + { p } + { \prg_return_true: } +``` + +### Function variant for an undefined conditional function {.e} +A variant is defined for an undefined conditional function. + +``` tex +\prg_new_conditional:Nnn + \module_foo:n + { p, T, F } + { \prg_return_true: } +\prg_generate_conditional_variant:Nnn % warning on this line + \module_foo:n + { TF } + { V } +\module_foo:nT + { bar } + { baz } +``` + +### Multiply defined function variant {.w} +A function or conditional function is defined multiple times. + +``` tex +\cs_new:Nn + \module_foo:n + { bar~#1 } +\cs_generate_variant:Nn + \module_foo:n + { V } +\cs_generate_variant:Nn % warning on this line + \module_foo:n + { o, V } +``` + +``` tex +\prg_new_conditional:Nnn + \module_foo:n + { p, T, F, TF } + { \prg_return_true: } +\prg_generate_conditional_variant:Nnn + \module_foo:n + { TF } + { V } +\prg_generate_conditional_variant:Nnn % warning on this line + \module_foo:n + { TF } + { o, V } +``` + +### Calling an undefined function {.e #calling-undefined-function} +A function is used but undefined. + +``` tex +\module_foo: % error on this line +``` + +### Calling an undefined function variant {.e #calling-undefined-function-variant} +A function or conditional function variant is used but undefined. + +``` tex +\cs_new:Nn + \module_foo:n + { bar~#1 } +\tl_set:Nn + \l_tmpa_tl + { baz } +\module_foo:V % error on this line + \l_tmpa_tl +``` + +``` tex +\prg_new_conditional:Nnn + \module_foo:n + { p, T, F, TF } + { \prg_return_true: } +\prg_generate_conditional_variant:Nnn + \module_foo:n + { TF } + { V } +\module_foo:VTF % error on this line + \l_tmpa_tl + { foo } + { bar } +``` + +## Variables and constants + +### Unused variable or constant {.w #unused-variable-or-constant} +A variable or a constant is declared and perhaps defined but unused. + +``` tex +\tl_new:N % warning on this line + \g_declared_but_undefined_tl +``` + +``` tex +\tl_new:N % warning on this line + \g_defined_but_unused_tl +\tl_gset:Nn + \g_defined_but_unused_tl + { foo } +``` + +``` tex +\tl_new:N + \g_defined_but_unused_tl +\tl_gset:Nn + \g_defined_but_unused_tl + { foo } +\tl_use:N + \g_defined_but_unused_tl +``` + +``` tex +\tl_const:Nn % warning on this line + \c_defined_but_unused_tl + { foo } +``` + +``` tex +\tl_const:Nn + \c_defined_but_unused_tl + { foo } +\tl_use:N + \c_defined_but_unused_tl +``` + +### Setting an undeclared variable {.w #setting-undeclared-variable} +An undeclared variable is set. + +``` tex +\tl_gset:Nn % warning on this line + \g_example_tl + { bar } +``` + +### Setting a constant {.e} +A constant is set. + +``` tex +\tl_gset:Nn % error on this line + \c_example_tl + { bar } +``` + +### Using a token list variable or constant without an accessor {.w} +A token list variable or constant is used without an accessor function. + +``` tex +\tl_set:Nn + \l_tmpa_tl + { world } +Hello,~\l_tmpa_tl! % warning on this line +Hello,~\tl_use:N \l_tmpa_tl ! +``` + +This also applies to subtypes of token lists such as strings +and comma-lists: + +``` tex +\str_set:Nn + \l_tmpa_str + { world } +Hello,~\l_tmpa_str! % warning on this line +Hello,~\str_use:N \l_tmpa_str ! +``` + +``` tex +\clist_set:Nn + \l_tmpa_clist + { world } +Hello,~\l_tmpa_clist! % warning on this line +Hello,~\clist_use:Nn \l_tmpa_clist { and } ! +``` + +### Using non-token-list variable or constant without an accessor {.e #using-variables-without-accessors} +A non-token-list variable or constant is used without an accessor function. + +``` tex +Hello,~\l_tmpa_seq! % error on this line +Hello,~\seq_use:Nn \l_tmpa_seq { and } ! +``` + +Note that boolean and integer variables may be used without accessor functions in boolean and integer expressions, respectively. Therefore, we may want to initially exclude them from this check to prevent false positives. + +### Multiply declared variable or constant {.e} +A variable or constant is declared multiple times. + +``` tex +\tl_new:N + \g_example_tl +\tl_new:N % error on this line + \g_example_tl +``` + +``` tex +\tl_const:Nn + \c_example_tl + { foo } +\tl_const:Nn % error on this line + \c_example_tl + { bar } +``` + +### Using an undefined variable or constant {.e #using-undefined-variable-or-constant} +A variable or constant is used but undeclared or undefined. + +``` tex +\tl_use:N % error on this line + \g_undeclared_tl +``` + +``` tex +\tl_new:N + \g_declared_but_undefined_tl +\tl_use:N % error on this line + \g_declared_but_undefined_tl +``` + +``` tex +\tl_new:N + \g_defined_tl +\tl_gset:Nn + \g_defined_tl + { foo } +\tl_use:N + \g_defined_tl +``` + +``` tex +\tl_use:N % error on this line + \c_undefined_tl +``` + +``` tex +\tl_const:Nn + \c_defined_tl + { foo } +\tl_use:N + \c_defined_tl +``` + +### Locally setting a global variable {.e} +A global variable is locally set. + +``` tex +\tl_new:N + \g_example_tl +\tl_set:Nn % error on this line + \g_example_tl + { foo } +``` + +### Globally setting a local variable {.e} +A local variable is globally set. + +``` tex +\tl_new:N + \l_example_tl +\tl_gset:Nn % error on this line + \l_example_tl + { foo } +``` + +### Using a variable of an incompatible type {.t} +A variable of one type is used where a variable of a different type should be used. + +``` tex +\tl_new:N + \l_example_str % error on this line +``` + +``` tex +\ior_new:N + \l_example_ior +\iow_open:Nn + \l_example_ior % error on this line + { example } +``` + +``` tex +\clist_new:N + \l_example_clist +\tl_set:Nn + \l_tmpa_tl + { foo } +\clist_set_eq:NN + \l_example_clist + \l_tmpa_tl % error on this line +``` + +``` tex +\tl_set:Nn + \l_tmpa_tl + { foo } +\seq_set_from_clist:NN + \l_tmpa_seq + \l_tmpa_tl % error on this line +``` + +``` tex +\tl_set:Nn + \l_tmpa_tl + { foo } +\regex_set:Nn + \l_tmpa_regex + { foo } +\int_set:Nn + \l_tmpa_int + { 1 + 2 } +\regex_show:N + \l_tmpa_tl +\regex_show:N + \l_tmpa_regex +\regex_show:N + \l_tmpa_int % error on this line +``` + +``` tex +\tl_set:Nn + \l_tmpa_tl + { foo } +\int_set_eq:NN + \l_tmpa_int + \l_tmpa_tl % error on this line +``` + +## Messages + +### Unused message {.w #unused-message} +A message is defined but unused. + +``` tex +\msg_new:nnn % warning on this line + { foo } + { bar } + { baz } +``` + +``` tex +\msg_new:nnn + { bar } + { bar } + { baz } +\msg_info:nn + { bar } + { bar } +``` + +### Setting an undefined message {.w #setting-undefined-message} +A message is set but undefined. + +``` tex +\msg_set:nnn % error on this line + { foo } + { bar } + { baz } +``` + +``` tex +\msg_new:nnn + { foo } + { bar } + { baz } +\msg_set:nnn + { foo } + { bar } + { baz } +``` + +### Multiply defined message {.e} +A message is defined multiple times. + +``` tex +\msg_new:nnn + { foo } + { bar } + { baz } +\msg_new:nnn % error on this line + { foo } + { bar } + { baz } +``` + +### Using an undefined message {.e #using-undefined-message} +A message is used but undefined. + +``` tex +\msg_info:nn + { foo } + { bar } +``` + +### Incorrect parameters in message text {.e #invalid-parameters-in-message-text} +Parameter tokens other than `#1`, `#2`, `#3`, and `#4` are specified in a message text. + +``` tex +\msg_new:nnn + { foo } + { bar } + { #5 } % error on this line +``` + +``` tex +\msg_new:nnnn + { foo } + { bar } + { #4 } + { #5 } % error on this line +``` + +## Sorting +### Comparison conditional without signature `:nnTF` {.e} +A sorting function is called with a conditional that has a signature different than `:nnTF` [@latexteam2024interfaces, Section 15.5.4]. + +``` tex +\cs_new:Nn + \example_foo: + { \prg_return_true: } +\tl_sort:nN + { { foo } { bar } } + \example_foo:TF +``` diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors-05-flow-analysis.md b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-05-flow-analysis.md new file mode 100644 index 00000000000..257b7407838 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-05-flow-analysis.md @@ -0,0 +1,651 @@ +# Flow analysis +In the flow analysis step, the expl3 analysis tool determines compiler-theoretic properties of functions, such as expandability, and variables, such as reaching definitions. + +## Functions and conditional functions + +### Multiply defined function {.e} +A function or conditional function is defined multiple times. + +``` tex +\cs_new:Nn + \module_foo: + { bar } +\cs_new:Nn % error on this line + \module_foo: + { bar } +``` + +``` tex +\cs_new:Nn + \module_foo: + { bar } +\cs_undefine:N + \module_foo: +\cs_new:Nn + \module_foo: + { bar } +``` + +``` tex +\prg_new_conditional:Nnn + \module_foo: + { p, T, F, TF } + { \prg_return_true: } +\prg_new_conditional:Nnn % error on this line + \module_foo: + { p, T, F, TF } + { \prg_return_true: } +``` + +``` tex +\prg_new_conditional:Nnn + \module_foo: + { p, T, F, TF } + { \prg_return_true: } +\cs_undefine:N + \module_foo_p: +\cs_undefine:N + \module_foo:T +\cs_undefine:N + \module_foo:F +\cs_undefine:N + \module_foo:TF +\prg_new_conditional:Nnn + \module_foo: + { p, T, F, TF } + { \prg_return_true: } +``` + +### Unreachable function {.w} +A private function or conditional function is defined but all its calls are unreachable.[^1] + + [^1]: Code is unreachable if it is only reachable through private functions which that are either unused or also unreachable. + +``` tex +\cs_new:Nn % warning on this line + \__module_foo: + { bar } +\cs_new:Nn + \__module_baz: + { \__module_foo: } +``` + +This check is a stronger version of <#unused-function> and should only be emitted if <#unused-function> has not previously been emitted for this function. + +### Unreachable function variant {.w} +A private function or conditional function variant is defined but all its calls are unreachable. + +``` tex +\cs_new:Nn + \__module_foo:n + { bar~#1 } +\cs_new:Nn + \__module_baz: + { + \tl_set:Nn + \l_tmpa_tl + { baz } + \__module_foo:V + \l_tmpa_tl + } +\cs_generate_variant:Nn % warning on this line + \__module_foo:n + { V } +\__module_foo:n + { baz } +``` + +This check is a stronger version of <#unused-function-variant> and should only be emitted if <#unused-function-variant> has not previously been emitted for this function variant. + +### Calling a function before definition {.e} +A function is used before it has been defined or after it has been undefined. + +``` tex +\module_foo: % error on this line +\cs_new:Nn + \module_foo: + { bar } +``` + +``` tex +\cs_new:Nn + \module_foo: + { bar } +\cs_undefine:N + \module_foo: +\module_foo: % error on this line +``` + +This check is a stronger version of <#calling-undefined-function> and should only be emitted if <#calling-undefined-function> has not previously been emitted for this function. + +### Calling a function variant before definition {.e} +A function or conditional function variant is used before it has been defined. + +``` tex +\cs_new:Nn + \module_foo:n + { bar~#1 } +\tl_set:Nn + \l_tmpa_tl + { baz } +\module_foo:V % error on this line + \l_tmpa_tl +\cs_generate_variant:Nn + \module_foo:n + { V } +``` + +This check is a stronger version of <#calling-undefined-function-variant> and should only be emitted if <#calling-undefined-function-variant> has not previously been emitted for this function variant. + +### Setting a function before definition {.w} +A function is set before it has been defined or after it has been undefined. + +``` tex +\cs_gset:N % warning on this line + \module_foo: + { foo } +\cs_new:Nn + \module_foo: + { bar } +``` + +``` tex +\cs_new:Nn + \module_foo: + { bar } +\cs_undefine:N + \module_foo: +\cs_gset:N % warning on this line + \module_foo: + { foo } +``` + +### Unexpandable or restricted-expandable boolean expression {.e} +A boolean expression [@latexteam2024interfaces, Section 9.2] is not fully-expandable. + +``` tex +\cs_new_protected:N + \example_unexpandable: + { + \tl_set:Nn + \l_tmpa_tl + { bar } + \c_true_bool + } +\cs_new:N + \example_restricted_expandable: + { + \bool_do_while:Nn + \c_false_bool + { } + \c_true_bool + } +\cs_new_protected:N + \example_expandable: + { \c_true_bool } +\bool_set:Nn + \l_tmpa_bool + { \example_unexpandable: } % error on this line +\bool_set:Nn + \l_tmpa_bool + { \example_restricted_expandable: } % error on this line +\bool_set:Nn + \l_tmpa_bool + { \example_expandable: } +``` + +### Expanding an unexpandable function {.e} +An unexpandable function or conditional function is called within an `x`-type, `e`-type, or `f`-type argument. + +``` tex +\cs_new_protected:N + \example_unexpandable: + { + \tl_set:Nn + \l_tmpa_tl + { bar } + } +\cs_new:N + \module_foo:n + { #1 } +\cs_generate_variant:Nn + \module_foo:n + { x, e, f } +\module_foo:n + { \example_unexpandable: } +\module_foo:x + { \example_unexpandable: } % error on this line +\module_foo:e + { \example_unexpandable: } % error on this line +\module_foo:f + { \example_unexpandable: } % error on this line +``` + +### Fully-expanding a restricted-expandable function {.e} +An restricted-expadable function or conditional function is called within an `f`-type argument. + +``` tex +\cs_new:N + \example_restricted_expandable: + { + \int_to_roman:n + { 1 + 2 } + } +\cs_new:N + \module_foo:n + { #1 } +\cs_generate_variant:Nn + \module_foo:n + { x, e, f } +\module_foo:n + { \example_restricted_expandable: } +\module_foo:x + { \example_restricted_expandable: } +\module_foo:e + { \example_restricted_expandable: } +\module_foo:f + { \example_restricted_expandable: } % error on this line +``` + +### Defined an expandable function as protected {.w} +A fully expandable function or conditional function is defined using a creator function `\cs_new_protected:*` or `\prg_new_protected_conditional:*`. [@latexteam2024style, Section 4] + +``` tex +\cs_new_protected:Nn % warning on this line + \example_expandable: + { foo } +``` + +``` tex +\prg_new_protected_conditional:Nnn % warning on this line + \example_expandable: + { T, F, TF } + { \prg_return_true: } +``` + +### Defined an unexpandable function as unprotected {.w} +An unexpandable or restricted-expandable function or conditional function is defined using a creator function `\cs_new:*` or `\prg_new_conditional:*`. [@latexteam2024style, Section 4] + +``` tex +\cs_new:Nn % warning on this line + \example_unexpandable: + { + \tl_set:Nn + \l_tmpa_tl + { bar } + } +``` + +``` tex +\prg_new_conditional:Nnn % warning on this line + \example_unexpandable: + { p, T, F, TF } + { + \tl_set:Nn + \l_tmpa_tl + { bar } + \prg_return_true: + } +``` + +### Conditional function with no return value {.e} +A conditional functions has no return value. + +``` tex +\prg_new_conditional:Nnn % error on this line + \example_no_return_value: + { p, T, F, TF } + { foo } +``` + +``` tex +\prg_new_conditional:Nnn + \example_has_return_value: + { p, T, F, TF } + { \example_foo: } +\cs_new:Nn + \example_foo: + { \prg_return_true: } +``` + +### Comparison code with no return value {.e} +A comparison code [@latexteam2024interfaces, Section 6.1] has no return value. + +``` tex +\clist_set:Nn + \l_foo_clist + { 3 , 01 , -2 , 5 , +1 } +\clist_sort:Nn % error on this line + \l_foo_clist + { foo } +``` + +``` tex +\clist_set:Nn + \l_foo_clist + { 3 , 01 , -2 , 5 , +1 } +\clist_sort:Nn + \l_foo_clist + { \example_foo: } +\cs_new:Nn + \example_foo: + { + \int_compare:nNnTF { #1 } > { #2 } + { \sort_return_swapped: } + { \sort_return_same: } + } +``` + +The above example has been taken from @latexteam2024interfaces [Chapter 6]. + +## Variables and constants + +### Unreachable variable or constant {.w} +A variable or a constant is declared and perhaps defined but all its uses are unreachable. + +``` tex +\tl_new:N % warning on this line + \g_defined_but_unreachable_tl +\tl_gset:Nn + \g_defined_but_unreachable_tl + { foo } +\cs_new:Nn + \__module_baz: + { + \tl_use:N + \g_defined_but_unreachable_tl + } +``` + +This check is a stronger version of <#unused-variable-or-constant> and should only be emitted if <#unused-variable-or-constant> has not previously been emitted for this variable or constant. + +### Setting a variable before declaration {.e} +A variable is set before it has been declared. + +``` tex +\tl_gset:Nn % error on this line + \g_example_tl + { bar } +\tl_new:N + \g_example_tl +``` + +This check is a stronger version of <#setting-undeclared-variable> and should prevent <#setting-undeclared-variable> from being emitted for this variable. + +### Using a variable or constant before definition {.e} +A variable or constant is used before it has been defined. + +``` tex +\tl_new:N + \g_example_tl +\tl_use:N % error on this line + \g_example_tl +\tl_gset:Nn + \g_example_tl + { foo } +``` + +This check is a stronger version of <#using-undefined-variable-or-constant> and should only be emitted if <#using-undefined-variable-or-constant> has not previously been emitted for this variable or constant. + +## Messages + +### Unreachable message {.w} +A message is defined but all its uses are unreachable. + +``` tex +\msg_new:nnn % warning on this line + { foo } + { bar } + { baz } +\cs_new:Nn + \__module_baz: + { + \msg_info:nn + { foo } + { bar } + } +``` + +This check is a stronger version of <#unused-message> and should only be emitted if <#unused-message> has not previously been emitted for this message. + +### Setting a message before definition {.e} +A message is set before it has been defined. + +``` tex +\msg_set:nnn % error on this line + { foo } + { bar } + { baz } +\msg_new:nnn + { foo } + { bar } + { baz } +``` + +This check is a stronger version of <#setting-undefined-message> and should prevent <#setting-undefined-message> from being emitted for this message. + +### Using a message before definition {.e} +A message is used before it has been defined. + +``` tex +\msg_info:nn % error on this line + { foo } + { bar } +\msg_new:nnn + { foo } + { bar } + { baz } +``` + +This check is a stronger version of <#using-undefined-message> and should only be emitted if <#using-undefined-message> has not previously been emitted for this message. + +### Too few arguments supplied to message {.e #too-few-arguments-supplied-to-message} +A message was supplied fewer arguments than there are parameters in the message text. + +``` tex +\msg_new:nnn + { foo } + { bar } + { #1~#2 } +\msg_info:nn % error on this line + { foo } + { bar } +\msg_info:nnn % error on this line + { foo } + { bar } + { baz } +\msg_info:nnnn + { foo } + { bar } + { baz } + { baz } +``` + +Since a message can be redefined, we need to track the (possibly many) definitions that can be active when we display a message. + +``` tex +\msg_new:nnn + { foo } + { bar } + { #1 } +\msg_set:nnn + { foo } + { bar } + { baz } +\msg_info:nnn % error on this line + { foo } + { bar } + { baz } +``` + +``` tex +\msg_new:nnn + { foo } + { bar } + { #1 } +\msg_info:nnn + { foo } + { bar } + { baz } +\msg_set:nnn + { foo } + { bar } + { baz } +``` + +## Input–output streams +### Using an unopened or closed stream {.e} +A stream is used before it has been opened or after it has been closed. + +``` tex +\ior_new:N + \l_example_ior +\ior_str_get:NN % error on this line + \l_example_ior + \l_tmpa_tl +\ior_open:Nn + \l_example_ior + { example } +``` + +``` tex +\ior_new:N + \l_example_ior +\ior_open:Nn + \l_example_ior + { example } +\ior_close:N + \l_example_ior +\ior_str_get:NN % error on this line + \l_example_ior + \l_tmpa_tl +``` + +### Multiply opened stream {.e} +A stream is opened a second time without closing the stream first. + +``` tex +\iow_new:N + \l_example_iow +\iow_open:Nn + \l_example_iow + { foo } +\iow_open:Nn % error on this line + \l_example_iow + { bar } +\iow_close:N + \l_example_iow +``` + +### Unclosed stream {.w} +A stream is opened but not closed. + +``` tex +% file-wide warning +\ior_new:N + \l_example_ior +\ior_open:Nn + \l_example_ior + { example } +``` + +## Piecewise token list construction +### Building on a regular token list {.t} +A token list variable is used with `\tl_build_*` functions before a function `\tl_build_*begin:N` has been called or after a function `\tl_build_*end:N` has been called. + +``` tex +\tl_new:N + \l_example_tl +\tl_build_put_right:Nn % error on this line + \l_example_tl + { foo } +\tl_build_begin:N + \l_example_tl +\tl_build_end:N + \l_example_tl +``` + +``` tex +\tl_new:N + \l_example_tl +\tl_build_begin:N + \l_example_tl +\tl_build_put_right:Nn + \l_example_tl + { foo } +\tl_build_end:N + \l_example_tl +``` + +``` tex +\tl_new:N + \l_example_tl +\tl_build_begin:N + \l_example_tl +\tl_build_end:N + \l_example_tl +\tl_build_put_right:Nn % error on this line + \l_example_tl + { foo } +``` + +### Using a semi-built token list {.t} +A token list variable is used where a regular token list is expected after a function `\tl_build_*begin:N` has been called and before a function `\tl_build_*end:N` has been called. + +``` tex +\tl_new:N + \l_example_tl +\tl_use:N + \l_example_tl +\tl_build_begin:N + \l_example_tl +\tl_build_end:N + \l_example_tl +``` + +``` tex +\tl_new:N + \l_example_tl +\tl_build_begin:N + \l_example_tl +\tl_use:N + \l_example_tl % error on this line +\tl_build_end:N + \l_example_tl +``` + +``` tex +\tl_new:N + \l_example_tl +\tl_build_begin:N + \l_example_tl +\tl_build_end:N + \l_example_tl +\tl_use:N + \l_example_tl +``` + +### Multiply started building a token list {.e} +A function `\tl_build_*begin:N` is called on a token list variable a second time without calling a function `\tl_build_*end:N` first. + +``` tex +\tl_new:N + \l_example_tl +\tl_build_begin:N + \l_example_tl +\tl_build_begin:N % error on this line + \l_example_tl +\tl_build_end:N + \l_example_tl +``` + +### Unfinished semi-built token list {.w} +A function `\tl_build_*begin:N` is called on a token list variable without calling a function `\tl_build_*end:N` later. + +``` tex +% file-wide warning +\tl_new:N + \l_example_tl +\tl_build_begin:N + \l_example_tl +``` diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors-99-caveats.md b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-99-caveats.md new file mode 100644 index 00000000000..47bd424a0e3 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors-99-caveats.md @@ -0,0 +1,141 @@ +# Caveats +The warnings and errors in this documents do not cover the complete expl3 language. The caveats currently include the following areas, among others: + +- Functions with “weird” (`w`) argument specifiers +- Verifying the `nopar` restriction on functions + [@latexteam2024interfaces, Section 4.3.1] +- Symbolic evaluation of expansion functions + [@latexteam2024interfaces, sections 5.4–5.10] +- Validation of parameters in (inline) functions + (c.f. <#invalid-parameters-in-message-text> + and <#too-few-arguments-supplied-to-message>) +- Shorthands such as `\~` and `\\` in message texts + [@latexteam2024interfaces, sections 11.4 and 12.1.3] +- Quotes in shell commands and file names + [@latexteam2024interfaces, Section 10.7 and Chapter 12] +- Functions used outside their intended context: + - `\sort_return_*:` outside comparison code + [@latexteam2024interfaces, Section 6.1] + - `\prg_return_*:` outside conditional functions + [@latexteam2024interfaces, Section 9.1] + - Predicates (`\*_p:*`) outside boolean expressions + [@latexteam2024interfaces, Section 9.3] + - `\*_map_break:*` outside a corresponding mapping + [@latexteam2024interfaces, sections 9.8] + - `\msg_line_*:`, `\iow_char:N`, and `\iow_newline:` + outside message text + [@latexteam2024interfaces, sections 11.3 and 12.1.3] + - `\iow_wrap_allow_break:` and `\iow_indent:n` + outside wrapped message text + [@latexteam2024interfaces, Section 12.1.4] + - Boolean variable without an accessor function + `\bool_to_str:N` outside boolean expressions + [@latexteam2024interfaces, Section 21.4] + (see <#using-variables-without-accessors>) + - Integer variable without an accessor function + `\int_use:N` outside integer or floating point + expressions [@latexteam2024interfaces, Section 21.4] + (see <#using-variables-without-accessors>) + - Dimension variable without an accessor function + `\dim_use:N` outside dimension or floating point + expressions [@latexteam2024interfaces, Section 26.7] + (see <#using-variables-without-accessors>) + - Skip variable without an accessor function + `\skip_use:N` outside skip or floating point expressions + [@latexteam2024interfaces, Section 26.14] + (see <#using-variables-without-accessors>) + - Muskip variable without an accessor function + `\muskip_use:N` outside muskip or floating point + expressions [@latexteam2024interfaces, Section 26.21] + (see <#using-variables-without-accessors>) + - Floating point variable without an accessor function + `\fp_use:N` outside floating point + expressions [@latexteam2024interfaces, Section 29.3] + (see <#using-variables-without-accessors>) + - Box variable without accessor functions + `\box_use(_drop)?:N` or `\[hv]box_unpack(_drop)?:N`, + or without a measuring function + `\box_(dp|ht|wd|ht_plus_dp):*` outside dimension or + floating point expressions + [@latexteam2024interfaces, sections 35.2 and 35.3] + - Coffin variable without accessor function + `\coffin_typeset:Nnnnn` outside dimension or + floating point expressions + [@latexteam2024interfaces, Section 36.4] +- Validation of literal expressions: + - Comparison expressions in functions + `\*_compare(_p:n|:nT?F?)` + - Regular expressions and replacement text + [@latexteam2024interfaces, sections 8.1 and 8.2] + - Boolean expressions + [@latexteam2024interfaces, Section 9.3] + - Integer expressions and bases + [@latexteam2024interfaces, sections 21.1 and 21.8] + - Dimension, skip, and muskip expressions + [@latexteam2024interfaces, Chapter 26] + - Floating point expressions + [@latexteam2024interfaces, Section 29.12] + - Color expressions + [@latexteam2024interfaces, Chapter 37.3] +- Validation of naming schemes and member access: + - String encoding and escaping + [@latexteam2024interfaces, Section 18.1] + - Key–value interfaces + [@latexteam2024interfaces, Chapter 27]: + - Are keys defined at the point of use or is the module + or its subdivision set up to accept unknown keys? + [@latexteam2024interfaces, sections 27.2, 27.5, + and 27.6] + - Are inheritance parents, choices, multi-choices, and + groups used in a key definition defined at points of + use? [@latexteam2024interfaces, sections 27.1, 27.3, + and 27.7] + - Floating-point symbolic expressions and user-defined + functions [@latexteam2024interfaces, sections 29.6 + and 29.7] + - Names of bitset indexes + [@latexteam2024interfaces, Section 31.1] + - BCP-47 language tags + [@latexteam2024interfaces, Section 34.2] + - Color support + [@latexteam2024interfaces, Chapter 37]: + - Named colors [@latexteam2024interfaces, Section 37.4] + - Color export targets [@latexteam2024interfaces, + Section 37.8] + - Color models and their families and params + [@latexteam2024interfaces, sections 37.2 and 37.9] +- Function `\file_input_stop:` not used on its own line + [@latexteam2024interfaces, Section 12.2.3] +- Exhaustively or fully expanding quarks and scan marks + [@latexteam2024interfaces, Chapter 19] +- Bounds checking for accessing constant sequences and other + sequences where the number of items can be easily bounded + such as integer and floating point arrays + [@latexteam2024interfaces, chapters 28 and 30]: + - Index checking functions `\*_range*:*` and `\*_item*:*` + - Endless loop checking in functions `\*_step_*:*` + [@latexteam2024interfaces, Section 21.7] + - Number of symbols in a value-to-symbol mapping + [@latexteam2024interfaces, Section 21.8] +- Applying functions `\clist_remove_duplicates:N` and + `\clist_if_in:*` to comma lists that contain `{`, `}`, or `*` + [@latexteam2024interfaces, sections 23.3 and 23.4] +- Incorrect parameters to function `\char_generate:nn` + [@latexteam2024interfaces, Section 24.1] +- Incorrect parameters to functions `\char_set_*code:nn` + [@latexteam2024interfaces, Section 24.2] +- Using implicit tokens `\c_catcode_(letter|other)_token` or + the token list `\c_catcode_active_tl` + [@latexteam2024interfaces, Section 24.3] +- Validation of key–value interfaces + [@latexteam2024interfaces, Chapter 27]: + - Setting a key with some properties `.*_g?(set|put)*:*` + should be validated similarly to calling the corresponding + functions directly: Have the variables been declared, do + they have the correct type, does the value have the + correct type? + - Do points of use always set keys with property + `.value_required:n` and never set keys with + property `.value_forbidden:n`? +- Horizontal box operation on a vertical box or vice + versa [@latexteam2024interfaces, Chapter 35] diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors.bib b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.bib new file mode 100644 index 00000000000..189014b1174 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.bib @@ -0,0 +1,55 @@ +@online{latexteam2024interfaces, + title = {The {\LaTeX}3 interfaces}, + author = {{The {\LaTeX} Project}}, + date = {2024-05-08}, + url = {http://mirrors.ctan.org/macros/latex/required/l3kernel/interface3.pdf}, + urldate = {2024-05-15}, + note = {The referenced version of the document is attached to this document.\embedfile{interface3.pdf}}, +} + +@online{latexteam2024style, + title = {The {\LaTeX}3 kernel}, + subtitle = {Style guide for code authors}, + author = {{The {\LaTeX} Project}}, + date = {2024-04-11}, + url = {http://mirrors.ctan.org/macros/latex/required/l3kernel/l3styleguide.pdf}, + urldate = {2024-05-08}, + note = {The referenced version of the document is attached to this document.\embedfile{l3styleguide.pdf}}, +} + +@online{latexteam2024programming, + title = {The expl3 package and {\LaTeX}3 programming}, + author = {{The {\LaTeX} Project}}, + date = {2024-04-11}, + url = {http://mirrors.ctan.org/macros/latex/required/l3kernel/expl3.pdf}, + urldate = {2024-05-08}, + note = {The referenced version of the document is attached to this document.\embedfile{expl3.pdf}}, +} + +@online{starynovotny2024static3, + title = {Static analysis of expl3 programs (3)}, + subtitle = {Design Specification}, + author = {Starý Novotný, Vít}, + url = {https://witiko.github.io/Expl3-Linter-3/}, +% urldate = {XXX}, + note = {To be released}, +} + +@online{josephwright2024obsolete, + author = {Joseph Wright}, + date = {2024-04-29}, + url = {https://github.com/latex3/latex3/pull/1542#issuecomment-2082352499}, + urldate = {2024-05-15}, +} + +@book{flemming1999principles, + author = {Flemming Nielson and Nielson, Hanne Riis and Chris Hankin}, + title = {Principles of Program Analysis}, + doi = {10.1007/978-3-662-03811-6}, + isbn = {978-3-540-65410-0}, + date = {1999-10-22}, + publisher = {Springer}, + location = {Berlin, Heidelberg}, + edition = {1}, + pagetotal = {452}, +} diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors.cls b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.cls new file mode 100644 index 00000000000..168488db23e --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.cls @@ -0,0 +1,24 @@ +\NeedsTeXFormat{LaTeX2e} +\ProvidesClass% + {warnings-and-errors}% + [2024-12-04 0.0.4 Document class for typesetting the documentation of the % + warnings and errors for the expl3 analysis tool] +\LoadClass{article} +\RequirePackage{lua-widow-control} +\RequirePackage{minted} +\setminted{breaklines, linenos} +\RequirePackage{markdown} +\markdownSetup { + import = { + witiko/expltools = metadata, + witiko/expltools/explcheck/warnings-and-errors = body, + } +} +\RequirePackage{biblatex} +\addbibresource{warnings-and-errors.bib} +\RequirePackage{embedfile} +\RequirePackage{imakeidx} +\makeindex[columns=3, intoc, options=-s warnings-and-errors.ist] +\RequirePackage{hyperref} +\def\meta#1{$\langle$\textrm{\textit{#1}}$\rangle$} +\RequirePackage{datetime2} diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors.ist b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.ist new file mode 100644 index 00000000000..cffb2b4873e --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.ist @@ -0,0 +1,4 @@ +headings_flag 1 + +heading_prefix "\n\\noindent\\textbf{" +heading_suffix "}\\par\\nopagebreak\n"
\ No newline at end of file diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors.pdf b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.pdf Binary files differnew file mode 100644 index 00000000000..73a77fe5bbb --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.pdf diff --git a/Master/texmf-dist/doc/support/expltools/warnings-and-errors.tex b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.tex new file mode 100644 index 00000000000..37b70748729 --- /dev/null +++ b/Master/texmf-dist/doc/support/expltools/warnings-and-errors.tex @@ -0,0 +1,24 @@ +\documentclass{warnings-and-errors} +\begin{yaml}[snippet=metadata] +title: Warnings and errors for the expl3 analysis tool +author: Vít Starý Novotný +date: `\today`{=tex} +\end{yaml} +\begin{document} +\tableofcontents +\clearpage +\setcounter{secnumdepth}{0} +\markdownSetup{snippet=body} +\markdownInput{warnings-and-errors-00-introduction.md} +\setcounter{secnumdepth}{1} +\markdownInput{warnings-and-errors-01-preprocessing.md} +\markdownInput{warnings-and-errors-02-lexical-analysis.md} +\markdownInput{warnings-and-errors-03-syntactic-analysis.md} +\setcounter{secnumdepth}{2} +\markdownInput{warnings-and-errors-04-semantic-analysis.md} +\markdownInput{warnings-and-errors-05-flow-analysis.md} +\setcounter{secnumdepth}{0} +\markdownInput{warnings-and-errors-99-caveats.md} +\printbibliography[heading=bibintoc] +\printindex +\end{document} diff --git a/Master/texmf-dist/scripts/expltools/explcheck-cli.lua b/Master/texmf-dist/scripts/expltools/explcheck-cli.lua new file mode 100755 index 00000000000..69e5e1a1cee --- /dev/null +++ b/Master/texmf-dist/scripts/expltools/explcheck-cli.lua @@ -0,0 +1,113 @@ +-- A command-line interface for the static analyzer explcheck. + +local new_issues = require("explcheck-issues") +local format = require("explcheck-format") + +local preprocessing = require("explcheck-preprocessing") +-- local lexical_analysis = require("explcheck-lexical-analysis") +-- local syntactic_analysis = require("explcheck-syntactic-analysis") +-- local semantic_analysis = require("explcheck-semantic-analysis") +-- local pseudo_flow_analysis = require("explcheck-pseudo-flow-analysis") + +-- Deduplicate pathnames. +local function deduplicate_pathnames(pathnames) + local deduplicated_pathnames = {} + local seen_pathnames = {} + for _, pathname in ipairs(pathnames) do + if seen_pathnames[pathname] ~= nil then + goto continue + end + seen_pathnames[pathname] = true + table.insert(deduplicated_pathnames, pathname) + ::continue:: + end + return deduplicated_pathnames +end + +-- Process all input files. +local function main(pathnames, warnings_are_errors, max_line_length) + local num_warnings = 0 + local num_errors = 0 + + print("Checking " .. #pathnames .. " files") + + for pathname_number, pathname in ipairs(pathnames) do + + -- Load an input file. + local file = assert(io.open(pathname, "r"), "Could not open " .. pathname .. " for reading") + local content = assert(file:read("*a")) + assert(file:close()) + local issues = new_issues() + + -- Run all processing steps. + local line_starting_byte_numbers, _ = preprocessing(issues, content, max_line_length) + if #issues.errors > 0 then + goto continue + end + -- lexical_analysis(issues) + -- syntactic_analysis(issues) + -- semantic_analysis(issues) + -- pseudo_flow_analysis(issues) + + -- Print warnings and errors. + ::continue:: + num_warnings = num_warnings + #issues.warnings + num_errors = num_errors + #issues.errors + format.print_results(pathname, issues, line_starting_byte_numbers, pathname_number == #pathnames) + end + + -- Print a summary. + format.print_summary(#pathnames, num_warnings, num_errors) + + if(num_errors > 0) then + return 1 + elseif(warnings_are_errors and num_warnings > 0) then + return 2 + else + return 0 + end +end + +local function print_usage() + print("Usage: " .. arg[0] .. " [OPTIONS] FILENAMES\n") + print("Run static analysis on expl3 files.\n") + print("Options:") + print("\t--max-line-length=N\tThe maximum line length before the warning S103 (Line too long) is produced.") + print("\t--warnings-are-errors\tProduce a non-zero exit code if any warnings are produced by the analysis.") +end + +if #arg == 0 then + print_usage() + os.exit(1) +else + -- Collect arguments. + local pathnames = {} + local warnings_are_errors = false + local only_pathnames_from_now_on = false + local max_line_length = nil + for _, argument in ipairs(arg) do + if only_pathnames_from_now_on then + table.insert(pathnames, argument) + elseif argument == "--" then + only_pathnames_from_now_on = true + elseif argument == "--help" then + print_usage() + os.exit(0) + elseif argument == "--warnings-are-errors" then + warnings_are_errors = true + elseif argument:sub(1, 18) == "--max-line-length=" then + max_line_length = tonumber(argument:sub(19)) + elseif argument:sub(1, 2) == "--" then + -- An unknown argument + print_usage() + os.exit(1) + else + table.insert(pathnames, argument) + end + end + pathnames = deduplicate_pathnames(pathnames) + + -- Run the analysis. + local exit_code = main(pathnames, warnings_are_errors, max_line_length) + os.exit(exit_code) +end diff --git a/Master/texmf-dist/scripts/expltools/explcheck-format.lua b/Master/texmf-dist/scripts/expltools/explcheck-format.lua new file mode 100755 index 00000000000..5b64ab51b90 --- /dev/null +++ b/Master/texmf-dist/scripts/expltools/explcheck-format.lua @@ -0,0 +1,233 @@ +-- Formatting for the command-line interface of the static analyzer explcheck. + +-- Transform a singular into plural if the count is zero or greater than two. +local function pluralize(singular, count) + if count == 1 then + return singular + else + return singular .. "s" + end +end + +-- Shorten a pathname, so that it does not exceed maximum length. +local function format_pathname(pathname, max_length) + -- First, replace path segments with `/.../`, keeping other segments. + local first_iteration = true + while #pathname > max_length do + local pattern + if first_iteration then + pattern = "([^/]*)/[^/]*/(.*)" + else + pattern = "([^/]*)/%.%.%./[^/]*/(.*)" + end + local prefix_start, _, prefix, suffix = pathname:find(pattern) + if prefix_start == nil or prefix_start > 1 then + break + end + pathname = prefix .. "/.../" .. suffix + first_iteration = false + end + -- If this isn't enough, remove the initial path segment and prefix the filename with `...`. + if #pathname > max_length then + local pattern + if first_iteration then + pattern = "([^/]*/?)(.*)" + else + pattern = "([^/]*/%.%.%./)(.*)" + end + local prefix_start, _, _, suffix = pathname:find(pattern) + if prefix_start == 1 then + pathname = ".../" .. suffix + if #pathname > max_length then + pathname = "..." .. suffix:sub(-(max_length - #("..."))) + end + end + end + return pathname +end + +-- Colorize a string using ASCII color codes. +local function colorize(text, ...) + local buffer = {} + for _, color_code in ipairs({...}) do + table.insert(buffer, "\27[") + table.insert(buffer, tostring(color_code)) + table.insert(buffer, "m") + end + table.insert(buffer, text) + table.insert(buffer, "\27[0m") + return table.concat(buffer, "") +end + +-- Remove ASCII color codes from a string. +local function decolorize(text) + return text:gsub("\27%[[0-9]+m", "") +end + +-- Convert a byte number in a file to a line and column number in a file. +local function convert_byte_to_line_and_column(line_starting_byte_numbers, byte_number) + local line_number = 0 + for _, line_starting_byte_number in ipairs(line_starting_byte_numbers) do + if line_starting_byte_number > byte_number then + break + end + line_number = line_number + 1 + end + assert(line_number > 0) + local line_starting_byte_number = line_starting_byte_numbers[line_number] + assert(line_starting_byte_number <= byte_number) + local column_number = byte_number - line_starting_byte_number + 1 + return line_number, column_number +end + +-- Print the results of analyzing a file. +local function print_results(pathname, issues, line_starting_byte_numbers, is_last_file) + -- Display an overview. + local all_issues = {} + local status + if(#issues.errors > 0) then + status = ( + colorize( + ( + tostring(#issues.errors) + .. " " + .. pluralize("error", #issues.errors) + ), 1, 31 + ) + ) + table.insert(all_issues, issues.errors) + if(#issues.warnings > 0) then + status = ( + status + .. ", " + .. colorize( + ( + tostring(#issues.warnings) + .. " " + .. pluralize("warning", #issues.warnings) + ), 1, 33 + ) + ) + table.insert(all_issues, issues.warnings) + end + elseif(#issues.warnings > 0) then + status = colorize( + ( + tostring(#issues.warnings) + .. " " + .. pluralize("warning", #issues.warnings) + ), 1, 33 + ) + table.insert(all_issues, issues.warnings) + else + status = colorize("OK", 1, 32) + end + + local max_overview_length = 72 + local prefix = "Checking " + local formatted_pathname = format_pathname( + pathname, + math.max( + ( + max_overview_length + - #prefix + - #(" ") + - #decolorize(status) + ), 1 + ) + ) + local overview = ( + prefix + .. formatted_pathname + .. (" "):rep( + math.max( + ( + max_overview_length + - #prefix + - #decolorize(status) + - #formatted_pathname + ), 1 + ) + ) + .. status + ) + io.write("\n" .. overview) + + -- Display the errors, followed by warnings. + if #all_issues > 0 then + for _, warnings_or_errors in ipairs(all_issues) do + print() + -- Display the warnings/errors. + for _, issue in ipairs(issues.sort(warnings_or_errors)) do + local code = issue[1] + local message = issue[2] + local range = issue[3] + local position = ":" + if range ~= nil then + local line_number, column_number = convert_byte_to_line_and_column(line_starting_byte_numbers, range[1]) + position = position .. tostring(line_number) .. ":" .. tostring(column_number) .. ":" + end + local max_line_length = 88 + local reserved_position_length = 10 + local reserved_suffix_length = 30 + local label_indent = (" "):rep(4) + local suffix = code:upper() .. " " .. message + formatted_pathname = format_pathname( + pathname, + math.max( + ( + max_line_length + - #label_indent + - reserved_position_length + - #(" ") + - math.max(#suffix, reserved_suffix_length) + ), 1 + ) + ) + local line = ( + label_indent + .. formatted_pathname + .. position + .. (" "):rep( + math.max( + ( + max_line_length + - #label_indent + - #formatted_pathname + - #decolorize(position) + - math.max(#suffix, reserved_suffix_length) + ), 1 + ) + ) + .. suffix + .. (" "):rep(math.max(reserved_suffix_length - #suffix, 0)) + ) + io.write("\n" .. line) + end + end + if(not is_last_file) then + print() + end + end +end + +-- Print the summary results of analyzing multiple files. +local function print_summary(num_pathnames, num_warnings, num_errors) + io.write("\n\nTotal: ") + + local errors_message = tostring(num_errors) .. " " .. pluralize("error", num_errors) + errors_message = colorize(errors_message, 1, (num_errors > 0 and 31) or 32) + io.write(errors_message .. ", ") + + local warnings_message = tostring(num_warnings) .. " " .. pluralize("warning", num_warnings) + warnings_message = colorize(warnings_message, 1, (num_warnings > 0 and 33) or 32) + io.write(warnings_message .. " in ") + + print(tostring(num_pathnames) .. " " .. pluralize("file", num_pathnames)) +end + +return { + convert_byte_to_line_and_column = convert_byte_to_line_and_column, + print_results = print_results, + print_summary = print_summary, +} diff --git a/Master/texmf-dist/scripts/expltools/explcheck-issues.lua b/Master/texmf-dist/scripts/expltools/explcheck-issues.lua new file mode 100755 index 00000000000..9f7c1be75b6 --- /dev/null +++ b/Master/texmf-dist/scripts/expltools/explcheck-issues.lua @@ -0,0 +1,81 @@ +-- A registry of warnings and errors identified by different processing steps. + +local Issues = {} + +function Issues.new(cls) + local new_object = {} + setmetatable(new_object, cls) + cls.__index = cls + new_object.errors = {} + new_object.warnings = {} + new_object.ignored_issues = {} + return new_object +end + +-- Convert an issue identifier to either a table of warnings or a table of errors. +function Issues:_get_issue_table(identifier) + local prefix = identifier:sub(1, 1) + if prefix == "s" or prefix == "w" then + return self.warnings + elseif prefix == "t" or prefix == "e" then + return self.errors + else + assert(false, 'Identifier "' .. identifier .. '" has an unknown prefix "' .. prefix .. '"') + end +end + +-- Add an issue to the table of issues. +function Issues:add(identifier, message, range_start, range_end) + if self.ignored_issues[identifier] then + return + end + local issue_table = self:_get_issue_table(identifier) + local range + if range_start == nil then + range = nil + else + range = {range_start, range_end} + end + table.insert(issue_table, {identifier, message, range}) +end + +-- Prevent an issue from being present in the table of issues. +function Issues:ignore(identifier) + -- Remove the issue if it has already been added. + local issue_table = self:_get_issue_table(identifier) + local updated_issues = {} + for _, issue in ipairs(issue_table) do + if issue[1] ~= identifier then + table.insert(updated_issues, issue) + end + end + for issue_index, issue in ipairs(updated_issues) do + issue_table[issue_index] = issue + end + for issue_index = #updated_issues + 1, #issue_table, 1 do + issue_table[issue_index] = nil + end + -- Prevent the issue from being added later. + self.ignored_issues[identifier] = true +end + +-- Sort the warnings/errors using location as the primary key. +function Issues.sort(warnings_and_errors) + local sorted_warnings_and_errors = {} + for _, issue in ipairs(warnings_and_errors) do + local code = issue[1] + local message = issue[2] + local range = issue[3] + table.insert(sorted_warnings_and_errors, {code, message, range}) + end + table.sort(sorted_warnings_and_errors, function(a, b) + local a_code, b_code = a[1], b[1] + local a_range, b_range = (a[3] and a[3][1]) or 0, (b[3] and b[3][1]) or 0 + return a_range < b_range or (a_range == b_range and a_code < b_code) + end) + return sorted_warnings_and_errors +end + +return function() + return Issues:new() +end diff --git a/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua b/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua new file mode 100755 index 00000000000..8ab1cdc3d06 --- /dev/null +++ b/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua @@ -0,0 +1,183 @@ +-- The preprocessing step of static analysis determines which parts of the input files contain expl3 code. + +local lpeg = require("lpeg") +local Cp, P, R, S, V = lpeg.Cp, lpeg.P, lpeg.R, lpeg.S, lpeg.V + +-- Define base parsers. +---- Generic +local any = P(1) +local eof = -any + +---- Tokens +local lbrace = P("{") +local rbrace = P("}") +local percent_sign = P("%") +local backslash = P([[\]]) +local letter = R("AZ","az") +local underscore = P("_") +local colon = P(":") + +---- Spacing +local newline = ( + P("\n") + + P("\r\n") + + P("\r") +) +local linechar = any - newline +local spacechar = S("\t ") +local optional_spaces = spacechar^0 +local optional_spaces_and_newline_after_cs = ( + optional_spaces + * ( + newline + * optional_spaces + )^-1 +) + +-- Define intermediate parsers. +---- Parts of TeX syntax +local comment = ( + percent_sign + * linechar^0 + * newline + * optional_spaces +) +local argument = ( + lbrace + * ( + comment + + (any - rbrace) + )^0 + * rbrace +) +local expl3like_control_sequence = ( + backslash + * (letter - underscore - colon)^1 + * (underscore + colon) + * letter^1 +) + +---- Standard delimiters +local provides = ( + P([[\ProvidesExpl]]) + * ( + P("Package") + + P("Class") + + P("File") + ) + * optional_spaces_and_newline_after_cs + * comment^0 + * argument + * comment^0 + * argument + * comment^0 + * argument + * comment^0 + * argument +) +local expl_syntax_on = P([[\ExplSyntaxOn]]) +local expl_syntax_off = P([[\ExplSyntaxOff]]) + +local function preprocessing(issues, content, max_line_length) + if max_line_length == nil then + max_line_length = 80 + end + + -- Determine the bytes where lines begin. + local line_starting_byte_numbers = {} + local function record_line(line_start) + table.insert(line_starting_byte_numbers, line_start) + end + local function line_too_long(range_start, range_end) + issues:add('s103', 'line too long', range_start, range_end + 1) + end + local line_numbers_grammar = ( + Cp() / record_line + * ( + ( + ( + Cp() * linechar^(max_line_length + 1) * Cp() / line_too_long + + linechar^0 + ) + * newline + * Cp() + ) / record_line + )^0 + ) + lpeg.match(line_numbers_grammar, content) + -- Determine which parts of the input files contain expl3 code. + local expl_ranges = {} + local function capture_range(range_start, range_end) + table.insert(expl_ranges, {range_start, range_end + 1}) + end + local function unexpected_pattern(pattern, code, message, test) + return Cp() * pattern * Cp() / function(range_start, range_end) + if test == nil or test() then + issues:add(code, message, range_start, range_end + 1) + end + end + end + local num_provides = 0 + local analysis_grammar = P{ + "Root"; + Root = ( + ( + V"NonExplPart" + * V"ExplPart" / capture_range + )^0 + * V"NonExplPart" + ), + NonExplPart = ( + ( + unexpected_pattern( + V"Closer", + "w101", + "unexpected delimiters" + ) + + unexpected_pattern( + expl3like_control_sequence, + "e102", + "expl3 control sequences in non-expl3 parts" + ) + + (any - V"Opener") + )^0 + ), + ExplPart = ( + V"Opener" + * Cp() + * ( + unexpected_pattern( + V"Opener", + "w101", + "unexpected delimiters" + ) + + (any - V"Closer") + )^0 + * Cp() + * (V"Closer" + eof) + ), + Opener = ( + expl_syntax_on + + unexpected_pattern( + provides, + "e104", + [[multiple delimiters `\ProvidesExpl*` in a single file]], + function() + num_provides = num_provides + 1 + return num_provides > 1 + end + ) + ), + Closer = expl_syntax_off, + } + lpeg.match(analysis_grammar, content) + -- If no parts were detected, assume that the whole input file is in expl3. + if(#expl_ranges == 0 and #content > 0) then + table.insert(expl_ranges, {0, #content}) + issues:add('w100', 'no standard delimiters') + issues:ignore('e102') + end + return line_starting_byte_numbers, expl_ranges +end + +return preprocessing diff --git a/Master/texmf-dist/scripts/expltools/explcheck.lua b/Master/texmf-dist/scripts/expltools/explcheck.lua new file mode 100755 index 00000000000..f2b1cfaa6d7 --- /dev/null +++ b/Master/texmf-dist/scripts/expltools/explcheck.lua @@ -0,0 +1,3 @@ +#!/usr/bin/env texlua +-- A command-line interface for the static analyzer explcheck. +require("explcheck-cli") diff --git a/Master/tlpkg/bin/tlpkg-ctan-check b/Master/tlpkg/bin/tlpkg-ctan-check index facacae406b..e764f309828 100755 --- a/Master/tlpkg/bin/tlpkg-ctan-check +++ b/Master/tlpkg/bin/tlpkg-ctan-check @@ -325,7 +325,7 @@ my @TLP_working = qw( examdesign example examplep examz exceltex excludeonly exercise exercisebank exercisepoints exercises exercisesheets exesheet exframe exp-testopt - expdlist expex expex-acro expex-glossonly expkv-bundle export + expdlist expex expex-acro expex-glossonly expkv-bundle expltools export expose-expl3-dunkerque-2019 expressg exsheets exsol extarrows exteps extpfeil extract extractbb extsizes ezedits diff --git a/Master/tlpkg/libexec/ctan2tds b/Master/tlpkg/libexec/ctan2tds index a3a4f397313..8862dc13a39 100755 --- a/Master/tlpkg/libexec/ctan2tds +++ b/Master/tlpkg/libexec/ctan2tds @@ -554,6 +554,7 @@ chomp (my $ctan_root = `tlpkginfo --ctan-root`); 'excel2latex', "die 'skipping, only supports excel?'", 'expkv', "die 'skipping, replaced by expkv-bundle'", 'expl3', "die 'skipping, now l3kernel, etc.'", + 'expltools', "&MAKEflatten", 'expressgx', "&MAKEexpressg", 'extractpdfmark', "die 'skipping, compiled program'", 'fac', "die 'skipping, noinfo license'", @@ -2233,6 +2234,7 @@ $standardtex 'europasscv', '(icon|logo).*\.pdf|' . $standardtex, 'europecv', 'EuropeFlag|europasslogo|\.cls|\.def', 'exam-n', '\.cls', # not .clo example file + 'expltools', 'NULL', # doc 'fancyqr', '\.code|' . $standardtex, 'fancytooltips', 'fancytipmark[.1-4].*|' . $standardtex, 'fewerfloatpages', '\.sty$', # not l3doc-TUB.cls @@ -3497,6 +3499,7 @@ $standardinsrunner = "latex -8bit"; 'bibleref' => 'NULL', # doc 'einfuehrung' => 'NULL', # doc 'einfuehrung2' => 'NULL', # doc + 'expltools' => 'NULL', # doc 'hyperxmp' => 'NULL', # doc 'istgame' => 'NULL', # doc 'labyrinth' => 'NULL', # doc @@ -3766,6 +3769,7 @@ $standardttf = '\.ttf|\.TTC'; 'changes' => '\.py$', 'cloze' => '\.lua$', 'epspdf' => '(epspdf(|\.help|\.tlu|.*tk.*)|\.rb|makegray\.pro)$', + 'expltools' => 'expl.*\.lua$', 'extractbb' => 'extractbb-.*\.lua$', 'latex2nemeth' => '\.jar$', 'latex-make' => '\.py$', @@ -3840,6 +3844,7 @@ $standardttf = '\.ttf|\.TTC'; 'eolang' => '\.pl$', 'epstopdf' => 'epstopdf\.pl', # doscripts() does r* 'exceltex' => 'exceltex$', + 'expltools' => 'explcheck\.lua$', 'extractbb' => 'extractbb\.lua$', 'fig4latex' => 'fig4latex', 'findhyph' => 'findhyph$', @@ -7299,7 +7304,6 @@ sub POSTfixme { print "POST$package - move subdirectories\n"; my $dstdir = "$DEST/tex/latex/$package"; &xmkdir ($dstdir); - &SYSTEM ("$MV themes layouts $dstdir/"); } sub POSTfont_change_xetex { @@ -8082,7 +8086,7 @@ sub POST_onelevel { my $foundry = $specialfoundry{$package} || $standardfoundry; $format = "$dir/$foundry"; } else { - die "$0: quitting, don't know about one-level directory: $dir\n"; + die "$0: quitting, don't know about one-level directory: $dir"; } &rename_with_mkdir ($dir, "$DEST/$destdir/$format/$package"); } diff --git a/Master/tlpkg/tlpsrc/collection-binextra.tlpsrc b/Master/tlpkg/tlpsrc/collection-binextra.tlpsrc index 35a54dad764..5eb1801661e 100644 --- a/Master/tlpkg/tlpsrc/collection-binextra.tlpsrc +++ b/Master/tlpkg/tlpsrc/collection-binextra.tlpsrc @@ -41,6 +41,7 @@ depend dvipng depend dvipos depend dvisvgm depend easydtx +depend expltools depend findhyph depend fragmaster depend git-latexdiff diff --git a/Master/tlpkg/tlpsrc/expltools.tlpsrc b/Master/tlpkg/tlpsrc/expltools.tlpsrc new file mode 100644 index 00000000000..2aad5d7a05c --- /dev/null +++ b/Master/tlpkg/tlpsrc/expltools.tlpsrc @@ -0,0 +1 @@ +binpattern f bin/${ARCH}/explcheck |