summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/prettytok
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex/contrib/prettytok')
-rw-r--r--macros/latex/contrib/prettytok/DEPENDS.txt1
-rw-r--r--macros/latex/contrib/prettytok/README7
-rwxr-xr-xmacros/latex/contrib/prettytok/prettytok-decode-8bit.py12
-rw-r--r--macros/latex/contrib/prettytok/prettytok-test.tex17
-rw-r--r--macros/latex/contrib/prettytok/prettytok.lua198
-rw-r--r--macros/latex/contrib/prettytok/prettytok.pdfbin369580 -> 485548 bytes
-rw-r--r--macros/latex/contrib/prettytok/prettytok.sty378
-rw-r--r--macros/latex/contrib/prettytok/prettytok.tex409
-rw-r--r--macros/latex/contrib/prettytok/prettytok_template.html76
9 files changed, 852 insertions, 246 deletions
diff --git a/macros/latex/contrib/prettytok/DEPENDS.txt b/macros/latex/contrib/prettytok/DEPENDS.txt
index dd4fa25323..a575b01313 100644
--- a/macros/latex/contrib/prettytok/DEPENDS.txt
+++ b/macros/latex/contrib/prettytok/DEPENDS.txt
@@ -1 +1,2 @@
precattl
+l3keys2e
diff --git a/macros/latex/contrib/prettytok/README b/macros/latex/contrib/prettytok/README
index 531d9833dd..421737ab54 100644
--- a/macros/latex/contrib/prettytok/README
+++ b/macros/latex/contrib/prettytok/README
@@ -1,6 +1,5 @@
prettytok -- Pretty-print token list
-Pretty-print token lists to HTML file for debugging purposes.
-Open the file in any browser to view the result.
+Pretty-print token lists for debugging purposes.
Can be used to replace |\tl_analysis_show:n|.
@@ -11,7 +10,7 @@ Report bugs at https://github.com/user202729/TeXlib
========
-Copyright 2022 user202729
+Copyright 2022-2023 user202729
This work may be distributed and/or modified under the conditions of the
LaTeX Project Public License (LPPL), either version 1.3c of this license or
@@ -24,4 +23,4 @@ This work has the LPPL maintenance status `maintained'.
The Current Maintainer of this work is user202729.
-This work consists of the files prettytok.sty, prettytok.lua, prettytok_template.html.
+This work consists of the files prettytok.sty, prettytok.lua, prettytok_template.html, prettytok-decode-8bit.py, prettytok-test.tex.
diff --git a/macros/latex/contrib/prettytok/prettytok-decode-8bit.py b/macros/latex/contrib/prettytok/prettytok-decode-8bit.py
new file mode 100755
index 0000000000..231cf84bd9
--- /dev/null
+++ b/macros/latex/contrib/prettytok/prettytok-decode-8bit.py
@@ -0,0 +1,12 @@
+#!/bin/python3
+"""
+A hack to allow decoding for example ^^[ to the proper escape sequence.
+"""
+
+import sys
+import re
+def replace_8bit(match_: re.Match) -> str:
+ return chr(ord(match_.group(1))^0x40)
+if __name__ == '__main__':
+ for line in sys.stdin:
+ sys.stdout.write(re.sub(r"\^\^(.)", replace_8bit, line))
diff --git a/macros/latex/contrib/prettytok/prettytok-test.tex b/macros/latex/contrib/prettytok/prettytok-test.tex
new file mode 100644
index 0000000000..cc3ad7e8fd
--- /dev/null
+++ b/macros/latex/contrib/prettytok/prettytok-test.tex
@@ -0,0 +1,17 @@
+\documentclass{article}
+\usepackage{prettytok}
+
+\prettyN{123{4 5}\6}
+\prettyN{#&^_:}
+\edef\mytest{\prettyeN{\error}}
+
+\prettyN {very long long long long long long long long long long long long long long argument \argument\argument\test123456}
+\prettyeN {very long long long long long long long long long long long long long long argument \argument\argument\test123456}
+
+\ExplSyntaxOn
+\pretty:n {#&^_:}
+\ExplSyntaxOff
+
+\begin {document}
+\prettyW abc \prettystop
+\end {document}
diff --git a/macros/latex/contrib/prettytok/prettytok.lua b/macros/latex/contrib/prettytok/prettytok.lua
index a0aba07243..0abb6595f0 100644
--- a/macros/latex/contrib/prettytok/prettytok.lua
+++ b/macros/latex/contrib/prettytok/prettytok.lua
@@ -6,8 +6,8 @@ end
-- public API :: once the package is loaded, this holds the .tok value of the frozen relax token
prettyprint_frozenrelaxtok=nil
-local function prettyprint_one_arg(tokenlist)
+local function prettyprint_one_arg(tokenlist)
if looks_like_token(tokenlist) then
return prettyprint_one_arg({tokenlist})
end
@@ -46,8 +46,9 @@ local function prettyprint_one_arg(tokenlist)
s=s.."csfrozenrelax(),"
else
s=s.."cs("
- for j=1, #t.csname do
- s=s..t.csname:byte(j)..","
+ -- TODO
+ for _, c in utf8.codes(t.csname) do
+ s=s..c..","
end
s=s.."),"
end
@@ -58,41 +59,192 @@ local function prettyprint_one_arg(tokenlist)
return s
end
-local function check_file_opened()
- -- if it's \relax then get_macro() also returns nil
- if token.get_macro("pretty_check_already_init:") == nil then
- error("Output file not initialized!")
- end
+local function get_output_format()
+ return token.get_macro "_prettytok_mode"
end
local function get_file_number()
- check_file_opened()
- return token.get_mode(token.create("__prettyh_file"))
+ return token.get_mode(token.create("_prettytok_file"))
end
function prettyprint(...)
- local prettyfilenumber=get_file_number()
-
- local s="print_tl("
+ local output_format=get_output_format()
local args={...}
- for i=1, select("#", ...) do
- s=s..prettyprint_one_arg(args[i])
- end
+ if output_format=="html" then
+ local prettyfilenumber=get_file_number()
+
+ local s="print_tl("
+ for i=1, select("#", ...) do
+ s=s..prettyprint_one_arg(args[i])
+ end
+
+ texio.write(prettyfilenumber, s..")//</script><script>\n")
+ else
+ assert(output_format=="term-8bit" or output_format=="term-shell")
+ texio.write_nl("")
+
+ local content=""
+ local content_len=0 -- only count visible content
+
+ local function typeout_content()
+ print(content)
+ content=""
+ content_len=0
+ end
+
+ local function append_content(s) -- only call this on content that takes visible width. For example don't use this to set color
+ assert(type(s)=="string")
+ content=content..s
+ content_len=content_len+utf8.len(s)
+ end
+
+ append_content(token.get_macro "prettytermprefix")
+ local wraplimit=tonumber(token.get_macro "prettytermwraplimit")
+
+ ---- color
+
+ --[[
+ local gray ='\x1b[90m'
+ local red ='\x1b[31m'
+ local green ='\x1b[38;5;121m'
+ local yellow='\x1b[93m'
+ local white ='\x1b[0m'
+ ]]
- texio.write(prettyfilenumber, s..")//</script><script>\n")
+ local gray =token.get_macro "_prettytok_gray"
+ local red =token.get_macro "_prettytok_red"
+ local green =token.get_macro "_prettytok_green"
+ local yellow=token.get_macro "_prettytok_yellow"
+ local white =token.get_macro "_prettytok_white"
+
+ local current_color=white
+ local function setcolor(c)
+ if current_color~=c then
+ current_color=c
+ content=content..c
+ end
+ end
+
+ local prettytermprefixmore=token.get_macro "prettytermprefixmore"
+
+ local function check_wrap()
+ if content_len>wraplimit then
+ if current_color~=white then
+ content=content..white
+ end
+ print(content)
+
+ content=""
+ content_len=0
+ append_content(prettytermprefixmore)
+ if current_color~=white then
+ content=content..current_color
+ end
+ end
+ end
+
+ local function prettyprint_term_one_arg(tokenlist)
+ if looks_like_token(tokenlist) then
+ return prettyprint_term_one_arg({tokenlist})
+ end
+
+
+ local function print_term_char(code)
+ if code<27 then
+ if code==9 then
+ append_content("⇥")
+ else
+ append_content("^^"..string.char(code+64))
+ if code==13 or code==10 then
+ typeout_content()
+ append_content(prettytermprefixmore) -- color the prefix with the same color
+ end
+ end
+ elseif code==32 then
+ append_content("␣")
+ else
+ append_content(utf8.char(code)) -- unlike TeX implementation currently this does not show ^^⟨hex digits⟩ representation
+ end
+ end
+
+
+ local function printstring(t)
+ -- this is just for debugging/convenience purpose. If the item is a string instead of a token...
+ t=tostring(t)
+ for _, c in utf8.codes(t) do
+ if c==32 then
+ setcolor(gray)
+ append_content("␣")
+ else
+ setcolor(white)
+ append_content(utf8.char(c))
+ end
+ end
+ end
+
+ if type(tokenlist)~="table" then
+ -- user give a string (or something similar). Print it as detokenized characters
+ printstring(tokenlist)
+ check_wrap()
+
+ else
+
+ -- normal print of tokenlist
+ for i=1, #tokenlist do
+ local t=tokenlist[i]
+ if not looks_like_token(t) then
+ printstring(t)
+ check_wrap()
+ elseif t.csname==nil then
+ setcolor(({left_brace=red, right_brace=red, math_shift=red, tab_mark=red, mac_param=red, sup_mark=red, sub_mark=red, spacer=gray, letter=green, other_char=white})[t.cmdname])
+ print_term_char(t.mode)
+ check_wrap()
+ else
+ if t.active then
+ setcolor(yellow)
+ print_term_char(utf8.codepoint(t.csname))
+ elseif t.tok==prettyprint_frozenrelaxtok then
+ setcolor(red)
+ append_content("\\relax ")
+ else
+ setcolor(yellow)
+ append_content("\\")
+ for _, c in utf8.codes(t.csname) do
+ print_term_char(c)
+ end
+ append_content(" ")
+ end
+ check_wrap()
+ end
+ end
+
+ end
+ end
+
+ for i=1, select("#", ...) do
+ prettyprint_term_one_arg(args[i])
+ end
+
+ setcolor(white)
+
+ typeout_content()
+ end
end
function prettyprintw()
- check_file_opened()
+ local callback=token.scan_toks()
local extra=token.scan_toks()
local s={}
+ local last_token
while true do
- local n=token.get_next()
- s[#s+1]=n
- if n.csname=="prettystop" then break end
+ last_token=token.get_next()
+ if last_token.csname=="prettystop" then break end
+ s[#s+1]=last_token
+ callback[#callback+1]=last_token
end
- prettyprint(extra, s)
- token.put_next(s)
+ prettyprint(extra, s) -- exclude the \prettystop token
+ callback[#callback+1]=last_token
+ token.put_next(callback)
end
end
diff --git a/macros/latex/contrib/prettytok/prettytok.pdf b/macros/latex/contrib/prettytok/prettytok.pdf
index 5b10ffaf19..87fda692ff 100644
--- a/macros/latex/contrib/prettytok/prettytok.pdf
+++ b/macros/latex/contrib/prettytok/prettytok.pdf
Binary files differ
diff --git a/macros/latex/contrib/prettytok/prettytok.sty b/macros/latex/contrib/prettytok/prettytok.sty
index 3acea38c62..d12abc727d 100644
--- a/macros/latex/contrib/prettytok/prettytok.sty
+++ b/macros/latex/contrib/prettytok/prettytok.sty
@@ -1,5 +1,5 @@
% File: prettytok.sty
-% Copyright 2022 user202729
+% Copyright 2022-2023 user202729
%
% This work may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this license or
@@ -12,33 +12,60 @@
%
% The Current Maintainer of this work is user202729.
-\ProvidesExplPackage{prettytok}{2022/07/08}{0.0.1}{Pretty-print token list}
+\ProvidesExplPackage{prettytok}{2023-04-18}{0.2.0}{Pretty-print token list}
\RequirePackage{precattl}
+\RequirePackage{l3keys2e}
-% ======== check tl_analysis bug.
-\msg_new:nnn {prettytok} {tl-analysis-bug}
- {
- Because~of~an~tl-analysis~bug~(see~https://github.com/latex3/latex3/issues/1073),~functions~will~not~be~usable~inside~tl-analysis~functions!~
- Upgrade~your~LaTeX~version,~or~
- define~macro~"prettyignoretlanalysisbug"~before~loading~the~package~to~ignore~this~error.
- }
+\keys_define:nn{prettytok}{
+ mode.choices:nn={ term-8bit, term-shell, html }{
+ \str_set:Nx \_prettytok_mode {\l_keys_choice_tl}
+ },
+ mode.initial:n = { term-8bit },
-\ifdefined \prettyignoretlanalysisbug
- \relax
-\else
-\begingroup
- \tl_set:Nn \__test {}
+ ignore-tlanalysis-bug.bool_set:N=\_prettytok_ignore_tlanalysis_bug,
+
+ term-shell-decode-cmd.tl_set:N=\_prettytok_term_shell_decode_cmd,
+ term-shell-decode-cmd.initial:n={},
+ term-shell-decode-cmd-print.bool_set:N=\_prettytok_term_shell_decode_cmd_print,
+
+ html-file-name.tl_set:N=\_prettytok_html_file_name,
+ html-file-name.initial:x={pretty-\jobname.html},
+ html-refresh-strategy.tl_set:N=\_prettytok_html_refresh_strategy,
+ html-refresh-strategy.initial:n=4,
+ html-refresh-duration.tl_set:N=\_prettytok_html_refresh_duration,
+ html-refresh-duration.initial:n=1000,
+
+ term-prefix.tl_set:N=\prettytermprefix,
+ term-prefix.initial:n={>~},
+ term-prefix-more.tl_set:N=\prettytermprefixmore,
+ term-prefix-more.initial:n={>~ ..~},
+ term-wrap-limit.tl_set:N=\prettytermwraplimit,
+ term-wrap-limit.initial:n=70,
+
+}
+\ProcessKeysOptions{prettytok}
+
+
+% ======== check tl_analysis bug.
+\bool_if:NF \_prettytok_ignore_tlanalysis_bug {
+ \tl_set:Nn \_prettytok_test {}
\tl_analysis_map_inline:nn {ab} {
\tl_analysis_map_inline:nn {cd} {
- \tl_put_right:Nx \__test {#1 ##1 .}
+ \tl_put_right:Nx \_prettytok_test {#1 ##1 .}
}
}
- \tl_if_eq:NnF \__test {ac.ad.bc.bd.} {
+ \tl_if_eq:NnF \_prettytok_test {ac.ad.bc.bd.} {
+ \msg_new:nnn {prettytok} {tl-analysis-bug}
+ {
+ Because~of~an~tl-analysis~bug~(see~https://github.com/latex3/latex3/issues/1073),~
+ functions~will~not~be~usable~inside~tl-analysis~functions!~
+ Upgrade~your~LaTeX~version,~or~
+ pass-option~"ignore-tlanalysis-bug"~to~ignore~this~error.
+ }
\msg_error:nn {prettytok} {tl-analysis-bug}
}
-\endgroup
-\fi
+}
% ======== load Lua module.
\sys_if_engine_luatex:T{
@@ -49,7 +76,6 @@
}
% ======== main code start.
-
\cs_generate_variant:Nn \exp_args:NNn {NNx}
\cs_generate_variant:Nn \iow_open:Nn {NV}
\cs_generate_variant:Nn \exp_args:NNn {NNV}
@@ -59,90 +85,241 @@
\cs_generate_variant:Nn \use:N {c}
\cs_generate_variant:Nn \str_map_inline:nn {xn}
\cs_generate_variant:Nn \tl_build_put_right:Nn {Nx}
+\cs_generate_variant:Nn \tl_build_put_right:Nn {NV}
\cs_generate_variant:Nn \iow_now:Nn {NV}
-\iow_new:N \_prettytok_file
-\ior_new:N \_prettytok_input_template_file
+\str_if_eq:VnTF \_prettytok_mode {html} {
+ \iow_new:N \_prettytok_file
+ \ior_new:N \_prettytok_input_template_file
+
+ \iow_open:NV \_prettytok_file \_prettytok_html_file_name
-\exp_args:NNx \providecommand \prettyfilename {pretty-\jobname.html}
-\providecommand \prettyrefreshstrategy {4}
-\providecommand \prettyrefreshduration {1000}
+ \precattl_exec:n{
+ % write the template
+ \ior_open:Nn \_prettytok_input_template_file {prettytok_template.html}
+ \ior_str_map_variable:NNn \_prettytok_input_template_file \_prettytok_line {
+ \str_replace_all:Nnn \_prettytok_line {\cO\^^I} {~} % workaround: XeTeX prints literal tab character as ^^I which obviously breaks JavaScript
+ \iow_now:NV \_prettytok_file \_prettytok_line
+ }
+ \ior_close:N \_prettytok_input_template_file
-\msg_new:nnn {prettytok} {already-initialized} {Output~file~is~already~initialized!}
+ % refresh strategy
+ \iow_now:Nx \_prettytok_file {set_refresh_strategy(\_prettytok_html_refresh_strategy,\_prettytok_html_refresh_duration)}
+ \cs_new_protected:Npn \pretty:n #1 {
-% See also Lua function: local function check_file_opened()
+ % ======== the actual content. Collect to one \write to put them on the same line (also maybe for efficiency???)
+ \tl_build_begin:N \_prettytok_content
+ \tl_build_put_right:Nn \_prettytok_content {print_tl(}
+ \tl_analysis_map_inline:nn {#1} {
+ % by documentation of \tl_analysis_map_inline:nn: #1=token, #2=char code, #3=catcode
+ \int_compare:nNnTF {##2} = {-1} {
+ % token is control sequence
+ \tl_if_eq:onTF {##1} {\cFrozenRelax} {
+ \tl_build_put_right:Nn \_prettytok_content {csfrozenrelax(),}
+ } {
+ \tl_build_put_right:Nn \_prettytok_content {cs(}
+ \tl_if_eq:xoF {\use:c {}} {##1} {
+ \str_map_inline:xn {\exp_after:wN \cs_to_str:N ##1} { % side note, must use str here to preserve spaces
+ \tl_build_put_right:Nx \_prettytok_content {\int_eval:n {`####1},}
+ }
+ }
+ \tl_build_put_right:Nn \_prettytok_content {),}
+ }
+ }
+ {
+ % token is not control sequence
+ \tl_build_put_right:Nn \_prettytok_content {token(##2, "##3"),}
+ }
+ }
+ \tl_build_put_right:Nn \_prettytok_content {)//</script><script>}
+ \tl_build_end:N \_prettytok_content
+ \iow_now:NV \_prettytok_file \_prettytok_content
+ }
+ }
-\global\let \pretty_check_already_init: \relax
+} {
+ \str_if_eq:VnTF \_prettytok_mode {term-shell} {
+ \newwrite \_prettytok_typeout_stream
+ \tl_if_empty:NTF \_prettytok_term_shell_decode_cmd {
+ \sys_get_shell:nnN {kpsewhich~ prettytok-decode-8bit.py} {\cctab_select:N \c_other_cctab} \_prettytok_term_shell_decode_cmd
+ } {
+ \begingroup
+ \escapechar=-1~
+ \str_gset:NV \_prettytok_term_shell_decode_cmd \_prettytok_term_shell_decode_cmd
+ \endgroup
+ }
+ \bool_if:NT \_prettytok_term_shell_decode_cmd_print {
+ \typeout{^^JThe~ value~ of~ term-shell-decode-cmd~ is:~ [[[\_prettytok_term_shell_decode_cmd]]]^^J}
+ }
+ \immediate \openout \_prettytok_typeout_stream = {|\_prettytok_term_shell_decode_cmd}
+ } {
+ \int_const:Nn \_prettytok_typeout_stream {128} % 16 works except on LuaLaTeX
+ }
-\precattl_exec:n {
- \cs_new_protected:Npn \_prettyinit_unchecked: {
- \global\protected\def \pretty_check_already_init: {\msg_error:nn {prettytok} {already-initialized}}
- \iow_open:NV \_prettytok_file \prettyfilename
+ \precattl_exec:n{
+ \str_gset:Nn \_prettytok_gray {\cO\^^[[90m}
+ \str_gset:Nn \_prettytok_red {\cO\^^[[31m}
+ \str_gset:Nn \_prettytok_green {\cO\^^[[38;5;121m} % this is the color vim use for green on my machine. Anyway~
+ %\str_gset:Nn \_prettytok_green {\cO\^^[[92m}
+ \str_gset:Nn \_prettytok_yellow {\cO\^^[[93m}
+ %\str_gset:Nn \_prettytok_white {\cO\^^[[0m} % i.e. the default
+ \str_gset:Nn \_prettytok_white {\cO\^^[[97m} % vim terminal has a quirk that if a line wraps, the status at the start of the line is considered "default" for that line, so need to explicitly specify color as white
- % write the template
- \begingroup
- \ior_open:Nn \_prettytok_input_template_file {prettytok_template.html}
- \ior_str_map_variable:NNn \_prettytok_input_template_file \_prettytok_line {
- \str_replace_all:Nnn \_prettytok_line {\cO\^^I} {~} % workaround: XeTeX prints literal tab character as ^^I which obviously breaks JavaScript
- \iow_now:NV \_prettytok_file \_prettytok_line
- }
- \ior_close:N \_prettytok_input_template_file
- \endgroup
+ % https://stackoverflow.com/q/4842424/5267751
- % refresh strategy
- \iow_now:Nx \_prettytok_file {set_refresh_strategy(\prettyrefreshstrategy,\prettyrefreshduration)}
+ \cs_new_protected:Npn \_prettytok_setcolor:N #1 {
+ \ifx \_prettytok_current_color #1
+ \else
+ \let \_prettytok_current_color #1
+ \tl_build_put_right:NV \_prettytok_content #1
+ \fi
}
- \cs_new_protected:Npn \prettyinit: {
- \pretty_check_already_init:
- \_prettyinit_unchecked:
+ % caller should call \_prettytok_check_wrap: when useful.
+ \cs_new_protected:Npn \_prettytok_term_prepare_print_char:n #1 {
+ \_prettytok_append_content:x {
+ \ifnum #1 < 27 ~
+ \ifnum #1 = 9 ~ % tab
+ \cO{⇥}
+ \else
+ \cO{\^\^} \char_generate:nn {#1+64} {12}
+ \bool_if:nT {\int_compare_p:nNn {#1} = {13} || \int_compare_p:nNn {#1} = {10}} {
+ \cO{^^J} \prettytermprefixmore
+ }
+ \fi
+ \else
+ \ifnum #1 = 32 ~
+ \cO{␣}
+ \else
+ \bool_if:nTF {\int_compare_p:nNn {#1} > {32} && \int_compare_p:nNn {#1} < {127}} { % printable ASCII character
+ \char_generate:nn {#1} {12}
+ } {
+ \ifnum #1 < "100 ~
+ ^ ^
+ \expandafter \@gobble \exp:w \exp_end_continue_f:w \int_to_hex:n {"100+#1} % convert to hex, exactly 2 digits (by adding 100 then delete first '1')
+ \else
+ \ifnum #1 < "10000 ~
+ ^ ^ ^ ^
+ \expandafter \@gobble \exp:w \exp_end_continue_f:w \int_to_hex:n {"10000+#1} % convert to hex, exactly 4 digits
+ \else
+ ^ ^ ^ ^ ^ ^
+ \expandafter \@gobble \exp:w \exp_end_continue_f:w \int_to_hex:n {"1000000+#1} % convert to hex, exactly 6 digits
+ \fi
+ \fi
+ }
+ \fi
+ \fi
+ }
}
-}
+ \cs_generate_variant:Nn \_prettytok_term_prepare_print_char:n {x}
-\cs_new_eq:NN \prettyinit \prettyinit:
+ \int_new:N \_prettytok_content_len
+ % only call this on content that takes visible width. For example don't use this to set color
+ \cs_new_protected:Npn \_prettytok_append_content:n #1 {
+ \tl_build_put_right:Nn \_prettytok_content {#1}
+ \int_add:Nn \_prettytok_content_len {\str_count:n {#1}}
+ }
-\precattl_exec:n {
+ \cs_generate_variant:Nn \_prettytok_append_content:n {V,x}
+
+ \cs_new_protected:Npn \_prettytok_check_wrap: {
+ \int_compare:nNnT {\_prettytok_content_len} > {\prettytermwraplimit} {
+ % be careful to not color the \prettytermprefixmore part
+ \ifx \_prettytok_current_color \_prettytok_white
+ \else
+ \tl_build_put_right:NV \_prettytok_content \_prettytok_white
+ \fi
+ \tl_build_end:N \_prettytok_content
+ \_prettytok_typeout_content:
+
+ \tl_build_begin:N \_prettytok_content
+ \int_zero:N \_prettytok_content_len
+ \_prettytok_append_content:V \prettytermprefixmore
+ \ifx \_prettytok_current_color \_prettytok_white
+ \else
+ \tl_build_put_right:NV \_prettytok_content \_prettytok_current_color
+ \fi
+ }
+ }
+ % \_prettytok_content must have been called \tl_build_end:N on before this function is called
+ \sys_if_engine_luatex:TF {
+ \cs_new_protected:Npn \_prettytok_typeout_content: {
+ \directlua{
+ texio.write_nl("")
+ print(token.get_macro("_prettytok_content"))
+ } % https://tex.stackexchange.com/questions/64462/lualatex-print-directlua-env-and-newlinehttps://tex.stackexchange.com/questions/64462/lualatex-print-directlua-env-and-newline
+ }
+ } {
+ \cs_new_protected:Npn \_prettytok_typeout_content: {
+ \immediate\write\_prettytok_typeout_stream{\_prettytok_content}
+ }
+ }
\cs_new_protected:Npn \pretty:n #1 {
+ \tl_build_begin:N \_prettytok_content
+ \int_zero:N \_prettytok_content_len
+ \_prettytok_append_content:V \prettytermprefix
- % ======== first write out the preamble if it isn't written. Only do once.
- \ifx \pretty_check_already_init: \relax \prettyinit: \fi
+ \let \_prettytok_current_color \_prettytok_white
- % ======== the actual content. Collect to one \write to put them on the same line (also maybe for efficiency???)
- \tl_build_begin:N \_prettytok_content
- \tl_build_put_right:Nn \_prettytok_content {print_tl(}
\tl_analysis_map_inline:nn {#1} {
% by documentation of \tl_analysis_map_inline:nn: #1=token, #2=char code, #3=catcode
\int_compare:nNnTF {##2} = {-1} {
% token is control sequence
\tl_if_eq:onTF {##1} {\cFrozenRelax} {
- \tl_build_put_right:Nn \_prettytok_content {csfrozenrelax(),}
+ \_prettytok_setcolor:N \_prettytok_red
+ \_prettytok_append_content:n { \cO\\relax~ }
} {
- \tl_build_put_right:Nn \_prettytok_content {cs(}
+ \_prettytok_setcolor:N \_prettytok_yellow
+ \_prettytok_append_content:n { \cO\\ }
\tl_if_eq:xoF {\use:c {}} {##1} {
\str_map_inline:xn {\exp_after:wN \cs_to_str:N ##1} { % side note, must use str here to preserve spaces
- \tl_build_put_right:Nx \_prettytok_content {\int_eval:n {`####1},}
+ \_prettytok_term_prepare_print_char:x {\int_eval:n {`####1}}
}
}
- \tl_build_put_right:Nn \_prettytok_content {),}
+ \_prettytok_append_content:n {~}
+ \_prettytok_check_wrap: % here we do not wrap in the middle of a csname
}
}
{
% token is not control sequence
- \tl_build_put_right:Nn \_prettytok_content {token(##2, "##3"),}
+ \if ##3 B % letter
+ \_prettytok_setcolor:N \_prettytok_green
+ \else
+ \if ##3 A % spacer
+ \_prettytok_setcolor:N \_prettytok_gray
+ \else
+ \if ##3 D % active
+ \_prettytok_setcolor:N \_prettytok_yellow
+ \else
+ \if ##3 C % other cat
+ \_prettytok_setcolor:N \_prettytok_white
+ \else % something else.
+ \_prettytok_setcolor:N \_prettytok_red
+ \fi
+ \fi
+ \fi
+ \fi
+
+ \_prettytok_term_prepare_print_char:n {##2}
+ \_prettytok_check_wrap:
}
}
- \tl_build_put_right:Nn \_prettytok_content {)//</script><script>}
+ \_prettytok_setcolor:N \_prettytok_white
\tl_build_end:N \_prettytok_content
- \iow_now:NV \_prettytok_file \_prettytok_content
+
+ \_prettytok_typeout_content:
+ }
}
+
}
+
% ======== Lua expandable variant.
\sys_if_engine_luatex:T {
\cs_new:Npn \prettye:n #1 {
@@ -150,65 +327,84 @@
}
\cs_new:Npn \prettye:w {
- \directlua{prettyprintw()} {}
+ \directlua{prettyprintw()} {} {}
}
\cs_new:Npn \prettye:nw #1 {
- \directlua{prettyprintw()} {#1}
+ \directlua{prettyprintw()} {#1} {}
+ }
+ \cs_new:Npn \prettye:nnw #1 #2 {
+ \directlua{prettyprintw()} {#1} {#2}
}
}
-\cs_new_protected:Npn \pretty:w #1 \prettystop {
- \pretty:n {#1} #1 \prettystop
-}
+% ======== w variant.
+\cs_new_protected:Npn \pretty:w { \pretty_aux:w \empty }
+\cs_new_protected:Npn \pretty_aux:w #1 \prettystop {
+ \pretty:o { #1 } % use o-expansion to remove the \empty
+ #1 \prettystop
+}
\cs_new:Npn \prettystop {}
+% ======== other variants.
\cs_generate_variant:Nn \pretty:n {x, o, V}
-\let \pretty:N \pretty:n
+\cs_new_eq:NN \pretty:N \pretty:n
+\cs_generate_variant:Nn \pretty:N {c}
-% ======== normal-catcode alias
-\let\prettyN\pretty:n
-\let\prettyX\pretty:x
-\let\prettyO\pretty:o
-\let\prettyV\pretty:V
-\let\prettyeN\prettye:n
-\let\prettyeW\prettye:w
+\cs_new_protected:Npn \prettythenrun:n #1 {
+ \pretty:n {running~#1}
+ #1
+ \pretty:n {done}
+}
+\cs_new_protected:Npn \prettythenrun:w #1 \prettythenrunstop {
+ \prettythenrun:n {#1}
+}
-\cs_generate_variant:Nn \pretty:N {c}
+\cs_new_protected:Nn \prettyshow:N { \pretty:o {\meaning #1} }
+\cs_generate_variant:Nn \prettyshow:N {c}
% ======== generate \pretty:nn, \pretty:nnn, etc.
\begingroup
- \def \__tmp:NN #1 #2 { % #1: example \pretty:nn, #2: example \pretty:n (the latter has one more n than the former)
+ \def \_prettytok_tmp:NN #1 #2 { % #1: example \pretty:nn, #2: example \pretty:n (the latter has one more n than the former)
\cs_new_protected:Npn #1 ##1 ##2 { #2 {##1 ##2} }
}
- \cs_generate_variant:Nn \__tmp:NN {cc}
+ \cs_generate_variant:Nn \_prettytok_tmp:NN {cc}
- \def \__tmp {pretty:n}
+ \def \_prettytok_tmp {pretty:n}
\int_step_inline:nn {9} {
- \__tmp:cc {\__tmp n} {\__tmp}
- \tl_put_right:Nn \__tmp {n}
+ \_prettytok_tmp:cc {\_prettytok_tmp n} {\_prettytok_tmp}
+ \tl_put_right:Nn \_prettytok_tmp {n}
}
\sys_if_engine_luatex:T {
- \def \__tmp:NN #1 #2 { % #1: example \prettye:nn, #2: example \prettye:n (the latter has one more n than the former)
+ \def \_prettytok_tmp:NN #1 #2 { % #1: example \prettye:nn, #2: example \prettye:n (the latter has one more n than the former)
\cs_new:Npn #1 ##1 ##2 { #2 {##1 ##2} }
}
- \def \__tmp {prettye:n}
+ \def \_prettytok_tmp {prettye:n}
+ \int_step_inline:nn {9} {
+ \_prettytok_tmp:cc {\_prettytok_tmp n} {\_prettytok_tmp}
+ \tl_put_right:Nn \_prettytok_tmp {n}
+ }
+
+ \def \_prettytok_tmp:NN #1 #2 { % #1: example \prettye:nnnw, #2: example \prettye:nnw (the latter has one more n than the former)
+ \cs_new:Npn #1 ##1 ##2 ##3 { #2 {##1} {##2 ##3} } % here ##1 is the callback (first ⟨n⟩), ##2 etc. are the additional-print things
+ }
+ \def \_prettytok_tmp {prettye:nn}
\int_step_inline:nn {9} {
- \__tmp:cc {\__tmp n} {\__tmp}
- \__tmp:cc {\__tmp nw} {\__tmp w}
- \tl_put_right:Nn \__tmp {n}
+ \_prettytok_tmp:cc {\_prettytok_tmp nw} {\_prettytok_tmp w}
+ \tl_put_right:Nn \_prettytok_tmp {n}
}
}
\endgroup
-
-\cs_new_protected:Nn \prettyshow:N {
- \exp_args:No \pretty:n {\meaning #1}
-}
-\cs_generate_variant:Nn \prettyshow:N {c}
+% ======== normal-catcode alias
+\let\prettyN\pretty:n
+\let\prettyX\pretty:x
+\let\prettyO\pretty:o
+\let\prettyV\pretty:V
+\let\prettyW\pretty:w
+\let\prettyeN\prettye:n
+\let\prettyeW\prettye:w
\let\prettyshowN\prettyshow:N
\let\prettyshowC\prettyshow:c
-
-
diff --git a/macros/latex/contrib/prettytok/prettytok.tex b/macros/latex/contrib/prettytok/prettytok.tex
index 96878ec457..ddd77dc494 100644
--- a/macros/latex/contrib/prettytok/prettytok.tex
+++ b/macros/latex/contrib/prettytok/prettytok.tex
@@ -1,14 +1,19 @@
%! TEX program = lualatex
-\ProvidesFile{prettytok.tex} [2022/07/08 v0.0.1 ]
+\ProvidesFile{prettytok.tex} [2023/04/18 v0.2.0 ]
+\RequirePackage{fvextra}
\documentclass{l3doc}
+\usepackage[svgnames]{xcolor}
\EnableCrossrefs
\CodelineIndex
-\RecordChanges
+\fvset{breaklines=true,gobble=0,tabsize=4,frame=single,numbers=left,numbersep=3pt}
+\AtBeginDocument{\DeleteShortVerb\"}
+\MakeOuterQuote{"}
+\usepackage{cleveref}
\begin{document}
+\hfuzz=1pt
\GetFileInfo{\jobname.tex}
-\DoNotIndex{\newcommand,\newenvironment}
-\title{\textsf{prettytok} --- Pretty-print token lists\thanks{This file
+\title{\pkg{prettytok} --- Pretty-print token lists\thanks{This file
describes version \fileversion, last revised \filedate.}
}
\author{user202729%
@@ -18,110 +23,300 @@
\maketitle
-\changes{v0.0.0}{2022/05/18}{First public release}
-
\begin{abstract}
-Pretty-print token lists to HTML file for debugging purposes.
-Open the file in any browser to view the result.
+Pretty-print token lists for debugging purposes.
Can be used to replace |\tl_analysis_show:n|.
\end{abstract}
+\section{Motivation}
+
+\TeX/\LaTeX's default mechanism for debug-printing -- that is, the following:
+\begin{itemize}
+ \item |\showtokens{...}|,
+ \item |\show...|,
+ \item |\typeout{\unexpanded{...}}|,
+ \item |\tl_show:n|,
+ \item etc.
+\end{itemize}
+has a few limitations:
+
+\begin{itemize}
+ \item |\show| and similar is considered an error and stops the \TeX\ run. (this point is partially fixable, see the package writer's question \url{https://tex.stackexchange.com/q/621919/250119})
+ \item If there's some unprintable character in the output (for example, |^^J|, |^^M|, |^^I| -- literal tab character), it's not easy to distinguish between them.
+ \item If some token has unexpected catcode (most commonly, letter versus other), it's not easy to distinguish as well.
+ \item They does not work in expansion-only context.
+
+ (apart from |\msg_expandable_error:nn|, but this one suffers from the first problem as well)
+\end{itemize}
+
+|\tl_analysis_show:n| attempts to fix the third problem, but is very, very verbose and does not fix the other problems.
+
+This package aims to fix all of them.
+
+(although the expandable debug printing functions are \LuaTeX-only.)
+
+And some more additional (expandable in \LuaTeX) functions to inspect the content of the input stream at a particular moment in time.
+
+
\section{Usage}
+\subsection{A complete example}
+
+For a full example, the following code, which prints several things, both using the \LaTeXe\ interface
+and using the \pkg{expl3}-style interface, using both expandable and unexpandable command (the former requires \LuaTeX, as mentioned in \ref{subsec:expandable-interface}):
+\VerbatimInput{prettytok-test.tex}
+The corresponding output (in the current version) is:
+\begingroup%
+\ExplSyntaxOn
+
+\definecolor{myyellow}{RGB}{173, 174, 0}
+\definecolor{mygreen}{RGB}{52, 174, 0}
+
+\let \\ \textbackslash
+\def \cmd#1{\textcolor{myyellow}{\textbf{\textbackslash}#1}}
+\def \spec#1 {\textcolor{red}{#1}}
+\def \letr#1 {\textcolor{mygreen}{#1}}
+\setlength{\parindent}{0pt}
+\frenchspacing % no double space after dot
+
+\catcode`␣\active \def␣{\textbf{\textcolor{gray!60}{\char32}}}
+
+\ExplSyntaxOff
+\small%
+\obeylines%
+\texttt{
+> 123\spec\{4␣5\spec\}\cmd{6}
+> \spec{\#\&\textasciicircum\_}:
+> \cmd{error}
+> \letr{very␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long}
+> .. \letr{␣long␣argument}␣\cmd{argument} \cmd{argument} \cmd{test} 123456
+> \letr{very␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long␣long}
+> .. \letr{␣long␣argument}␣\cmd{argument} \cmd{argument} \cmd{test} 123456
+> \spec{\#\&\textasciicircum}\letr{\_:}
+> \letr{abc}␣
+}\endgroup
+
+\subsection{Loading the package}
+
+\begin{verbatim}
+\usepackage{prettytok}
+\end{verbatim}
+
+\subsection{Options}
+
+There are several options that can be passed to the package.
+
+Usage example: |\usepackage[mode=term-shell]{prettytok}|.
+
+\begin{variable}{mode=}
+ Specify the working mode of the package, that is, where the output is displayed. It can either be:
+\begin{itemize}
+ \item |term-8bit|: this is the default.
+
+ Assume XTerm-compatible system, output to the terminal.
+
+ Requires |-8bit| option on engines other than \LuaTeX, see the following link: \\
+ \url{https://tex.stackexchange.com/q/168460/250119}.
+
+ Besides, this drops the distinction between the catcode of some tokens (for example \verb|{}$#&^_| are all shown as the same color as "special catcode"),
+ which is available in the tooltip in the HTML version.
+
+ This might work on Windows, although the package writer have not tested it.
+ Refer to \url{https://stackoverflow.com/q/2048509/5267751}
+ and \url{https://superuser.com/q/413073/577463}.
+
+ \item |term-shell|:
+ To output colored text to the terminal, |--8bit| flag is needed otherwise the terminal escape codes will be
+ changed to |^^[| etc.
+ A workaround, using \TeX\ Live's behavior when the file name has the form \verb+|...+,
+ is provided with this option.
+
+ Requires |--shell-escape| flag. May not be very reliable.
+
+ Refer to \url{https://tex.stackexchange.com/a/670572/250119} for more details.
+
+ \item |html|: output to a HTML file named for example |pretty-abc.html| (although this can be customized,
+ refer to \hyperref[opt:html-file-name]{|html-file-name=|}%
+ ) if the main \TeX\ file is named |abc.tex|.
+
+ Open the file in any browser to view the result.
+
+ Using this option, the output will not be cluttered with the traceback/other \TeX\ default output.
+
+By default, the output refreshes whenever the \TeX\ file is recompiled. The behavior
+can be customized with \hyperref[opt:html-refresh-strategy]{|html-refresh-strategy=|} and \hyperref[opt:html-refresh-duration]{|html-refresh-duration=|}.
+
+\end{itemize}
+\end{variable}
+
+Currently, it's not supported to print the debug output to the PDF itself,
+because if the \TeX\ program stops with error / has some error that corrupts the PDF output, the output will even with corrupted more by the debug print.
+
+\subsubsection{HTML configuration}
+
+These options are only meaningful if |mode=html|.
+
+\begin{variable}{html-file-name=}\label{opt:html-file-name}
+ The output file name. Defaults to |pretty-|\meta{jobname}|.html|, as mentioned above.
+\end{variable}
+
+\begin{variable}{html-refresh-strategy=}\label{opt:html-refresh-strategy}
+ The auto-refresh strategy. Allowed values are 0-4. 0 is no refresh. \\
+ Which value works best depends on the particular browser. \\
+ If you're using Google Chrome to view the output HTML,
+ invoking the browser from the command-line with |--allow-file-access-from-files| flag may be useful.
+\end{variable}
+
+\begin{variable}{html-refresh-duration=}\label{opt:html-refresh-duration}
+ The duration between two consecutive refresh check, in milliseconds. Defaults to 1000.
+\end{variable}
+
+\subsubsection{Terminal configuration}
+
+These options are only meaningful if |mode=term-8bit| or |mode=term-shell|.
+
+\begin{variable}{term-prefix=,term-prefix-more=}
+ Strings consist of prefixes to be output before each terminal line.
+
+ This might be useful for log-filtering/output-filtering tools such as |texfot| to recognize the output line.
+
+ Defaults to \verb*|> | and \verb*|> .. | respectively.
+\end{variable}
+
+\begin{variable}{term-wrap-limit=}
+ Estimated line length limit. Set this to a little smaller than your terminal width.
+
+ Defaults to 70.
+\end{variable}
+
+\begin{variable}{term-shell-decode-cmd=}
+ Only meaningful with |mode=term-shell|.
+
+ Normally you would not need to explicitly pass this option, unless something does not work.
+
+ By default, a file named \file{prettytok-decode-8bit.py}
+ should be included in your \TeX\ distribution, and
+ the package runs |kpsewhich prettytok-decode-8bit.py| to find the location
+ of that file in order to execute it.
+ However, if by any reason this does not work, you can specify an explicit command such as
+ |python3 /full/path/to/prettytok-decode-8bit.py| to override it.
+
+ Passing blank value invokes the default behavior (runs |kpsewhich|).
+
+ Alternatively, you can also choose to explicitly pass the path in order to save a call to |kpsewhich|
+ to make the program a bit faster.
+
+ If you \emph{really} want to, special characters may be passed by prefixing them with |\|.
+ \DisableQuotes
+ But |\"| won't work anyway (as far as the package writer know, this is impossible in non-\LuaTeX\ engines).
+ \EnableQuotes
+ \begin{texnote} The path is interpreted by detokenizing the value in |escapechar=-1|. \end{texnote}
+\end{variable}
+
+\begin{variable}{term-shell-decode-cmd-print=}
+ If |mode=term-shell|, print out the command correspond to |term-shell-decode-cmd| on the console, for debugging purpose.
+
+ Example output:
+ |The value of term-shell-decode-cmd is: [[[./prettytok-decode-8bit.py]]]|
+\end{variable}
+
+
\subsection{Main function}
\begin{function}{\pretty:n,\pretty:x,\pretty:o,\pretty:V}
\begin{syntax}
\cs{pretty:n} \Arg{token list}
\end{syntax}
Print the content of \meta{token list}.
+
+ This is a simple replacement of the functions above. (|\tl_analysis_show:n|, etc.)
\end{function}
-\begin{function}{\pretty:N,\pretty:c}
+\begin{function}{\pretty:w}
\begin{syntax}
- \cs{pretty:N} \meta{token}
- \cs{pretty:c} \Arg{control sequence name}
+ \cs{pretty:w} \meta{token list} \cs{prettystop}
\end{syntax}
- Print \meta{token}.
-
- This function is not very useful. Usually it's preferable to use |\pretty:V| to print a token list variable's value,
- or |\prettyshow:N| to print a control sequence's meaning.
+ Print the content of \meta{token list}.
+
+The purpose of this function is that
+it can be inserted "anywhere" in order to \emph{inspect the input stream}
+without affecting how the function works.
+
+Note that the input stream will be tokenized and has catcode frozen.
+
+For example
+\begin{verbatim}
+\ExplSyntaxOn
+\def \f #1 {\prettye:w 789}
+\f 123456 \prettystop
+\end{verbatim}
+will print out |78923456|.
+
+For now, it must be brace-balanced. Use \cs{prettye:w} instead if this is a problem.
\end{function}
-\begin{function}{\prettyinit:}
+\begin{function}[EXP]{\prettystop}
\begin{syntax}
- \cs{prettyinit:}
+ \cs{prettystop}
\end{syntax}
- Do necessary initialization.
-
- This should be called right after the package is loaded, after setting the configuration variables described below.
+ Only used as a delimiter for |:w| functions. For convenience, this function is defined to do nothing.
\end{function}
-\subsection{Expandable interface (\LuaTeX\ only)}
-
-\cs{prettyinit:} must be called before calling the functions in this section.
-
-\begin{function}[EXP]{\prettye:n}
+\begin{function}{\prettyshow:N,\prettyshow:c}
\begin{syntax}
- \cs{prettye:n} \Arg{token list}
+ \cs{prettyshow:N} \meta{token}
+ \cs{prettyshow:c} \Arg{control sequence name}
\end{syntax}
- Print the token list, similar to \cs{pretty:n}, but is fully expandable.
+ Show the meaning of a |N|-type argument.
\end{function}
-\begin{function}[EXP]{\prettye:w}
+\begin{function}{\pretty:N,\pretty:c}
\begin{syntax}
- \cs{prettye:w} \meta{tokens} \cs{prettystop}
+ \cs{pretty:N} \meta{token}
+ \cs{pretty:c} \Arg{control sequence name}
\end{syntax}
- Print the tokens until \cs{prettystop} is seen. Useful for inspecting the content of the input stream.
+ Print \meta{token}.
- Note that the input stream will be tokenized and has catcode frozen. Use with care.
+ It may also be useful to use |\pretty:V| to print a token list variable's value,
+ or |\prettyshow:N| to print a control sequence's meaning.
\end{function}
-\subsection{Lua programming interface}
-
-\cs{prettyinit:} must also be called before calling the functions in this section.
-
-\begin{function}{prettyprint}
+\begin{function}{\pretty:nn, \pretty:nnn}
\begin{syntax}
- |prettyprint(|\meta{content}|)|
+ \cs{pretty:nn} \Arg{token list} \Arg{token list}
+ \cs{pretty:nnn} \Arg{token list} \Arg{token list} \Arg{token list}
\end{syntax}
- Print the content, which should be a table of token objects.
-
- For convenience, you can pass multiple arguments. Strings are also supported.
+ Print multiple token lists. Its effect is similar to multiple consecutive calls to \cs{pretty:n}.
\end{function}
-\subsection{View result}
-The result is printed to a HTML file named |pretty-|\meta{jobname}|.html| by default.
-Open the file in any browser to view the result.
-Note that the HTML file will not be touched if nothing is printed, as such it's encouraged to add a line such as |\pretty:n {start}| right after loading the file.
-Why HTML file?
-\begin{itemize}
- \item If the \TeX\ program stops with error / has some error that corrupts the PDF output, the output will even with corrupted more by the debug print.
- \item Printing to the console is extremely limited (difficult syntax highlighting/scrolling/line wrapping), and most likely cluttered with the traceback/other \TeX\ default output.
+\subsection{Expandable interface (\LuaTeX\ only)}
+\label{subsec:expandable-interface}
- (nevertheless, there might be plan to support output to console with syntax highlighting later.)
- \item Output to another \TeX\ file to be compiled works as well; however it's necessary to run another instance of |latexmk| (or other tool used to compile \TeX\ code)
- and compiling \TeX\ file takes longer than loading a HTML file.
-\end{itemize}
+\begin{function}[EXP]{\prettye:n}
+ \begin{syntax}
+ \cs{prettye:n} \Arg{token list}
+ \end{syntax}
+ Print the token list, similar to \cs{pretty:n}, but is fully expandable.
+\end{function}
-By default, the output refreshes whenever the \TeX\ file is recompiled. The behavior
-can be customized with \cs{prettyrefreshstrategy} and \cs{prettyrefreshduration}.
+\begin{function}[EXP]{\prettye:w}
+ \begin{syntax}
+ \cs{prettye:w} \meta{tokens} \cs{prettystop}
+ \end{syntax}
+ Print the tokens until \cs{prettystop} is seen. Useful for inspecting the content of the input stream.
-\subsection{Additional functions}
+ As a debugging tool, it's possible to execute |\everyeof{\prettystop}| to avoid runaway printing in weird catcode environments.
-There are also these functions, for convenience.
+ Currently some implementation details (it can be fixed, but the package writer does not
+have an immediate need for it, see \url{https://tex.stackexchange.com/q/335994/250119}) means control sequences not
+in the hash table will be destroyed.
+Use with care.
-\begin{function}{\pretty:nn, \pretty:nnn}
- \begin{syntax}
- \cs{pretty:nn} \Arg{token list} \Arg{token list}
- \cs{pretty:nnn} \Arg{token list} \Arg{token list} \Arg{token list}
- \end{syntax}
- Print multiple token lists. Its effect is similar to multiple consecutive calls to \cs{pretty:n}.
\end{function}
\begin{function}[EXP]{\prettye:nn, \prettye:nnn}
@@ -132,49 +327,63 @@ There are also these functions, for convenience.
Similar to multiple consecutive calls to \cs{prettye:n}.
\end{function}
-\begin{function}[EXP]{\prettye:nw, \prettye:nnw}
+
+
+\begin{function}[EXP]{\prettye:nw}
\begin{syntax}
- \cs{prettye:nw} \Arg{token list} \meta{tokens} \cs{prettystop}
- \cs{prettye:nnw} \Arg{token list} \Arg{token list} \meta{tokens} \cs{prettystop}
+ \cs{prettye:nw} \Arg{callback} \meta{tokens} \cs{prettystop}
\end{syntax}
- Similar to call(s) to \cs{prettye:n} followed by a call to \cs{prettye:w}.
+ Same as above, but has a callback, that is, code that is put in the input stream after the content is printed.
+
+ Useful if you want to fine-tune what is printed exactly.
+ (\texttt{\cs{prettye:w} \meta{callback} \meta{tokens}} is functionally the same, but the callback
+ is also printed, which will clutter the debug output)
+
+ For example
+\begin{verbatim}
+\prettye:nw {\somecode ...} 123 \prettystop
+\end{verbatim}
+will print |123|, then after some expansion steps results in the input stream contain
+|\somecode ... 123 \prettystop|.
\end{function}
-\begin{function}{\pretty:w}
+\begin{function}[EXP]{\prettye:nnw, \prettye:nnnw}
\begin{syntax}
- \cs{pretty:w} \meta{token list} \cs{prettystop}
+ \cs{prettye:nnw} \Arg{callback} \Arg{token list} \meta{tokens} \cs{prettystop}
+ \cs{prettye:nnnw} \Arg{callback} \Arg{token list} \Arg{token list} \meta{tokens} \cs{prettystop}
\end{syntax}
- Print the content of \meta{token list}.
-
- Same restriction applies -- the catcode is frozen.
+ Similar to call(s) to \cs{prettye:n} followed by a call to \cs{prettye:nw}.
- In addition -- for now, it must be brace-balanced and if \meta{token list} has a surrounding brace group it will be removed. Use \cs{prettye:w} instead if possible.
+ For example
+\begin{verbatim}
+\prettye:nnw {\somecode ...} {123} 456 \prettystop
+\end{verbatim}
+will print |123456|, then, after some expansion steps, |\somecode ... 456 \prettystop| remains in the input stream.
\end{function}
-\begin{function}[EXP]{\prettystop}
- \begin{syntax}
- \cs{prettystop}
- \end{syntax}
- Only used as a delimiter for |:w| functions. For convenience, this function is defined to do nothing.
-\end{function}
+\subsection{Lua programming interface}
-\begin{function}{\prettyshow:N,\prettyshow:c}
+\begin{function}{prettyprint}
\begin{syntax}
- \cs{prettyshow:N} \meta{token}
- \cs{prettyshow:c} \Arg{control sequence name}
+ |prettyprint(|\meta{content}|)|
\end{syntax}
- Show the meaning of a |N|-type argument.
+ Print the content, which should be a table of token objects.
+
+ For convenience, you can pass multiple arguments. Strings are also supported.
\end{function}
+
+
+
\subsection{\LaTeXe\ interface}
-\begin{function}{\prettyN,\prettyX,\prettyO,\prettyV,\prettyinit,\prettyshowN,\prettyshowC}
+\begin{function}{\prettyN,\prettyX,\prettyO,\prettyV,\prettyW,\prettyshowN,\prettyshowC}
\begin{syntax}
\tn{prettyN} \Arg{token list}
\tn{prettyX} \Arg{token list}
\tn{prettyO} \Arg{token list}
\tn{prettyV} \meta{tl var}
- \tn{prettyinit}
+ \cs{prettyW} \meta{token list} \cs{prettystop}
\tn{prettyshowN} \meta{token}
\tn{prettyshowC} \Arg{control sequence name}
\end{syntax}
@@ -190,33 +399,19 @@ There are also these functions, for convenience.
\end{function}
-\subsection{Customization}
-
-These variables can be redefined before the
-first call to \cs{pretty:n} or \cs{prettyinit:} to customize the behavior.
-
-\begin{variable}{\prettyfilename}
- The output file name. Defaults to |pretty-|\meta{jobname}|.html|, as mentioned above.
-\end{variable}
-
-\begin{variable}{\prettyrefreshstrategy}
- The auto-refresh strategy. Allowed values are 0-4. 0 is no refresh. \\
- Which value works best depends on the particular browser. \\
- On Google Chrome, passing |--allow-file-access-from-files| may be useful.
-\end{variable}
-
-\begin{variable}{\prettyrefreshduration}
- The duration between two consecutive refresh check, in milliseconds. Defaults to 1000.
-\end{variable}
-
\section{Implementation}
Unfortunately, the implementation is not typesetted in \TeX. Read the |.sty| file.
-\StopEventually{^^A
- \PrintChanges
- \PrintIndex
-}
+Remark: it's possible to do expandable printing in other engines as well by, for example, turning on |\tracingmacros|, parse the token list somehow
+(and use some not-always-exact logic to distinguish normal character and active character with same meaning;
+then grep the resulting log file for special markers.
+
+But that would be very, very slow and slows down everything else by turning on logging. Just use \LuaTeX\ for debugging.
+
+There's another option of recompiling the engine and adding some expandable primitive for debug logging...
+\PrintChanges
+\PrintIndex
\Finale
\end{document}
diff --git a/macros/latex/contrib/prettytok/prettytok_template.html b/macros/latex/contrib/prettytok/prettytok_template.html
index 8764d49809..5b0d988a4c 100644
--- a/macros/latex/contrib/prettytok/prettytok_template.html
+++ b/macros/latex/contrib/prettytok/prettytok_template.html
@@ -37,10 +37,9 @@ iframe{
.tokenlist, .fileline{
border: 1px solid gray;
- overflow: scroll;
- white-space: nowrap;
+ padding: 2px;
+ line-height: 1.5em;
}
-.tokenlist::-webkit-scrollbar, .fileline::-webkit-scrollbar { width: 0 !important }
.fileline{
background-color: #404040;
@@ -57,12 +56,6 @@ iframe{
.cs, .active_char{
color: #FFFF00;
}
-.cs::before{
- content: "\\";
-}
-.cs::after{
- content: " ";
-}
#popup{
display: none;
@@ -106,6 +99,18 @@ input[type=button]{
flex: 0 0 30%;
overflow: scroll;
}
+
+/*
+ hack to make it display as ␣ but copy-paste (or search) as space
+*/
+.space_charcode{
+ font-size: 0pt;
+}
+
+.space_charcode::before{
+ font-size: medium;
+ content: "␣";
+}
</style>
</head>
<body>
@@ -117,22 +122,29 @@ input[type=button]{
</div>
<script>
"use strict"
-function show_html(s){
- return s.replace(/ /g, "␣")
+
+function show_html_except_space(s){
+ return s
.replace(/\r|\n/g, "↵")
.replace(/\t/g, "⇥")
}
+function show_html(s){
+ return show_html_except_space(s).replace(/ /g, "␣")
+}
+
output.innerHTML=""
function print_tl(...tokens){
const tl_div=document.createElement("div")
- tl_div.className="tokenlist"
+ tl_div.classList.add("tokenlist")
for(let token of tokens){
tl_div.appendChild(token)
if({13: true, 10: true}[token.data_charcode]){
tl_div.appendChild(document.createElement("br"))
- }
+ }else{
+ tl_div.appendChild(document.createElement("wbr"))
+ }
}
output.appendChild(tl_div)
}
@@ -140,10 +152,13 @@ function print_tl(...tokens){
function token(charcode /* int */, catcode /* single character string in hex */){
const token_span=document.createElement("span")
const ch=String.fromCodePoint(charcode)
- token_span.innerText=show_html(ch)
+ token_span.innerText=show_html_except_space(ch)
token_span.data_charcode=charcode
+ if(charcode==32){ // special handle space here
+ token_span.classList.add("space_charcode")
+ }
if(catcode==='D'){
- token_span.className="active_char"
+ token_span.classList.add("active_char")
token_span.title=`active ${ch} # ${charcode}`
}else{
const cls={
@@ -158,7 +173,7 @@ function token(charcode /* int */, catcode /* single character string in hex */)
B: 'cmd__letter_cmd',
C: 'cmd__other_char_cmd',
}[catcode]
- token_span.className="token "+cls
+ token_span.classList.add("token", cls)
token_span.title=`${cls} ${ch} # ${charcode}`
}
return token_span
@@ -170,18 +185,37 @@ function cs(...codepoints){
const control_sequence_name=String.fromCodePoint(...codepoints)
//const meaning_str=readstr()
- token_span.innerText=show_html(control_sequence_name)
- //token_span.title=`cs '${control_sequence_name}' → ${meaning_str}`
- token_span.className="cs"
+ /*
+ token_span.innerText='\\' + show_html(control_sequence_name) + ' '
+ */
+
+ var items=control_sequence_name.split(' ')
+ items[0]="\\"+items[0]
+ items[items.length-1]+=' '
+ for(var i=0; i<items.length; i++){
+ if(i!=0){ // append a '␣' span that copy-paste as a space
+ var space_span=document.createElement("span")
+ space_span.innerText=' '
+ space_span.classList.add("space_charcode")
+ token_span.appendChild(space_span)
+ }
+
+ // append the text itself
+ if(items[i]!="")
+ token_span.appendChild(new Text(show_html_except_space(items[i])))
+ }
+
+ //token_span.title=`cs '${control_sequence_name}' → ${meaning_str}`
+ token_span.classList.add("cs")
return token_span
}
function csfrozenrelax(){
const token_span=document.createElement("span")
- token_span.innerText="relax"
+ token_span.innerText="\\relax"
token_span.title="frozen relax"
- token_span.className="cs frozenrelax"
+ token_span.classList.add("cs", "frozenrelax")
return token_span
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-09 15:21:17 +0000