From 31e47b2b4385332095798ee4d2c322bdcefbff54 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Sun, 14 Jul 2024 20:19:05 +0000 Subject: luaxml (14jul24) git-svn-id: svn://tug.org/texlive/trunk@71803 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/luatex/luaxml/README | 2 +- Master/texmf-dist/doc/luatex/luaxml/luaxml.pdf | Bin 119150 -> 124725 bytes Master/texmf-dist/doc/luatex/luaxml/luaxml.tex | 92 +- .../tex/luatex/luaxml/luaxml-cssquery.lua | 14 +- .../tex/luatex/luaxml/luaxml-domobject.lua | 95 +- .../tex/luatex/luaxml/luaxml-encodings.lua | 88 + .../tex/luatex/luaxml/luaxml-mod-html.lua | 221 +- .../tex/luatex/luaxml/luaxml-namedentities.lua | 4193 ++++++++++---------- .../tex/luatex/luaxml/luaxml-transform.lua | 13 +- 9 files changed, 2586 insertions(+), 2132 deletions(-) create mode 100644 Master/texmf-dist/tex/luatex/luaxml/luaxml-encodings.lua (limited to 'Master/texmf-dist') diff --git a/Master/texmf-dist/doc/luatex/luaxml/README b/Master/texmf-dist/doc/luatex/luaxml/README index 5df074c453b..6a81ab25ce0 100644 --- a/Master/texmf-dist/doc/luatex/luaxml/README +++ b/Master/texmf-dist/doc/luatex/luaxml/README @@ -32,7 +32,7 @@ Author ------ Michal Hoftich Email: michal.h21@gmail.com -Version: v0.1r, 2024-02-23 +Version: v0.2, 2024-07-14 Original authors: Paul Chakravarti and Manoel Campos (http://manoelcampos.com) diff --git a/Master/texmf-dist/doc/luatex/luaxml/luaxml.pdf b/Master/texmf-dist/doc/luatex/luaxml/luaxml.pdf index f574a0a5287..071fa4e4cd0 100644 Binary files a/Master/texmf-dist/doc/luatex/luaxml/luaxml.pdf and b/Master/texmf-dist/doc/luatex/luaxml/luaxml.pdf differ diff --git a/Master/texmf-dist/doc/luatex/luaxml/luaxml.tex b/Master/texmf-dist/doc/luatex/luaxml/luaxml.tex index 5031a0468e2..bf3b8fbaa74 100644 --- a/Master/texmf-dist/doc/luatex/luaxml/luaxml.tex +++ b/Master/texmf-dist/doc/luatex/luaxml/luaxml.tex @@ -7,7 +7,7 @@ \usepackage{framed} % Version is defined in the makefile, use default values when compiled directly \ifdefined\version\else -\def\version{v0.1r} +\def\version{v0.2} \let\gitdate\date \fi \newcommand\modulename[1]{\subsection{#1}\label{sec:#1}} @@ -99,6 +99,67 @@ end \end{luacode*} \end{framed} +\subsection{HTML parsing} + +You can parse HTML documents using the \verb|DOM_Object.html_parse| function. This parser is slower +than the default XML parser, but it can load files that would cause errors in the XML mode. +It can handle wrongly nested HTML tags, inline JavaScript and CSS styles, and other HTML features +that would cause XML errors. + +\begin{verbatim} +dom = require "luaxml-domobject" +local document = [[ + +sample + +

test

+

hello +

another paragraph +

+ + + ]] + +-- dom.html_parse returns the DOM_Object +obj = dom.html_parse(document) +-- print names of all elements contained in body +for _, x in ipairs(obj:query_selector("body *")) do + tex.print(x:get_element_name().. "\\par") +end +\end{verbatim} + + +\begin{framed} +\begin{luacode*} +dom = require "luaxml-domobject" +local document = [[ + +sample + +

test

+

hello +

another paragraph +

+ + + ]] + +-- dom.html_parse returns the DOM_Object +obj = dom.html_parse(document) +-- print names of all elements contained in body +for _, x in ipairs(obj:query_selector("body *")) do + tex.print(x:get_element_name().. "\\par") +end +\end{luacode*} +\end{framed} + + \subsection{Void elements} The \verb|DOM_Object.parse| function tries to support the HTML void elements, @@ -641,6 +702,35 @@ transform.print_tex("\\verb|" .. result .. "|") \end{luacode*} \end{framed} +\section{Character sets handling} + +The \texttt{luaxml-encodings} library provides functions to convert texts in legacy 8-bit encodings such as WINDOWS-1250 +or ISO-8859-2 to UTF-8. This can be useful in fixing document encoding before HTML parsing using the \texttt{luaxml-mod-html} +library. + +\subsection{Example} + + +\begin{verbatim} +kpse.set_program_name "luatex" +local encodings = require "luaxml-encodings" + +--read HTML page from the standard input +local text = io.read("*all") +-- find the character encoding in HTML metadata +local enc = encodings.find_html_encoding(text) +if enc then + -- local conversion table for the found encoding + local mapping = encodings.load_mapping(enc) + if mapping then + -- if the mapping exists, recode the HTML input and print it + local converted = encodings.recode(text, mapping) + print(converted) + end +end +\end{verbatim} + + \section{The API documentation} diff --git a/Master/texmf-dist/tex/luatex/luaxml/luaxml-cssquery.lua b/Master/texmf-dist/tex/luatex/luaxml/luaxml-cssquery.lua index ebb29abb15c..452656845c4 100644 --- a/Master/texmf-dist/tex/luatex/luaxml/luaxml-cssquery.lua +++ b/Master/texmf-dist/tex/luatex/luaxml/luaxml-cssquery.lua @@ -156,7 +156,7 @@ local function cssquery() -- make sure we deal with a string value = tostring(value) -- make the search string safe for pattern matching - local escaped_search = search:gsub("([%(%)%.%%%+%–%*%?%[%^%$])", "%%%1") + local escaped_search = search:gsub("([%(%)%.%%%+%-%#%*%?%[%^%$])", "%%%1") if modifier == "" then return value == search elseif modifier == "|" then @@ -434,6 +434,18 @@ local function cssquery() return querylist end + --- Remove selector from the CSS list object. + -- All actions that literaly match the given selector will be removed. + function CssQuery:remove_selector( + selector -- CSS selector to be removed + ) + for pos, obj in ipairs(self.querylist) do + if obj.source == selector then + table.remove(self.querylist, pos) + end + end + end + --- It tests list of queries agaings a DOM element and executes the --- coresponding function that is saved for the matched query. -- @return nothing diff --git a/Master/texmf-dist/tex/luatex/luaxml/luaxml-domobject.lua b/Master/texmf-dist/tex/luatex/luaxml/luaxml-domobject.lua index 73689add5a1..dca38df0db5 100644 --- a/Master/texmf-dist/tex/luatex/luaxml/luaxml-domobject.lua +++ b/Master/texmf-dist/tex/luatex/luaxml/luaxml-domobject.lua @@ -6,16 +6,20 @@ local dom = {} local xml local handler local css_query +local html if kpse then xml = require("luaxml-mod-xml") handler = require("luaxml-mod-handler") css_query = require("luaxml-cssquery") + html = require("luaxml-mod-html") else xml = require("luaxml.mod-xml") handler = require("luaxml.mod-handler") css_query = require("luaxml.cssquery") + html = require("luaxml.mod-html") end +local HtmlParser = html.HtmlParser local void = {area = true, base = true, br = true, col = true, hr = true, img = true, input = true, link = true, meta = true, param = true} @@ -91,9 +95,14 @@ local function serialize_dom(parser, current,level, output) local format = get_action(typ, "start") insert(format, el, prepare_attributes(attr)) end - local function text(typ, text) + local function text(typ, text, parent) + local parent = parent or {} local format = get_action(typ, "text") - insert(format, escape_element(text)) + if parent.verbatim then + insert(format, text) + else + insert(format, escape_element(text)) + end end local function stop(typ, el) local format = get_action(typ, "stop") @@ -141,7 +150,7 @@ local function serialize_dom(parser, current,level, output) end start(xtype, name, attributes) - text(xtype,text_content) + text(xtype,text_content, (current or {})._parent) local children = root._children or {} for _, child in ipairs(children) do output = serialize_dom(parser,child, level + 1, output) @@ -411,8 +420,8 @@ local parse = function( return strings end - --- Get table with the inner text of an element -- leading and trailing spaces are removed and elements that contain only white space are ignored. - --- @return table + --- Get table with the inner text of an element - leading and trailing spaces are removed and elements that contain only white space are ignored. + -- @return table function DOM_Object:stripped_strings( current --- [optional] element to be selected ) @@ -612,11 +621,83 @@ local parse = function( -- include the methods to all xml nodes save_methods(parser._handler.root) -- parser: - return parser + return parser, DOM_Object +end + +-- table of elements that should be kept without XML escaping in the DOM serialization +local verbatim_elements = {script=true, style=true} + +local function html_to_dom(html_object) + -- convert parsed HTML DOM to the XML DOM + local dom, DOM_Object = parse("") -- use empty text to just initialize the DOM object + -- use root of the DOM object as the original parent + local current_parent = dom._handler.root + + local function create_node(tbl) + -- create node suitable for LuaXML DOM object + tbl._children = {} + -- this should copy methods from the DOM object to the newly created object + tbl.__index = DOM_Object + return setmetatable(tbl, DOM_Object) + end + + local function build_tree(object) + -- convert tree produced by the HTML parser to LuaXML DOM + local typ = object._type + -- process particular node types from the HTML parser + if typ == "doctype" then + current_parent:add_child_node(create_node {_name=object.name, _type="DTD"}) + elseif typ == "comment" then + current_parent:add_child_node(create_node {_text=object.text, _type="COMMENT"}) + elseif typ == "element" then + local attributes = {} + -- convert attributes to the form expected by the DOM object + for _, attr in ipairs(object.attr) do + attributes[attr.name] = attr.value + end + local element = current_parent:create_element(object.tag, attributes) + -- disable escaping of text in dom:serialize() for