ConTeXt 2012.05.30 11:26

git-svn-id: svn://tug.org/texlive/trunk@26861 c570f23f-e606-0410-a88d-b1316a301751

19 files changed, 1786 insertions, 57 deletions

diff --git a/Master/bin/alpha-linux/mtxrun b/Master/bin/alpha-linux/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/alpha-linux/mtxrun
+++ b/Master/bin/alpha-linux/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/amd64-freebsd/mtxrun b/Master/bin/amd64-freebsd/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/amd64-freebsd/mtxrun
+++ b/Master/bin/amd64-freebsd/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/amd64-kfreebsd/mtxrun b/Master/bin/amd64-kfreebsd/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/amd64-kfreebsd/mtxrun
+++ b/Master/bin/amd64-kfreebsd/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/armel-linux/mtxrun b/Master/bin/armel-linux/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/armel-linux/mtxrun
+++ b/Master/bin/armel-linux/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/i386-cygwin/mtxrun b/Master/bin/i386-cygwin/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/i386-cygwin/mtxrun
+++ b/Master/bin/i386-cygwin/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/i386-freebsd/mtxrun b/Master/bin/i386-freebsd/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/i386-freebsd/mtxrun
+++ b/Master/bin/i386-freebsd/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/i386-kfreebsd/mtxrun b/Master/bin/i386-kfreebsd/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/i386-kfreebsd/mtxrun
+++ b/Master/bin/i386-kfreebsd/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/i386-linux/mtxrun b/Master/bin/i386-linux/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/i386-linux/mtxrun
+++ b/Master/bin/i386-linux/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/i386-solaris/mtxrun b/Master/bin/i386-solaris/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/i386-solaris/mtxrun
+++ b/Master/bin/i386-solaris/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/mips-irix/mtxrun b/Master/bin/mips-irix/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/mips-irix/mtxrun
+++ b/Master/bin/mips-irix/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/mipsel-linux/mtxrun b/Master/bin/mipsel-linux/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/mipsel-linux/mtxrun
+++ b/Master/bin/mipsel-linux/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/powerpc-aix/mtxrun b/Master/bin/powerpc-aix/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/powerpc-aix/mtxrun
+++ b/Master/bin/powerpc-aix/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/powerpc-linux/mtxrun b/Master/bin/powerpc-linux/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/powerpc-linux/mtxrun
+++ b/Master/bin/powerpc-linux/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/sparc-solaris/mtxrun b/Master/bin/sparc-solaris/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/sparc-solaris/mtxrun
+++ b/Master/bin/sparc-solaris/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/universal-darwin/mtxrun b/Master/bin/universal-darwin/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/universal-darwin/mtxrun
+++ b/Master/bin/universal-darwin/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/win32/mtxrun.lua b/Master/bin/win32/mtxrun.lua
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/win32/mtxrun.lua
+++ b/Master/bin/win32/mtxrun.lua
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/x86_64-darwin/mtxrun b/Master/bin/x86_64-darwin/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/x86_64-darwin/mtxrun
+++ b/Master/bin/x86_64-darwin/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/x86_64-linux/mtxrun b/Master/bin/x86_64-linux/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/x86_64-linux/mtxrun
+++ b/Master/bin/x86_64-linux/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure
 
diff --git a/Master/bin/x86_64-solaris/mtxrun b/Master/bin/x86_64-solaris/mtxrun
index d2f7e408f0d..108f2a8a112 100755
--- a/Master/bin/x86_64-solaris/mtxrun
+++ b/Master/bin/x86_64-solaris/mtxrun
@@ -4087,8 +4087,10 @@ local one  = P(1)
 local two  = C(1) * C(1)
 local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
 
+-- actually one of them is already utf ... sort of useless this one
+
 local pattern = P("\254\255") * Cs( (
-                    four   / function(a,b,c,d)
+                    four  / function(a,b,c,d)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4099,7 +4101,7 @@ local pattern = P("\254\255") * Cs( (
                   + one
                 )^1 )
               + P("\255\254") * Cs( (
-                    four   / function(b,a,d,c)
+                    four  / function(b,a,d,c)
                                 local ab = 0xFF * byte(a) + byte(b)
                                 local cd = 0xFF * byte(c) + byte(d)
                                 return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
@@ -4114,6 +4116,22 @@ function string.toutf(s)
     return lpegmatch(pattern,s) or s -- todo: utf32
 end
 
+local validatedutf = Cs (
+    (
+        patterns.utf8one
+      + patterns.utf8two
+      + patterns.utf8three
+      + patterns.utf8four
+      + P(1) / "�"
+    )^0
+)
+
+patterns.validatedutf = validatedutf
+
+function string.validutf(str)
+    return lpegmatch(validatedutf,str)
+end
+
 
 end -- of closure
 
@@ -8174,6 +8192,34 @@ function xml.makestandalone(root)
     return root
 end
 
+function xml.kind(e)
+    local dt = e and e.dt
+    if dt then
+        local n = #dt
+        if n == 1 then
+            local d = dt[1]
+            if d.special then
+                local tg = d.tg
+                if tg == "@cd@" then
+                    return "cdata"
+                elseif tg == "@cm" then
+                    return "comment"
+                elseif tg == "@pi@" then
+                    return "instruction"
+                elseif tg == "@dt@" then
+                    return "declaration"
+                end
+            elseif type(d) == "string" then
+                return "text"
+            end
+            return "element"
+        elseif n > 0 then
+            return "mixed"
+        end
+    end
+    return "empty"
+end
+
 
 end -- of closure
 
@@ -9601,7 +9647,7 @@ local xmlfilter = xml.filter
 
 local type, setmetatable, getmetatable = type, setmetatable, getmetatable
 local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat
-local gmatch, gsub, format = string.gmatch, string.gsub, string.format
+local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip
 local utfbyte = utf.byte
 
 local function report(what,pattern,c,e)
@@ -10293,6 +10339,51 @@ function xml.setcdata(e,str) -- also setcomment
     } }
 end
 
+-- maybe helpers like this will move to an autoloader
+
+function xml.separate(x,pattern)
+    local collected = xmlapplylpath(x,pattern)
+    if collected then
+        for c=1,#collected do
+            local e = collected[c]
+            local d = e.dt
+            if d == x then
+                report_xml("warning: xml.separate changes root")
+                x = d
+            end
+            local t, n = { "\n" }, 1
+            local i, nd = 1, #d
+            while i <= nd do
+                while i <= nd do
+                    local di = d[i]
+                    if type(di) == "string" then
+                        if di == "\n" or find(di,"^%s+$") then -- first test is speedup
+                            i = i + 1
+                        else
+                            d[i] = strip(di)
+                            break
+                        end
+                    else
+                        break
+                    end
+                end
+                if i > nd then
+                    break
+                end
+                t[n+1] = "\n"
+                t[n+2] = d[i]
+                t[n+3] = "\n"
+                n = n + 3
+                i = i + 1
+            end
+            t[n+1] = "\n"
+            setmetatable(t,getmetatable(d))
+            e.dt = t
+        end
+    end
+    return x
+end
+
 
 end -- of closure