updated luatex reference manual with luaharfbuzz doc.

git-svn-id: svn://tug.org/texlive/trunk@54290 c570f23f-e606-0410-a88d-b1316a301751

11 files changed, 367 insertions, 32 deletions

diff --git a/Master/texmf-dist/doc/luatex/base/graphics/luaharfbuzz.pdf b/Master/texmf-dist/doc/luatex/base/graphics/luaharfbuzz.pdf
new file mode 100644
index 00000000000..72a9455ad56
--- /dev/null
+++ b/Master/texmf-dist/doc/luatex/base/graphics/luaharfbuzz.pdf
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-backend.tex b/Master/texmf-dist/doc/luatex/base/luatex-backend.tex
index e99a95ed26f..2606ff38475 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex-backend.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex-backend.tex
@@ -627,7 +627,7 @@ pdfe.unencrypt(<pdfe document>,userpassword,ownerpassword)
 
 \stopsubsection
 
-\startsubsection[title={\type {getsize}, \type {getversion}, \type {getnofobjects}, \type {getnofpages}}]
+\startsubsection[title={\type {getsize}, \type {getversion}, \type {getnofobjects}, \type {getnofpages}, \type{getmemoryusage}}]
 
 \libindex {getsize}
 \libindex {getversion}
@@ -641,7 +641,7 @@ bytes = getsize(<pdfe document>)
 major, minor = getversion(<pdfe document>)
 n = getnofobjects(<pdfe document>)
 n = getnofpages(<pdfe document>)
-bytes, waste = getnofpages(<pdfe document>)
+bytes, waste  = getmemoryusage(<pdfe document>)
 \stoptyping
 
 \stopsubsection
@@ -677,7 +677,7 @@ A specific page can conveniently be reached with the next command, which
 returns a dictionary. The first argument is to be a page dictionary.
 
 \starttyping
-<pdfe dictionary> = pdfe.getpage(<pdfe dictionary>,pagenumber)
+<pdfe dictionary> = pdfe.getpage(<pdfe document>,pagenumber)
 \stoptyping
 
 Another convenience command gives you the (bounding) box of a (normally page)
@@ -685,7 +685,7 @@ which can be inheritted from the document itself. An example of a valid box name
 is \type {MediaBox}.
 
 \starttyping
-pages = pdfe.getbox(<pdfe document>,boxname)
+pages = pdfe.getbox(<pdfe dictionary>,boxname)
 \stoptyping
 
 \stopsubsection
@@ -747,7 +747,7 @@ print(pdfe.open("foo.pdf").Pages[1])
 
 \stopsubsection
 
-\startsubsection[title={\type {[open|close|readfrom][whole|]stream}}]
+\startsubsection[title={\type {[open|close|readfrom|whole|]stream}}]
 
 \libindex {openstream}
 \libindex {closestream}
@@ -797,8 +797,8 @@ type, value, detail = getfromarray(<pdfe array>,index)
 \DB type       \BC meaning    \BC value            \BC detail \NC \NR
 \NC \type {0}  \NC none       \NC nil              \NC \NC \NR
 \NC \type {1}  \NC null       \NC nil              \NC \NC \NR
-\NC \type {2}  \NC boolean    \NC boolean          \NC \NC \NR
-\NC \type {3}  \NC boolean    \NC integer          \NC \NC \NR
+\NC \type {2}  \NC boolean    \NC 1 or 0           \NC \NC \NR
+\NC \type {3}  \NC integer    \NC integer          \NC \NC \NR
 \NC \type {4}  \NC number     \NC float            \NC \NC \NR
 \NC \type {5}  \NC name       \NC string           \NC \NC \NR
 \NC \type {6}  \NC string     \NC string           \NC hex \NC \NR
@@ -843,10 +843,11 @@ Because you can have unresolved references, a reference object can be resolved
 with:
 
 \starttyping
-<pdfe dictionary|array|stream> = getfromreference(<pdfe reference>)
+type, <pdfe dictionary|array|stream>, detail = getfromreference(<pdfe reference>)
 \stoptyping
 
-So, you get back a new \type {pdfe} userdata object that you can query.
+So, as second value you get back a new \type {pdfe} userdata object that you can
+query.
 
 \stopsubsection
 
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-callbacks.tex b/Master/texmf-dist/doc/luatex/base/luatex-callbacks.tex
index 965fba80629..6bacfe7f059 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex-callbacks.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex-callbacks.tex
@@ -619,7 +619,7 @@ end
 It is ok to return nothing in which case you also need to flush the box or deal
 with it yourself. The prevdepth is also optional. Locations are \type {box},
 \type {alignment}, \type {equation}, \type {equation_number} and \type
-{post_linebreak}.
+{post_linebreak}. You can pass \type {nil} instead of a node.
 
 \subsection{\cbk {post_linebreak_filter}}
 
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-fonts.tex b/Master/texmf-dist/doc/luatex/base/luatex-fonts.tex
index e5b6e1cb294..400550a96dd 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex-fonts.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex-fonts.tex
@@ -728,14 +728,18 @@ font.setfont(<number> n, <table> f)
 
 Note that at the moment, each access to the \type {font.fonts} or call to \type
 {font.getfont} creates a \LUA\ table for the whole font unless you cached it.
-This process can be quite slow.
+If you want a copy of the internal data you can use \type {font.copyfont}:
 
 \startfunctioncall
-<table> p = font.getparameters(<number> n)
+<table> f = font.copyfont(<number> n)
 \stopfunctioncall
 
 This one will return a table of the parameters as known to \TEX. These can be
-different from the ones in the cached table.
+different from the ones in the cached table:
+
+\startfunctioncall
+<table> p = font.getparameters(<number> n)
+\stopfunctioncall
 
 Also note the following: assignments can only be made to fonts that have already
 been defined in \TEX, but have not been accessed {\it at all\/} since that
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-harfbuzz.tex b/Master/texmf-dist/doc/luatex/base/luatex-harfbuzz.tex
new file mode 100644
index 00000000000..c7ed992405a
--- /dev/null
+++ b/Master/texmf-dist/doc/luatex/base/luatex-harfbuzz.tex
@@ -0,0 +1,317 @@
+% language=uk
+
+\environment luatex-style
+
+\startcomponent luatex-harfbuzz
+
+\definelayer
+  [hbpage]
+  [width=\paperwidth,
+   height=\paperheight]
+
+
+\setupbackgrounds
+  [page]
+  [background=hbpage,
+   state=start]
+
+
+\startchapter[reference=harfbuzz,title={The HarfBuzz libraries}]
+
+
+\startsection[title={The \type {luaharfbuzz} library}][library=luaharfbuzz]
+
+At the moment the documentation of the library is a raw "html-to-pdf" 
+rendering of the \type{index.html} file under \type{luaharfbuzz/docs} 
+folder of the source code. The example is almost the verbatim copy
+of the file under \type{luaharfbuzz/examples}.
+\page
+
+
+\dorecurse{17}{%
+\setlayer[hbpage][x=0mm,y=0mm,]{\externalfigure[graphics/luaharfbuzz.pdf][page=\recurselevel]}
+\blank
+\page[empty]
+}
+
+
+\startsubsection[title={Example}]
+
+The example is (a small modification of)  the file \type{core_types.lua}
+and it requires the file \type{harfbuzz.lua}; both are 
+under \type{luaharfbuzz}  folder of the source code.
+The fonts \type{notonastaliq.ttf} and \type{amiri-regular.ttf'} are under 
+\type{luaharfbuzz/fonts}.
+
+
+\blank[3*big]
+
+\startbuffer[hbmod]
+local hb = require("luaharfbuzz")
+
+-- special tags
+hb.Tag.NONE = hb.Tag.new()
+
+-- special script codes (ISO 15924)
+hb.Script.COMMON    = hb.Script.new("Zyyy")
+hb.Script.INHERITED = hb.Script.new("Zinh")
+hb.Script.UNKNOWN   = hb.Script.new("Zzzz")
+hb.Script.INVALID   = hb.Script.from_iso15924_tag(hb.Tag.NONE)
+
+-- directions
+hb.Direction.INVALID = hb.Direction.new("invalid")
+hb.Direction.LTR = hb.Direction.new("ltr")
+hb.Direction.RTL = hb.Direction.new("rtl")
+hb.Direction.TTB = hb.Direction.new("ttb")
+hb.Direction.BTT = hb.Direction.new("btt")
+
+-- special languages
+hb.Language.INVALID = hb.Language.new()
+
+hb.shape = function(font, buf, options)
+  options = options or { }
+
+  -- Apply options to buffer if they are set.
+  if options.language then buf:set_language(options.language) end
+  if options.script then buf:set_script(options.script) end
+  if options.direction then buf:set_direction(options.direction) end
+
+  -- Guess segment properties, in case all steps above have failed
+  -- to set the right properties.
+  buf:guess_segment_properties()
+
+  local features = {}
+
+  -- Parse features
+  if type(options.features) == "string" then
+    for fs in string.gmatch(options.features, '([^,]+)') do
+      local feature = hb.Feature.new(fs)
+      if feature then
+        table.insert(features, hb.Feature.new(fs))
+      else
+        error(string.format("Invalid feature string: '%s'", fs))
+      end
+    end
+  elseif type(options.features) == "table" then
+    features = options.features
+  elseif options.features then -- non-nil but not a string or table
+    error("Invalid features option")
+  end
+
+  return hb.shape_full(font,buf,features,options.shapers or {})
+end
+
+-- For backward compatibility
+hb.Buffer.HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES  = hb.Buffer.CLUSTER_LEVEL_MONOTONE_GRAPHEMES
+hb.Buffer.HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS = hb.Buffer.CLUSTER_LEVEL_MONOTONE_CHARACTERS
+hb.Buffer.HB_BUFFER_CLUSTER_LEVEL_CHARACTERS          = hb.Buffer.CLUSTER_LEVEL_CHARACTERS
+hb.Buffer.HB_BUFFER_CLUSTER_LEVEL_DEFAULT             = hb.Buffer.CLUSTER_LEVEL_DEFAULT
+
+hb.Tag.HB_TAG_NONE = hb.Tag.NONE
+
+hb.Script.HB_SCRIPT_COMMON    = hb.Script.COMMON
+hb.Script.HB_SCRIPT_INHERITED = hb.Script.INHERITED
+hb.Script.HB_SCRIPT_UNKNOWN   = hb.Script.UNKNOWN
+hb.Script.HB_SCRIPT_INVALID   = hb.Script.INVALID
+
+hb.Language.HB_LANGUAGE_INVALID = hb.Language.INVALID
+
+hb.Direction.HB_DIRECTION_INVALID = hb.Direction.INVALID
+hb.Direction.HB_DIRECTION_LTR = hb.Direction.LTR
+hb.Direction.HB_DIRECTION_RTL = hb.Direction.RTL
+hb.Direction.HB_DIRECTION_TTB = hb.Direction.TTB
+hb.Direction.HB_DIRECTION_BTT = hb.Direction.BTT
+
+hb.Direction.HB_DIRECTION_IS_VALID = hb.Direction.is_valid
+hb.Direction.HB_DIRECTION_IS_HORIZONTAL = hb.Direction.is_horizontal
+hb.Direction.HB_DIRECTION_IS_VERTICAL = hb.Direction.is_vertical
+hb.Direction.HB_DIRECTION_IS_FORWARD = hb.Direction.is_forward
+hb.Direction.HB_DIRECTION_IS_BACKWARD = hb.Direction.is_backward
+
+hb.Buffer.get_glyph_infos_and_positions = hb.Buffer.get_glyphs
+
+return hb
+
+\stopbuffer
+
+\blank[3*big]
+
+The file \type{harfbuzz.lua}:
+\typebuffer[hbmod]
+
+
+\startbuffer[hbex]
+local harfbuzz = require('harfbuzz')
+
+-- Harfbuzz API Version
+print("Harfbuzz API version", harfbuzz.version())
+
+-- Shapers available
+print("Shapers:", harfbuzz.shapers())
+
+
+-- harfbuzz.Face
+local face = harfbuzz.Face.new('notonastaliq.ttf')
+print('\nFace upem = '..face:get_upem())
+
+-- harfbuzz.Font
+local font = harfbuzz.Font.new(face)
+local xs, xy = font:get_scale()
+print("\nDefault font scale = X: "..xs..", Y: "..xy)
+
+-- harfbuzz.Buffer
+local text = "یہ" -- U+06CC U+06C1
+local buf = harfbuzz.Buffer.new()
+buf:add_utf8(text)
+
+-- harfbuzz.shape (Shapes text)
+print("\nShaping '"..text.."' set with Noto Nastaliq Urdu")
+harfbuzz.shape(font, buf, { language = harfbuzz.Language.new("urd"), 
+script = harfbuzz.Script.new("Arab"), direction = harfbuzz.Direction.RTL})
+
+local glyphs = buf:get_glyphs()
+print("No. of glyphs", #glyphs)
+for k,v in pairs(glyphs) do 
+  print(k) 
+  for k1,v1 in pairs(v) do 
+   print("",k1,v1) 
+  end 
+end 
+
+local opts = { language = harfbuzz.Language.new("eng"), 
+script = harfbuzz.Script.new("Latn"), direction = harfbuzz.Direction.LTR }
+
+local amiri_face = harfbuzz.Face.new('amiri-regular.ttf')
+local amiri_font = harfbuzz.Font.new(amiri_face)
+
+-- shaping '123' w/o features
+print("\nShaping '123' set with Amiri Regular and no features")
+buf= harfbuzz.Buffer.new()
+buf:add_utf8("123")
+harfbuzz.shape(amiri_font, buf, opts)
+glyphs = buf:get_glyphs()
+for k,v in pairs(glyphs) do 
+  print(k) 
+  for k1,v1 in pairs(v) do 
+   print("",k1,v1) 
+  end 
+end 
+
+
+-- shaping '123' with '+numr' (numerators)
+print("\nShaping '123' set with Amiri Regular with 'numr' feature turned on")
+buf= harfbuzz.Buffer.new()
+buf:add_utf8("123")
+opts.features = "+numr"
+harfbuzz.shape(amiri_font, buf, opts)
+glyphs = buf:get_glyphs()
+for k,v in pairs(glyphs) do 
+  print(k) 
+  for k1,v1 in pairs(v) do 
+   print("",k1,v1) 
+  end 
+end 
+\stopbuffer
+
+The example:
+\typebuffer[hbex]
+
+
+\startbuffer[hbout]
+Harfbuzz API version	2.6.4
+Shapers:	graphite2	ot	fallback
+
+Face upem = 2048
+
+Default font scale = X: 2048, Y: 2048
+
+Shaping 'یہ' set with Noto Nastaliq Urdu
+No. of glyphs	4
+1
+    flags	1.0
+    cluster	2
+    codepoint	277
+    x_advance	472.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+2
+    cluster	0
+    codepoint	19
+    x_advance	0.0
+    y_advance	0.0
+    x_offset	310.0
+    y_offset	-383.0
+3
+    cluster	0
+    codepoint	985
+    x_advance	0.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+4
+    cluster	0
+    codepoint	316
+    x_advance	731.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	-68.0
+
+Shaping '123' set with Amiri Regular and no features
+1
+    cluster	0
+    codepoint	20
+    x_advance	1090.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+2
+    cluster	1
+    codepoint	21
+    x_advance	1090.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+3
+    cluster	2
+    codepoint	22
+    x_advance	1090.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+
+Shaping '123' set with Amiri Regular with 'numr' feature turned on
+1
+    cluster	0
+    codepoint	6673
+    x_advance	600.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+2
+    cluster	1
+    codepoint	6674
+    x_advance	600.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+3
+    cluster	2
+    codepoint	6675
+    x_advance	600.0
+    y_advance	0.0
+    x_offset	0.0
+    y_offset	0.0
+
+\stopbuffer
+
+\blank[3*big]
+The result:
+\typebuffer[hbout]
+
+
+\stopsubsection
+\stopsection
+
+\stopchapter
+\stopcomponent
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-math.tex b/Master/texmf-dist/doc/luatex/base/luatex-math.tex
index 4623ce706d7..e5999e72fd1 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex-math.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex-math.tex
@@ -1454,7 +1454,7 @@ $
 $
 \stopbuffer
 
-\typebuffer
+\typebuffer[script]
 
 results in \inlinebuffer[script].
 
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-modifications.tex b/Master/texmf-dist/doc/luatex/base/luatex-modifications.tex
index b7ce6ca2bd7..747945f55f7 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex-modifications.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex-modifications.tex
@@ -262,7 +262,10 @@ backend are decoupled as much as possible.
 \startitem
     When \lpr {adjustspacing} has value~2, hz optimization will be applied to
     glyphs and kerns. When the value is~3, only glyphs will be treated. A value
-    smaller than~2 disables this feature.
+    smaller than~2 disables this feature. With value of 1, font expansion is
+    applied after \TEX's normal paragraph breaking routines have broken the
+    paragraph into lines. In this case, line breaks are identical to standard
+    \TEX\ behavior (as with \PDFTEX).
 \stopitem
 
 \startitem
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-preamble.tex b/Master/texmf-dist/doc/luatex/base/luatex-preamble.tex
index 8293179776d..e6389528414 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex-preamble.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex-preamble.tex
@@ -11,7 +11,7 @@
 \topicindex{\LUA}
 
 This is a reference manual, not a tutorial. This means that we discuss changes
-relative to traditonal \TEX\ and also present new functionality. As a consequence
+relative to traditional \TEX\ and also present new functionality. As a consequence
 we will refer to concepts that we assume to be known or that might be explained
 later.
 
diff --git a/Master/texmf-dist/doc/luatex/base/luatex-tex.tex b/Master/texmf-dist/doc/luatex/base/luatex-tex.tex
index 0924896288f..4e70243c6d2 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex-tex.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex-tex.tex
@@ -113,18 +113,6 @@ checking some mechanism going haywire.
 
 \stopsubsection
 
-\startsubsection[title={Code page (Windows only)}]
-
-\libindex{getcodepage}
-
-The \type {getcodepage}  returns the value from  \type{GetOEMCP()} 
-(the current original equipment manufacturer (OEM) code page identifier for the operating system)
-and the value from \type{GetACP()} (the current Windows ANSI code page identifier for the operating system).
-
-\stopsubsection
-
-
-
 \stopsection
 
 \startsection[title={The \type {status} library}][library=status]
@@ -1488,6 +1476,23 @@ tex.runtoks(<lua function>)
 When the \prm {tracingnesting} parameter is set to a value larger than~2 some
 information is reported about the state of the local loop.
 
+This function has two optional arguments in case a token register is passed:
+
+\startfunctioncall
+tex.runtoks(<token register>,force,grouped)
+\stopfunctioncall
+
+Inside for instance an \type {\edef} the \type {runtoks} function behaves (at
+least tries to) like it were an \type {\the}. This prevents unwanted side
+effects: normally in such an definition tokens remain tokens and (for instance)
+characters don't become nodes. With the second argument you can force the local
+main loop, no matter what. The third argument adds a level of grouping.
+
+You can quit the local loop with \type {\endlocalcontrol} or from the \LUA\ end
+with \type {tex.quittoks}. In that case you end one level up! Of course in the
+end that can mean that you arrive at the main level in which case an extra end
+will trigger a redundancy warning (not an abort!).
+
 \subsubsection{\type {forcehmode}}
 
 \libindex{forcehmode}
@@ -1952,15 +1957,16 @@ and|/|or console.
 \libindex{write}
 
 \startfunctioncall
-texio.write(<string> target, <string> s, ...)
+texio.write(<string> target | <integer> stream, <string> s, ...)
 texio.write(<string> s, ...)
 \stopfunctioncall
 
 Without the \type {target} argument, writes all given strings to the same
 location(s) \TEX\ writes messages to at this moment. If \prm {batchmode} is in
 effect, it writes only to the log, otherwise it writes to the log and the
-terminal. The optional \type {target} can be one of three possibilities: \type
-{term}, \type {log} or \type {term and log}.
+terminal. The optional \type {target} can be one of four  possibilities: \type
+{term}, \type {log}, \type {term and log} or a valid integer of a stream 
+opened for writing.
 
 Note: If several strings are given, and if the first of these strings is or might
 be one of the targets above, the \type {target} must be specified explicitly to
diff --git a/Master/texmf-dist/doc/luatex/base/luatex.pdf b/Master/texmf-dist/doc/luatex/base/luatex.pdf
index 3112101266b..45255f6d3d3 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex.pdf
+++ b/Master/texmf-dist/doc/luatex/base/luatex.pdf
diff --git a/Master/texmf-dist/doc/luatex/base/luatex.tex b/Master/texmf-dist/doc/luatex/base/luatex.tex
index 413af2b4a63..fb331cabffb 100644
--- a/Master/texmf-dist/doc/luatex/base/luatex.tex
+++ b/Master/texmf-dist/doc/luatex/base/luatex.tex
@@ -44,6 +44,9 @@
 
 % after adding primitives: context mult-prm.mkiv
 
+% \enabledirectives[pdf.stripzeros]
+% \enabledirectives[metapost.stripzeros]
+
 \environment luatex-style
 
 \startmode[book]
@@ -63,7 +66,7 @@
 \startdocument
   [manual=Lua\TeX,
    status=stable,
-   version=1.11]
+   version=1.12]
 
 \startnotmode[*export]
     \component luatex-titlepage
@@ -92,6 +95,7 @@
     \component luatex-tex
     \component luatex-graphics
     \component luatex-fontloader
+    \component luatex-harfbuzz
     \component luatex-backend
 \stopbodymatter