Implement generation of language.dat.lua by extending execute AddHyphen.

git-svn-id: svn://tug.org/texlive/trunk@18379 c570f23f-e606-0410-a88d-b1316a301751

6 files changed, 194 insertions, 48 deletions

diff --git a/Master/texmf/scripts/texlive/tlmgr.pl b/Master/texmf/scripts/texlive/tlmgr.pl
index bf26049cd96..f63b531361a 100755
--- a/Master/texmf/scripts/texlive/tlmgr.pl
+++ b/Master/texmf/scripts/texlive/tlmgr.pl
@@ -669,23 +669,27 @@ sub handle_execute_actions
     }
 
     if ($do_something) {
-      for my $ext ("dat", "def") {
+      for my $ext ("dat", "def", "dat.lua") {
         my $lang = "language.$ext";
         info("regenerating $lang\n");
 
         my $arg1 = "$TEXMFSYSVAR/tex/generic/config/language.$ext";
-        my $arg2 = "$TEXMFLOCAL/tex/generic/config/language-local.dat";
+        my $arg2 = "$TEXMFLOCAL/tex/generic/config/language-local.$ext";
         if ($ext eq "dat") {
           TeXLive::TLUtils::create_language_dat($localtlpdb, $arg1, $arg2);
-        } else {
+        } elsif ($ext eq "def") {
           TeXLive::TLUtils::create_language_def($localtlpdb, $arg1, $arg2);
+        } else {
+          TeXLive::TLUtils::create_language_lua($localtlpdb, $arg1, $arg2);
         }
 
         if (! TeXLive::TLUtils::win32()) {
           # Use full path for external command, except on Windows.
           $lang = "$TEXMFSYSVAR/tex/generic/config/$lang";
         }
-        if ($localtlpdb->option("create_formats") && !$::regenerate_all_formats) {
+        if ($localtlpdb->option("create_formats")
+            && !$::regenerate_all_formats
+            && $ext ne "dat.lua") {
           $errors += do_cmd_and_check("fmtutil-sys --byhyphen $lang");
         }
       }
@@ -3200,7 +3204,7 @@ sub action_generate {
   chomp (my $TEXMFSYSCONFIG = `kpsewhich -var-value=TEXMFSYSCONFIG`);
   chomp (my $TEXMFLOCAL = `kpsewhich -var-value=TEXMFLOCAL`);
 
-  if ($what =~ m/^language(\.dat|\.def)?$/i) {
+  if ($what =~ m/^language(\.dat|\.def|\.dat\.lua)?$/i) {
     #
     # if --rebuild-sys is given *and* --dest we warn that this might not
     # work if the destination is not the default one
@@ -3240,6 +3244,14 @@ sub action_generate {
              . " run fmtutil-sys --byhyphen $dest.\n");
       }
     }
+    if ($what =~ m/^language(\.dat\.lua)?$/i) {
+      my $dest = $opts{"dest"} ? $opts{"dest"} . '.dat.lua' :
+        "$TEXMFSYSVAR/tex/generic/config/language.dat.lua";
+      my $localcfg = $opts{"localcfg"} ||
+        "$TEXMFLOCAL/tex/generic/config/language-local.dat.lua";
+      debug("$0: writing language.dat.lua data to $dest\n");
+      TeXLive::TLUtils::create_language_lua($localtlpdb, $dest, $localcfg);
+    }
 
   } elsif ($what =~ m/^fmtutil$/i) {
     my $dest = $opts{"dest"} || "$TEXMFSYSVAR/web2c/fmtutil.cnf";
@@ -5365,6 +5377,8 @@ Do not ask for confirmation, remove immediately.
 
 =item B<generate language.def>
 
+=item B<generate language.dat.lua>
+
 =item B<generate fmtutil>
 
 =item B<generate updmap>
@@ -5392,31 +5406,33 @@ C<updmap-local.cfg>, there is nothing wrong with using C<generate
 updmap>.  Indeed, we use it ourselves to generate the C<updmap.cfg> file
 in the live source repository.
 
-In more detail: C<generate> remakes any of the four config files
-C<language.dat>, C<language.def>, C<fmtutil.cnf>, and C<updmap.cfg> from
-the information present in the local TLPDB, plus locally-maintained
-files.
+In more detail: C<generate> remakes any of the five config files
+C<language.dat>, C<language.def>, C<language.dat.lua>, C<fmtutil.cnf>,
+and C<updmap.cfg> from the information present in the local TLPDB, plus
+locally-maintained files.
 
 The locally-maintained files are C<language-local.dat>,
-C<language-local.def>, C<fmtutil-local.cnf>, or C<updmap-local.cfg>,
-searched for in C<TEXMFLOCAL> in the respective directories.  If they
-are present, the final file is made by starting with the main file,
-omitting any entries that the local file specifies to be disabled, and
-finally appending the local file.
+C<language-local.def>, C<language-local.dat.lua>, C<fmtutil-local.cnf>,
+or C<updmap-local.cfg>, searched for in C<TEXMFLOCAL> in the respective
+directories.  If they are present, the final file is made by starting
+with the main file, omitting any entries that the local file specifies
+to be disabled, and finally appending the local file.
 
 Local files specify entries to be disabled with a comment line like this:
 
   #!NAME
   %!NAME
+  --!NAME
 
-where C<fmtutil.cnf> and C<updmap.cfg> use C<#>, and C<language.dat> and
-C<language.def> use C<%>.  In any case, the I<name> is the respective
-format name, map file name (include the C<.map> extension), or
-hyphenation pattern identifier.  Examples:
+where C<fmtutil.cnf> and C<updmap.cfg> use C<#>, C<language.dat> and
+C<language.def> use C<%>, and C<language.dat.lua> use C<-->.  In any
+case, the I<name> is the respective format name, map file name (include
+the C<.map> extension), or hyphenation pattern identifier.  Examples:
 
   #!pdflatex
   #!lm.map
   %!german
+  --!usenglishmax
 
 (Of course, you're not likely to actually want to disable those
 particular items.  They're just examples.)
@@ -5426,9 +5442,9 @@ for the same item, if a different definition is desired.  In general,
 except for the special disabling lines, the local files follow the same
 syntax as the master files.
 
-The form C<generate language> recreates both the C<language.dat> and the
-C<language.def> files, while the forms with extension recreates only
-the given language file.
+The form C<generate language> recreates both the C<language.dat>, the
+C<language.def> and the C<language.dat.lua>files, while the forms with
+extension recreates only the given language file.
 
 Special consideration for updmap.cfg: in addition to font map files,
 this file specifies the setting of five options: C<dvipsPreferOutline>,
@@ -5450,10 +5466,11 @@ Options:
 specifies the output file (defaults to the respective location in
 C<TEXMFSYSVAR> for C<language*> and C<fmtutil>, and C<TEXMFSYSCONFIG>
 for C<updmap>).  If C<--dest> is given to C<generate language>, its
-value will be used for the C<language.dat> output, and C<.def> will be
-appended to the value for the name of the C<language.def> output file.
-(This is just to avoid overwriting; if you want a specific name for each
-output file, we recommend invoking C<tlmgr> twice.)
+value will be used for the C<language.dat> output, C<.def> will be
+appended to the value for the name of the C<language.def> output file,
+and C<.dat.lua> to the name of the C<language.dat.lua> file.  (This is
+just to avoid overwriting; if you want a specific name for each output
+file, we recommend invoking C<tlmgr> twice.)
 
 =item B<--localcfg> I<local conf file>
 
@@ -5464,7 +5481,7 @@ location in C<TEXMFLOCAL>).
 
 tells tlmgr to run necessary programs after config files have been
 regenerated. These are: C<updmap-sys> after C<generate updmap>,
-C<fmtutil-sys --all> after C<generate fmtutil>, 
+C<fmtutil-sys --all> after C<generate fmtutil>,
 C<fmtutil-sys --byhyphen .../language.dat> after C<generate language.dat>,
 and
 C<fmtutil-sys --byhyphen .../language.def> after C<generate language.def>.
@@ -5480,6 +5497,7 @@ The respective locations are as follows:
 
   tex/generic/config/language.dat (and language-local.dat);
   tex/generic/config/language.def (and language-local.def);
+  tex/generic/config/language.dat.lua (and language-local.dat.lua);
   web2c/fmtutil.cnf (and fmtutil-local.cnf);
   web2c/updmap.cfg (and updmap-local.cfg).
 
diff --git a/Master/texmf/tex/generic/config/language.us.lua b/Master/texmf/tex/generic/config/language.us.lua
new file mode 100644
index 00000000000..2328b8b4916
--- /dev/null
+++ b/Master/texmf/tex/generic/config/language.us.lua
@@ -0,0 +1,25 @@
+-- $Id: $
+--[[
+language.us.dat (and the start of language.dat.lua), used by:
+- a special luatex version of hyphen.cfg (derived from the babel system);
+- a special luatex version of etex.src (from the e-TeX distributon).
+
+See luatex-hyphen.pdf (currently part of the hyph-utf8 package) for details.
+]]
+return {
+	["english"]={
+		loader="hyphen.tex",
+		special="language0", -- should be dumped in the format
+		lefthyphenmin=2,
+		righthyphenmin=3,
+		synonyms={"usenglish","USenglish","american"},
+	},
+	["usenglishmax"]={
+		loader="ushyphmax.tex",
+		patterns="hyph-en-US.pat.txt",
+		hyphenation="hyph-en-US.hyp.txt",
+		lefthyphenmin=2,
+		righthyphenmin=3,
+		synonyms={},
+	},
+-- END of language.us.lua
diff --git a/Master/tlpkg/TeXLive/TLPDB.pm b/Master/tlpkg/TeXLive/TLPDB.pm
index 811556c77bc..8993105264c 100644
--- a/Master/tlpkg/TeXLive/TLPDB.pm
+++ b/Master/tlpkg/TeXLive/TLPDB.pm
@@ -52,6 +52,7 @@ C<TeXLive::TLPDB> -- A database of TeX Live Packages
   $tlpdb->fmtutil_cnf_lines;
   $tlpdb->language_dat_lines;
   $tlpdb->language_def_lines;
+  $tlpdb->language_lua_lines;
   $tlpdb->package_revision("packagename");
   $tlpdb->location;
   $tlpdb->config_src_container;
@@ -1346,7 +1347,28 @@ sub language_def_lines {
   }
   return(@lines);
 }
-    
+
+=item C<< $tlpdb->language_lua_lines ( [@disabled_hyphen_names] ) >>
+
+The function C<language_lua_lines> returns the list of all
+lines for language.dat.lua that can be generated from the tlpdb.
+
+Every hyphenation pattern listed in the tlpdb but listed in the arguments
+will not be included in the list of lines returned.
+
+=cut
+
+sub language_lua_lines {
+  my $self = shift;
+  my @lines;
+  foreach my $p ($self->list_packages) {
+    my $obj = $self->get_package ($p);
+    die "$0: No TeX Live package named $p, strange" if ! $obj;
+    push @lines, $obj->language_lua_lines(@_);
+  }
+  return(@lines);
+}
+
 =back
 
 =pod
diff --git a/Master/tlpkg/TeXLive/TLPOBJ.pm b/Master/tlpkg/TeXLive/TLPOBJ.pm
index b867ab7b4a0..2ad420f657c 100644
--- a/Master/tlpkg/TeXLive/TLPOBJ.pm
+++ b/Master/tlpkg/TeXLive/TLPOBJ.pm
@@ -1024,15 +1024,15 @@ sub updmap_cfg_lines {
 sub language_dat_lines {
   my $self = shift;
   local @disabled = @_;  # we use @disabled in the nested sub
-  my @lines = $self->_parse_hyphen_execute(\&make_dat_lines);
+  my @lines = $self->_parse_hyphen_execute(\&make_dat_lines, 'dat');
   return @lines;
 
   sub make_dat_lines {
-    my ($name, $lhm, $rhm, $file, @syn) = @_;
+    my ($name, $lhm, $rhm, $file, $syn) = @_;
     my @ret;
     return if TeXLive::TLUtils::member($name, @disabled);
     push @ret, "$name $file\n";
-    foreach (@syn) {
+    foreach (@$syn) {
       push @ret, "=$_\n";
     }
     return @ret;
@@ -1043,16 +1043,16 @@ sub language_dat_lines {
 sub language_def_lines {
   my $self = shift;
   local @disabled = @_;  # we use @disabled in the nested sub
-  my @lines = $self->_parse_hyphen_execute(\&make_def_lines);
+  my @lines = $self->_parse_hyphen_execute(\&make_def_lines, 'def');
   return @lines;
 
   sub make_def_lines {
-    my ($name, $lhm, $rhm, $file, @syn) = @_;
+    my ($name, $lhm, $rhm, $file, $syn) = @_;
     return if TeXLive::TLUtils::member($name, @disabled);
     my $exc = "";
     my @ret;
     push @ret, "\\addlanguage\{$name\}\{$file\}\{$exc\}\{$lhm\}\{$rhm\}\n";
-    foreach (@syn) {
+    foreach (@$syn) {
       # synonyms in language.def ???
       push @ret, "\\addlanguage\{$_\}\{$file\}\{$exc\}\{$lhm\}\{$rhm\}\n";
       #debug("Ignoring synonym $_ for $name when creating language.def\n");
@@ -1060,10 +1060,36 @@ sub language_def_lines {
     return @ret;
   }
 }
-    
+
+
+sub language_lua_lines {
+  my $self = shift;
+  local @disabled = @_;  # we use @disabled in the nested sub
+  my @lines = $self->_parse_hyphen_execute(\&make_lua_lines, 'lua', '--');
+  return @lines;
+
+  sub make_lua_lines {
+    my ($name, $lhm, $rhm, $file, $syn, $patt, $hyph, $special) = @_;
+    return if TeXLive::TLUtils::member($name, @disabled);
+    my @syn = (@$syn); # avoid modifying the original
+    map { $_ = "'$_'" } @syn;
+    my @ret;
+    push @ret, "['$name'] = {", "\tloader = '$file',",
+               "\tlefthyphenmin = $lhm,", "\trighthyphenmin = $rhm,",
+               "\tsynonyms = { " . join(', ', @syn) . " },";
+    push @ret, "\tpatterns = '$patt'," if defined $patt;
+    push @ret, "\thyphenation = '$hyph'," if defined $hyph;
+    push @ret, "\tspecial = '$special'," if defined $special;
+    push @ret, '},';
+    map { $_ = "\t$_\n" } @ret;
+    return @ret;
+  }
+}
+
 
 sub _parse_hyphen_execute {
-  my ($obj, $coderef) = @_;
+  my ($obj, $coderef, $db, $cc) = @_;
+  $cc ||= '%'; # default comment char
   my @langlines = ();
   my $pkg = $obj->name;
   my $first = 1;
@@ -1073,14 +1099,16 @@ sub _parse_hyphen_execute {
       if (defined($r{"error"})) {
         die "$r{'error'}, package $pkg, execute $e";
       }
+      if (not TeXLive::TLUtils::member($db, @{$r{"databases"}})) {
+        next;
+      }
       if ($first) {
-        push @langlines, "% from $pkg:\n";
+        push @langlines, "$cc from $pkg:\n";
         $first = 0;
       }
-      my @syns;
-      @syns = @{$r{"synonyms"}} if (defined($r{"synonyms"}));
-      my @foo = &$coderef ($r{"name"}, $r{"lefthyphenmin"}, 
-                           $r{"righthyphenmin"}, $r{"file"}, @syns);
+      my @foo = &$coderef ($r{"name"}, $r{"lefthyphenmin"},
+                           $r{"righthyphenmin"}, $r{"file"}, $r{"synonyms"},
+                           $r{"txtpatt"}, $r{"txthyph"}, $r{"luaspecial"});
       push @langlines, @foo;
     }
   }
@@ -1638,6 +1666,11 @@ lines for language.dat that can be generated from the tlpobj
 The function C<language_def_lines> returns the list of all
 lines for language.def that can be generated from the tlpobj.
 
+=item C<< $tlpobj->language_lua_lines >>
+
+The function C<language_lua_lines> returns the list of all
+lines for language.dat.lua that can be generated from the tlpobj.
+
 =back
 
 =head1 SEE ALSO
diff --git a/Master/tlpkg/TeXLive/TLPSRC.pm b/Master/tlpkg/TeXLive/TLPSRC.pm
index b5f044402b0..d3b870c490d 100644
--- a/Master/tlpkg/TeXLive/TLPSRC.pm
+++ b/Master/tlpkg/TeXLive/TLPSRC.pm
@@ -791,8 +791,15 @@ the font cache for luatex.
 
 activates the hyphenation pattern with name I<texlang> and load the file
 I<file> for that language.  The additional variables I<var> are:
-C<lefthyphenmin>, C<righthyphenmin> (both integers), and C<synonyms> (a
-comma-separated list of alias names for that hyphenation).
+C<lefthyphenmin>, C<righthyphenmin> (both integers), C<synonyms> (a
+comma-separated list of alias names for that hyphenation), C<databases>
+(a comma-separated list of databases the entry should go in; currently
+recognized are: C<dat> (C<language.dat>), C<def> (C<language.def>) and
+C<lua> (C<language.dat.lua>)), C<txtpatt> and C<txthyph> (files with the
+patterns (resp. exceptions) in plain txt), and C<luaspecial> (string).
+
+The variable C<databases> defaults to C<dat,def>, or C<dat,def,lua> if
+one of the keys C<txtpatt>, C<txthyph> or C<luaspecial> is used.
 
 =item C<execute AddFormat name=I<fmt> engine=I<eng> [I<var>...]>
 
diff --git a/Master/tlpkg/TeXLive/TLUtils.pm b/Master/tlpkg/TeXLive/TLUtils.pm
index b416a008242..f2504f62d5c 100644
--- a/Master/tlpkg/TeXLive/TLUtils.pm
+++ b/Master/tlpkg/TeXLive/TLUtils.pm
@@ -69,6 +69,7 @@ C<TeXLive::TLUtils> -- utilities used in the TeX Live infrastructure
   TeXLive::TLUtils::create_updmap($tlpdb,$dest,$localconf);
   TeXLive::TLUtils::create_language_dat($tlpdb,$dest,$localconf);
   TeXLive::TLUtils::create_language_def($tlpdb,$dest,$localconf);
+  TeXLive::TLUtils::create_language_lua($tlpdb,$dest,$localconf);
   TeXLive::TLUtils::time_estimate($totalsize, $donesize, $starttime)
   TeXLive::TLUtils::install_packages($from_tlpdb,$media,$to_tlpdb,$what,$opt_src, $opt_doc)>);
   TeXLive::TLUtils::install_package($what, $filelistref, $target, $platform);
@@ -129,6 +130,7 @@ BEGIN {
     &create_updmap
     &create_language_dat
     &create_language_def
+    &create_language_lua
     &parse_AddFormat_line
     &parse_AddHyphen_line
     &sort_uniq
@@ -2344,11 +2346,13 @@ sub make_local_skeleton {
 
 =item C<create_language_def($tlpdb, $dest, $localconf)>
 
-These four functions create C<fmtutil.cnf>, C<updmap.cfg>, C<language.dat>,
-and C<language.def> respectively, in C<$dest> (which by default is below
-C<$TEXMFSYSVAR>).  These functions merge the information present in the
-TLPDB C<$tlpdb> (formats, maps, hyphenations) with local configuration
-additions: C<$localconf>.
+=item C<create_language_lua($tlpdb, $dest, $localconf)>
+
+These five functions create C<fmtutil.cnf>, C<updmap.cfg>, C<language.dat>,
+C<language.def>, and C<language.dat.lua> respectively, in C<$dest> (which by
+default is below C<$TEXMFSYSVAR>).  These functions merge the information
+present in the TLPDB C<$tlpdb> (formats, maps, hyphenations) with local
+configuration additions: C<$localconf>.
 
 Currently the merging is done by omitting disabled entries specified
 in the local file, and then appending the content of the local
@@ -2497,6 +2501,16 @@ sub create_language_def {
   _create_config_files ($tlpdb, "texmf/tex/generic/config/language.us.def", $dest, $localconf, 1, '%', \@lines, @postlines);
 }
 
+sub create_language_lua {
+  my ($tlpdb,$dest,$localconf) = @_;
+  # no checking for disabled stuff for language.dat and .lua
+  my @lines = $tlpdb->language_lua_lines(
+                         get_disabled_local_configs($localconf, '--'));
+  my @postlines = ("}\n");
+  _create_config_files ($tlpdb, "texmf/tex/generic/config/language.us.lua",
+    $dest, $localconf, 0, '--', \@lines, @postlines);
+}
+
 sub _create_config_files {
   my ($tlpdb, $headfile, $dest,$localconf, $keepfirstline, $cc, $tlpdblinesref, @postlines) = @_;
   my $root = $tlpdb->root;
@@ -2533,11 +2547,13 @@ sub _create_config_files {
 sub parse_AddHyphen_line {
   my $line = shift;
   my %ret;
+  # default values
   my $default_lefthyphenmin = 2;
   my $default_righthyphenmin = 3;
   $ret{"lefthyphenmin"} = $default_lefthyphenmin;
   $ret{"righthyphenmin"} = $default_righthyphenmin;
-  for my $p (split(' ', $line)) {
+  $ret{"synonyms"} = [];
+  for my $p (quotewords('\s+', 0, "$line")) {
     my ($a, $b) = split /=/, $p;
     if ($a eq "name") {
       if (!$b) {
@@ -2563,6 +2579,22 @@ sub parse_AddHyphen_line {
       $ret{"file"} = $b;
       next;
     }
+    if ($a eq "txtpatt") {
+        $ret{"txtpatt"} = $b;
+        next;
+    }
+    if ($a eq "txthyph") {
+        $ret{"txthyph"} = $b;
+        next;
+    }
+    if ($a eq "luaspecial") {
+        $ret{"luaspecial"} = $b;
+        next;
+    }
+    if ($a eq "databases") {
+      @{$ret{"databases"}} = split /,/, $b;
+      next;
+    }
     if ($a eq "synonyms") {
       @{$ret{"synonyms"}} = split /,/, $b;
       next;
@@ -2571,6 +2603,15 @@ sub parse_AddHyphen_line {
     $ret{"error"} = "Unknown language directive $e";
     return %ret;
   }
+  # this default value couldn't be set earlier
+  if (not defined($ret{"databases"})) {
+    if (defined $ret{"txtpatt"} or defined $ret{"txthyph"}
+        or defined $ret{"luaspecial"}) {
+      @{$ret{"databases"}} = qw(dat def lua);
+    } else {
+      @{$ret{"databases"}} = qw(dat def);
+    }
+  }
   return %ret;
 }