From d3807e8026e4eefdf5f0f8863039f2d26fffa97b Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Mon, 3 Feb 2020 22:33:15 +0000 Subject: babel (3feb20) git-svn-id: svn://tug.org/texlive/trunk@53660 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/source/latex/babel/babel.dtx | 578 ++++++++++++++-------- Master/texmf-dist/source/latex/babel/babel.ins | 2 +- Master/texmf-dist/source/latex/babel/bbcompat.dtx | 2 +- Master/texmf-dist/source/latex/babel/locale.zip | Bin 713940 -> 728098 bytes 4 files changed, 365 insertions(+), 217 deletions(-) (limited to 'Master/texmf-dist/source/latex/babel') diff --git a/Master/texmf-dist/source/latex/babel/babel.dtx b/Master/texmf-dist/source/latex/babel/babel.dtx index 0d4360f493e..d0a6314f27d 100644 --- a/Master/texmf-dist/source/latex/babel/babel.dtx +++ b/Master/texmf-dist/source/latex/babel/babel.dtx @@ -1,7 +1,7 @@ % \iffalse % This document requires lualatex %% -%% Copyright (C) 2012-2019 Javier Bezos and Johannes L. Braams. +%% Copyright (C) 2012-2020 Javier Bezos and Johannes L. Braams. %% Copyright (C) 1989-2012 Johannes L. Braams and %% any individual authors listed elsewhere in this file. %% All rights reserved. @@ -31,7 +31,7 @@ % % \iffalse %<*filedriver> -\ProvidesFile{babel.dtx}[2020/01/15 v3.38 The Babel package] +\ProvidesFile{babel.dtx}[2020/02/03 v3.39 The Babel package] \documentclass{ltxdoc} \GetFileInfo{babel.dtx} \usepackage{fontspec} @@ -235,21 +235,25 @@ Xe\largetex \part{User guide} \begin{itemize} -\item This user guide focuses on \LaTeX. There are also some notes on - its use with Plain \TeX. +\item This user guide focuses on internationalization and localization + with \LaTeX. There are also some notes on its use with Plain \TeX. \item Changes and new features with relation to version 3.8 are - highlighted with \New{X.XX}\hspace{-.5em}. The most recent features + highlighted with \New{X.XX}\hspace{-.5em}, and there are some notes + for the latest versions in + \href{https://github.com/latex3/babel/wiki}{the \babel{} wiki}. + The most recent features could be still unstable. Please, report any issues you find in - \texttt{https://github.com/latex3/babel/issues}, which is better + \href{https://github.com/latex3/babel/issues}{GitHub}, which is better than just complaining on an e-mail list or a web forum. \item If you are interested in the \TeX{} multilingual support, please - join the \textsf{kadingira} list on - \texttt{http://tug.org/mailman/listinfo/kadingira}. You can follow + join the \href{http://tug.org/mailman/listinfo/kadingira}{kadingira + mail list}. You can follow the development of \babel{} in - \texttt{https://github.com/latex3/babel} (which provides some sample - files, too). + \href{https://github.com/latex3/babel}{GitHub} (which provides many + \href{https://github.com/latex3/babel/tree/master/samples}{sample + files}, too). \item See section \ref{contribute} for contributing a language. @@ -492,6 +496,40 @@ _\usepackage[vietnamese,danish]{babel}_ \end{verbatim} \end{example} +\subsection{Mostly monolingual documents} + +\New{3.39} Very often, multilingual documents consist of a main +language with small pieces of text in another languages (words, idioms, +short sentences). Typically, all you need is to set the line breaking +rules and, perhaps, the font. In such a case, \babel{} now does not +require declaring these secondary languages explicitly, because the +basic settings are loaded on the fly when the language is selected (and +also when provided in the optional argument of |\babelfont|, if used.) + +This is particularly useful, too, when there are short texts of this +kind coming from an external source whose contents are not known on +beforehand (for example, titles in a bibliography). At this regard, it +is worth remembering that |\babelfont| does not load any font until +required, so that it can be used just in case. + +\begin{example} + A trivial document is: +\setengine{luatex/xetex} +\begin{verbatim} +\documentclass{article} +\usepackage[english]{babel} + +\babelfont[russian]{rm}{FreeSerif} + +\begin{document} + +English. \foreignlanguage{russian}{Русский}. +\foreignlanguage{spanish}{Español} + +\end{document} +\end{verbatim} +\end{example} + \subsection{Modifiers} \New{3.9c} The basic behavior of some languages can be modified when @@ -541,6 +579,7 @@ modifiers are a more general mechanism. cases, the error vanishes when the document is typeset again, but in more severe ones you will need to remove the |aux| file. \end{itemize} + \subsection{Plain} In Plain, load languages styles with |\input| and then use @@ -1188,9 +1227,17 @@ be the same as the option name as set in |\usepackage|!). \subsection{\texttt{ini} files} -An alternative approach to define a language is by means of an -\texttt{ini} file. Currently \babel{} provides about 200 of these -files containing the basic data required for a language. +An alternative approach to define a language (or, more precisely, a +\textit{locale}) is by means of an \texttt{ini} file. Currently +\babel{} provides about 200 of these files containing the basic data +required for a locale. + +|ini| files are not meant only for \babel, and they has been devised as +a resource for other packages. To easy interoperability between \TeX{} +and other systems, they are identified with the BCP 47 codes as +preferred by the Unicode Common Language Data Repository, which was +used as source for most of the data provided by these files, too (the +main exception being the |\...name| strings). Most of them set the date, and many also the captions (Unicode and LICR). They will be evolving with the time to add more features @@ -1280,6 +1327,19 @@ loads \textsf{luatexja}: \end{description} \end{note} +\begin{note} + Wikipedia defines a \textit{locale} as follows: “In computing, a + locale is a set of parameters that defines the user’s language, + region and any special variant preferences that the user wants to see + in their user interface. Usually a locale identifier consists of at + least a language code and a country/region code.” \Babel{} is moving + gradually from the old and fuzzy concept of \textit{language} to the + more modern of \textit{locale}. Note each locale is by itself a + separate “language”, which explains why there are so many files. This + is on purpose, so that possible variants can be created and/or + redefined easily. +\end{note} + Here is the list (u means Unicode captions, and l means LICR captions): @@ -1891,6 +1951,26 @@ zulu \endgroup \hrule +\bigskip + +\textbf{Modifying and adding values to |ini| files} + +\New{3.39} There is a way to modify the values of |ini| files when they +get loaded with |\babelprovide| and |import|. To set, say, +|digits.native| in the |numbers| section, use something like +|numbers/digits.native=abcdefghij|. Keys may be added, too. Without +|import| you may modify the identification keys. + +This can be used to create private variants easily. All you need is to +import the same |ini| file with a different locale name and different +parameters. + +% \begin{example} +% Let's assume you need only the basic features of `spanish`, so that +% the |ini| file is enough, and also a variant with your own +% hyphenation with +% \end{example} + \subsection{Selecting fonts} \New{3.15} Babel provides a high level interface on top of |fontspec| @@ -1988,22 +2068,18 @@ to select fonts in addition to the three basic families. \babelfont[hindi]{rm}{Shobhika} \end{verbatim} This makes sure the OpenType script for Devanagari is |deva| and not - |dev2| (\luatex{} does not detect automatically the correct - script\footnote{And even with the correct code some fonts could be - rendered incorrectly by \textsf{fontspec}, so double-check the - results. \xetex{} fares better, but some fonts are still - problematic.}). You may also pass some options to \textsf{fontspec}: - with |silent|, the warnings about unavailable scripts or languages - are not shown (they are only really useful when the document format - is being set up). + |dev2|, in case it is not detected correctly. You may also pass some + options to \textsf{fontspec}: with |silent|, the warnings about + unavailable scripts or languages are not shown (they are only really + useful when the document format is being set up). \end{note} \begin{note} Directionality is a property affecting margins, indentation, column - order, etc., not just text. Therefore, it is under the direct - control of the language, which applies both the script and the - direction to the text. As a consequence, there is no need to set - \texttt{Script} when declaring a font (nor \texttt{Language}). In + order, etc., not just text. Therefore, it is under the direct control + of the language, which applies both the script and the direction to + the text. As a consequence, there is no need to set \texttt{Script} + when declaring a font with |\babelfont| (nor \texttt{Language}). In fact, it is even discouraged. \end{note} @@ -2012,10 +2088,10 @@ to select fonts in addition to the three basic families. (|rm|, |sf|, |tt|, and the like). If a language is switched when an \textit{ad hoc} font is active, or you select the font with this command, neither the script nor the language is passed. You must - add them by hand. This is by design, for several reasons (for + add them by hand. This is by design, for several reasons —for example, each font has its own set of features and a generic setting for several of them could be problematic, and also a “lower-level” - font selection is useful). + font selection is useful. \end{note} \begin{note} @@ -2028,53 +2104,45 @@ to select fonts in addition to the three basic families. \end{note} \begin{warning} - Do not use |\set|\textit{xxxx}|font| and |\babelfont| at the same - time. |\babelfont| follows the standard \LaTeX{} conventions to set - the basic families -- define |\|\textit{xx}|default|, and activate - it with |\|\textit{xx}|family|. On the other hand, - |\set|\textit{xxxx}|font| in \textsf{fontspec} takes a different - approach, because |\|\textit{xx}|family| is redefined with the - family name hardcoded (so that |\|\textit{xx}|default| becomes - no-op). Of course, both methods are incompatible, and if you use - |\set|\textit{xxxx}|font|, font switching with |\babelfont| just - does \textit{not} work (nor the standard |\|\textit{xx}|default|, - for that matter). As of \New{3.34} there is an attempt to make them - compatible, but the language system will not be set by \babel and - should be set with |fontspec| if necessary. + Using |\set|\textit{xxxx}|font| and |\babelfont| at the same time is + discouraged, but very often works as expected. However, be aware with + |\set|\textit{xxxx}|font| the language system will not be set by + \babel{} and should be set with |fontspec| if necessary. \end{warning} \begin{troubleshooting} -\trouble{Package fontspec Warning: 'Language 'LANG' not available for -font 'FONT' with script 'SCRIPT' 'Default' language used instead'} -\textit{Package fontspec Warning: 'Language 'LANG' not available for -font 'FONT' with script 'SCRIPT' 'Default' language used instead'}. -\textbf{This is \textit{not} and error.} -This warning is shown by \textsf{fontspec}, not by \babel. It could be -irrelevant for English, but not for many other languages, including -Urdu and Turkish. This is a useful and harmless warning, and if -everything is fine with your document the best thing you can do is just -to ignore it altogether. + \trouble{Package fontspec Warning: 'Language 'LANG' not available for + font 'FONT' with script 'SCRIPT' 'Default' language used instead'} + \textit{Package fontspec Warning: 'Language 'LANG' not available for + font 'FONT' with script 'SCRIPT' 'Default' language used instead'}. + + \textbf{This is \textit{not} and error.} This warning is shown by + \textsf{fontspec}, not by \babel. It could be irrelevant for English, + but not for many other languages, including Urdu and Turkish. This is + a useful and harmless warning, and if everything is fine with your + document the best thing you can do is just to ignore it altogether. \end{troubleshooting} \begin{troubleshooting} -\trouble{Package babel Info: The following fonts are not babel standard families} -\textit{Package babel Info: The following fonts are not babel -standard families}. - -\textbf{This is \textit{not} and error.} \babel{} assumes that if you -are using |\babelfont| for a family, very likely you want to define the -rest of them. If you don't, you can find some inconsistencies between -families. This checking is done at the beginning of the document, at a -point where we cannot know which families will be used. - -Actually, there is no real need to use |\babelfont| in a monolingual -document, if you set the language system in |\setmainfont| (or not, -depending on what you want). - -As the message explains, \textit{there is nothing intrinsically wrong} -with not defining all the families. In fact, there is nothing -intrinsically wrong with not using |\babelfont| at all. But you must be -aware that this may lead to some problems. + \trouble{Package babel Info: The following fonts are not babel + standard families} \textit{Package babel Info: The following fonts + are not babel standard families}. + + \textbf{This is \textit{not} and error.} \babel{} assumes that if you + are using |\babelfont| for a family, very likely you want to define + the rest of them. If you don't, you can find some inconsistencies + between families. This checking is done at the beginning of the + document, at a point where we cannot know which families will be + used. + + Actually, there is no real need to use |\babelfont| in a monolingual + document, if you set the language system in |\setmainfont| (or not, + depending on what you want). + + As the message explains, \textit{there is nothing intrinsically + wrong} with not defining all the families. In fact, there is nothing + intrinsically wrong with not using |\babelfont| at all. But you must + be aware that this may lead to some problems. \end{troubleshooting} \subsection{Modifying a language} @@ -2090,7 +2158,7 @@ sometimes necessary, but not always trivial. } \end{verbatim} As of 3.15, there is no need to hide spaces with \texttt{\%% -} (\babel{} removes them), but it is advisable to do it. +} (\babel{} removes them), but it is advisable to do so. \item The new way, which is found in |bulgarian|, |azerbaijani|, |spanish|, |french|, |turkish|, |icelandic|, |vietnamese| and a few more, as well as in languages created with |\babelprovide| and its @@ -2177,6 +2245,17 @@ In most cases, you will only need to define a few macros. \end{verbatim} \end{example} +\begin{example} + Locales with names based on BCP 47 codes can be created with + something like: +\begin{verbatim} + \babelprovide[import=en-US]{enUS} +\end{verbatim} + Note, however, mixing ways to identify locales can lead to problems. + For example, is \verb|yi| the name of the language spoken by the Yi + people or is it the code for Yiddish? +\end{example} + The main language is not changed (\texttt{danish} in this example). So, you must add |\selectlanguage{arhinish}| or other selectors where necessary. @@ -2206,8 +2285,7 @@ There are about 200 |ini| files, with data taken from the |ldf| files and the CLDR provided by Unicode. Not all languages in the latter are complete, and therefore neither are the |ini| files. A few languages will show a warning about the current lack of suitability of the date -format (\textsf{hindi}, \textsf{french}, \textsf{breton}, and -\textsf{occitan}). +format (\textsf{french}, \textsf{breton}, and \textsf{occitan}). Besides |\today|, this option defines an additional command for dates: |\date|, which takes three arguments, namely, year, month and @@ -2267,23 +2345,25 @@ the script, no matter which language is active. Although somewhat inconsistent, this makes setting a language up easier in most typical cases. +\Describe{onchar=}{\texttt{ids} $\string|$ \texttt{fonts}} +\New{3.38} This option is much like an ‘event’ called with a character +belonging to the script of this locale is found. There are currently +two ‘actions’, which can be used at the same time (separated by a +space): with |ids| the |\language| and the |\localeid| are set to the +values of this locale; with |fonts|, the fonts are changed to those of +the this locale (as set with |\babelfont|). This option is not +compatible with |mapfont|. Characters can be added with +|\babelcharproperty|. + \Describe{mapfont=}{\texttt{direction}} Assigns the font for the writing direction of this language (only with -|bidi=basic|). More precisely, what |mapfont=direction| means is, ‘when -a character has the same direction as the script for the “provided” -language, then change its font to that set for this language’. There -are 3 directions, following the bidi Unicode algorithm, namely, -Arabic-like, Hebrew-like and left to right. So, there should be at most -3 directives of this kind. - -\Describe{onchar=}{\texttt{ids} $\string|$ \texttt{fonts}} -\New{3.38} This options is much like an ‘event’ called with a character -belonging to the script of the current locale is found. There are two -action, which can be used at the same time (separated by a space): with -|ids| the |\language| and the |\localeid| are set to the values of this -locale; with |fonts|, the fonts are changed to those of the current -locale (as set with |\babelfont|). This option is not compatible with -|mapfont|. +|bidi=basic|). Whenever possible, instead of this option use |onchar|, +based on the script, which usually makes more sense. More precisely, +what |mapfont=direction| means is, ‘when a character has the same +direction as the script for the “provided” language, then change its +font to that set for this language’. There are 3 directions, following +the bidi Unicode algorithm, namely, Arabic-like, Hebrew-like and left +to right. So, there should be at most 3 directives of this kind. \Describe{intraspace=}{\meta{base} \meta{shrink} \meta{stretch}} Sets the interword space for the writing system of the language, in em @@ -2393,6 +2473,12 @@ conditions are not met, write |\BabelEnsureInfo|. \subsection{Hyphenation and line breaking} +\Babel{} deals with three kinds of line breaking rules: Western, +typically the LGC group, South East Asian, like Thai, and CJK, but +support depends on the engine: \textsf{pdftex} only deals with the +former, \xetex{} also with the second one, while \luatex{} provides +basic rules for the latter, too. + \Describe{\babelhyphen}{% \colorbox{thegrey}{\ttfamily\hskip-.2em*\hskip-.2em}\marg{type}} \DescribeOther\babelhyphen{% @@ -2515,6 +2601,55 @@ Unicode engines, spacing is based on the ``current'' em unit (the size of the previous char in \luatex, and the font size set by the last |\selectfont| in \xetex). +\Describe{\babelposthyphenation}{\marg{hyphenrules-name}% + \marg{lua-pattern}\marg{replacement}} + +\New{3.37-3.39} With \luatex{} it is now possible to define +non-standard hyphenation rules, like |f-f| $\to$ |ff-f|, repeated +hyphens, ranked ruled (or more precisely, ‘penalized’ hyphenation +points), and so on. No rules are currently provided by default, but +they can be defined as shown in the following example, where |{1}| is +the first captured char (between |()| in the pattern): +\begin{verbatim} +\babelposthyphenation{german}{([fmtrp]) | {1}} +{ + { no = {1}, pre = {1}{1}- }, % Replace first char with disc + remove, % Remove automatic disc (2nd node) + {} % Keep last char, untouched +} +\end{verbatim} + +In the replacements, a captured char may be mapped to another, too. For +example, if the first capture reads |([ΐΰ])|, the replacement could be +\verb+{1|ΐΰ|ίύ}+, which maps \textit{ΐ} to \textit{ί}, and \textit{ΰ} +to \textit{ύ}, so that the diaeresis is removed. + +This feature is activated with the first |\babelposthyphenation|. + +See the \href{https://github.com/latex3/babel/wiki}{\babel\ wiki} for a +more detailed description and some examples. It also describes an +additional replacement type with the key |string|. + +\begin{example} + Although the main purpose of this command is non-standard + hyphenation, it may actually be used for other transformations (after + hyphenation is applied, so you must take discretionaries into + account). For example, you can use the |string| replacement to + replace a character (or series of them) by another character (or + series of them). Thus, to enter \textit{ž} as |zh| and \textit{š} as + |sh| in a newly created locale for transliterated Russian: +\begin{verbatim} +\babelprovide[hyphenrules=+]{russian-latin} % Create locale +\babelposthyphenation{russian-latin}{([sz])h} % Create rule +{ + { string = {1|sz|šž} }, + remove +} +\end{verbatim} + In other words, it is a quite general tool. (A counterpart + |\babelprehyphenation| is on the way.) +\end{example} + \subsection{Selecting scripts} Currently \babel{} provides no standard interface to select @@ -2589,6 +2724,14 @@ differ in the way `weak' numeric characters are ordered (eg, Arabic options described below). \end{warning} +\begin{warning} + If characters to be mirrored are shown without changes with \luatex, + try with the following line: +\begin{verbatim} +\babeladjust{bidi.mirroring=off} +\end{verbatim} +\end{warning} + There are some package options controlling bidi writing. \Describe{bidi=}{\texttt{default} $\string|$ \texttt{basic} @@ -2612,20 +2755,13 @@ consider the intrinsic direction of scripts and weak directionality.) tentative, but it mostly works. For RL documents use the former, and for LR ones use the latter. -\New{3.37} There is some experimental support for \textsf{luahbtex} -(with |lualatex-dev|) and the latest releases of \textsf{luaotfload} -(3.11), with |Renderer = Harfbuzz| in \textsf{fontspec}. Since it is -based on \luatex, the option |basic| mostly works (You may need -deactivate the |rtlm| or the |rtla| font features, or alternatively -deactive mirroring in \babel{} with |\babeladjust|.) - There are samples on GitHub, under \texttt{/required/babel/samples}. See particularly |lua-bidibasic.tex| and |lua-secenum.tex|. \begin{example} The following text comes from the Arabic Wikipedia (article about Arabia). Copy-pasting some text from the Wikipedia is a good way to - test this feature. Remember |basic-r| is available in \luatex{} + test this feature. Remember |basic| is available in \luatex{} only. \begingroup % If you are looking at the code to see how it has been written, you @@ -2638,7 +2774,7 @@ See particularly |lua-bidibasic.tex| and |lua-secenum.tex|. \begin{verbatim} \documentclass{article} -\usepackage[_bidi=basic-r_]{babel} +\usepackage[_bidi=basic_]{babel} _\babelprovide[import, main]{arabic}_ @@ -2676,7 +2812,7 @@ _\babelprovide[import, main]{arabic}_ \usepackage[english, _bidi=basic_]{babel} -\babelprovide[_mapfont=direction_]{arabic} +\babelprovide[_onchar=ids fonts_]{arabic} \babelfont{rm}{Crimson} \babelfont[*arabic]{rm}{FreeSerif} @@ -2691,7 +2827,7 @@ Arabic as @1فصحى العصر@0 \textit{fuṣḥā l-ʻaṣr} (MSA) and \end{document} \end{verbatim} \endgroup - In this example, and thanks to |mapfont=direction|, any Arabic letter + In this example, and thanks to |onchar=ids fonts|, any Arabic letter (because the language is |arabic|) changes its font to that set for this language (here defined via |*arabic|, because Crimson does not provide Arabic letters). @@ -2892,6 +3028,7 @@ command is available always in bidi documents, even without \subsection{Language attributes} \DescribeMacro{\languageattribute} + This is a user-level command, to be used in the preamble of a document (after |\usepackage[...]{babel}|), that declares which attributes are to be used for a given language. It takes two @@ -3088,12 +3225,14 @@ tables, which you can modify with the following macro (for example, to set them for glyphs in the PUA). \Describe{\babelcharproperty}{\marg{char-code}\oarg{to-char-code}% - \marg{propertry}\marg{value}} + \marg{property}\marg{value}} \New{3.32} Here, \marg{char-code} is a number (with \TeX{} syntax). With the optional argument, you can set a range of values. There are three properties (with a short name, taken from Unicode): |direction| -(|bc|), |mirror| (|bmg|), |linebreak| (|lb|). The settings are global. +(|bc|), |mirror| (|bmg|), |linebreak| (|lb|). The settings are global, +and this command is allowed only in vertical mode (the preamble or +between paragraphs). For example: \begin{verbatim} @@ -3102,8 +3241,12 @@ For example: \babelcharproperty{`)}{linebreak}{cl} % or id, op, cl, ns, ex, in, hy \end{verbatim} -This command is allowed only in vertical mode (the preamble or between -paragraphs). +\New{3.39} Another property is |locale|, which adds characters to the +list used by |onchar| in |\babelprovide|, or, if the last argument is +empty, removes them. The last argument is the locale name: +\begin{verbatim} +\babelcharproperty{`,}{locale}{english} +\end{verbatim} \subsection{Tweaking some features} @@ -3242,41 +3385,6 @@ for a small example (|xe-bidi|). See the code section for |\foreignlanguage*| (a new starred version of |\foreignlanguage|). -\medskip -\textbf{Modifying, and adding, values of |ini| files} - -\New{3.37} There is a way to modify the values of |ini| files when they -get loaded with |\babelprovide|. To set, say, |digits.native| in the -|numbers| section, use something like -|numbers..digits.native=abcdefghij| (note the double dot between the -section and the key name). New keys may be added, too. - -\medskip -\textbf{Non-standard hyphenation} - -\New{3.37} With \luatex{} it is now possible to define non-standard -hyohenation rules, like |f-f| $\to$ |ff-f|. No rules are currently -provided by default, but they can be defined as shown in the following -example: -\begin{verbatim} -\babelposthyphenation{ngerman}{([fmtrp]) | {1}} -{ - { no = {1}, pre = {1}{1}-}, % Replace first char with disc - remove, % Remove automatic disc - {} % Keep last char, untouched -} -\end{verbatim} - -This feature must be explicitly activated with: -\begin{verbatim} -\babeladjust{ hyphenation.extra = on } -\end{verbatim} - -See the \babel{} wiki for a more detailed description and some examples: -\begin{verbatim} -https://github.com/latex3/babel/wiki -\end{verbatim} - \medskip \textbf{Old and deprecated stuff} @@ -4222,8 +4330,8 @@ help from Bernd Raichle, for which I am grateful. % \section{Tools} % % \begin{macrocode} -%<> -%<> +%<> +%<> % \end{macrocode} % % \textbf{Do not use the following macros in \texttt{ldf} files. They @@ -4715,7 +4823,6 @@ help from Bernd Raichle, for which I am grateful. % % \changes{babel~3.30}{2019/04/22}{Callbacks aren't specific to % bidi any more, so they are moved.} -% \changes{babel~3.32}{2019/05/30}{Consider Harf.} % % \begin{macrocode} \ifx\bbl@languages\@undefined\else @@ -4737,8 +4844,6 @@ help from Bernd Raichle, for which I am grateful. \bbl@languages \fi \ifodd\bbl@engine - % Harftex is evolving, so the callback is not harcoded, just in case - \def\bbl@harfpreline{Harf pre_linebreak_filter callback}% \def\bbl@activate@preotf{% \let\bbl@activate@preotf\relax % only once \directlua{ @@ -4768,23 +4873,14 @@ help from Bernd Raichle, for which I am grateful. Babel.pre_otfload_v, 'Babel.pre_otfload_v', luatexbase.priority_in_callback('pre_linebreak_filter', - '\bbl@harfpreline') - or luatexbase.priority_in_callback('pre_linebreak_filter', - 'luaotfload.node_processor') - or nil) + 'luaotfload.node_processor') or nil) % luatexbase.add_to_callback('hpack_filter', Babel.pre_otfload_h, 'Babel.pre_otfload_h', luatexbase.priority_in_callback('hpack_filter', - '\bbl@harfpreline') - or luatexbase.priority_in_callback('hpack_filter', - 'luaotfload.node_processor') - or nil) - }% - \@ifpackageloaded{harfload}% - {\directlua{ Babel.mirroring_enabled = false }}% - {}} + 'luaotfload.node_processor') or nil) + }} \let\bbl@tempa\relax \@ifpackagewith{babel}{bidi=basic}% {\def\bbl@tempa{basic}}% @@ -6771,20 +6867,8 @@ help from Bernd Raichle, for which I am grateful. \ifx\bbl@known@attribs\@undefined \in@false \else -% \end{macrocode} -% -% Now we need to see if the attribute occurs in the list of -% already selected attributes. -% -% \begin{macrocode} \bbl@xin@{,\bbl@tempc-##1,}{,\bbl@known@attribs,}% \fi -% \end{macrocode} -% -% When the attribute was in the list we issue a warning; this might -% not be the users intention. -% -% \begin{macrocode} \ifin@ \bbl@warning{% You have more than once selected the attribute '##1'\\% @@ -7349,6 +7433,7 @@ help from Bernd Raichle, for which I am grateful. \edef\bbl@L{\zap@space#1 \@empty}% \edef\bbl@G{\zap@space#2 \@empty}% \bbl@startcmds@ii} +\let\bbl@startcommands\StartBabelCommands % \end{macrocode} % % Parse the encoding info to get the label, input, and font parts. @@ -7474,6 +7559,7 @@ help from Bernd Raichle, for which I am grateful. \endgroup \endgroup \bbl@scafter} +\let\bbl@endcommands\EndBabelCommands % \end{macrocode} % % Now we define commands to be used inside |\StartBabelCommands|. @@ -7602,7 +7688,7 @@ help from Bernd Raichle, for which I am grateful. \newcommand\SetHyphenMap[1]{% \bbl@forlang\bbl@tempa{% \expandafter\bbl@stringdef - \csname\bbl@tempa @bbl@hyphenmap\endcsname{##1}}} + \csname\bbl@tempa @bbl@hyphenmap\endcsname{##1}}}% %<> % \end{macrocode} % @@ -8213,7 +8299,7 @@ help from Bernd Raichle, for which I am grateful. \let\bbl@savelangname\languagename \edef\bbl@savelocaleid{\the\localeid}% % Set name and locale id - \def\languagename{#2}% + \edef\languagename{#2}% % \global\@namedef{bbl@lcname@#2}{#2}% \bbl@id@assign \let\bbl@KVP@captions\@nil @@ -8230,7 +8316,7 @@ help from Bernd Raichle, for which I am grateful. \let\bbl@KVP@onchar\@nil \let\bbl@KVP@chargroups\@nil \bbl@forkv{#1}{% TODO - error handling - \in@{..}{##1}% + \in@{/}{##1}% \ifin@ \bbl@renewinikey##1\@@{##2}% \else @@ -8279,8 +8365,10 @@ help from Bernd Raichle, for which I am grateful. \bbl@read@ini{##1}{basic data}% \bbl@exportkey{chrng}{characters.ranges}{}% \bbl@exportkey{dgnat}{numbers.digits.native}{}% + \bbl@exportkey{lnbrk}{typography.linebreaking}{h}% \bbl@exportkey{hyphr}{typography.hyphenrules}{}% \bbl@exportkey{intsp}{typography.intraspace}{}% + \endinput \endgroup}% boxed, to avoid extra spaces: {\setbox\z@\hbox{\InputIfFileExists{babel-#2.tex}{}{}}}}% {}% @@ -8301,10 +8389,15 @@ help from Bernd Raichle, for which I am grateful. Babel.locale_mapped = true Babel.linebreaking.add_before(Babel.locale_map) Babel.loc_to_scr = {} - Babel.chr_to_loc = {} + Babel.chr_to_loc = Babel.chr_to_loc or {} end}% \bbl@xin@{ ids }{ \bbl@KVP@onchar\space}% \ifin@ + \ifx\bbl@starthyphens\@undefined % Needed if no explicit selection + \AddBabelHook{babel-onchar}{beforestart}{{\bbl@starthyphens}}% + \fi + \bbl@exp{\\\bbl@add\\\bbl@starthyphens + {\\\bbl@patterns@lua{\languagename}}}% % TODO - error/warning if no script \directlua{ if Babel.script_blocks['\bbl@cs{sbcp@\languagename}'] then @@ -8323,7 +8416,7 @@ help from Bernd Raichle, for which I am grateful. if Babel.script_blocks['\bbl@cs{sbcp@\languagename}'] then Babel.loc_to_scr[\the\localeid] = Babel.script_blocks['\bbl@cs{sbcp@\languagename}'] - end} + end}% \ifx\bbl@mapselect\@undefined \AtBeginDocument{% \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}% @@ -8343,9 +8436,6 @@ help from Bernd Raichle, for which I am grateful. \fi % TODO - catch non-valid values \fi -% \ifx\bbl@KVP@chargroups\@nil\else -% \bbl@chargroups -% \fi % == mapfont == % For bidi texts, to switch the font based on direction \ifx\bbl@KVP@mapfont\@nil\else @@ -8430,7 +8520,7 @@ help from Bernd Raichle, for which I am grateful. return head end end - }} + }}% \fi \fi % == require.babel in ini == @@ -8497,7 +8587,7 @@ help from Bernd Raichle, for which I am grateful. \@namedef{date#1}{}% marks lang exists - required by \StartBabelCommands \@namedef{extras#1}{}% \@namedef{noextras#1}{}% - \StartBabelCommands*{#1}{captions}% + \bbl@startcommands*{#1}{captions}% \ifx\bbl@KVP@captions\@nil % and also if import, implicit \def\bbl@tempb##1{% elt for \bbl@captionslist \ifx##1\@empty\else @@ -8520,7 +8610,7 @@ help from Bernd Raichle, for which I am grateful. \bbl@savetoday \bbl@savedate \fi - \EndBabelCommands + \bbl@endcommands \bbl@exp{% \def\<#1hyphenmins>{% {\bbl@ifunset{bbl@lfthm@#1}{2}{\@nameuse{bbl@lfthm@#1}}}% @@ -8620,7 +8710,7 @@ help from Bernd Raichle, for which I am grateful. \def\bbl@inipreread##1=##2\@@{% \bbl@trim@def\bbl@tempa{##1}% Redundant below !! % Move trims here ?? - \bbl@ifunset{bbl@KVP@\bbl@section..\bbl@tempa}% + \bbl@ifunset{bbl@KVP@\bbl@section/\bbl@tempa}% {\expandafter\bbl@inireader\bbl@tempa=##2\@@}% {}}% \let\bbl@inireader\bbl@iniskip @@ -8643,6 +8733,7 @@ help from Bernd Raichle, for which I am grateful. \@nameuse{bbl@renew@\bbl@section}% \global\bbl@csarg\let{renew@\bbl@section}\relax \@nameuse{bbl@secpost@\bbl@section}% +% \bbl@csarg\bbl@toglobal{inikeys@\languagename}% \fi} \def\bbl@iniline#1\bbl@iniline{% \@ifnextchar[\bbl@inisec{\@ifnextchar;\bbl@iniskip\bbl@inipreread}#1\@@}% ] @@ -8665,14 +8756,14 @@ help from Bernd Raichle, for which I am grateful. \@nameuse{bbl@secpost@\bbl@section}% ends previous section \def\bbl@section{#1}% starts current section \def\bbl@elt##1##2{% - \@namedef{bbl@KVP@#1..##1}{}}% + \@namedef{bbl@KVP@#1/##1}{}}% \@nameuse{bbl@renew@#1}% \@nameuse{bbl@secpre@#1}% pre-section `hook' \bbl@ifunset{bbl@inikv@#1}% {\let\bbl@inireader\bbl@iniskip}% {\bbl@exp{\let\\\bbl@inireader\}}} \let\bbl@renewlist\@empty -\def\bbl@renewinikey#1..#2\@@#3{% +\def\bbl@renewinikey#1/#2\@@#3{% \bbl@ifunset{bbl@renew@#1}% {\bbl@add@list\bbl@renewlist{#1}}% {}% @@ -8732,6 +8823,7 @@ help from Bernd Raichle, for which I am grateful. \bbl@exportkey{lfthm}{typography.lefthyphenmin}{2}% \bbl@exportkey{rgthm}{typography.righthyphenmin}{3}% \bbl@exportkey{prehc}{typography.prehyphenchar}{}% + \bbl@exportkey{lnbrk}{typography.linebreaking}{h}% \bbl@exportkey{hyphr}{typography.hyphenrules}{}% \bbl@exportkey{intsp}{typography.intraspace}{}% \bbl@exportkey{jstfy}{typography.justify}{w}% @@ -9016,9 +9108,7 @@ help from Bernd Raichle, for which I am grateful. \@namedef{bbl@ADJ@layout.lists@on}{% \bbl@adjust@layout{\let\list\bbl@OL@list}} \@namedef{bbl@ADJ@hyphenation.extra@on}{% - \directlua{ - Babel.linebreaking.add_after(Babel.post_hyphenate_replace) - }} + \bbl@activateposthyphen} % \end{macrocode} % % \section{The kernel of Babel (\texttt{babel.def} for \LaTeX only)} @@ -9851,11 +9941,7 @@ help from Bernd Raichle, for which I am grateful. % and so on, but bidi text does not work out of the box and some % development is necessary. It also provides tools to properly % set left-to-right and right-to-left page layouts. As Lua\TeX-ja -% shows, vertical typesetting is possible, too. Its main drawback -% is font handling is often considered to be less mature than -% \xetex{}, mainly in Indic scripts (but there are steps to make -% HarfBuzz, the \xetex{} font engine, available in \luatex{}; see -% ). +% shows, vertical typesetting is possible, too. % \end{itemize} % % \changes{babel~3.15}{2017/10/30}{Use an attribute instead of tex language @@ -10014,18 +10100,20 @@ help from Bernd Raichle, for which I am grateful. \newtoks\everypar \everypar=\bbl@severypar \bbl@severypar{\bbl@xeeverypar\the\everypar}} - \@ifpackagewith{babel}{bidi=bidi}% - {\let\bbl@textdir@i\@gobbletwo - \let\bbl@xebidipar\@empty - \AddBabelHook{bidi}{foreign}{% - \def\bbl@tempa{\def\BabelText####1}% - \ifcase\bbl@thetextdir - \expandafter\bbl@tempa\expandafter{\BabelText{\LR{##1}}}% - \else - \expandafter\bbl@tempa\expandafter{\BabelText{\RL{##1}}}% - \fi} - \def\bbl@pardir#1{\ifcase#1\relax\setLR\else\setRL\fi}} - {}% + \def\bbl@tempb{% + \let\bbl@textdir@i\@gobbletwo + \let\bbl@xebidipar\@empty + \AddBabelHook{bidi}{foreign}{% + \def\bbl@tempa{\def\BabelText########1}% + \ifcase\bbl@thetextdir + \expandafter\bbl@tempa\expandafter{\BabelText{\LR{####1}}}% + \else + \expandafter\bbl@tempa\expandafter{\BabelText{\RL{####1}}}% + \fi} + \def\bbl@pardir##1{\ifcase##1\relax\setLR\else\setRL\fi}} + \@ifpackagewith{babel}{bidi=bidi}{\bbl@tempb}{}% + \@ifpackagewith{babel}{bidi=bidi-l}{\bbl@tempb}{}% + \@ifpackagewith{babel}{bidi=bidi-r}{\bbl@tempb}{}% \fi % \end{macrocode} % @@ -10406,7 +10494,6 @@ help from Bernd Raichle, for which I am grateful. \edef\languagename{% \ifnum\escapechar=\expandafter`\string#1\@empty \else\string#1\@empty\fi}% - % \@namedef{bbl@lcname@#1}{#1}% \select@language{\languagename}% % write to auxs \expandafter\ifx\csname date\languagename\endcsname\relax\else @@ -10423,8 +10510,13 @@ help from Bernd Raichle, for which I am grateful. % set name \edef\languagename{#1}% \bbl@fixname\languagename + \expandafter\ifx\csname date\languagename\endcsname\relax + \IfFileExists{babel-\languagename.tex}% + {\babelprovide{\languagename}}% + {}% + \fi \bbl@iflanguage\languagename{% - \expandafter\ifx\csname date\languagename\endcsname\relax + \expandafter\ifx\csname date\languagename\endcsname\relax \bbl@error {Unknown language `#1'. Either you have\\% misspelled its name, it has not been installed,\\% @@ -10725,8 +10817,12 @@ help from Bernd Raichle, for which I am grateful. \def\foreign@language#1{% % set name \edef\languagename{#1}% - % \@namedef{bbl@lcname@#1}{#1}% \bbl@fixname\languagename + \expandafter\ifx\csname date\languagename\endcsname\relax + \IfFileExists{babel-\languagename.tex}% + {\babelprovide{\languagename}}% + {}% + \fi \bbl@iflanguage\languagename{% \expandafter\ifx\csname date\languagename\endcsname\relax \bbl@warning % TODO - why a warning, not an error? @@ -10945,6 +11041,7 @@ help from Bernd Raichle, for which I am grateful. \let\uselocale\setlocale \let\locale\setlocale \let\selectlocale\setlocale +\let\localename\setlocale \let\textlocale\setlocale \let\textlanguage\setlocale \let\languagetext\setlocale @@ -11570,6 +11667,12 @@ help from Bernd Raichle, for which I am grateful. \bbl@trace{Font handling with fontspec} \@onlypreamble\babelfont \newcommand\babelfont[2][]{% 1=langs/scripts 2=fam + \bbl@foreach{#1}{% + \expandafter\ifx\csname date##1\endcsname\relax + \IfFileExists{babel-##1.tex}% + {\babelprovide{##1}}% + {}% + \fi}% \edef\bbl@tempa{#1}% \def\bbl@tempb{#2}% Used by \bbl@bblfont \ifx\fontspec\@undefined @@ -12457,6 +12560,9 @@ help from Bernd Raichle, for which I am grateful. % \changes{babel~3.37}{2019/12/07}{Added code for non-standard % hyphenation.} % +% First, some general code for line breaking, used by +% |\babelposthyphenation|. +% % \textit{In progress.} Replace regular (ie, implicit) discretionaries % by spaceskips, based on the previous glyph (which I think makes % sense, because the hyphen and the previous char go always together). @@ -12641,7 +12747,7 @@ help from Bernd Raichle, for which I am grateful. \def\bbl@provide@intraspace{% \bbl@ifunset{bbl@intsp@\languagename}{}% {\expandafter\ifx\csname bbl@intsp@\languagename\endcsname\@empty\else - \bbl@xin@{\bbl@cs{sbcp@\languagename}}{Hant,Hans,Jpan,Kore,Kana}% + \bbl@xin@{\bbl@cs{lnbrk@\languagename}}{c}% \ifin@ % cjk \bbl@cjkintraspace \directlua{ @@ -12738,7 +12844,7 @@ Babel.script_blocks = { {0x2B740, 0x2B81F}, {0x2B820, 0x2CEAF}, {0x2CEB0, 0x2EBEF}, {0x2F800, 0x2FA1F}}, ['Hebr'] = {{0x0590, 0x05FF}}, - ['Japa'] = {{0x3000, 0x303F}, {0x3040, 0x309F}, {0x30A0, 0x30FF}, + ['Jpan'] = {{0x3000, 0x303F}, {0x3040, 0x309F}, {0x30A0, 0x30FF}, {0x4E00, 0x9FAF}, {0xFF00, 0xFFEF}}, ['Khmr'] = {{0x1780, 0x17FF}, {0x19E0, 0x19FF}}, ['Knda'] = {{0x0C80, 0x0CFF}}, @@ -12760,10 +12866,11 @@ Babel.script_blocks = { ['Thai'] = {{0x0E00, 0x0E7F}}, ['Tibt'] = {{0x0F00, 0x0FFF}}, ['Vaii'] = {{0xA500, 0xA63F}}, - ['Yiii']= {{0xA490, 0xA4CF}, {0xA000, 0xA48F}} + ['Yiii'] = {{0xA000, 0xA48F}, {0xA490, 0xA4CF}} } Babel.script_blocks.Hant = Babel.script_blocks.Hans +Babel.script_blocks.Kana = Babel.script_blocks.Jpan function Babel.locale_map(head) if not Babel.locale_mapped then return head end @@ -12789,7 +12896,7 @@ function Babel.locale_map(head) end end % Now, take action - if toloc then + if toloc and toloc > -1 then if Babel.locale_props[toloc].lg then item.lang = Babel.locale_props[toloc].lg node.set_attribute(item, LOCALE, toloc) @@ -12856,6 +12963,12 @@ end Babel.Babel.cjk_characters[\the\count@]['c'] = '#1' }} \let\bbl@chprop@lb\bbl@chprop@linebreak +\def\bbl@chprop@locale#1{% + \directlua{ + Babel.chr_to_loc = Babel.chr_to_loc or {} + Babel.chr_to_loc[\the\count@] = + \bbl@ifblank{#1}{-1000}{\the\@nameuse{bbl@id@@#1}}\space + }} % \end{macrocode} % % Post-handling hyphenation patterns for non-standard rules, like |ff| @@ -13050,13 +13163,40 @@ end return head end - &% Used below + &% The following functions belong to the next macro + + &% This table stores capture maps, numbered consecutively + Babel.capture_maps = {} + function Babel.capture_func(key, cap) local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" + ret = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) ret = ret:gsub("%[%[%]%]%.%.", '') ret = ret:gsub("%.%.%[%[%]%]", '') return key .. [[=function(m) return ]] .. ret .. [[ end]] end + + function Babel.capt_map(from, mapno) + return Babel.capture_maps[mapno][from] or from + end + + &% Handle the {n|abc|ABC} syntax in captures + function Babel.capture_func_map(capno, from, to) + local froms = {} + for s in string.utfcharacters(from) do + table.insert(froms, s) + end + local cnt = 1 + table.insert(Babel.capture_maps, {}) + local mlen = table.getn(Babel.capture_maps) + for s in string.utfcharacters(to) do + Babel.capture_maps[mlen][froms[cnt]] = s + cnt = cnt + 1 + end + return "]]..Babel.capt_map(m[" .. capno .. "]," .. + (mlen) .. ").." .. "[[" + end + } % \end{macrocode} % @@ -13064,7 +13204,10 @@ end % defined above for converting strings to functions returning a string. % These functions handle the |{|\textit{n}|}| syntax. For example, % |pre={1}{1}-| becomes |function(m) return m[1]..m[1]..'-' end|, where -% |m| are the matches returned after applying the pattern. The way it +% |m| are the matches returned after applying the pattern. With a +% mapped capture the functions are similar to +% |function(m) return Babel.capt_map(m[1],1) end|, where the last +% argument identifies the mapping to be applied to |m[1]|. The way it % is carried out is somewhat tricky, but the effect in not dissimilar % to lua |load| – save the code as string in a TeX macro, and expand % this macro at the appropriate place. As |\directlua| does not take @@ -13074,6 +13217,7 @@ end % \begin{macrocode} \catcode`\#=6 \gdef\babelposthyphenation#1#2#3{&% + \bbl@activateposthyphen \begingroup \def\babeltempa{\bbl@add@list\babeltempb}&% \let\babeltempb\@empty @@ -13106,6 +13250,11 @@ end }&% \endgroup} \endgroup +\def\bbl@activateposthyphen{% + \let\bbl@activateposthyphen\relax + \directlua{ + Babel.linebreaking.add_after(Babel.post_hyphenate_replace) + }} % \end{macrocode} % % \subsection{Layout} @@ -19714,12 +19863,6 @@ Babel.fontmap[2] = {} -- al/an Babel.bidi_enabled = true Babel.mirroring_enabled = true --- Temporary: - -if harf then - Babel.mirroring_enabled = false -end - require('babel-data-bidi.lua') local characters = Babel.characters @@ -20010,7 +20153,10 @@ function Babel.bidi(head, ispar, hdir) item = nodes[r][1] -- MIRRORING if Babel.mirroring_enabled and item.id == GLYPH and temp == 'r' and characters[item.char] then - item.char = characters[item.char].m or item.char + local font_mode = font.fonts[item.font].properties.mode + if font_mode ~= 'harf' and font_mode ~= 'plug' then + item.char = characters[item.char].m or item.char + end end end first_on = nil @@ -20321,6 +20467,8 @@ Babel.cjk_class = setmetatable ( Babel.cjk_characters, { __index = function(_, k) if (k >= 0xAC00 and k <= 0xD7A3) -- H2/H3 or (k >= 0x2E80 and k <= 0x9FFF) + or (k >= 0xA000 and k <= 0xA48F) -- Yi + or (k >= 0xA490 and k <= 0xA4CF) -- Yi or (k >= 0xF900 and k <= 0xFAFF) or (k >= 0xFE10 and k <= 0xFE1F) or (k >= 0xFE30 and k <= 0xFE6F) diff --git a/Master/texmf-dist/source/latex/babel/babel.ins b/Master/texmf-dist/source/latex/babel/babel.ins index dc9587f35c0..86e70a5fc62 100644 --- a/Master/texmf-dist/source/latex/babel/babel.ins +++ b/Master/texmf-dist/source/latex/babel/babel.ins @@ -26,7 +26,7 @@ %% and covered by LPPL is defined by the unpacking scripts (with %% extension .ins) which are part of the distribution. %% -\def\filedate{2020/01/15} +\def\filedate{2020/02/03} \def\batchfile{babel.ins} \input docstrip.tex diff --git a/Master/texmf-dist/source/latex/babel/bbcompat.dtx b/Master/texmf-dist/source/latex/babel/bbcompat.dtx index b9ca5b9d880..610eb840459 100644 --- a/Master/texmf-dist/source/latex/babel/bbcompat.dtx +++ b/Master/texmf-dist/source/latex/babel/bbcompat.dtx @@ -30,7 +30,7 @@ % % \iffalse %<*dtx> -\ProvidesFile{bbcompat.dtx}[2020/01/15 v3.38] +\ProvidesFile{bbcompat.dtx}[2020/02/03 v3.39] % % %% File 'bbcompat.dtx' diff --git a/Master/texmf-dist/source/latex/babel/locale.zip b/Master/texmf-dist/source/latex/babel/locale.zip index c899ace8a8e..8177913f9e8 100644 Binary files a/Master/texmf-dist/source/latex/babel/locale.zip and b/Master/texmf-dist/source/latex/babel/locale.zip differ -- cgit v1.2.3