From b33e18a5efa7b022c42aee2adec507dd6917ec28 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Thu, 16 Sep 2021 20:18:12 +0000 Subject: lua-uca (16sep21) git-svn-id: svn://tug.org/texlive/trunk@60523 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/support/lua-uca/CHANGELOG.md | 9 + Master/texmf-dist/doc/support/lua-uca/HACKING.md | 98 ++ Master/texmf-dist/doc/support/lua-uca/README.md | 38 +- .../texmf-dist/doc/support/lua-uca/lua-uca-doc.pdf | Bin 55816 -> 72038 bytes .../texmf-dist/doc/support/lua-uca/lua-uca-doc.tex | 11 +- .../texmf-dist/scripts/lua-uca/lua-uca-ducet.lua | 2 +- .../scripts/lua-uca/lua-uca-languages.lua | 1014 ++++++++++++++++++++ .../scripts/lua-uca/lua-uca-reordering-table.lua | 2 +- Master/texmf-dist/source/support/lua-uca/Makefile | 67 ++ 9 files changed, 1212 insertions(+), 29 deletions(-) create mode 100644 Master/texmf-dist/doc/support/lua-uca/HACKING.md create mode 100644 Master/texmf-dist/source/support/lua-uca/Makefile diff --git a/Master/texmf-dist/doc/support/lua-uca/CHANGELOG.md b/Master/texmf-dist/doc/support/lua-uca/CHANGELOG.md index 5f005ad64dd..2d5816ebfc8 100644 --- a/Master/texmf-dist/doc/support/lua-uca/CHANGELOG.md +++ b/Master/texmf-dist/doc/support/lua-uca/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +2021-09-16 + + - added sorting rules for all languages contained in CLDR collation files. + +2020-06-09 + + - moved development information that depends on files not distributed on CTAN to `HACKING.md`. + - extended documentation. + 2020-03-24 - version `0.1` released diff --git a/Master/texmf-dist/doc/support/lua-uca/HACKING.md b/Master/texmf-dist/doc/support/lua-uca/HACKING.md new file mode 100644 index 00000000000..e0b7802ef58 --- /dev/null +++ b/Master/texmf-dist/doc/support/lua-uca/HACKING.md @@ -0,0 +1,98 @@ +# Lua-UCA hacking + +You need the full installation from +[Github](https://github.com/michal-h21/lua-uca) in order to do stuff described +in this section. Package distributed on CTAN doesn't contain all necessary +files. + +## Install + +The package needs to download Unicode collation data and convert it to a Lua +table. It depends on `wget` and `unzip` utilities. All files can be downloaded +using Make: + + make + +To install the package in the local TEXMF tree, run: + + make install + +## New language support + +To add a new language, add new function to `src/lua-uca/lua-uca-languages.lua` +file. The function name should be short language code. Example function for +the Russian language: + + languages.ru = function(collator_obj) + collator_obj:reorder{ "cyrillic" } + return collator_obj + end + +The language function takes the Collator object as a parameter. Methods showed +in the *Change sorting rules* section can be used with this object. + +The `data/common/collation/` directory in the source repository contains files from the `CLDR` project. +They contain rules for many languages. The files needs to be normalized to the +[NFC form](https://en.wikipedia.org/wiki/Unicode_equivalence), for example using: + + cat cs.xml | uconv -x any-nfc -o cs.xml + +The `uconv` utility is a part of the [ICU Project](http://userguide.icu-project.org/). + +Sorting rules for a language are placed in the `` element. Multiple +`` elements may be present in the XML file. It is usually best to chose the one with attribute +`type="standard"`. + +The following example contains code from `da.xml`: + + + [caseFirst upper] + &D<<đ<<<Đ<<ð<<<Ð + &th<<<þ + &TH<<<Þ + &Y< chochol > jasan -More samples of use can be found in the `spec` directory. -`tools/indexing-sample.lua` is a simple indexing processor. +More samples of the library usage can be found in the source repository of this package on [Github](https://github.com/michal-h21/lua-uca). +% See `HACKING.md` file in the repo for more information. ## Use with Xindex processor [Xindex](https://www.ctan.org/pkg/xindex) is flexible index processor written -in Lua by Herbert Voß. It supports Lua configuration files, which enables use -of Lua-UCA for sorting of the index entries, as shown in [this -example](https://tex.stackexchange.com/a/524014/2891) for Norwegian text. +in Lua by Herbert Voß. It has built-in `Lua-UCA` support starting with version +`0.23`. The support can be requested using the `-u` option: + + xindex -u -l no -c norsk filename.idx -The `xindex` directory in the [source repository](https://github.com/michal-h21/lua-uca/tree/master/xindex) contains more advanced version of such configuration -file together with several examples. Run `make xindex` command to compile them. ## Change sorting rules @@ -90,7 +83,7 @@ It is also possible to expand a letter to multiple letters, like this example fo tailoring "&Ö=Oe" tailoring "&ö=oe" -Some languages, like Norwegian sort uppercase letters before lowercase. This +Some languages, like Norwegian, sort uppercase letters before lowercase. This can be enabled using `collator_obj:uppercase_first()` function: local tailoring = function(s) collator_obj:tailor_string(s) end @@ -102,14 +95,8 @@ can be enabled using `collator_obj:uppercase_first()` function: tailoring("&ǀ<æ<<<Æ<<ä<<<Ä<ø<<<Ø<<ö<<<Ö<<ő<<<Ő<å<<<Å<</dev/null)),true) + VERSION:= $(shell git --no-pager describe --abbrev=0 --tags --always ) + DATE:= $(firstword $(shell git --no-pager show --date=short --format="%ad" --name-only)) +endif + + +all: data/allkeys.txt data/common src/lua-uca/lua-uca-ducet.lua src/lua-uca/lua-uca-reordering-table.lua $(DOC_PDF) + + + +data/allkeys.txt: + mkdir data + wget http://www.unicode.org/Public/UCA/latest/allkeys.txt + mv allkeys.txt data + +data/common: + wget http://unicode.org/Public/cldr/33/cldr-common-33.0.zip + unzip cldr-common-33.0.zip "common/collation/*" -d data/ + rm cldr-common-33.0.zip + +src/lua-uca/lua-uca-ducet.lua: + mkdir -p src/lua-uca + texlua tools/make_ducet.lua > src/lua-uca/lua-uca-ducet.lua + +src/lua-uca/lua-uca-reordering-table.lua: + mkdir -p src/lua-uca + texlua tools/make_reordering_table.lua > $@ + +.PHONY: test install + +test: + busted spec/ducet-spec.lua + busted spec/collator_spec.lua + busted spec/languages-spec.lua + busted spec/tailoring-spec.lua + +$(DOC_PDF): $(DOC_SOURCES) + lualatex '\def\version{${VERSION}}\def\gitdate{${DATE}}\input{$<}' + + +install: + mkdir -p ${INSTALL_DIR} + cp ${LUA_DIR}*.lua ${INSTALL_DIR} + +xindex: + $(MAKE) -C $@ + +build: $(DOC_PDF) $(DOC_SOURCES) $(lua_content) + mkdir -p $(BUILD_DIR)/lua-uca/lua-uca + cp $(lua_content) $(BUILD_DIR)/lua-uca/lua-uca/ + cp $(DOC_PDF) $(DOC_SOURCES) Makefile $(BUILD_DIR)/lua-uca/ + @cd $(BUILD_DIR) && zip -r lua-uca.zip lua-uca + + +.PHONY: all xindex build -- cgit v1.2.3