diff options
Diffstat (limited to 'Master/texmf-dist/scripts/bibtexperllibs/LaTeX/ToUnicode.pm')
-rw-r--r-- | Master/texmf-dist/scripts/bibtexperllibs/LaTeX/ToUnicode.pm | 124 |
1 files changed, 82 insertions, 42 deletions
diff --git a/Master/texmf-dist/scripts/bibtexperllibs/LaTeX/ToUnicode.pm b/Master/texmf-dist/scripts/bibtexperllibs/LaTeX/ToUnicode.pm index cf62cf5a29d..50ae29399a8 100644 --- a/Master/texmf-dist/scripts/bibtexperllibs/LaTeX/ToUnicode.pm +++ b/Master/texmf-dist/scripts/bibtexperllibs/LaTeX/ToUnicode.pm @@ -2,9 +2,9 @@ use strict; use warnings; package LaTeX::ToUnicode; BEGIN { - $LaTeX::ToUnicode::VERSION = '0.05'; + $LaTeX::ToUnicode::VERSION = '0.11'; } -#ABSTRACT: Convert LaTeX commands to Unicode +#ABSTRACT: Convert LaTeX commands to Unicode (simplistically) require Exporter; @@ -22,29 +22,12 @@ sub convert { $string = _convert_german( $string ) if $options{german}; $string = _convert_symbols( $string ); $string = _convert_specials( $string ); + $string = _convert_ligatures( $string ); $string = _convert_markups( $string ); $string =~ s/{(\w*)}/$1/g; $string; } -sub _convert_accents { - my $string = shift; - $string =~ s/(\{\\(.)\{(\\?\w{1,2})\}\})/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # {\"{a}} - $string =~ s/(\{\\(.)(\\?\w{1,2})\})/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # {\"a} - $string =~ s/(\\(.)(\\?\w{1,2}))/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # \"a - $string =~ s/(\\(.)\{(\\?\w{1,2})\})/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # \"{a} - $string; -} - -sub _convert_specials { - my $string = shift; - my $specials = join( '|', @LaTeX::ToUnicode::Tables::SPECIALS ); - my $pattern = qr/\\($specials)/o; - $string =~ s/$pattern/$1/g; - $string =~ s/\\\$/\$/g; - $string; -} - sub _convert_commands { my $string = shift; @@ -56,6 +39,15 @@ sub _convert_commands { $string; } +sub _convert_accents { + my $string = shift; + $string =~ s/(\{\\(.)\{(\\?\w{1,2})\}\})/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # {\"{a}} + $string =~ s/(\{\\(.)(\\?\w{1,2})\})/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # {\"a} + $string =~ s/(\\(.)(\\?\w{1,2}))/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # \"a + $string =~ s/(\\(.)\{(\\?\w{1,2})\})/$LaTeX::ToUnicode::Tables::ACCENTS{$2}{$3} || $1/eg; # \"{a} + $string; +} + sub _convert_german { my $string = shift; @@ -75,18 +67,55 @@ sub _convert_symbols { $string; } -sub _convert_markups { +# Replace \<specialchar> with <specialchar>. +sub _convert_specials { + my $string = shift; + my $specials = join( '|', @LaTeX::ToUnicode::Tables::SPECIALS ); + my $pattern = qr/\\($specials)/o; + $string =~ s/$pattern/$1/g; + $string =~ s/\\\$/\$/g; + $string; +} + +sub _convert_ligatures { my $string = shift; + # have to convert these in order specified. + my @ligs = @LaTeX::ToUnicode::Tables::LIGATURES; + for (my $i = 0; $i < @ligs; $i+=2) { + my $in = $ligs[$i]; + my $out = $ligs[$i+1]; + $string =~ s/\Q$in\E/$out/g; + } + $string; +} + +# +sub _convert_markups { + my $string = shift; + my $orig_string = $string; + my $markups = join( '|', @LaTeX::ToUnicode::Tables::MARKUPS ); - $string =~ s/(\{[^{}]+)\\(?:$markups)\s+([^{}]+\})/$1$2/g; # { ... \command ... } - my $pattern = qr/\{\\(?:$markups)\s+([^{}]*)\}/o; - $string =~ s/$pattern/$1/g; + + # Remove \textMARKUP{...}, leaving just the {...} + $string =~ s/\\text($markups)\b\s*//g; + + # Remove braces and \command in: {... \command ...} + $string =~ s/(\{[^{}]+)\\(?:$markups)\s+([^{}]+\})/$1$2/g; + # + # Remove braces and \command in: {\command ...} + $string =~ s/\{\\(?:$markups)\s+([^{}]*)\}/$1/g; + # + # Remove: {\command + # Although this will leave unmatched } chars behind, there's no + # alternative without full parsing, since the bib entry will often + # look like: {\em {The TeX{}book}}. Also might, in principle, be + # at the end of a line. + $string =~ s/\{\\(?:$markups)\b\s*//g; + + # Ultimately we remove all braces in ltx2crossrefxml SanitizeText fns, + # so the unmatched braces don't matter ... that code should be moved here. - $string =~ s/``/“/g; - $string =~ s/`/”/g; - $string =~ s/''/‘/g; - $string =~ s/'/’/g; $string; } @@ -103,7 +132,7 @@ LaTeX::ToUnicode - Convert LaTeX commands to Unicode =head1 VERSION -version 0.05 +version 0.11 =head1 SYNOPSIS @@ -111,49 +140,60 @@ version 0.05 convert( '{\"a}' ) eq 'ä'; # true convert( '"a', german => 1 ) eq 'ä'; # true, `german' package syntax - convert( '"a', ) eq '"a'; # not enabled by default + convert( '"a', ) eq '"a'; # not enabled by default + + # more generally: + my $latexstr; + my $unistr = convert($latexstr); =head1 DESCRIPTION This module provides a method to convert LaTeX-style markups for accents etc. into their Unicode equivalents. It translates commands for special characters or accents into their Unicode equivalents and removes formatting commands. +It is not at all bulletproof or complete. -I use this module to convert values from BibTeX files into plain text, if your +This module converts values from BibTeX files into plain text. If your use case is different, YMMV. -In contrast to L<TeX::Encode>, this module does not create HTML of any kind. +In contrast to L<TeX::Encode>, this module does not create HTML of any +kind, including for HTML/XML metacharacters such as E<lt>, E<gt>, C<&>, +which can appear literally in the output. Entities are other handling +for these has to happen at another level, if need be. =head1 FUNCTIONS -=head2 convert( $string, %options ) +=head2 convert( $latex_string, %options ) -Convert the text in C<$string> that contains LaTeX into a plain(er) Unicode -string. All escape sequences for special characters (e.g. \i, \"a, ...) are -converted, formatting commands (e.g. {\it ...}) are removed. +Convert the text in C<$string> that contains LaTeX into a plain(er) +Unicode string. All escape sequences for accented and special characters +(e.g., \i, \"a, ...) are converted. Basic formatting commands (e.g. {\it +...}) are removed. -C<%options> allows you to enable additional translations. This values are +C<%options> allows you to enable additional translations. These keys are recognized: =over =item C<german> -If true, the commands introduced by the package `german' (e.g. C<"a> eq C<ä>, -note the missing backslash) are also handled. +If this option is set, the commands introduced by the package `german' +(e.g. C<"a> eq C<ä>, note the missing backslash) are also +handled. =back =head1 AUTHOR -Gerhard Gossen <gerhard.gossen@googlemail.com> and Boris Veytsman <boris@varphi.com> +Gerhard Gossen <gerhard.gossen@googlemail.com> and +Boris Veytsman <boris@varphi.com> +L<https://github.com/borisveytsman/bibtexperllibs> =head1 COPYRIGHT AND LICENSE -This software is copyright (c) 2010-2015 by Gerhard Gossen and Boris Veytsman +This software is copyright (c) 2010-2020 by Gerhard Gossen and Boris Veytsman This is free software; you can redistribute it and/or modify it under the same terms as the Perl 5 programming language system itself. =cut - |