blob: 0680525efd19942f55da8d01aac1904e6c4897dd (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
#
# Copyright (C) 2002-2006, International Business Machines Corporation and others.
# All Rights Reserved.
#
# file: sent.txt
#
# ICU Sentence Break Rules
# See Unicode Standard Annex #29.
# These rules are based on SA 29 version 5.0.0
# Includes post 5.0 changes to treat Japanese half width voicing marks
# as Grapheme Extend.
#
$VoiceMarks = [\uff9e\uff9f];
#
# Character categories as defined in TR 29
#
$Sep = [\p{Sentence_Break = Sep}];
$Format = [\p{Sentence_Break = Format}];
$Sp = [\p{Sentence_Break = Sp}];
$Lower = [\p{Sentence_Break = Lower}];
$Upper = [\p{Sentence_Break = Upper}];
$OLetter = [\p{Sentence_Break = OLetter}-$VoiceMarks];
$Numeric = [\p{Sentence_Break = Numeric}];
$ATerm = [\p{Sentence_Break = ATerm}];
$STerm = [\p{Sentence_Break = STerm}];
$Close = [\p{Sentence_Break = Close}];
#
# Define extended forms of the character classes,
# incorporate grapheme cluster + format chars.
# Rules 4 and 5.
$CR = \u000d;
$LF = \u000a;
$Extend = [[:Grapheme_Extend = TRUE:]$VoiceMarks];
$SpEx = $Sp ($Extend | $Format)*;
$LowerEx = $Lower ($Extend | $Format)*;
$UpperEx = $Upper ($Extend | $Format)*;
$OLetterEx = $OLetter ($Extend | $Format)*;
$NumericEx = $Numeric ($Extend | $Format)*;
$ATermEx = $ATerm ($Extend | $Format)*;
$STermEx = $STerm ($Extend | $Format)*;
$CloseEx = $Close ($Extend | $Format)*;
## -------------------------------------------------
!!chain;
!!forward;
# Rule 3 - break after separators. Keep CR/LF together.
#
$CR $LF;
# Rule 4 - Break after $Sep.
# Rule 5 - Ignore $Format and $Extend
#
[^$Sep]? ($Extend | $Format)*;
# Rule 6
$ATermEx $NumericEx;
# Rule 7
$UpperEx $ATermEx $UpperEx;
#Rule 8
# Note: follows errata for Unicode 5.0 boundary rules.
$NotLettersEx = [^$OLetter $Upper $Lower $Sep $ATerm $STerm] ($Extend | $Format)*;
$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower;
# Rule 8a
($STermEx | $ATermEx) $CloseEx* $SpEx* ($STermEx | $ATermEx);
#Rule 9, 10, 11
($STermEx | $ATermEx) $CloseEx* $SpEx* $Sep?;
#Rule 12
[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* .;
[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep{eof}] | $CR $LF){100};
## -------------------------------------------------
!!reverse;
$SpEx_R = ($Extend | $Format)* $Sp;
$ATermEx_R = ($Extend | $Format)* $ATerm;
$STermEx_R = ($Extend | $Format)* $STerm;
$CloseEx_R = ($Extend | $Format)* $Close;
#
# Reverse rules.
# For now, use the old style inexact reverse rules, which are easier
# to write, but less efficient.
# TODO: exact reverse rules. It appears that exact reverse rules
# may require improving support for look-ahead breaks in the
# builder. Needs more investigation.
#
[{bof}] (.? | $LF $CR) [^$Sep]* [$Sep {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*;
#.*;
# Explanation for this rule:
#
# It needs to back over
# The $Sep at which we probably begin
# All of the non $Sep chars leading to the preceding $Sep
# The preceding $Sep, which will be the second one that the rule matches.
# Any immediately preceding STerm or ATerm sequences. We need to see these
# to get the correct rule status when moving forwards again.
#
# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match
# the entire string.
#
# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be
# at the beginning of the string at this point, and we don't want to fail.
# Can only use {eof} once, and it is used later.
#
|
