blob: 1fe15764bd1ef2d7cef4f58eddbe91c70e4092d0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
#
# Copyright (C) 2002-2005, International Business Machines Corporation and others.
# All Rights Reserved.
#
# file: sent.txt
#
# ICU Sentence Break Rules
# See Unicode Standard Annex #29.
# These rules are based on TR 29 version 4.0.0
#
#
# Character categories as defined in TR 29
#
$Sep = [\p{Sentence_Break = Sep}];
$Format = [\p{Sentence_Break = Format}];
$Sp = [\p{Sentence_Break = Sp}];
$Lower = [\p{Sentence_Break = Lower}];
$Upper = [\p{Sentence_Break = Upper}];
$OLetter = [\p{Sentence_Break = OLetter}];
$Numeric = [\p{Sentence_Break = Numeric}];
$ATerm = [\p{Sentence_Break = ATerm}];
$Term = [\p{Sentence_Break = STerm}];
$Close = [\p{Sentence_Break = Close}];
#
# Define extended forms of the character classes,
# incorporate grapheme cluster + format chars.
$Extend = [[:Grapheme_Extend = TRUE:]];
$ATermEx = $ATerm $Extend* $Format*;
$NumericEx = $Numeric $Extend* $Format*;
$UpperEx = $Upper $Extend* $Format*;
$TermEx = $Term $Extend* $Format*;
#
# $SepSeq keeps together CRLF as a separator. (CRLF is a grapheme cluster)
#
$SepSeq = $Sep | \u000d\u000a;
# $InteriorChars are those that never trigger a following break.
$InteriorChars = [^$Term $ATerm $Sep]; #Note: includes Extend and Format chars
## -------------------------------------------------
!!forward;
# Rule 6. Match an ATerm (.) that does not cause a break because a number immediately follows it.
$NumberFollows = $InteriorChars* $ATermEx $NumericEx;
# Rule 7. $UppersSurround Match a no-break sentence fragment containing a . surrounded by Uppers
$UppersSurround = $InteriorChars* $UpperEx $ATermEx $UpperEx;
# Rule 8 Matches a sentence fragment containing "." that should not cause a sentence break,
# because a lower case word follows the period.
$LowerWordFollows = $InteriorChars* $ATermEx $Close* $Sp* [^$OLetter $Upper $Lower $Sep]* $Lower;
# Rules 3, 9, 10, 11
# Matches a simple sentence, or the trailing part of a complex sentence,
# where a simple sentence contains no interior "."s.
$TermEndSequence = $InteriorChars* ($TermEx | $ATermEx) $Close* $Sp* $SepSeq?;
$EndSequence = $InteriorChars* $SepSeq?;
# Put them all together.
($NumberFollows | $UppersSurround | $LowerWordFollows)* $TermEndSequence{0}; # status = UBRK_SENTENCE_TERM
($NumberFollows | $UppersSurround | $LowerWordFollows)* $EndSequence{100}; # status = UBRK_SENTENCE_SEP
## -------------------------------------------------
!!reverse;
# rule 6
$RULE6 = $Numeric $Format* $Extend* $ATerm;
# rule 7
$RULE7 = $Upper $Format* $Extend* $ATerm $Format* $Extend* $Upper;
# rule 8
$RULE8 = $Lower ($Format* $Extend* [^$OLetter $Upper $Lower $Sep])*
($Format* $Extend* $Sp)* ($Format* $Extend* $Close)*
$Format* $Extend* $ATerm;
# rule 9, 10, 11
# $CR $LF
$End = $Sep | \u000a\u000d
| $Format* $Extend* $Sp* $Format* $Extend* $Close* $Format*
$Extend* ($Term | $ATerm)
| $Sep $Format* $Extend* $Sp* $Format* $Extend* $Close* $Format*
$Extend* ($Term | $ATerm);
# rule 12
$RULE12 = [^$Sep $Term $ATerm];
$Join = ($RULE6 | $RULE7 | $RULE8 | $RULE12)*;
$End;
$End? $Join [$RULE12 - $Sp - $Close];
# forces a break at the beginning of text "$Sp blah blah blah"
# remember the break iterators takes the longest match
$NOT_T_A_S_C = [^$Term $ATerm $Sp $Close];
$End? $Join $Sp / [$NOT_T_A_S_C {eof}];
# forces a break at the beginning of text "$Close blah blah blah"
$NOT_T_A_C = [^$Term $ATerm $Close];
$End? $Join $Close / [$NOT_T_A_C {eof}];
## -------------------------------------------------
!!safe_reverse;
# rule 4
$Extend+ [^$Extend];
# rule 7
$Extend* $ATerm $Format* $Extend* $Upper;
# rule 8
($Extend* $Term)+ ($Extend* $Sp $Format*)* ($Extend* $Close $Format*)* $Extend* $ATerm;
# rule 11
($Extend* $Sp $Format*)* ($Extend* $Close $Format*)*;
($Extend* $Sp $Format*)* ($Extend* $Close $Format*)* $Extend* ($Term | $ATerm);
## -------------------------------------------------
!!safe_forward;
# rule 7
$ATerm $Extend* $Format* $Upper;
# rule 8
$Lower .;
# rule 11
($Close $Extend* $Format*)* ($Sp $Extend* $Format*)*;
|