1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
|
/*------------------------------------------------------------------------
Copyright (C) 2002-2014 SIL International. All rights reserved.
Distributable under the terms of either the Common Public License or the
GNU Lesser General Public License, as specified in the LICENSING.txt file.
File: TECkit_Format.h
Responsibility: Jonathan Kew
Last reviewed: Not yet.
Description:
Definitions used in the TECkit binary table format
2006-06-02 jk added support for extended string rules (>255 per initial char)
-------------------------------------------------------------------------*/
#ifndef __TECkit_Format_H__
#define __TECkit_Format_H__
#include "TECkit_Common.h"
#define kMagicNumber 0x714d6170 /* 'qMap' */
#define kMagicNumberCmp 0x7a516d70 /* 'zQmp' */
#define kFileVersion2_1 0x00020001 /* version before tables with ExtStringRules */
#define kCurrentFileVersion 0x00030000 /* current version */
#define kTableVersion2 0x00020000
#define kCurrentTableVersion 0x00030000 /* actually, the engine doesn't check this,
it only looks at the file version */
struct NameRec {
UInt16 nameID;
UInt16 nameLength;
/*
Byte data[nameLength];
pad to 2-byte boundary
*/
};
struct FileHeader {
UInt32 type; /* magic number = 'qMap' */
UInt32 version; /* version = kFileCurrentVersion */
UInt32 headerLength; /* length of this header including offset arrays and name records */
UInt32 formFlagsLHS; /* flags for normalization form, Unicode/byte encoding on LHS of mapping */
UInt32 formFlagsRHS; /* flags for normalization form, Unicode/byte encoding on RHS of mapping */
UInt32 numNames; /* number of strings in the names table */
UInt32 numFwdTables; /* number of tables in forward pipeline */
UInt32 numRevTables; /* number of tables in reverse pipeline */
#if 0
UInt32 nameOffsets[numNames]; /* offsets from FileHeader to each NameRec */
UInt32 fwdBase[numFwdTables]; /* offsets from FileHeader to forward tables */
UInt32 revBase[numRevTables]; /* offsets from FileHeader to reverse tables */
NameRec names[numNames]; /* the name records */
#endif
};
struct TableHeader {
UInt32 type; /* type = 'B->B', 'B->U', 'U->B', 'U->U' */
/* or type = 'NFC ', 'NFD ', and no additional header fields are present */
UInt32 version; /* version = kCurrentTableVersion */
UInt32 length; /* total length of this table */
UInt32 flags; /* flags:
0x00000001: supplementary-plane Unicode characters supported in mapping and classes
0x00000002: DBCS support (BB/BU tables only) in lookup table
*/
UInt32 pageBase; /* offset from table header to page table (Ux tables) or dbcsPage table (Bx tables) */
UInt32 lookupBase; /* offset from table header to lookup table(s) */
UInt32 matchClassBase; /* offset from table header to match class definitions */
UInt32 repClassBase; /* offset from table header to replacement class definitions */
UInt32 stringListBase; /* offset from table header to string rule lists */
UInt32 stringRuleData; /* offset from table header to string rule data */
UInt8 maxMatch; /* max number of input code units matched by a rule */
UInt8 maxPre; /* max number of input code units matched by pre-context */
UInt8 maxPost; /* max number of input code units matched by post-context */
UInt8 maxOutput; /* max number of output code units generated by a rule */
UInt32 replacementChar; /* default output for unmapped codes */
};
#ifndef __cplusplus
typedef struct TableHeader TableHeader;
#endif
#define kTableType_BB 0x422d3e42
#define kTableType_BU 0x422d3e55
#define kTableType_UB 0x552d3e42
#define kTableType_UU 0x552d3e55
#define kTableType_NFC 0x4e464320
#define kTableType_NFD 0x4e464420
#define kTableFlags_Supplementary 0x0001
#define kTableFlags_DBCS 0x0002
union Lookup {
/* for any table when string rules are used */
struct {
UInt8 type; /*
0xff: use string rules
0xfe: illegal DBCS trailing byte
0xfd: unmapped character: copy (BB/UU) or output default (UB/BU)
0x00-0x03: direct lookup
*/
UInt8 ruleCount; /* number of rules for this code */
UInt16 ruleIndex; /* index into stringList of start of rule list for this code */
} rules;
/* for UB and BB tables with direct byte output */
struct {
UInt8 count; /* count of bytes present in data[]: 0-3 */
UInt8 data[3];
} bytes;
/* for BU and UU tables with direct Unicode output */
UInt32 usv; /* unicode scalar value */
};
#ifndef __cplusplus
typedef union Lookup Lookup;
#endif
#define kLookupType_StringRules 0xff
#define kLookupType_IllegalDBCS 0xfe
#define kLookupType_Unmapped 0xfd
#define kLookupType_RuleTypeMask 0xc0
#define kLookupType_ExtStringRules 0x80
#define kLookupType_ExtRuleCountMask 0x3f
/*
/rules.ruleOffset/ points to an array of /rules.ruleCount/ UInt32 values which are the offsets
from stringRuleData to each rule to test for this character
*/
struct StringRule {
UInt8 matchLength; /* length of match string in matchElements */
UInt8 postLength; /* length of post-context in matchElements */
UInt8 preLength; /* length of pre-context in matchElements */
UInt8 repLength; /* length of replacement string in repElements */
#if 0
MatchElem matchString[];
MatchElem postContext[];
MatchElem preContext[]; /* reversed */
RepElem repString[];
#endif
};
#ifndef __cplusplus
typedef struct StringRule StringRule;
#endif
union MatchElem {
#ifdef __cplusplus
MatchElem()
{ }
#endif
struct {
UInt8 repeat; /* repeat count: (min << 4) + max */
UInt8 type; /*
0x80: negate flag (not allowed with group)
0x40: non-literal flag--if set, bits 0x3f indicate specific type (value must not be zero)
Note that if 'non-literal' flag is NOT set, remaining bits are not used as type code
but are part of a USV value (or must be set to zero for literal byte data).
*/
UInt16 reserved;
} flags;
union {
struct {
UInt16 reserved;
UInt8 dNext; /* offset to following OR or EGroup element */
UInt8 dAfter; /* offset to element after the group for BGroup */
} bgroup;
struct {
UInt16 reserved;
UInt8 dNext; /* offset to following OR or EGroup element (for OR only) */
UInt8 dStart; /* reverse offset to corresponding BGroup */
} egroup; /* (also used for OR elements) */
struct {
UInt16 reserved;
UInt16 index; /* index of character class */
} cls;
struct {
UInt8 reserved[3];
UInt8 data; /* literal byte */
} byte;
struct {
UInt32 data; /* literal Unicode scalar: must mask with kUSVMask, as top bits overlap flags.repeat and "negate" bit in flags.type */
} usv;
} value;
};
#ifndef __cplusplus
typedef union MatchElem MatchElem;
#endif
#define kMatchElem_Negate 0x80 /* negated test */
#define kMatchElem_NonLit 0x40 /* test value is not a literal character; need to check type */
#define kMatchElem_TypeMask 0x3f /* Mask for type value. Note that type 0 must not be used (=literal) */
#define kMatchElem_Type_Class 0x01 /* class match */
#define kMatchElem_Type_BGroup 0x02 /* begin group */
#define kMatchElem_Type_EGroup 0x03 /* end group */
#define kMatchElem_Type_OR 0x04 /* special code: OR */
#define kMatchElem_Type_ANY 0x05 /* special code: ANY */
#define kMatchElem_Type_EOS 0x06 /* special code: EOS */
#define kMatchElem_Type_Copy 0x07 /* copy matched item (invalid; for internal compiler use) */
#define kUSVMask 0x001fffff
union RepElem {
struct {
UInt8 type; /* see kRepElem_... below */
UInt8 matchIndex; /* index of corresponding item in matchString for type == kRepElem_Class or kRepElem_Copy */
UInt16 repClass; /* repClass if type == kRepElem_Class */
} flags;
UInt32 value; /* literal value (mask with kUSVMask) if flags.type == kRepElem_Literal */
};
#ifndef __cplusplus
typedef union RepElem RepElem;
#endif
#define kRepElem_Literal 0x00
#define kRepElem_Class kMatchElem_Type_Class
#define kRepElem_Copy kMatchElem_Type_Copy
#define kRepElem_Unmapped 0x0f /* used in default terminator rules */
#endif /* __TECkit_Format_H__ */
|