summaryrefslogtreecommitdiff
path: root/Build/source/libs/teckit/TECkit-2.5.1/source/TECkit_Format.h
blob: 1ce178a7f3a56773fb8c315f43624ab61214dff0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
/*------------------------------------------------------------------------
Copyright (C) 2002-2006 SIL International. All rights reserved.

Distributable under the terms of either the Common Public License or the
GNU Lesser General Public License, as specified in the LICENSING.txt file.

File: TECkit_Format.h
Responsibility: Jonathan Kew
Last reviewed: Not yet.

Description:
    Definitions used in the TECkit binary table format

	2006-06-02	jk	added support for extended string rules (>255 per initial char)
-------------------------------------------------------------------------*/

#ifndef __TECkit_Format_H__
#define __TECkit_Format_H__

#include "TECkit_Common.h"

#define kMagicNumber			0x714d6170	/* 'qMap' */
#define kMagicNumberCmp			0x7a516d70	/* 'zQmp' */

#define kFileVersion2_1			0x00020001	/* version before tables with ExtStringRules */
#define kCurrentFileVersion		0x00030000	/* current version */

#define kTableVersion2			0x00020000
#define kCurrentTableVersion	0x00030000	/* actually, the engine doesn't check this,
												it only looks at the file version */

struct NameRec {
	UInt16	nameID;
	UInt16	nameLength;
/*
	Byte	data[nameLength];
	pad to 2-byte boundary
*/
};

struct FileHeader {
	UInt32	type;					/* magic number = 'qMap' */
	UInt32	version;				/* version = kFileCurrentVersion */
	UInt32	headerLength;			/* length of this header including offset arrays and name records */
	UInt32	formFlagsLHS;			/* flags for normalization form, Unicode/byte encoding on LHS of mapping */
	UInt32	formFlagsRHS;			/* flags for normalization form, Unicode/byte encoding on RHS of mapping */
	UInt32	numNames;				/* number of strings in the names table */
	UInt32	numFwdTables;			/* number of tables in forward pipeline */
	UInt32	numRevTables;			/* number of tables in reverse pipeline */
#if 0
	UInt32	nameOffsets[numNames];	/* offsets from FileHeader to each NameRec */
	UInt32	fwdBase[numFwdTables];	/* offsets from FileHeader to forward tables */
	UInt32	revBase[numRevTables];	/* offsets from FileHeader to reverse tables */
	NameRec	names[numNames];		/* the name records */
#endif
};

struct TableHeader {
	UInt32	type;					/* type = 'B->B', 'B->U', 'U->B', 'U->U' */
									/* or type = 'NFC ', 'NFD ', and no additional header fields are present */
	UInt32	version;				/* version = kCurrentTableVersion */
	UInt32	length;					/* total length of this table */
	UInt32	flags;					/* flags:
										0x00000001:	supplementary-plane Unicode characters supported in mapping and classes
										0x00000002:	DBCS support (BB/BU tables only) in lookup table
									*/
	UInt32	pageBase;				/* offset from table header to page table (Ux tables) or dbcsPage table (Bx tables) */
	UInt32	lookupBase;				/* offset from table header to lookup table(s) */
	UInt32	matchClassBase;			/* offset from table header to match class definitions */
	UInt32	repClassBase;			/* offset from table header to replacement class definitions */
	UInt32	stringListBase;			/* offset from table header to string rule lists */
	UInt32	stringRuleData;			/* offset from table header to string rule data */
	UInt8	maxMatch;				/* max number of input code units matched by a rule */
	UInt8	maxPre;					/* max number of input code units matched by pre-context */
	UInt8	maxPost;				/* max number of input code units matched by post-context */
	UInt8	maxOutput;				/* max number of output code units generated by a rule */
	UInt32	replacementChar;		/* default output for unmapped codes */
};
#ifndef __cplusplus
typedef struct TableHeader		TableHeader;
#endif

#define kTableType_BB				0x422d3e42
#define kTableType_BU				0x422d3e55
#define kTableType_UB				0x552d3e42
#define kTableType_UU				0x552d3e55

#define kTableType_NFC				0x4e464320
#define kTableType_NFD				0x4e464420

#define	kTableFlags_Supplementary	0x0001
#define	kTableFlags_DBCS			0x0002

union Lookup {
	/* for any table when string rules are used */
	struct {
		UInt8	type;				/*
										0xff: use string rules
										0xfe: illegal DBCS trailing byte
										0xfd: unmapped character: copy (BB/UU) or output default (UB/BU)
										0x00-0x03: direct lookup
									*/
		UInt8	ruleCount;			/* number of rules for this code */
		UInt16	ruleIndex;			/* index into stringList of start of rule list for this code */
	}	rules;
	/* for UB and BB tables with direct byte output */
	struct {
		UInt8	count;				/* count of bytes present in data[]: 0-3 */
		UInt8	data[3];
	}	bytes;
	/* for BU and UU tables with direct Unicode output */
	UInt32		usv;				/* unicode scalar value */
};
#ifndef __cplusplus
typedef union Lookup			Lookup;
#endif

#define kLookupType_StringRules		0xff
#define kLookupType_IllegalDBCS		0xfe
#define kLookupType_Unmapped		0xfd

#define kLookupType_RuleTypeMask		0xc0
#define kLookupType_ExtStringRules		0x80
#define kLookupType_ExtRuleCountMask	0x3f

/*
	/rules.ruleOffset/ points to an array of /rules.ruleCount/ UInt32 values which are the offsets
	from stringRuleData to each rule to test for this character
*/

struct StringRule {
	UInt8	matchLength;			/* length of match string in matchElements */
	UInt8	postLength;				/* length of post-context in matchElements */
	UInt8	preLength;				/* length of pre-context in matchElements */
	UInt8	repLength;				/* length of replacement string in repElements */
#if 0
	MatchElem	matchString[];
	MatchElem	postContext[];
	MatchElem	preContext[];		/* reversed */
	RepElem		repString[];
#endif
};
#ifndef __cplusplus
typedef struct StringRule		StringRule;
#endif

union MatchElem {
#ifdef __cplusplus
				MatchElem()
					{ }
#endif
	struct {
		UInt8	repeat;				/* repeat count: (min << 4) + max */
		UInt8	type;				/* 
										0x80:	negate flag (not allowed with group)
										0x40:	non-literal flag--if set, bits 0x3f indicate specific type (value must not be zero)
												Note that if 'non-literal' flag is NOT set, remaining bits are not used as type code
												but are part of a USV value (or must be set to zero for literal byte data).
									*/
		UInt16	reserved;
	}	flags;
	union {
		struct {
			UInt16		reserved;
			UInt8		dNext;		/* offset to following OR or EGroup element */
			UInt8		dAfter;		/* offset to element after the group for BGroup */
		}	bgroup;
		struct {
			UInt16		reserved;
			UInt8		dNext;		/* offset to following OR or EGroup element (for OR only) */
			UInt8		dStart;		/* reverse offset to corresponding BGroup */
		}	egroup;					/* (also used for OR elements) */
		struct {
			UInt16		reserved;
			UInt16		index;		/* index of character class */
		}	cls;
		struct {
			UInt8		reserved[3];
			UInt8		data;		/* literal byte */
		}	byte;
		struct {
			UInt32		data;		/* literal Unicode scalar: must mask with kUSVMask, as top bits overlap flags.repeat and "negate" bit in flags.type */
		}	usv;
	}	value;
};
#ifndef __cplusplus
typedef union MatchElem			MatchElem;
#endif

#define	kMatchElem_Negate			0x80	/* negated test */
#define kMatchElem_NonLit			0x40	/* test value is not a literal character; need to check type */

#define kMatchElem_TypeMask			0x3f	/* Mask for type value. Note that type 0 must not be used (=literal) */
#define	kMatchElem_Type_Class		0x01	/* class match */
#define kMatchElem_Type_BGroup		0x02	/* begin group */
#define kMatchElem_Type_EGroup		0x03	/* end group */
#define	kMatchElem_Type_OR			0x04	/* special code: OR */
#define	kMatchElem_Type_ANY			0x05	/* special code: ANY */
#define	kMatchElem_Type_EOS			0x06	/* special code: EOS */
#define kMatchElem_Type_Copy		0x07	/* copy matched item (invalid; for internal compiler use) */

#define kUSVMask					0x001fffff

union RepElem {
	struct {
		UInt8	type;				/* see kRepElem_... below */
		UInt8	matchIndex;			/* index of corresponding item in matchString for type == kRepElem_Class or kRepElem_Copy */
		UInt16	repClass;			/* repClass if type == kRepElem_Class */
	}	flags;
	UInt32	value;					/* literal value (mask with kUSVMask) if flags.type == kRepElem_Literal */
};
#ifndef __cplusplus
typedef union RepElem			RepElem;
#endif

#define kRepElem_Literal			0x00
#define kRepElem_Class				kMatchElem_Type_Class
#define kRepElem_Copy				kMatchElem_Type_Copy
#define kRepElem_Unmapped			0x0f	/* used in default terminator rules */

#endif	/* __TECkit_Format_H__ */