diff options
Diffstat (limited to 'Build/source/libs/teckit/TECkit-2.5.1/source/Compiler.h')
-rw-r--r-- | Build/source/libs/teckit/TECkit-2.5.1/source/Compiler.h | 349 |
1 files changed, 349 insertions, 0 deletions
diff --git a/Build/source/libs/teckit/TECkit-2.5.1/source/Compiler.h b/Build/source/libs/teckit/TECkit-2.5.1/source/Compiler.h new file mode 100644 index 00000000000..039ad1a1bde --- /dev/null +++ b/Build/source/libs/teckit/TECkit-2.5.1/source/Compiler.h @@ -0,0 +1,349 @@ +/*------------------------------------------------------------------------ +Copyright (C) 2002-2006 SIL International. All rights reserved. + +Distributable under the terms of either the Common Public License or the +GNU Lesser General Public License, as specified in the LICENSING.txt file. + +File: Engine.h +Responsibility: Jonathan Kew +Last reviewed: Not yet. + +Description: + + +Changes: + + 2008-01-23 jk revised endian-ness stuff to allow Universal build + 2006-06-02 jk added support for extended string rules (>255 per initial char) + 24-May-2005 change from Ulrik to work around MS VC++ 6 issues + 21-May-2005 changes based on Ulrik Petersen's patch for MS VC++ 6 + +-------------------------------------------------------------------------*/ + +#ifndef __Compiler_H__ +#define __Compiler_H__ + +#ifdef HAVE_CONFIG_H +# include "config.h" /* a Unix-ish setup where we have config.h available */ +#endif + +#if (defined(__dest_os) && (__dest_os == __win32_os)) || defined(WIN32) /* Windows target: little-endian */ +# undef WORDS_BIGENDIAN +#endif + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif + +#if defined(TARGET_RT_BIG_ENDIAN) /* the CodeWarrior prefix files or Apple TargetConditionals.h sets this */ +# if TARGET_RT_BIG_ENDIAN +# undef WORDS_BIGENDIAN +# define WORDS_BIGENDIAN 1 +# else +# undef WORDS_BIGENDIAN +# endif +#endif + +#include "TECkit_Format.h" +#include "TECkit_Compiler.h" + +#include "TECkit_Engine.h" + +#ifndef __MWERKS__ +# include "ulong_chartraits.h" +#endif + +#include <string> +#include <vector> +#include <map> + +using namespace std; + +class Compiler +{ +public: + Compiler(const char* txt, UInt32 len, char inForm, bool cmp, bool genXML, TECkit_ErrorFn errFunc, void* userData); + ~Compiler(); + + void GetCompiledTable(Byte*& table, UInt32& len) const; + void DetachCompiledTable(); + + enum { kInvalidRuleOffset = 0xffffffffUL }; + +protected: + typedef enum { + // general token types recognized by the compiler + tok_Newline = 256, + tok_Map, + tok_Ellipsis, + tok_Number, + tok_USV, + tok_Identifier, + tok_String, + tok_Unknown, + // then we have the TECkit language keywords: + tok_Name, + tok_Flags, + tok_FlagValue, + tok_Pass, + tok_PassType, + tok_Class, + tok_Default, + tok_Define + } tokenType; + + Byte* compiledTable; + UInt32 compiledSize; + + TECkit_ErrorFn errorFunction; + void* errFuncUserData; + + typedef basic_string<UInt32> string32; + + struct Token { + tokenType type; + UInt32 val; + const char* str; + string32 strval; + }; + + struct Keyword { + const char *keyword; + tokenType token; + UInt32 refCon; + }; + static Keyword keywords[]; + + const unsigned char* textEnd; + const unsigned char* textPtr; + + char idBuffer[256]; + + // used by the front end parser + UInt32 ungotten; + Token tok; + const unsigned char* tokStart; + UInt32 errorCount; + UInt32 lineNumber; + char inputForm; + bool errorState; + bool generateXML; + bool usedExtStringRules; + + // used in compiling passes + enum { + notInRule, + inLHSString, + inLHSPreContext, + inLHSPostContext, + inRHSString, + inRHSPreContext, + inRHSPostContext + } ruleState; + char ruleType; + + struct Item { + UInt8 type; // 0: literal; kMatchElem_Type_XXXX; 0xff: copy + UInt8 negate; + UInt8 repeatMin; + UInt8 repeatMax; + UInt32 val; // class index or literal value + UInt8 start; // OR/EGroup: index of BGroup + UInt8 next; // BGroup/OR: index of next OR/EGroup + UInt8 after; // BGroup: index of EGroup + 1 + UInt8 index; // Class/Copy: index of corresponding item in match + string tag; + }; + + struct Rule { + Rule( + const vector<Item>& mat, + const vector<Item>& pre, + const vector<Item>& post, + const vector<Item>& rep, + UInt32 line + ) : matchStr(mat) + , preContext(pre) + , postContext(post) + , replaceStr(rep) + , lineNumber(line) + , offset(kInvalidRuleOffset) + , sortKey(0) + { } + vector<Item> matchStr; + vector<Item> preContext; + vector<Item> postContext; + vector<Item> replaceStr; + UInt32 lineNumber; + UInt32 offset; // offset of the packed form in the StringRuleData block + UInt16 sortKey; + UInt16 reserved; + }; + + struct CurrRule { + void clear(); + void setLineNo(UInt32 lineNo); + UInt32 startingLine; + vector<Item> lhsString; + vector<Item> lhsPreContext; + vector<Item> lhsPostContext; + vector<Item> rhsString; + vector<Item> rhsPreContext; + vector<Item> rhsPostContext; + }; + + CurrRule currentRule; // the current rule being parsed + + UInt32 classLine; + + typedef vector<UInt32> Class; + + struct MatClass { + MatClass(UInt32 m) + : membersClass(m) + { } + UInt32 membersClass; + }; + struct RepClass { + RepClass(UInt32 m, UInt32 s) + : membersClass(m) + , sortLikeClass(s) + { } + UInt32 membersClass; + UInt32 sortLikeClass; + }; + + struct Pass { + void clear(); + void setLineNo(UInt32 lineNo); + UInt32 startingLine; + vector<Rule> fwdRules; + vector<Rule> revRules; + vector<string> xmlRules; + map<string,string> xmlContexts; + + map<string,UInt32> byteClassNames; // map name to byteClassMembers index + map<string,UInt32> uniClassNames; + + vector<Class> byteClassMembers; // the actual members of each byte class + vector<Class> uniClassMembers; + vector<UInt32> byteClassLines; + vector<UInt32> uniClassLines; + + UInt32 passType; + UInt32 uniDefault; + UInt8 byteDefault; + bool supplementaryChars; + }; + + Pass currentPass; // the current pass being built + + struct BuildVars { + void clear(); + string planeMap; + vector<string> pageMaps; + vector< vector<UInt16> > charMaps; + UInt8 maxMatch; + UInt8 maxPre; + UInt8 maxPost; + UInt8 maxOutput; + }; + + BuildVars buildVars; // variables used during pass compilation + + vector<string> fwdTables; // binary forms of compiled tables + vector<string> revTables; + + UInt32 lhsFlags; + UInt32 rhsFlags; + + map<UInt16,string> names; // map name IDs to name texts (NB: utf8) + + typedef vector<Token> tokListT; + tokListT::const_iterator defIter; + tokListT::const_iterator defEnd; + map<string,tokListT> defines; + + string xmlRepresentation; + + UInt32 getChar(void); + void ungetChar(UInt32 c); + + void SkipSpaces(void); + tokenType IDlookup(const char* str, UInt32 len); + bool GetNextToken(); + bool ExpectToken(tokenType type, const char* errMsg); + bool ExpectToken(char c, const char* errMsg) + { return ExpectToken((tokenType)c, errMsg); } + void Error(const char* errMsg, const char* s = 0, UInt32 line = 0xffffffff); + void StartDefaultPass(); + void AppendLiteral(UInt32 val, bool negate = false); + void AppendUSV(UInt32 val, bool negate = false); + void AppendSpecial(UInt8 type, bool negate = false); + void AppendClass(const string& className, bool negate = false); + void AppendToRule(const Item& item); + bool tagExists(bool rhs, const string& tag); + void AssignTag(const string& tag); + void SetMinMax(int repeatMin, int repeatMax); + void FinishPass(); + string asUTF8(const string32 s); + void ReadNameString(UInt16 nameID); + + UInt32 charLimit(); + static int ruleKeyComp(const Rule& a, const Rule& b); + int findTag(const string& tag, const vector<Item>& str); + void associateItems(vector<Rule>& rules, bool fromUni, bool toUni); + void setGroupPointers(vector<Item>::iterator b, vector<Item>::iterator e, int startIndex, bool isReversed = false); + void setGroupPointers(vector<Rule>& rules); + void sortRules(vector<Rule>& rules); + int calcMaxLen(vector<Item>::iterator b, vector<Item>::iterator e); + int calcMaxOutLen(Rule& rule); + bool findInitialItems(const Rule& rule, vector<Item>::const_iterator b, vector<Item>::const_iterator e, + vector<Item>& initialItems); + void findInitialItems(const Rule& rule, vector<Item>& initialItems); + void addToCharMap(UInt32 ch, UInt16 index); + void buildTable(vector<Rule>& rules, bool fromUni, bool toUni, string& table); + long classIndex(UInt32 charCode, const Class& classMembers); + long uniClassIndex(UInt32 charCode, UInt32 classIndex); + long byteClassIndex(UInt8 charCode, UInt32 classIndex); + bool isSingleCharRule(const Rule& rule); + void appendMatchElem(string& packedRule, Item& item, int index, + vector<MatClass>& matchClasses); + void appendReplaceElem(string& packedRule, Item& item, + vector<Item>& matchStr, vector<RepClass>& repClasses); + void appendToTable(string& s, const char* ptr, UInt32 len); + template <class T> + void appendToTable(string& table, T x) { +#ifdef WORDS_BIGENDIAN + const char* xp = (const char*)&x; + table.append(xp, sizeof(x)); +#else + /* split into separate statements to work around VC++6 problems */ + const char* xp = (const char*)&x; + xp = xp + sizeof(T); + for (unsigned int i = 0; i < sizeof(T); ++i) { + xp = xp - 1; + table.append(1, *xp); + } +#endif + } + + vector<Item> reverseContext(const vector<Item>& ctx); + void align(string& table, int alignment); + + void xmlOut(const char* s); + void xmlOut(const string& s); + void xmlOut(char c); + string xmlString(vector<Item>::const_iterator b, vector<Item>::const_iterator e, bool isUnicode); + string getContextID(const vector<Item>& ctx, bool isUnicode); +}; + +extern "C" { + struct CharName { + unsigned int usv; + const char* name; + }; + extern CharName gUnicodeNames[]; +} + +#endif /* __Compiler_H__ */ |