%% options copyright owner = Dirk Krause copyright year = 2015-xxxx license = bsd %% header #include "dk3conf.h" #include "dk3types.h" /** @file dk3bom.h Byte order marker detection. The functions in this module check for a byte order marker at the start of a data stream. First use dk3bom_detect_init() to initialize a detector structure. Use dk3bom_detect_add() to add the first byte to the dectector. As long as the result is DK3_STM_ACCEPT, continue adding bytes. On DK3_STM_FINISHED use dk3bom_detect_results() to retrieve the detected encoding and the array of rejected bytes. Process the rejected bytes before processing the remaining bytes from the data stream. */ /** State machine to detect a BOM (byte order marker) at the start of a data stream. */ typedef struct { int defenc; int found; int state; size_t rjb; unsigned char rj[4]; } dk3_bom_detector_t; #ifdef __cplusplus extern "C" { #endif /** Initialize detector structure. @param ptr Detector structure to initialize. */ void dk3bom_detect_init(dk3_bom_detector_t *ptr, int de); /** Add one byte to detector. @param ptr Detector structure. @param b Byte to add. @return DK3_STM_ACCEPT if there is no result yet, DK3_STM_FINISHED if a result was found or DK3_STM_ERROR on attempts to add further bytes after a result was found. */ int dk3bom_detect_add(dk3_bom_detector_t *ptr, unsigned char b); /** Retrieve results from detector structure. @param enc Pointer to variable receiving encoding. @param rjptr Address of pointer receiving address of rejected bytes array. @param rjsz Pointer to variable receiving size of rejected bytes array. @param ptr Detector structure (data source). */ void dk3bom_detect_results( int *enc, const unsigned char **rjptr, size_t *rjsz, dk3_bom_detector_t *ptr ); #ifdef __cplusplus } #endif %% module #include "dk3const.h" #include "dk3stm.h" #include "dk3bom.h" /** Initialize detector structure. @param ptr Detector structure to initialize. */ void dk3bom_detect_init(dk3_bom_detector_t *ptr, int de) { if (NULL != ptr) { ptr->defenc = de; ptr->found = de; ptr->state = 0; ptr->rjb = 0; ptr->rj[0] = 0x00; ptr->rj[1] = 0x00; ptr->rj[2] = 0x00; ptr->rj[3] = 0x00; } } /* BOMS and states --------------- UTF-32 LSB: 0xFF (1) 0xFE (2) 0x00 (3) 0x00 UTF-32 MSB: 0x00 (4) 0x00 (5) 0xFE (6) 0xFF UTF-16 LSB: 0xFF (1) 0xFE (2) UTF-16 MSB: 0xFE (7) 0xFF UTF-8: 0xEF (8) 0xBB (9) 0xBF */ int dk3bom_detect_add(dk3_bom_detector_t *ptr, unsigned char b) { int back = DK3_STM_ERROR; if (NULL != ptr) { switch (ptr->state) { case 0: { ptr->rj[0] = b; ptr->rjb = 1; switch ((int)(b)) { case 0x00: { ptr->state = 4; back = DK3_STM_ACCEPT; } break; case 0xFF: { ptr->state = 1; back = DK3_STM_ACCEPT; } break; case 0xFE: { ptr->state = 7; back = DK3_STM_ACCEPT; } break; case 0xEF: { ptr->state = 8; back = DK3_STM_ACCEPT; } break; default: { ptr->state = -1; back = DK3_STM_FINISHED; } break; } } break; case 1: { ptr->rj[1] = b; ptr->rjb = 2; if (0xFE == b) { ptr->state = 2; back = DK3_STM_ACCEPT; ptr->found = DK3_FILE_ENCODING_UTF16_LSB_FIRST; ptr->rjb = 0; } else { ptr->state = -1; back = DK3_STM_FINISHED; } } break; case 2: { ptr->rj[0] = b; ptr->rjb = 1; if (0x00 == b) { ptr->state = 3; back = DK3_STM_ACCEPT; } else { ptr->state = -1; back = DK3_STM_FINISHED; ptr->found = DK3_FILE_ENCODING_UTF16_LSB_FIRST; } } break; case 3: { ptr->rj[1] = b; ptr->rjb = 2; if (0x00 == b) { ptr->state = -1; back = DK3_STM_FINISHED; ptr->found = DK3_FILE_ENCODING_UNICODE_LSB_FIRST; ptr->rjb = 0; } else { ptr->state = -1; back = DK3_STM_FINISHED; ptr->found = DK3_FILE_ENCODING_UTF16_LSB_FIRST; } } break; case 4: { ptr->rj[1] = b; ptr->rjb = 2; if (0x00 == b) { ptr->state = 5; back = DK3_STM_ACCEPT; } else { ptr->state = -1; back = DK3_STM_FINISHED; } } break; case 5: { ptr->rj[2] = b; ptr->rjb = 3; if (0xFE == b) { ptr->state = 6; back = DK3_STM_ACCEPT; } else { ptr->state = -1; back = DK3_STM_FINISHED; } } break; case 6: { ptr->rj[3] = b; ptr->rjb = 4; if (0xFF == b) { ptr->state = -1; back = DK3_STM_FINISHED; ptr->found = DK3_FILE_ENCODING_UNICODE_MSB_FIRST; ptr->rjb = 0; } else { ptr->state = -1; back = DK3_STM_FINISHED; } } break; case 7: { ptr->rj[1] = b; ptr->rjb = 2; if (0xFF == b) { ptr->state= -1; back = DK3_STM_FINISHED; ptr->found = DK3_FILE_ENCODING_UTF16_MSB_FIRST; ptr->rjb = 0; } else { ptr->state = -1; back = DK3_STM_FINISHED; } } break; case 8: { ptr->rj[1] = b; ptr->rjb = 2; if (0xBB == b) { ptr->state = 9; back = DK3_STM_ACCEPT; } else { ptr->state = -1; back = DK3_STM_FINISHED; } } break; case 9: { ptr->rj[2] = b; ptr->rjb = 3; if (0xBF == b) { ptr->state = -1; back = DK3_STM_FINISHED; ptr->found = DK3_FILE_ENCODING_UTF8; ptr->rjb = 0; } else { ptr->state = -1; back = DK3_STM_FINISHED; } } break; /* Default not necessary due to initialization of return variable. */ } } return back; } void dk3bom_detect_results( int *enc, const unsigned char **rjptr, size_t *rjsz, dk3_bom_detector_t *ptr ) { if (NULL != ptr) { if (NULL != enc) { *enc = ptr->found; } if ((NULL != rjptr) && (NULL != rjsz)) { *rjptr = ptr->rj; *rjsz = ptr->rjb; } else { if (NULL != rjptr) { *rjptr = NULL; } if (NULL != rjsz ) { *rjsz = 0; } } } }