%% options copyright owner = Dirk Krause copyright year = 2015-xxxx license = bsd %% header /** @file Detect BOM (byte order marker) at start of data stream, keep track of unprocessed characters. First, use dk4bom_detect_init() to initialize the state machine. Now use dk4bom_detect_add() to add the data streams contents byte by byte to the state machine as long as the dk4bom_detect_add() function returns DK4_EDSTM_ACCEPT. If the function returns DK4_EDSTM_ERROR, an error occured, you should stop processing. If the function returns DK4_EDSTM_FINISHED or DK4_EDSTM_FINISHED_WITH_UNUSED a BOM was found or definitely not found. Use dk4bom_detect_get_encoding() to obtain the decoding found and process the remaining data stream in that encoding. If the function returned DK4_EDSTM_FINISHED_WITH_UNUSED, use dk4bom_detect_num_unused_bytes() to find the number of unused bytes saved in the detector and dk4bom_detect_unused_byte() to retrieve and process the unused bytes before processing the remaining bytes from the data stream. For a very short data stream, the detection may not be finished at the end of the stream. Use dk4bom_detect_in_detection() to find out whether the detection was not yet finished. If the function returns 1, the detection is not finished and unused bytes are stored in the detector. Use dk4bom_detect_num_unused_bytes() and dk4bom_detect_unused_byte() to obtain the unused bytes. CRT on Windows: Not used. */ #ifndef DK4CONF_H_INCLUDED #include "dk4conf.h" #endif #ifndef DK4TYPES_H_INCLUDED #include "dk4types.h" #endif #ifndef DK4EDSTM_H_INCLUDED #include "dk4edstm.h" #endif #if DK4_HAVE_SYS_TYPES_H #ifndef SYS_TYPES_H_INCLUDED #include #define SYS_TYPES_H_INCLUDED 1 #endif #endif /** State machine to detect a BOM (byte order marker) at the beginning of a data stream. */ typedef struct { int defenc; /**< Default encoding, used if no BOM detected. */ int found; /**< Encoding from the BOM found. */ int state; /**< Current state of BOM detection. */ size_t rjb; /**< Number of unused bytes. */ unsigned char c1; /**< Rejected byte 1. */ unsigned char c2; /**< Rejected byte 2. */ unsigned char c3; /**< Rejected byte 3. */ unsigned char c4; /**< Rejected byte 4. */ } dk4_bom_detector_t; #ifdef __cplusplus extern "C" { #endif /** Initialize the BOM detector. @param ptr BOM detector to initialize. @param defenc Encoding to use if not BOM at beginning of data. */ void dk4bom_detect_init(dk4_bom_detector_t *ptr, int defenc); /** Add one byte to the BOM detector. @param ptr Detector to use. @param byte The data byte to add. @return Operation result, one from: DK4_EDSTM_ACCEPT if you can continue feeding bytes into the detector, DK4_EDSTM_FINISHED if a BOM was detected (do not feed further bytes), DK4_EDSTM_FINISHED_WITH_UNUSED if BOM detection was finished and there are unused bytes stored in the detector), or DK4_EDSTM_ERROR if you attempt to add further bytes after the detection was finished. On DK4_EDSTM_FINISHED and DK4_EDSTM_FINISHED_WITH_UNUSED you can use dk4bom_detect_get_encoding() to retrieve the encoding. On DK4_EDSTM_FINISHED_WITH_UNUSED use dk4bom_detect_num_unused_bytes() and dk4bom_detect_unused_byte() to retrieve the unused bytes from the detector and process them before processing further data from the data stream. */ int dk4bom_detect_add(dk4_bom_detector_t *ptr, unsigned char byte); /** Retrieve the encoding from the detector. @param ptr Detector to use. @return A file encoding, one from: DK4_FILE_ENCODING_ASCII, DK4_FILE_ENCODING_ANSI, DK4_FILE_ENCODING_UTF8, DK4_FILE_ENCODING_UTF16_LE, DK4_FILE_ENCODING_UTF16_BE, DK4_FILE_ENCODING_32_LE, or DK4_FILE_ENCODING_32_BE. */ int dk4bom_detect_get_encoding(dk4_bom_detector_t *ptr); /** Retrieve the number of cted bytes stored in the detector. @param ptr Detector to use. @return Number of unused bytes stored in the detector. */ size_t dk4bom_detect_num_unused_bytes(dk4_bom_detector_t *ptr); /** Retrieve one unused byte stored in the detector. @param ptr Detector to use. @param num Byte number. @return The byte value. */ unsigned char dk4bom_detect_unused_byte(dk4_bom_detector_t *ptr, size_t num); /** Check if BOM detection is still in progress. @param ptr Detector to check. @return 1 if detection is in progress and bytes are stored in the detector, 0 otherwise. */ int dk4bom_detect_in_detection(dk4_bom_detector_t *ptr); #ifdef __cplusplus } #endif %% module #include "dk4conf.h" #include "dk4types.h" #include "dk4enc.h" #include "dk4bom.h" $!trace-include void dk4bom_detect_init(dk4_bom_detector_t *ptr, int defenc) { if (NULL != ptr) { ptr->defenc = defenc; ptr->found = defenc; ptr->state = 0; ptr->rjb = 0; ptr->c1 = 0x00; ptr->c2 = 0x00; ptr->c3 = 0x00; } } /* BOMS and states --------------- UTF-32 LSB: 0xFF (1) 0xFE (2) 0x00 (3) 0x00 UTF-32 MSB: 0x00 (4) 0x00 (5) 0xFE (6) 0xFF UTF-16 LSB: 0xFF (1) 0xFE (2) UTF-16 MSB: 0xFE (7) 0xFF UTF-8: 0xEF (8) 0xBB (9) 0xBF */ int dk4bom_detect_add(dk4_bom_detector_t *ptr, unsigned char byte) { int back = DK4_EDSTM_ERROR; if (NULL != ptr) { switch (ptr->state) { case 0: { ptr->c1 = byte; ptr->rjb = 1; switch ( (int)byte ) { case 0x00: { ptr->state = 4; back = DK4_EDSTM_ACCEPT; } break; case 0xFF: { ptr->state = 1; back = DK4_EDSTM_ACCEPT; } break; case 0xFE: { ptr->state = 7; back = DK4_EDSTM_ACCEPT; } break; case 0xEF: { ptr->state = 8; back = DK4_EDSTM_ACCEPT; } break; default: { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } break; } } break; case 1: { ptr->c2 = byte; ptr->rjb = 2; if (0xFE == byte) { ptr->state = 2; back = DK4_EDSTM_ACCEPT; ptr->found = DK4_FILE_ENCODING_UTF16_LE; ptr->rjb = 0; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } } break; case 2: { ptr->c1 = byte; ptr->rjb = 1; if (0x00 == byte) { ptr->state = 3; back = DK4_EDSTM_ACCEPT; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; ptr->found = DK4_FILE_ENCODING_UTF16_LE; } } break; case 3: { ptr->c2 = byte; ptr->rjb = 2; if (0x00 == byte) { ptr->state = -1; back = DK4_EDSTM_FINISHED; ptr->found = DK4_FILE_ENCODING_32_LE; ptr->rjb = 0; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; ptr->found = DK4_FILE_ENCODING_UTF16_LE; } } break; case 4: { ptr->c2 = byte; ptr->rjb = 2; if (0x00 == byte) { ptr->state = 5; back = DK4_EDSTM_ACCEPT; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } } break; case 5: { ptr->c3 = byte; ptr->rjb = 3; if (0xFE == byte) { ptr->state = 6; back = DK4_EDSTM_ACCEPT; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } } break; case 6: { ptr->c4 = byte; ptr->rjb = 4; if (0xFF == byte) { ptr->state = -1; back = DK4_EDSTM_FINISHED; ptr->found = DK4_FILE_ENCODING_32_BE; ptr->rjb = 0; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } } break; case 7: { ptr->c2 = byte; ptr->rjb = 2; if (0xFF == byte) { ptr->state = -1; back = DK4_EDSTM_FINISHED; ptr->found = DK4_FILE_ENCODING_UTF16_BE; ptr->rjb = 0; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } } break; case 8: { ptr->c2 = byte; ptr->rjb = 2; if (0xBB == byte) { ptr->state = 9; back = DK4_EDSTM_ACCEPT; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } } break; case 9: { ptr->c3 = byte; ptr->rjb = 3; if (0xBF == byte) { ptr->state = -1; back = DK4_EDSTM_FINISHED; ptr->found = DK4_FILE_ENCODING_UTF8; ptr->rjb = 0; } else { ptr->state = -1; back = DK4_EDSTM_FINISHED_WITH_UNUSED; } } break; } } return back; } int dk4bom_detect_get_encoding(dk4_bom_detector_t *ptr) { int back = DK4_ENCODING_ASCII; if (NULL != ptr) { back = ptr->found; } return back; } size_t dk4bom_detect_num_unused_bytes(dk4_bom_detector_t *ptr) { size_t back = 0; if (NULL != ptr) { back = ptr->rjb; } return back; } unsigned char dk4bom_detect_unused_byte(dk4_bom_detector_t *ptr, size_t num) { unsigned char back = 0x00; if (NULL != ptr) { if (0 == num) { back = ptr->c1; } else { if (1 == num) { back = ptr->c2; } else { if (2 == num) { back = ptr->c3; } else { if (3 == num) { back = ptr->c4; } } } } } return back; } int dk4bom_detect_in_detection(dk4_bom_detector_t *ptr) { int back = 0; if (NULL != ptr) { if (-1 != ptr->state) { back = 1; } } return back; }