%% options copyright owner = Dirk Krause copyright year = 2015-xxxx SPDX-License-Identifier: BSD-3-Clause %% header /** @file Text stream processing for 32 bit characters. CRT on Windows: Optional. */ #ifndef DK4CONF_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4conf.h" #else #include #endif #endif #ifndef DK4TYPES_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4types.h" #else #include #endif #endif #ifndef DK4ERROR_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4error.h" #else #include #endif #endif #ifndef DK4BOM_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4bom.h" #else #include #endif #endif #ifndef DK4TSP_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4tsp.h" #else #include #endif #endif #ifndef DK4UTF8_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4utf8.h" #else #include #endif #endif #ifndef DK4UTF16_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4utf16.h" #else #include #endif #endif #ifndef DK4C32_H_INCLUDED #if DK4_BUILDING_DKTOOLS4 #include "dk4c32.h" #else #include #endif #endif /** Handler function for single characters. @param obj Object to modify while processing the character. @param c Character to process. @param pos Current position in file or data stream. @param erp Error report, may be NULL. @return DK4_TSP_RES_OK if the character was processed successfully, DK4_TSP_RES_ERROR if there was an error but we can continue, DK4_TSP_RES_FATAL if there was a fatal error so we should abort processing. */ typedef int dk4_c32_handler_t( void *obj, dk4_c32_t c, dk4_text_stream_position_t *pos, dk4_er_t *erp ); /** Handler function for text lines. @param obj Object to modify while processing the character. @param line Text line to process. @param lineno Current line number. @param erp Error report, may be NULL. @return DK4_TSP_RES_OK if the character was processed successfully, DK4_TSP_RES_ERROR if there was an error but we can continue, DK4_TSP_RES_FATAL if there was a fata error so we should abort processing. */ typedef int dk4_c32_line_handler_t( void *obj, dk4_c32_t *line, dk4_um_t lineno, dk4_er_t *erp ); /** Structure for 32 bit character processing. */ typedef struct { union { dk4_utf8_decoder_t u08; /**< UTF-8 decoder. */ dk4_utf16_byte_decoder_t u16; /**< UTF-16 decoder. */ dk4_c32_byte_decoder_t c32; /**< 32 bit char decoder. */ } dec; /**< Input decoder. */ dk4_bom_detector_t bomd; /**< BOM detector. */ dk4_text_stream_position_t pos; /**< Current position. */ dk4_er_t er_en; /**< Errors in input decoding. */ dk4_er_t er_pr; /**< Errors in processing. */ union { dk4_c32_line_handler_t *lh; /**< Handler function for lines. */ dk4_c32_handler_t *ch; /**< Handler function for char. */ } fct; /**< Handler function. */ dk4_c32_t *inbuf; /**< Buffer for input line. */ void *obj; /**< Object to modify in processing. */ size_t in_sz; /**< Size of input line buffer. */ size_t in_us; /**< Used bytes in input line buffer. */ int ief; /**< Input encoding found. */ int iee; /**< Input encoding expected. */ int pst; /**< Processing stage. */ } dk4_tsp32_t; #ifdef __cplusplus extern "C" { #endif /** Set up processor for byte by byte processing. @param tsp Processor to set up. @param obj Object to modify when processing input, may be NULL. @param fct Handler function to call for each character. @param eie Expected input encoding. @param erp Error report, may be NULL. @return 1 on success, 0 on error. */ int dk4tsp32_setup_char( dk4_tsp32_t *tsp, void *obj, dk4_c32_handler_t *fct, int eie, dk4_er_t *erp ); /** Set up processor for line processing. @param tsp Processor to set up. @param obj Object to modify when processing input, may be NULL. @param fct Handler function to invoke for each line. @param inbuf Input line buffer. @param szin Size of input line buffer (number of dk4_c32_t). @param eie Expected input encoding. @param erp Error report, may be NULL. @return 1 on success, 0 on error. */ int dk4tsp32_setup_line( dk4_tsp32_t *tsp, void *obj, dk4_c32_line_handler_t *fct, dk4_c32_t *inbuf, size_t szin, int eie, dk4_er_t *erp ); /** Add one single byte. @param tsp Text stream processor. @param inbyte Byte to process. @return DK4_TSP_RES_OK if the character was processed successfully, DK4_TSP_RES_ERROR if there was an error but we can continue, DK4_TSP_RES_FATAL if there was a fata error so we should abort processing. */ int dk4tsp32_add_one_byte( dk4_tsp32_t *tsp, unsigned char inbyte ); /** Add multiple bytes. @param tsp Text stream processor. @param buffer Buffer start address. @param sz Number of bytes in buffer. @return DK4_TSP_RES_OK if the character was processed successfully, DK4_TSP_RES_ERROR if there was an error but we can continue, DK4_TSP_RES_FATAL if there was a fata error so we should abort processing. */ int dk4tsp32_add_bytes( dk4_tsp32_t *tsp, const unsigned char *buffer, size_t sz ); /** Finish processing. @param tsp Text stream processor. @return DK4_TSP_RES_OK if processing was finished successfully, DK4_TSP_RES_ERROR if there was an error, DK4_TSP_RES_FATAL if there was a fatal error. */ int dk4tsp32_finish(dk4_tsp32_t *tsp); /** Retrieve error reports for encoding/decoding and processing. @param er_en Destination error report buffer for encoding/decoding. @param er_pr Destination error report buffer for processing. @param tsp Text stream processor to retrieve errors from. */ void dk4tsp32_get_errors(dk4_er_t *er_en, dk4_er_t *er_pr, dk4_tsp32_t const *tsp); #ifdef __cplusplus } #endif %% module #include "dk4conf.h" #include "dk4tsp32.h" #include "dk4enc.h" #include "dk4mem.h" #include "dk4ansi.h" #include "dk4utf8.h" #include "dk4utf16.h" #include "dk4c32.h" #if DK4_HAVE_ASSERT_H #ifndef ASSERT_H_INCLUDED #include #define ASSERT_H_INCLUDED 1 #endif #endif $!trace-include /** Initialize text stream processing structure. @param tsp Text stream processor. @param eie Expected input encoding. */ static void dk4tsp32_init(dk4_tsp32_t *tsp, int eie) { #if DK4_USE_ASSERT assert(NULL != tsp); #endif DK4_MEMRES(tsp, sizeof(dk4_tsp32_t)); dk4bom_detect_init(&(tsp->bomd), eie); dk4error_init(&(tsp->er_en)); dk4error_init(&(tsp->er_pr)); tsp->inbuf = NULL; tsp->obj = NULL; tsp->in_sz = 0; tsp->in_us = 0; tsp->ief = eie; tsp->iee = eie; tsp->pst = 0; (tsp->pos).bytes = (dk4_um_t)0UL; (tsp->pos).chars = (dk4_um_t)1UL; (tsp->pos).lineno = (dk4_um_t)1UL; (tsp->pos).charil = (dk4_um_t)1UL; } /** Process a 32 bit character. @param tsp Text stream processor. @param chr Character to process. @return DK4_TSP_RES_OK if the characters were processed successfully, DK4_TSP_RES_ERROR if there was an error but we can continue, DK4_TSP_RES_FATAL if there was a fatal error so we should abort processing. */ static int dk4tsp32_process_character( dk4_tsp32_t *tsp, dk4_c32_t chr ) { int back = DK4_TSP_RES_FATAL; $? "+ dk4tsp32_process_character" #if DK4_USE_ASSERT assert(NULL != tsp); #endif /* Increase position */ (tsp->pos).chars += (dk4_um_t)1UL; (tsp->pos).charil += (dk4_um_t)1UL; /* Check for line buffering or direct processing */ if ((NULL != tsp->inbuf) && (0 < tsp->in_sz) && (NULL != (tsp->fct).lh)) { $? ". line buffering %lu %lu", (unsigned long)(tsp->in_us), (unsigned long)(tsp->in_sz) if (tsp->in_us < tsp->in_sz) { $? ". used size < buffer size" (tsp->inbuf)[tsp->in_us] = chr; tsp->in_us += 1; back = DK4_TSP_RES_OK; if ((dk4_c32_t)'\n' == chr) { back = DK4_TSP_RES_FATAL; if (tsp->in_us < tsp->in_sz) { (tsp->inbuf)[tsp->in_us] = (dk4_c32_t)0UL; back = (*((tsp->fct).lh))( tsp->obj,tsp->inbuf,(tsp->pos).lineno,&(tsp->er_pr) ); } else { dk4error_set_with_position( &(tsp->er_en), DK4_E_BUFFER_TOO_SMALL, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } /* 2015-07-18 Bugfix: We must reset the number of used characters to 0 after processing and flushing the line buffer. */ tsp->in_us = 0; } } else { $? "! line buffer full" dk4error_set_with_position( &(tsp->er_en), DK4_E_BUFFER_TOO_SMALL, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } } else { $? ". direct char processing" if (NULL != (tsp->fct).ch) { back = (*((tsp->fct).ch))(tsp->obj, chr, &(tsp->pos), &(tsp->er_pr)); } } if ((dk4_c32_t)('\n') == chr) { (tsp->pos).lineno += (dk4_um_t)1UL; (tsp->pos).charil = (dk4_um_t)1UL; } $? "- dk4tsp32_process_character %d", back return back; } /** Normal processing for one byte. Retrieve a 32 bit character first, either by decoding directly or by adding to a decoder. @param tsp Text stream processor. @param inbyte Byte to process. @return Operation result, one from DK4_TSP_RES_OK, DK4_TSP_RES_ERROR or DK4_TSP_RES_FATAL. */ static int dk4tsp32_process_byte( dk4_tsp32_t *tsp, unsigned char inbyte ) { dk4_c32_t c32 = (dk4_c32_t)0UL; /* 32 bit character */ int back = DK4_TSP_RES_FATAL; /* Function result */ int cuc32 = 0; /* Flag: Have 32 bit char */ int res = 0; /* Operation result */ #if DK4_USE_ASSERT assert(NULL != tsp); #endif switch (tsp->ief) { case DK4_FILE_ENCODING_PLAIN: { c32 = (dk4_c32_t)inbyte; cuc32 = 1; } break; case DK4_FILE_ENCODING_WIN1252: { if (0 != dk4ansi_decode(&c32, inbyte)) { cuc32 = 1; } else { dk4error_set_with_position( &(tsp->er_en), DK4_E_DECODING_FAILED, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } } break; case DK4_FILE_ENCODING_UTF8: { res = dk4utf8_add(&((tsp->dec).u08), inbyte); switch (res) { case DK4_EDSTM_ERROR: { dk4error_set_with_position( &(tsp->er_en), DK4_E_DECODING_FAILED, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } break; case DK4_EDSTM_FINISHED: { c32 = dk4utf8_get(&((tsp->dec).u08)); cuc32 = 1; dk4utf8_init(&((tsp->dec).u08)); } break; case DK4_EDSTM_ACCEPT: { back = DK4_TSP_RES_OK; } break; } } break; case DK4_FILE_ENCODING_UTF16_LE: { res = dk4utf16_byte_add(&((tsp->dec).u16), inbyte); switch (res) { case DK4_EDSTM_ERROR: { dk4error_set_with_position( &(tsp->er_en), DK4_E_DECODING_FAILED, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } break; case DK4_EDSTM_FINISHED: { c32 = dk4utf16_byte_get(&((tsp->dec).u16)); cuc32 = 1; dk4utf16_byte_init(&((tsp->dec).u16), 0); } break; case DK4_EDSTM_ACCEPT: { back = DK4_TSP_RES_OK; } break; } } break; case DK4_FILE_ENCODING_UTF16_BE: { res = dk4utf16_byte_add(&((tsp->dec).u16), inbyte); switch (res) { case DK4_EDSTM_ERROR: { dk4error_set_with_position( &(tsp->er_en), DK4_E_DECODING_FAILED, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } break; case DK4_EDSTM_FINISHED: { c32 = dk4utf16_byte_get(&((tsp->dec).u16)); cuc32 = 1; dk4utf16_byte_init(&((tsp->dec).u16), 1); } break; case DK4_EDSTM_ACCEPT: { back = DK4_TSP_RES_OK; } break; } } break; case DK4_FILE_ENCODING_32_LE: { res = dk4c32_decoder_add(&((tsp->dec).c32), inbyte); switch (res) { case DK4_EDSTM_ERROR: { dk4error_set_with_position( &(tsp->er_en), DK4_E_DECODING_FAILED, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } break; case DK4_EDSTM_FINISHED: { c32 = dk4c32_decoder_get(&((tsp->dec).c32)); cuc32 = 1; dk4c32_decoder_init(&((tsp->dec).c32), 0); } break; case DK4_EDSTM_ACCEPT: { back = DK4_TSP_RES_OK; } break; } } break; case DK4_FILE_ENCODING_32_BE: { res = dk4c32_decoder_add(&((tsp->dec).c32), inbyte); switch (res) { case DK4_EDSTM_ERROR: { dk4error_set_with_position( &(tsp->er_en), DK4_E_DECODING_FAILED, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } break; case DK4_EDSTM_FINISHED: { c32 = dk4c32_decoder_get(&((tsp->dec).c32)); cuc32 = 1; dk4c32_decoder_init(&((tsp->dec).c32), 1); } break; case DK4_EDSTM_ACCEPT: { back = DK4_TSP_RES_OK; } break; } } break; } if (0 != cuc32) { back = dk4tsp32_process_character(tsp, c32); } if (DK4_TSP_RES_FATAL == back) { tsp->pst = 2; } return back; } /** Initialize decoder for found input encoding. @param tsp Text stream processor. */ static void dk4tsp32_initialize_decoder(dk4_tsp32_t *tsp) { #if DK4_USE_ASSERT assert(NULL != tsp); #endif switch (tsp->ief) { case DK4_FILE_ENCODING_UTF8: { dk4utf8_init(&((tsp->dec).u08)); } break; case DK4_FILE_ENCODING_UTF16_LE: { dk4utf16_byte_init(&((tsp->dec).u16), 0); } break; case DK4_FILE_ENCODING_UTF16_BE: { dk4utf16_byte_init(&((tsp->dec).u16), 1); } break; case DK4_FILE_ENCODING_32_LE: { dk4c32_decoder_init(&((tsp->dec).c32), 0); } break; case DK4_FILE_ENCODING_32_BE: { dk4c32_decoder_init(&((tsp->dec).c32), 1); } break; } } #if TRACE_DEBUG static unsigned long byte_number = 0UL; #endif /** Add one single byte. @param tsp Text stream processor. @param inbyte Byte to process. @return DK4_TSP_RES_OK if the character was processed successfully, DK4_TSP_RES_ERROR if there was an error but we can continue, DK4_TSP_RES_FATAL if there was a fata error so we should abort processing. */ static int dk4tsp32_i_add_one_byte( dk4_tsp32_t *tsp, unsigned char inbyte ) { size_t nrej; /* Number of unused bytes */ size_t i; /* Current unused byte index */ int back = DK4_TSP_RES_FATAL; /* Function result */ int res; /* Operation result */ unsigned char uc; /* Current unused byte */ $? "+ dk4tsp32_i_add_one_byte index=%lu '%!8c' 0x%02x", byte_number++, inbyte, (unsigned)inbyte #if DK4_USE_ASSERT assert(NULL != tsp); #endif (tsp->pos).bytes += (dk4_um_t)1UL; switch (tsp->pst) { case 0: { res = dk4bom_detect_add(&(tsp->bomd), inbyte); switch (res) { case DK4_EDSTM_ACCEPT: { back = DK4_TSP_RES_OK; } break; case DK4_EDSTM_FINISHED: case DK4_EDSTM_FINISHED_WITH_UNUSED: { back = DK4_TSP_RES_OK; tsp->ief = dk4bom_detect_get_encoding(&(tsp->bomd)); dk4tsp32_initialize_decoder(tsp); tsp->pst = 1; if (DK4_EDSTM_FINISHED_WITH_UNUSED == res) { nrej = dk4bom_detect_num_unused_bytes(&(tsp->bomd)); if (0 < nrej) { for (i = 0; ((i < nrej) && (DK4_TSP_RES_FATAL != back)); i++) { uc = dk4bom_detect_unused_byte(&(tsp->bomd), i); switch (dk4tsp32_process_byte(tsp, uc)) { case DK4_TSP_RES_FATAL: { back = DK4_TSP_RES_FATAL; } break; case DK4_TSP_RES_ERROR: { if (DK4_TSP_RES_OK == back) { back = DK4_TSP_RES_ERROR; } } break; } } } } } break; } } break; case 1: { back = dk4tsp32_process_byte(tsp, inbyte); } break; /* Processing stage 2 indicates there was a serious error before, we must skip further processing. This situation is covered by the initialization value DK4_TSP_RES_FATAL, so we do not need a case branch here. */ } $? "- dk4tsp32_i_add_one_byte %d", back return back; } int dk4tsp32_setup_char( dk4_tsp32_t *tsp, void *obj, dk4_c32_handler_t *fct, int eie, dk4_er_t *erp ) { int back = 0; #if DK4_USE_ASSERT assert(NULL != tsp); #endif if (NULL != tsp) { dk4tsp32_init(tsp, eie); if (NULL != fct) { tsp->obj = obj; (tsp->fct).ch = fct; tsp->iee = eie; tsp->ief = eie; tsp->pst = 0; back = 1; } else { dk4error_set_simple_error_code(erp, DK4_E_INVALID_ARGUMENTS); tsp->pst = 2; } } else { dk4error_set_simple_error_code(erp, DK4_E_INVALID_ARGUMENTS); } return back; } int dk4tsp32_setup_line( dk4_tsp32_t *tsp, void *obj, dk4_c32_line_handler_t *fct, dk4_c32_t *inbuf, size_t szin, int eie, dk4_er_t *erp ) { int back = 0; #if DK4_USE_ASSERT assert(NULL != tsp); assert(NULL != inbuf); assert(0 < szin); #endif if (NULL != tsp) { dk4tsp32_init(tsp, eie); if ((NULL != fct) && (NULL != inbuf) && (0 < szin)) { tsp->obj = obj; (tsp->fct).lh = fct; tsp->inbuf = inbuf; tsp->in_sz = szin; tsp->in_us = 0; tsp->iee = eie; tsp->ief = eie; tsp->pst = 0; back = 1; } else { dk4error_set_simple_error_code(erp, DK4_E_INVALID_ARGUMENTS); tsp->pst = 2; } } else { dk4error_set_simple_error_code(erp, DK4_E_INVALID_ARGUMENTS); } return back; } int dk4tsp32_add_one_byte( dk4_tsp32_t *tsp, unsigned char inbyte ) { int back = DK4_TSP_RES_FATAL; $? "+ dk4tsp32_add_one_byte" #if DK4_USE_ASSERT assert(NULL != tsp); #endif if (NULL != tsp) { if (2 > tsp->pst) { back = dk4tsp32_i_add_one_byte(tsp, inbyte); } } $? "- dk4tsp32_add_one_byte %d", back return back; } int dk4tsp32_add_bytes( dk4_tsp32_t *tsp, const unsigned char *buffer, size_t sz ) { int back = DK4_TSP_RES_FATAL; /* Function result */ int res; /* Processing result */ $? "+ dk4tsp32_add_bytes" #if DK4_USE_ASSERT assert(NULL != tsp); assert(NULL != buffer); assert(0 < sz); #endif if ((NULL != tsp) && (NULL != buffer) && (0 < sz)) { if (2 > tsp->pst) { back = DK4_TSP_RES_OK; while ((sz--) && (2 > tsp->pst)) { res = dk4tsp32_i_add_one_byte(tsp, *(buffer++)); switch (res) { case DK4_TSP_RES_FATAL: { back = DK4_TSP_RES_FATAL; } break; case DK4_TSP_RES_ERROR: { if (DK4_TSP_RES_OK == back) { back = DK4_TSP_RES_ERROR; } } break; } } } } $? "- dk4tsp32_add_bytes %d", back return back; } int dk4tsp32_finish(dk4_tsp32_t *tsp) { size_t nrej = 0; /* Number of unused bytes */ size_t i = 0; /* Current unused byte index */ int back = DK4_TSP_RES_FATAL; /* Function result */ int res; /* Operation result */ unsigned char uc; /* Current unused byte */ $? "+ dk4tsp32_finish" #if DK4_USE_ASSERT assert(NULL != tsp); #endif if (NULL != tsp) { /* Flush all unprocessed output 1. If there are unprocessed bytes in the BOM detector as BOM detection was not completed, process these bytes. 2. If there are characters in the line buffer, finalize the line buffer text and process it. */ back = DK4_TSP_RES_OK; /* Retrieve an process bytes stored in BOM detector */ if (0 == tsp->pst) { tsp->pst = 1; nrej = dk4bom_detect_num_unused_bytes(&(tsp->bomd)); if (0 < nrej) { dk4tsp32_initialize_decoder(tsp); for (i = 0; ((i < nrej) && (2 > tsp->pst)); i++) { uc = dk4bom_detect_unused_byte(&(tsp->bomd), i); switch (dk4tsp32_process_byte(tsp, uc)) { case DK4_TSP_RES_FATAL: { back = DK4_TSP_RES_FATAL; tsp->pst = 2; } break; case DK4_TSP_RES_ERROR: { if (DK4_TSP_RES_OK == back) { back = DK4_TSP_RES_ERROR; } } break; } } } } /* Process final line */ if ((NULL != tsp->inbuf) && (0 < tsp->in_sz) && (NULL != (tsp->fct).lh)) { if (0 < tsp->in_us) { if (2 > tsp->pst) { if (tsp->in_us < tsp->in_sz) { (tsp->inbuf)[tsp->in_us] = (dk4_c32_t)0UL; res = (*((tsp->fct).lh))( tsp->obj,tsp->inbuf,(tsp->pos).lineno,&(tsp->er_pr) ); switch (res) { case DK4_TSP_RES_FATAL: { back = DK4_TSP_RES_FATAL; tsp->pst = 2; } break; case DK4_TSP_RES_ERROR: { if (DK4_TSP_RES_OK == back) { back = DK4_TSP_RES_ERROR; } } break; } } else { back = DK4_TSP_RES_FATAL; tsp->pst = 2; dk4error_set_with_position( &(tsp->er_en), DK4_E_BUFFER_TOO_SMALL, (tsp->pos).bytes, (tsp->pos).lineno, (tsp->pos).chars, (tsp->pos).charil ); } } } } } $? "- dk4tsp32_finish %d", back return back; } void dk4tsp32_get_errors(dk4_er_t *er_en, dk4_er_t *er_pr, dk4_tsp32_t const *tsp) { #if DK4_USE_ASSERT assert(NULL != tsp); #endif if (NULL != tsp) { if (NULL != er_en) { DK4_MEMCPY(er_en, &(tsp->er_en), sizeof(dk4_er_t)); } if (NULL != er_pr) { DK4_MEMCPY(er_pr, &(tsp->er_pr), sizeof(dk4_er_t)); } } }