%% options copyright owner = Dirk Krause copyright year = 2016-xxxx license = bsd %% header /** @file dk4strmco.h Compression streams. A compression stream is an additional dk4_stream_t object on top of an existring stream (referred to as downside stream). Data written to the compression stream is compressed using one or multiple algorithms: - Run-length encoding as preprocessing. - Flate compression or LZW compression. - ASCII-Hex or ASCII-85 output encoding. The results of the compression operations are passed to the downside stream. Note: Closing the compression stream does not close the downside stream. So you create a mixed stream (i.e. for PDF files) containing both plain text and compressed chunks. To add a compressed chunk open a compression stream on top of the existing output stream, pass data to the compression stream, close the compression stream and continue writing to the output stream. */ #ifndef DK4CONF_H_INCLUDED #include "dk4conf.h" #endif #ifndef DK4ERROR_H_INCLUDED #include "dk4error.h" #endif #ifndef DK4STRM_H_INCLUDED #include "dk4strm.h" #endif /** Preprocessing before compression. */ enum dk4_strmco_pp { DK4_STRMCO_PP_NONE = 0 , /**< No preprocessing. */ DK4_STRMCO_PP_RUNLENGTH /**< Use run-length before real compression. */ }; /** Compression method. */ enum dk4_strmco_co { DK4_STRMCO_CO_NONE = 0 , /**< No compression. */ DK4_STRMCO_CO_FLATE , /**< Use flate compression. */ DK4_STRMCO_CO_LZW /**< Use LZW compression for PS level 2. */ }; /** Output encoding. */ enum dk4_strmco_oe { DK4_STRMCO_OE_NONE = 0 , /**< No output encoding. */ DK4_STRMCO_OE_HEX , /**< Hexadecimal encoding. */ DK4_STRMCO_OE_ASCII85 /**< ASCII85 encoding. */ }; /** Flags for compression stream. */ enum { DK4_STRMCO_FL_EOD_PP = 1 , /**< Write EOD marker at end of preproc. */ DK4_STRMCO_FL_EOD_OE = 2 , /**< Write EOD marker at end of out enc. */ DK4_STRMCO_FL_NL_OE = 4 , /**< Break output lines before 80. */ DK4_STRMCO_FL_CR_NL = 8 , /**< Use CR/NL combination. */ DK4_STRMCO_FL_UPPER_OE = 16 , /**< Upper case characters in hex digits. */ }; #ifdef __cplusplus extern "C" { #endif /** Open a compressing stream on top of an existing stream. Only write operations can be performed on the new stream, no read operations. @param strm Low-level destination stream, must be writable. @param pp Preprocessing method. @param co Compression method. @param oe Output encoding. @param fl Flags. @param erp Error report, may be NULL. @return Valid pointer to new stream on success, NULL on error. */ dk4_stream_t * dk4strmco_open( dk4_stream_t *strm, enum dk4_strmco_pp pp, enum dk4_strmco_co co, enum dk4_strmco_oe oe, int fl, dk4_er_t *erp ); #ifdef __cplusplus } #endif /* vim: set ai sw=4 ts=4 : */ %% module #include "dk4conf.h" #if DK4_HAVE_ZLIB_H #ifndef ZLIB_H_INCLUDED #include #define ZLIB_H_INCLUDED 1 #endif #endif #include "dk4rle.h" #include "dk4a85e.h" #include "dk4strmco.h" #include "dk4mem.h" #include "dk4memrs.h" #include "dk4lzwe.h" $!trace-include #ifndef DK4_STRMCO_ZBUF_SIZE #define DK4_STRMCO_ZBUF_SIZE 4096 #endif /** Compressor data structure. */ typedef struct { char zbuf[DK4_STRMCO_ZBUF_SIZE]; /**< Output buffer. */ union { #if DK4_HAVE_ZLIB_H z_stream zs; /**< Flate compression structure. */ #endif dk4_lzwe_t lzw; /**< LZW compression structure. */ } u_co; /**< Compression structures. */ union { dk4_rl_enc_t rle; /**< Run-length encoder. */ } u_pp; /**< Preprocessing structures. */ union { dk4_a85_enc_t a85; /**< ASCII-85 encoder. */ } u_oe; /**< Output encoding structures. */ dk4_stream_t *dststrm; /**< Destination stream. */ size_t ll; /**< Line length. */ int pp; /**< Preprocessing. */ int co; /**< Compression. */ int oe; /**< Output encoding. */ int fl; /**< Flags. */ } compressor_t; /** Newline. */ static char nl[] = { '\n', '\0' }; /** Carriage return and newline. */ static char crnl[] = { '\r', '\n', '\0' }; /** Finalizer for ASCII-Hex encoding. */ static char ahfin[] = { '>', '\0' }; /** Finalizer for ASCII-85 encoding. */ static char a85fin[] = { '~', '>', '\0' }; /** Hexadecimal digits using lower case characters. */ static char lower[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', '\0' }; /** Hexadecimal digits using upper case characters. */ static char upper[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', '\0' }; /** Write newline to destination stream. @param co Compressor structure. @return 1 on success, 0 on error. */ static int dk4strmco_nl(compressor_t *co) { int back = 0; $? "+ dk4strmco_nl" if (0 != (DK4_STRMCO_FL_CR_NL & (co->fl))) { back = dk4stream_write(co->dststrm, crnl, 2, NULL); } else { back = dk4stream_write(co->dststrm, nl, 1, NULL); } $? "- dk4strmco_nl %d", back return back; } /** Write bytes to compressor at output encoding level. @param co Compressor to write to. @param b Buffer start address. @param sz Number of bytes to write. @param szo Address of variable for number of bytes written, may be NULL. @return 1 on success, 0 on error. */ static int dk4strmco_co_oe( compressor_t *co, const char *b, size_t sz, size_t *szo ) { char buf[8]; /* Buffer for hexadecimal digits. */ const char *dptr; /* Pointer to A85 encoding result. */ size_t osz = 0; /* Number of bytes processed. */ size_t szd = 0; /* A85 encoding result length. */ size_t i = 0; /* Index of current byte to process. */ size_t j = 0; /* Index of cur out char. */ int back = 0; /* Function result. */ int res; /* Result from various operations. */ unsigned us; /* Index in hex digits array. */ unsigned char uc; /* UC version of current character */ $? "+ dk4strmco_co_oe" switch (co->oe) { case DK4_STRMCO_OE_ASCII85 : { $? ". ASCII-85 encoding" back = 1; for (i = 0; ((i < sz) && (1 == back)); i++) { switch (dk4a85_enc_add(&(co->u_oe.a85), b[i], NULL)) { case DK4_EDSTM_ACCEPT : { osz++; } break; case DK4_EDSTM_FINISHED : { dptr = NULL; szd = 0; res = dk4a85_enc_output( &dptr, &szd, &(co->u_oe.a85), NULL ); if (0 < res) { if (0 < szd) { #if VERSION_BEFORE_20161019 res = dk4stream_write( co->dststrm, dptr, szd, NULL ); if (0 < res) { osz++; co->ll += szd; if (0 != (DK4_STRMCO_FL_NL_OE & (co->fl))) { if (74 <= co->ll) { res = dk4strmco_nl(co); if (0 < res) { co->ll = 0; } else { back = 0; } } } } else { back = 0; } #else back = 1; for (j = 0; ((j < szd) && (1 == back)); j++) { res = dk4stream_write_byte( co->dststrm, dptr[j], NULL ); if (0 < res) { osz++; co->ll += 1; if (75 <= co->ll) { res = dk4strmco_nl(co); if (0 < res) { co->ll = 0; } else { back = 0; } } } else { back = 0; } } #endif } else { osz++; } } else { back = 0; } } break; default : { back = 0; } break; } } } break; case DK4_STRMCO_OE_HEX : { $? ". ASCII-Hex encoding" back = 1; buf[2] = '\0'; for (i = 0; ((i < sz) && (1 == back)); i++) { uc = (unsigned char)(b[i]); us = 0x000FU & ((unsigned char)uc); if (0 != (DK4_STRMCO_FL_UPPER_OE & (co->fl))) { buf[1] = upper[us]; } else { buf[1] = lower[us]; } us = 0x000FU & (((unsigned char)uc) >> 4); if (0 != (DK4_STRMCO_FL_UPPER_OE & (co->fl))) { buf[0] = upper[us]; } else { buf[0] = lower[us]; } res = dk4stream_write(co->dststrm, buf, 2, NULL); co->ll += 2; if (0 < res) { osz++; if (0 != (DK4_STRMCO_FL_NL_OE & (co->fl))) { if (78 <= co->ll) { res = dk4strmco_nl(co); if (0 < res) { co->ll = 0; } else { back = 0; } } } } else { back = 0; } } } break; default : { $? ". no encoding" back = dk4stream_write(co->dststrm, b, sz, NULL); if (0 != back) { osz = sz; } } break; } if (NULL != szo) { *szo = osz; } $? "- dk4strmco_co_oe %d", back return back; } /** Write bytes to compressor at compression level. @param co Compressor to write to. @param b Buffer start address. @param sz Number of bytes to write. @param szo Address of variable for number of bytes written, may be NULL. @return 1 on success, 0 on error. */ static int dk4strmco_co_co( compressor_t *co, const char *b, size_t sz, size_t *szo ) { const unsigned char *cptr; size_t osz = 0; /* Number of bytes processed successfully. */ size_t used = 0; /* Used bytes in output buffer. */ size_t i = 0; /* Index of current LZW byte to process. */ size_t lzwsz = 0; /* Bytes from LZW compression. */ int back = 0; /* Function result. */ int res = 0; /* Result from various operations. */ $? "+ dk4strmco_co_co" switch (co->co) { case DK4_STRMCO_CO_FLATE : { $? ". flate compression" back = 1; co->u_co.zs.next_in = (unsigned char *)b; co->u_co.zs.avail_in = sz; do { co->u_co.zs.next_out = (unsigned char *)(&(co->zbuf[0])); co->u_co.zs.avail_out = DK4_STRMCO_ZBUF_SIZE; res = deflate(&(co->u_co.zs), Z_NO_FLUSH); switch (res) { case Z_OK : { used = DK4_STRMCO_ZBUF_SIZE - co->u_co.zs.avail_out; if (0 < used) { res = dk4strmco_co_oe(co,&(co->zbuf[0]),used,NULL); if (0 < res) { osz = sz - co->u_co.zs.avail_in; } else { back = 0; } } else { osz = sz - co->u_co.zs.avail_in; } } break; default : { back = 0; } break; } } while ((0 < co->u_co.zs.avail_in) && (1 == back)); if (NULL != szo) { *szo = osz; } } break; case DK4_STRMCO_CO_LZW : { back = 1; for (i = 0; ((1 == back) && (i < sz)); i++) { res = dk4lzwe_add(&(co->u_co.lzw), (unsigned char)(b[i]), NULL); switch (res) { case DK4_EDSTM_ACCEPT : { osz++; } break; case DK4_EDSTM_FINISHED : { cptr = NULL; lzwsz = 0; res = dk4lzwe_output( &cptr, &lzwsz, &(co->u_co.lzw), NULL ); if ((0 != res) && (NULL != cptr) && (0 < lzwsz)) { back = dk4strmco_co_oe( co, (const char *)cptr, lzwsz, NULL ); if (0 != back) { osz++; } } else { back = 0; } } break; case DK4_EDSTM_ERROR : { back = 0; } break; } } if (NULL != szo) { *szo = osz; } } break; default : { $? ". no compression" back = dk4strmco_co_oe(co, b, sz, szo); } break; } $? "- dk4strmco_co_co %d", back return back; } /** Write bytes to compressor at preprocessor level. @param co Compressor to write to. @param b Buffer start address. @param sz Number of bytes to write. @param szo Address of variable for number of bytes written, may be NULL. @return 1 on success, 0 on error. */ static int dk4strmco_co_write( compressor_t *co, const char *b, size_t sz, size_t *szo ) { const unsigned char *rleptr; /* RL encoding result */ size_t osz = 0; /* Bytes processed successfully */ size_t i; /* Running index of current byte */ size_t rlesz; /* Size of RL encoding result */ int back = 0; /* Function result */ int res; /* Result from various operations */ $? "+ dk4strmco_co_write" switch (co->pp) { case DK4_STRMCO_PP_RUNLENGTH : { $? ". run-length preprocessing" back = 1; for (i = 0; ((i < sz) && (0 != back)); i++) { switch (dk4rle_add(&(co->u_pp.rle),(unsigned char)(b[i]),NULL)) { case DK4_EDSTM_FINISHED : { rleptr = NULL; rlesz = 0; res = dk4rle_output( &rleptr, &rlesz, &(co->u_pp.rle), NULL ); if (0 != res) { res = dk4strmco_co_co( co, (const char *)rleptr, rlesz, NULL ); if (0 != res) { osz++; } else { back = 0; } } else { back = 0; } } break; case DK4_EDSTM_ERROR : { back = 0; } break; default : { osz++; } break; } } if (NULL != szo) { *szo = osz; } } break; default : { $? ". no preprocessing" back = dk4strmco_co_co(co, b, sz, szo); } break; } $? "- dk4strmco_co_write %d", back return back; } /** Flush data in output encoders if used. @param co Compressor structure. @return 1 on success, 0 on error. */ static int dk4strmco_co_oe_close(compressor_t *co) { const char *dptr = NULL; size_t sz = 0; size_t j = 0; int back = 1; int res = 0; $? "+ dk4strmco_co_oe_close" switch (co->oe) { case DK4_STRMCO_OE_ASCII85 : { $? ". ASCII-85" switch (dk4a85_enc_finish(&(co->u_oe.a85), NULL)) { $? ". handle final A85 data" case DK4_EDSTM_FINISHED : { if (0 < dk4a85_enc_output(&dptr,&sz,&(co->u_oe.a85),NULL)) { if (0 < sz) { #if VERSION_BEFORE_20161019 back = dk4stream_write(co->dststrm, dptr, sz, NULL); #else back = 1; for (j = 0; ((j < sz) && (1 == back)); j++) { res = dk4stream_write_byte( co->dststrm, dptr[j], NULL ); if (0 < res) { co->ll += 1; if (75 <= co->ll) { res = dk4strmco_nl(co); if (0 < res) { co->ll = 0; } else { back = 0; } } } else { back = 0; } } #endif } } else { back = 0; } } break; } if (0 != (DK4_STRMCO_FL_EOD_OE & (co->fl))) { $? ". write EOD marker" res = dk4stream_write(co->dststrm, a85fin, 2, NULL); if (0 == res) { back = 0; } } if (0 != (DK4_STRMCO_FL_NL_OE & (co->fl))) { $? ". line end" if (0 == dk4strmco_nl(co)) { back = 0; } } } break; case DK4_STRMCO_OE_HEX : { if (0 != (DK4_STRMCO_FL_EOD_OE & (co->fl))) { $? ". write EOD marker" back = dk4stream_write(co->dststrm, ahfin, 1, NULL); } if (0 != (DK4_STRMCO_FL_NL_OE & (co->fl))) { $? ". line end" if (0 == dk4strmco_nl(co)) { back = 0; } } } break; } $? "- dk4strmco_co_oe_close %d", back return back; } /** Flush data from flate compressor if used. @param co Compressor structure. @return 1 on success, 0 on error. */ static int dk4strmco_co_co_close(compressor_t *co) { const unsigned char *cptr; size_t used = 0; int back = 1; int zres = Z_OK; int res = 0; $? "+ dk4strmco_co_co_close" switch (co->co) { case DK4_STRMCO_CO_FLATE : { $? ". flate compression used" co->u_co.zs.next_in = (unsigned char *)nl; co->u_co.zs.avail_in = 0; do { co->u_co.zs.next_out = (unsigned char *)(&(co->zbuf[0])); co->u_co.zs.avail_out = DK4_STRMCO_ZBUF_SIZE; zres = deflate(&(co->u_co.zs), Z_FINISH); $? ". zres = %d", zres switch (zres) { case Z_OK : case Z_STREAM_END : { used = DK4_STRMCO_ZBUF_SIZE - co->u_co.zs.avail_out; $? ". used = %u", (unsigned)used if (0 < used) { res = dk4strmco_co_oe(co,&(co->zbuf[0]),used,NULL); if (0 == res) { back = 0; } } } break; default : { back = 0; } break; } } while ((Z_OK == zres) && (1 == back)); $? ". deflateEnd" if (Z_OK != deflateEnd(&(co->u_co.zs))) { back = 0; } } break; case DK4_STRMCO_CO_LZW : { res = dk4lzwe_finish(&(co->u_co.lzw), NULL); switch (res) { case DK4_EDSTM_FINISHED : { cptr = NULL; used = 0; res = dk4lzwe_output( &cptr, &used, &(co->u_co.lzw), NULL ); if ((0 != res) && (NULL != cptr) && (0 < used)) { back = dk4strmco_co_oe( co, (const char *)cptr, used, NULL ); } else { back = 0; } } break; case DK4_EDSTM_ERROR : { back = 0; } break; } } break; } $? "- dk4strmco_co_co_close %d", back return back; } /** Flush data from run-length encoder if used. @param co Compressor structure. @return 1 on success, 0 on error. */ static int dk4strmco_co_pp_close(compressor_t *co) { const unsigned char *dptr = NULL; size_t szd = 0; int back = 1; int res; $? "+ dk4strmco_co_pp_close" switch (co->pp) { case DK4_STRMCO_PP_RUNLENGTH : { $? ". run-length preprocessing used" switch (dk4rle_finish(&(co->u_pp.rle), NULL)) { case DK4_EDSTM_ACCEPT : { } break; case DK4_EDSTM_FINISHED : { $? ". data in encoder" if (0 < dk4rle_output(&dptr, &szd, &(co->u_pp.rle), NULL)) { res = dk4strmco_co_write( co, (const char *)dptr, szd, NULL ); if (0 == res) { $? "! failed to write final data" back = 0; } } else { $? "! failed to retrieve final data" back = 0; } } break; default : { } break; } } break; } $? "- dk4strmco_co_pp_close %d", back return back; } /** Finish output (flush data stored in the encoders). @param co Compressor structure. @return 1 on success, 0 on error. */ static int dk4strmco_co_close(compressor_t *co) { int back = 1; $? "+ dk4strmco_co_close" /* Flush preprocessor (run-length encoder). */ if (0 == dk4strmco_co_pp_close(co)) { back = 0; } /* Flush compressor (flate stream). */ if (0 == dk4strmco_co_co_close(co)) { back = 0; } /* Flush output encoder (ASCII-Hex or ASCII-85 encoder). */ if (0 == dk4strmco_co_oe_close(co)) { back = 0; } /* Release memory. */ $? ". reset memory" if (0 == dk4mem_reset_secure(co, sizeof(compressor_t), NULL)) { back = 0; } $? ". release memory" dk4mem_free(co); $? "- dk4strmco_co_close %d", back return back; } void dk4strmco_handler(dk4_stream_api_t *api) { compressor_t *co = NULL; $? "+ dk4strmco_handler %d", api->cmd if (NULL != api) { api->res = 0; api->sz_out = 0; co = (compressor_t *)(api->d); switch (api->cmd) { case DK4_STREAM_API_WRITE : { $? ". write" api->res = dk4strmco_co_write( co, api->b, api->sz_in, &(api->sz_out) ); } break; case DK4_STREAM_API_FLUSH : { $? ". flush (ignored)" /* Do nothing but indicate success. Otherwise closing a stream will return an error. */ api->res = 1; } break; case DK4_STREAM_API_CLOSE : { $? ". close" api->res = dk4strmco_co_close(co); } break; case DK4_STREAM_API_ZERO_READ_IS_EOF : { $? ". zero read is eof" api->res = 0; } break; default : { $? "! any else" /* Error: Functions not available */ } break; } } $? "- dk4strmco_handler %d", api->res } dk4_stream_t * dk4strmco_open( dk4_stream_t *strm, enum dk4_strmco_pp pp, enum dk4_strmco_co co, enum dk4_strmco_oe oe, int fl, dk4_er_t *erp ) { dk4_stream_t *back = NULL; compressor_t *cc = NULL; $? "+ dk4strmco_open" /* Allocate compressor and set up. */ $? ". allocate memory" cc = dk4mem_new(compressor_t,1,erp); if (NULL != cc) { cc->dststrm = strm; cc->pp = pp; cc->co = co; cc->oe = oe; cc->fl = fl; cc->ll = 0; switch (pp) { case DK4_STRMCO_PP_RUNLENGTH : { $? ". initialize run-length encoder" dk4rle_init( &(cc->u_pp.rle), ((0 != (DK4_STRMCO_FL_EOD_PP & fl)) ? 1 : 0), erp ); } break; default : { } break; } switch (co) { case DK4_STRMCO_CO_FLATE : { $? ". initialize flate compression" cc->u_co.zs.zalloc = Z_NULL; cc->u_co.zs.zfree = Z_NULL; cc->u_co.zs.opaque = Z_NULL; $? ". deflateInit" if (Z_OK != deflateInit(&(cc->u_co.zs), 9)) { $? ". release memory" dk4mem_free(cc); cc = NULL; $? "! failed to initialize flate compression" } } break; case DK4_STRMCO_CO_LZW : { $? ". initialize LZW compression" if (0 == dk4lzwe_init(&(cc->u_co.lzw), NULL)) { $? ". release memory" dk4mem_free(cc); cc = NULL; $? "! failed to initialize LZW compression" } } break; default : { } break; } switch (oe) { case DK4_STRMCO_OE_ASCII85 : { $? ". initialize ASCII-85 output encoding" dk4a85_enc_init(&(cc->u_oe.a85), 1, erp); } break; default : { } break; } } #if TRACE_DEBUG else { $? "! memory" } #endif /* If the compressor was set up successfully, create stream. */ if (NULL != cc) { $? ". create stream" back = dk4stream_open(cc,dk4strmco_handler,DK4_STREAM_WRITE,0,4096,erp); /* When failed to create stream, we must release the compressor. */ if (NULL == back) { /* Release dynamic resources assigned to components. */ $? "! failed to create stream" switch (co) { case DK4_STRMCO_CO_FLATE : { $? ". deflateEnd" (void)deflateEnd(&(cc->u_co.zs)); } break; case DK4_STRMCO_CO_LZW : { $? ". LZW finish" (void)dk4lzwe_finish(&(cc->u_co.lzw), NULL); } break; default : { } break; } $? ". release memory" dk4mem_free(cc); } #if TRACE_DEBUG else { $? ". stream created" } #endif } $? "- dk4strmco_open %d", TR_IPTR(back) return back; } /* vim: set ai sw=4 ts=4 : */