diff options
author | Luigi Scarso <luigi.scarso@gmail.com> | 2020-04-21 18:43:36 +0000 |
---|---|---|
committer | Luigi Scarso <luigi.scarso@gmail.com> | 2020-04-21 18:43:36 +0000 |
commit | 7c0b908f1a6e1489834fbdb0789766eed8a37b49 (patch) | |
tree | cb55d631b861bfcf95fe853713af6f760227ba0f /Build/source/libs/pplib/pplib-src/src/util | |
parent | e783b071ded7eef421d0333416b47142bc5542cb (diff) |
pplib under libs -- WORK IN PROGRSS grep '?' out
git-svn-id: svn://tug.org/texlive/trunk@54824 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/libs/pplib/pplib-src/src/util')
34 files changed, 14278 insertions, 0 deletions
diff --git a/Build/source/libs/pplib/pplib-src/src/util/README.md b/Build/source/libs/pplib/pplib-src/src/util/README.md new file mode 100644 index 00000000000..209bced0082 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/README.md @@ -0,0 +1,8 @@ +# pplib util
+
+This part is a toolbox. Contains utilities that are used by `pplib`
+but aren't tightly related to `PDF`. I use the toolbox in different
+projects and repos. It is important to me to keep this part in a perfect
+sync, at the cost of some redundant code (not used in `pplib`).
+`pplib` is hopefully not a subject for eternal development, so once
+it become final, we will make some cleanups here.
diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilbasexx.c b/Build/source/libs/pplib/pplib-src/src/util/utilbasexx.c new file mode 100644 index 00000000000..cfe14884071 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilbasexx.c @@ -0,0 +1,1742 @@ + +#include "utilnumber.h" +#include "utilmem.h" +#include "utilbasexx.h" + +/* filters state structs */ + +struct basexx_state { + size_t line, maxline; + size_t left; + int tail[5]; + int flush; +}; + +struct runlength_state { + int run; + int flush; + int c1, c2; + uint8_t *pos; +}; + +typedef union { basexx_state *basexxstate; runlength_state *runlengthstate; void *voidstate; } basexx_state_pointer; // to avoid 'dereferencing type-puned ...' warnings + +/* config */ + +#if defined(BASEXX_PDF) +# define ignored(c) (c == 0x20 || c == 0x0A || c == 0x0C || c == 0x0D || c == 0x09 || c == 0x00) +# define base16_eof(c) (c == '>' || c < 0) +# define base85_eof(c) (c == '~' || c < 0) +#else +# define ignored(c) (c == 0x20 || c == 0x0A || c == 0x0D || c == 0x09) +# define base16_eof(c) (c < 0) +# define base85_eof(c) (c < 0) +#endif + +#define base64_eof(c) (c == '=' || c < 0) + +#define basexx_nl '\x0A' +//#define put_nl(O, line, maxline, n) ((void)((line += n) > maxline && ((line = n), iof_set(O, basexx_nl)))) // assignment in conditional warning +#define put_nl(O, line, maxline, n) do { line += n; if (line > maxline) { line = n; iof_set(O, basexx_nl); }} while (0) + +/* tail macros */ + +#define set_tail1(state, c1) (state->left = 1, state->tail[0] = c1) +#define set_tail2(state, c1, c2) (state->left = 2, state->tail[0] = c1, state->tail[1] = c2) +#define set_tail3(state, c1, c2, c3) (state->left = 3, state->tail[0] = c1, state->tail[1] = c2, state->tail[2] = c3) +#define set_tail4(state, c1, c2, c3, c4) (state->left = 4, state->tail[0] = c1, state->tail[1] = c2, state->tail[2] = c3, state->tail[3] = c4) +#define set_tail5(state, c1, c2, c3, c4, c5) \ + (state->left = 5, state->tail[0] = c1, state->tail[1] = c2, state->tail[2] = c3, state->tail[3] = c4, state->tail[4] = c5) + +#define get_tail1(state, c1) (state->left = 0, c1 = state->tail[0]) +#define get_tail2(state, c1, c2) (state->left = 0, c1 = state->tail[0], c2 = state->tail[1]) +#define get_tail3(state, c1, c2, c3) (state->left = 0, c1 = state->tail[0], c2 = state->tail[1], c3 = state->tail[2]) +#define get_tail4(state, c1, c2, c3, c4) (state->left = 0, c1 = state->tail[0], c2 = state->tail[1], c3 = state->tail[2], c4 = state->tail[3]) + +/* basexx state initialization */ + +void basexx_state_init_ln (basexx_state *state, size_t line, size_t maxline) +{ + state->line = line; + state->maxline = maxline; + state->left = 0; + state->flush = 0; +} + +/* base 16; xxxx|xxxx */ + +iof_status base16_encoded_uc (const void *data, size_t size, iof *O) +{ + const uint8_t *s, *e; + for (s = (const uint8_t *)data, e = s + size; s < e; ++s) + { + if (!iof_ensure(O, 2)) + return IOFFULL; + iof_set_uc_hex(O, *s); + } + return IOFEOF; +} + +iof_status base16_encoded_lc (const void *data, size_t size, iof *O) +{ + const uint8_t *s, *e; + for (s = (const uint8_t *)data, e = s + size; s < e; ++s) + { + if (!iof_ensure(O, 2)) + return IOFFULL; + iof_set_lc_hex(O, *s); + } + return IOFEOF; +} + +iof_status base16_encoded_uc_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline) +{ + const uint8_t *s, *e; + for (s = (const uint8_t *)data, e = s + size; s < e; ++s) + { + if (!iof_ensure(O, 3)) + return IOFFULL; + put_nl(O, line, maxline, 2); + iof_set_uc_hex(O, *s); + } + return IOFFULL; +} + +iof_status base16_encoded_lc_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline) +{ + const uint8_t *s, *e; + for (s = (const uint8_t *)data, e = s + size; s < e; ++s) + { + if (!iof_ensure(O, 3)) + return IOFFULL; + put_nl(O, line, maxline, 2); + iof_set_lc_hex(O, *s); + } + return IOFFULL; +} + +iof_status base16_encode_uc (iof *I, iof *O) +{ + register int c; + while (iof_ensure(O, 2)) + { + if ((c = iof_get(I)) < 0) + return IOFEOF; + iof_set_uc_hex(O, c); + } + return IOFFULL; +} + +iof_status base16_encode_state_uc (iof *I, iof *O, basexx_state *state) +{ + register int c; + while (iof_ensure(O, 2)) + { + if ((c = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + iof_set_uc_hex(O, c); + } + return IOFFULL; +} + +iof_status base16_encode_lc (iof *I, iof *O) +{ + register int c; + while (iof_ensure(O, 2)) + { + if ((c = iof_get(I)) < 0) + return IOFEOF; + iof_set_lc_hex(O, c); + } + return IOFFULL; +} + +iof_status base16_encode_state_lc (iof *I, iof *O, basexx_state *state) +{ + register int c; + while (iof_ensure(O, 2)) + { + if ((c = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + iof_set_lc_hex(O, c); + } + return IOFFULL; +} + +iof_status base16_encode_uc_ln (iof *I, iof *O, size_t line, size_t maxline) +{ + register int c; + while (iof_ensure(O, 3)) + { + if ((c = iof_get(I)) < 0) + return IOFEOF; + put_nl(O, line, maxline, 2); + iof_set_uc_hex(O, c); + } + return IOFFULL; +} + +iof_status base16_encode_state_uc_ln (iof *I, iof *O, basexx_state *state) +{ + register int c; + while (iof_ensure(O, 3)) + { + if ((c = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + put_nl(O, state->line, state->maxline, 2); + iof_set_uc_hex(O, c); + } + return IOFFULL; +} + +iof_status base16_encode_lc_ln (iof *I, iof *O, size_t line, size_t maxline) +{ + register int c; + while (iof_ensure(O, 3)) + { + if ((c = iof_get(I)) < 0) + return IOFEOF; + put_nl(O, line, maxline, 2); + iof_set_lc_hex(O, c); + } + return IOFFULL; +} + +iof_status base16_encode_state_lc_ln (iof *I, iof *O, basexx_state *state) +{ + register int c; + while (iof_ensure(O, 3)) + { + if ((c = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + put_nl(O, state->line, state->maxline, 2); + iof_set_lc_hex(O, c); + } + return IOFFULL; +} + +int base16_getc (iof *I) +{ + register int c1, c2; + do { c1 = iof_get(I); } while (ignored(c1)); + if (base16_eof(c1)) + return IOFEOF; + do { c2 = iof_get(I); } while (ignored(c2)); + if (base16_eof(c2)) + { + if ((c1 = base16_value(c1)) < 0) + return IOFERR; + return c1<<4; + } + if ((c1 = base16_value(c1)) < 0 || (c2 = base16_value(c2)) < 0) + return IOFERR; + return (c1<<4)|c2; +} + +int base16_lc_putc (iof *O, int c) +{ + if (iof_ensure(O, 2)) + iof_set_lc_hex(O, c); + return IOFFULL; +} + +int base16_uc_putc (iof *O, int c) +{ + if (iof_ensure(O, 2)) + iof_set_uc_hex(O, c); + return IOFFULL; +} + + +iof_status base16_decode (iof *I, iof *O) +{ + register int c1, c2; + while (iof_ensure(O, 1)) + { + do { c1 = iof_get(I); } while (ignored(c1)); + if (base16_eof(c1)) + return IOFEOF; + do { c2 = iof_get(I); } while (ignored(c2)); + if (base16_eof(c2)) + { + if ((c1 = base16_value(c1)) < 0) + return IOFERR; + iof_set(O, c1<<4); // c2 := '0' + return IOFEOF; + } + if ((c1 = base16_value(c1)) < 0 || (c2 = base16_value(c2)) < 0) + return IOFERR; + iof_set(O, (c1<<4)|c2); + } + return IOFFULL; +} + +iof_status base16_decode_state (iof *I, iof *O, basexx_state *state) +{ + register int c1, c2, d1, d2; + if (!(iof_ensure(O, 1))) + return IOFFULL; + switch(state->left) + { + case 0: goto byte0; + case 1: get_tail1(state, c1); goto byte1; + } + while (iof_ensure(O, 1)) + { + byte0: + do { c1 = iof_get(I); } while (ignored(c1)); + if (base16_eof(c1)) + return (state->flush ? IOFEOF : IOFEMPTY); + byte1: + do { c2 = iof_get(I); } while (ignored(c2)); + if (base16_eof(c2)) + { + set_tail1(state, c1); /* set tail to let the caller display invalid chars */ + if (state->flush) + { + if ((c1 = base16_value(c1)) < 0) + return IOFERR; + iof_set(O, c1<<4); // c2 := '0' + return IOFEOF; + } + return IOFEMPTY; + } + if ((d1 = base16_value(c1)) < 0 || (d2 = base16_value(c2)) < 0) + { + set_tail2(state, c1, c2); + return IOFERR; + } + iof_set(O, (d1<<4)|d2); + } + return IOFFULL; +} + +/* base 64; xxxxxx|xx xxxx|xxxx xx|xxxxxx */ + +const char base64_alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +const int base64_lookup[] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,62,-1,-1,-1,63, + 52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-1,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8,9 ,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1, + -1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, + 41,42,43,44,45,46,47,48,49,50,51,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +}; + +#define base64_value(c) base64_lookup[(uint8_t)(c)] + +#define base64_digit1(c1) base64_alphabet[c1>>2] +#define base64_digit2(c1, c2) base64_alphabet[((c1&3)<<4)|(c2>>4)] +#define base64_digit3(c2, c3) base64_alphabet[((c2&15)<<2)|(c3>>6)] +#define base64_digit4(c3) base64_alphabet[c3&63] + +#define base64_encode_word(O, c1, c2, c3) \ + iof_set4(O, base64_digit1(c1), base64_digit2(c1, c2), base64_digit3(c2, c3), base64_digit4(c3)) + +#define base64_encode_tail2(O, c1, c2) \ + iof_set3(O, base64_digit1(c1), base64_digit2(c1, c2), base64_digit3(c2, 0)) + +#define base64_encode_tail1(O, c1) \ + iof_set2(O, base64_digit1(c1), base64_digit2(c1, 0)) + +iof_status base64_encoded (const void *data, size_t size, iof *O) +{ + const uint8_t *s, *e; + uint8_t c1, c2, c3; + for (s = (const uint8_t *)data, e = s + size; s + 2 < e; ) + { + if (!iof_ensure(O, 4)) + return IOFFULL; + c1 = *s++; + c2 = *s++; + c3 = *s++; + base64_encode_word(O, c1, c2, c3); + } + switch (e - s) + { + case 0: + break; + case 1: + if (!iof_ensure(O, 2)) + return IOFFULL; + c1 = *s; + base64_encode_tail1(O, c1); + break; + case 2: + if (!iof_ensure(O, 3)) + return IOFFULL; + c1 = *s++; + c2 = *s; + base64_encode_tail2(O, c1, c2); + break; + } + return IOFEOF; +} + +iof_status base64_encoded_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline) +{ + const uint8_t *s, *e; + uint8_t c1, c2, c3; + for (s = (const uint8_t *)data, e = s + size; s + 2 < e; ) + { + if (!iof_ensure(O, 5)) + return IOFFULL; + c1 = *s++; + c2 = *s++; + c3 = *s++; + put_nl(O, line, maxline, 4); + base64_encode_word(O, c1, c2, c3); + } + switch (e - s) + { + case 0: + break; + case 1: + if (!iof_ensure(O, 3)) + return IOFFULL; + c1 = *s; + put_nl(O, line, maxline, 2); + base64_encode_tail1(O, c1); + break; + case 2: + if (!iof_ensure(O, 4)) + return IOFFULL; + c1 = *s++; + c2 = *s; + put_nl(O, line, maxline, 3); + base64_encode_tail2(O, c1, c2); + break; + } + return IOFEOF; +} + +iof_status base64_encode (iof *I, iof *O) +{ + register int c1, c2, c3; + while(iof_ensure(O, 4)) + { + if ((c1 = iof_get(I)) < 0) + return IOFEOF; + if ((c2 = iof_get(I)) < 0) + { + base64_encode_tail1(O, c1); + return IOFEOF; + } + if ((c3 = iof_get(I)) < 0) + { + base64_encode_tail2(O, c1, c2); + return IOFEOF; + } + base64_encode_word(O, c1, c2, c3); + } + return IOFFULL; +} + +iof_status base64_encode_state (iof *I, iof *O, basexx_state *state) +{ + register int c1, c2, c3; + if (!(iof_ensure(O, 4))) + return IOFFULL; + switch(state->left) + { + case 0: goto byte0; + case 1: get_tail1(state, c1); goto byte1; + case 2: get_tail2(state, c1, c2); goto byte2; + } + while(iof_ensure(O, 4)) + { + byte0: + if ((c1 = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + byte1: + if ((c2 = iof_get(I)) < 0) + return (state->flush ? (base64_encode_tail1(O, c1), IOFEOF) : (set_tail1(state, c1), IOFEMPTY)); + byte2: + if ((c3 = iof_get(I)) < 0) + return (state->flush ? (base64_encode_tail2(O, c1, c2), IOFEOF) : (set_tail2(state, c1, c2), IOFEMPTY)); + base64_encode_word(O, c1, c2, c3); + } + return IOFFULL; +} + +iof_status base64_encode_ln (iof *I, iof *O, size_t line, size_t maxline) +{ + register int c1, c2, c3; + while(iof_ensure(O, 5)) + { + if ((c1 = iof_get(I)) < 0) + return IOFEOF; + if ((c2 = iof_get(I)) < 0) + { + put_nl(O, line, maxline, 2); + base64_encode_tail1(O, c1); + return IOFEOF; + } + if ((c3 = iof_get(I)) < 0) + { + put_nl(O, line, maxline, 3); + base64_encode_tail2(O, c1, c2); + return IOFEOF; + } + put_nl(O, line, maxline, 4); + base64_encode_word(O, c1, c2, c3); + } + return IOFFULL; +} + +iof_status base64_encode_state_ln (iof *I, iof *O, basexx_state *state) +{ + register int c1, c2, c3; + if (!(iof_ensure(O, 5))) + return IOFFULL; + switch(state->left) + { + case 0: goto byte0; + case 1: get_tail1(state, c1); goto byte1; + case 2: get_tail2(state, c1, c2); goto byte2; + } + while(iof_ensure(O, 5)) + { + byte0: + if ((c1 = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + byte1: + if ((c2 = iof_get(I)) < 0) + { + if (state->flush) + { + put_nl(O, state->line, state->maxline, 2); + base64_encode_tail1(O, c1); + return IOFEOF; + } + set_tail1(state, c1); + return IOFEMPTY; + } + byte2: + if ((c3 = iof_get(I)) < 0) + { + if (state->flush) + { + put_nl(O, state->line, state->maxline, 3); + base64_encode_tail2(O, c1, c2); + return IOFEOF; + } + set_tail2(state, c1, c2); + return IOFEMPTY; + } + put_nl(O, state->line, state->maxline, 4); + base64_encode_word(O, c1, c2, c3); + } + return IOFFULL; +} + +// #define base64_code(c1, c2, c3, c4) ((c1<<18)|(c2<<12)|(c3<<6)|c4) + +#define base64_decode_word(O, c1, c2, c3, c4) \ + iof_set3(O, (c1<<2)|(c2>>4), ((c2&15)<<4)|(c3>>2), ((c3&3)<<6)|c4) + +#define base64_decode_tail3(O, c1, c2, c3) \ + iof_set2(O, (c1<<2)|(c2>>4), ((c2&15)<<4)|(c3>>2)) + +#define base64_decode_tail2(O, c1, c2) \ + iof_set(O, (c1<<2)|(c2>>4)) + +iof_status base64_decode (iof *I, iof *O) +{ + register int c1, c2, c3, c4; + while(iof_ensure(O, 3)) + { + do { c1 = iof_get(I); } while (ignored(c1)); + if (base64_eof(c1)) + return IOFEOF; + do { c2 = iof_get(I); } while (ignored(c2)); + if (base64_eof(c2)) + return IOFERR; + do { c3 = iof_get(I); } while (ignored(c3)); + if (base64_eof(c3)) + { + if ((c1 = base64_value(c1)) < 0 || (c2 = base64_value(c2)) < 0) + return IOFERR; + base64_decode_tail2(O, c1, c2); + return IOFEOF; + } + do { c4 = iof_get(I); } while (ignored(c4)); + if (base64_eof(c4)) + { + if ((c1 = base64_value(c1)) < 0 || (c2 = base64_value(c2)) < 0 || (c3 = base64_value(c3)) < 0) + return IOFERR; + base64_decode_tail3(O, c1, c2, c3); + return IOFEOF; + } + if ((c1 = base64_value(c1)) < 0 || (c2 = base64_value(c2)) < 0 || + (c3 = base64_value(c3)) < 0 || (c4 = base64_value(c4)) < 0) + return IOFERR; + base64_decode_word(O, c1, c2, c3, c4); + } + return IOFFULL; +} + +iof_status base64_decode_state (iof *I, iof *O, basexx_state *state) +{ + register int c1, c2, c3, c4; + register int d1, d2, d3, d4; + switch(state->left) + { + case 0: goto byte0; + case 1: get_tail1(state, c1); goto byte1; + case 2: get_tail2(state, c1, c2); goto byte2; + case 3: get_tail3(state, c1, c2, c3); goto byte3; + } + while(iof_ensure(O, 3)) + { + byte0: + do { c1 = iof_get(I); } while (ignored(c1)); + if (base64_eof(c1)) + return (state->flush ? IOFEOF : IOFEMPTY); + byte1: + do { c2 = iof_get(I); } while (ignored(c2)); + if (base64_eof(c2)) + { + set_tail1(state, c1); /* set tail to let the caller make padding or display invalid char in case of error */ + return (state->flush ? IOFERR : IOFEMPTY); /* if state->flush then error; tail must have at least two bytes */ + } + byte2: + do { c3 = iof_get(I); } while (ignored(c3)); + if (base64_eof(c3)) + { + set_tail2(state, c1, c2); + if (state->flush) + { + if ((c1 = base64_value(c1)) < 0 || (c2 = base64_value(c2)) < 0) + return IOFERR; + base64_decode_tail2(O, c1, c2); + return IOFEOF; + } + else + return IOFEMPTY; + } + byte3: + do { c4 = iof_get(I); } while (ignored(c4)); + if (base64_eof(c4)) + { + set_tail3(state, c1, c2, c3); + if (state->flush) + { + if ((c1 = base64_value(c1)) < 0 || (c2 = base64_value(c2)) < 0 || (c3 = base64_value(c3)) < 0) + return IOFERR; + base64_decode_tail3(O, c1, c2, c3); + return IOFEOF; + } + else + return IOFEMPTY; + } + if ((d1 = base64_value(c1)) < 0 || (d2 = base64_value(c2)) < 0 || + (d3 = base64_value(c3)) < 0 || (d4 = base64_value(c4)) < 0) + { + set_tail4(state, c1, c2, c3, c4); + return IOFERR; + } + base64_decode_word(O, d1, d2, d3, d4); + } + return IOFFULL; +} + +/* base85 */ + +const char base85_alphabet[] = "!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu"; /* for completness, not used below */ + +const int base85_lookup[] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30, + 31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46, + 47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62, + 63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78, + 79,80,81,82,83,84,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +}; + +#define base85_value(c) base85_lookup[(uint8_t)(c)] + +#define base85_encode_word(O, code) \ + (*(O->pos+4) = '!' + code%85, code /= 85, *(O->pos+3) = '!' + code%85, code /= 85, \ + *(O->pos+2) = '!' + code%85, code /= 85, *(O->pos+1) = '!' + code%85, code /= 85, \ + *(O->pos) = '!' + code, \ + O->pos += 5) + +#define base85_encode_tail3(O, code) \ + (*(O->pos+3) = '!' + code%85, code /= 85, *(O->pos+2) = '!' + code%85, code /= 85, \ + *(O->pos+1) = '!' + code%85, code /= 85, *(O->pos) = '!' + code, \ + O->pos += 4) + +#define base85_encode_tail2(O, code) \ + (*(O->pos+2) = '!' + code%85, code /= 85, *(O->pos+1) = '!' + code%85, code /= 85, \ + *(O->pos) = '!' + code, \ + O->pos += 3) + +#define base85_encode_tail1(O, code) \ + (*(O->pos+1) = '!' + code%85, code /= 85, *(O->pos) = '!' + code, \ + O->pos += 2) + +iof_status base85_encoded (const void *data, size_t size, iof *O) +{ + unsigned int code; + const uint8_t *s, *e; + uint8_t c1, c2, c3, c4; + for (s = (const uint8_t *)data, e = s + size; s + 3 < e; ) + { + if (!iof_ensure(O, 5)) + return IOFFULL; + c1 = *s++; + c2 = *s++; + c3 = *s++; + c4 = *s++; + code = (c1<<24)|(c2<<16)|(c3<<8)|c4; + if (code == 0) + { + iof_set(O, 'z'); + continue; + } + base85_encode_word(O, code); + } + switch (e - s) + { + case 0: + break; + case 1: + if (!iof_ensure(O, 2)) + return IOFFULL; + c1 = *s; + code = (c1<<24)/85/85/85; + base85_encode_tail1(O, code); + break; + case 2: + if (!iof_ensure(O, 3)) + return IOFFULL; + c1 = *s++; + c2 = *s; + code = ((c1<<24)|(c2<<16))/85/85; + base85_encode_tail2(O, code); + break; + case 3: + if (!iof_ensure(O, 4)) + return IOFFULL; + c1 = *s++; + c2 = *s++; + c3 = *s; + code = ((c1<<24)|(c2<<16)|(c3<<8))/85; + base85_encode_tail3(O, code); + break; + } + return IOFEOF; +} + +iof_status base85_encoded_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline) +{ + unsigned int code; + const uint8_t *s, *e; + uint8_t c1, c2, c3, c4; + for (s = (const uint8_t *)data, e = s + size; s + 3 < e; ) + { + if (!iof_ensure(O, 6)) + return IOFFULL; + c1 = *s++; + c2 = *s++; + c3 = *s++; + c4 = *s++; + code = (c1<<24)|(c2<<16)|(c3<<8)|c4; + if (code == 0) + { + put_nl(O, line, maxline, 1); + iof_set(O, 'z'); + continue; + } + put_nl(O, line, maxline, 5); + base85_encode_word(O, code); + } + switch (e - s) + { + case 0: + break; + case 1: + if (!iof_ensure(O, 3)) + return IOFFULL; + c1 = *s; + code = (c1<<24)/85/85/85; + put_nl(O, line, maxline, 2); + base85_encode_tail1(O, code); + break; + case 2: + if (!iof_ensure(O, 4)) + return IOFFULL; + c1 = *s++; + c2 = *s; + code = ((c1<<24)|(c2<<16))/85/85; + put_nl(O, line, maxline, 3); + base85_encode_tail2(O, code); + break; + case 3: + if (!iof_ensure(O, 5)) + return IOFFULL; + c1 = *s++; + c2 = *s++; + c3 = *s; + code = ((c1<<24)|(c2<<16)|(c3<<8))/85; + put_nl(O, line, maxline, 4); + base85_encode_tail3(O, code); + break; + } + return IOFEOF; +} + +iof_status base85_encode (iof *I, iof *O) +{ + register int c1, c2, c3, c4; + register unsigned int code; + while(iof_ensure(O, 5)) + { + if ((c1 = iof_get(I)) < 0) + return IOFEOF; + if ((c2 = iof_get(I)) < 0) + { + code = (c1<<24)/85/85/85; + base85_encode_tail1(O, code); + return IOFEOF; + } + if ((c3 = iof_get(I)) < 0) + { + code = ((c1<<24)|(c2<<16))/85/85; + base85_encode_tail2(O, code); + return IOFEOF; + } + if ((c4 = iof_get(I)) < 0) + { + code = ((c1<<24)|(c2<<16)|(c3<<8))/85; + base85_encode_tail3(O, code); + return IOFEOF; + } + code = (c1<<24)|(c2<<16)|(c3<<8)|c4; + if (code == 0) + { + iof_set(O, 'z'); + continue; + } + /* in btoa 'y' character stays for 0x20202020, but pdf does not support this */ + /* if (code == 0x20202020) + { + iof_set(O, 'y'); + continue; + } */ + base85_encode_word(O, code); + } + return IOFFULL; +} + +iof_status base85_encode_state (iof *I, iof *O, basexx_state *state) +{ + register int c1, c2, c3, c4; + register unsigned int code; + if (!(iof_ensure(O, 5))) + return IOFFULL; + switch(state->left) + { + case 0: goto byte0; + case 1: get_tail1(state, c1); goto byte1; + case 2: get_tail2(state, c1, c2); goto byte2; + case 3: get_tail3(state, c1, c2, c3); goto byte3; + } + while(iof_ensure(O, 5)) + { + byte0: + if ((c1 = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + byte1: + if ((c2 = iof_get(I)) < 0) + { + set_tail1(state, c1); + if (state->flush) + { + code = (c1<<24)/85/85/85; + base85_encode_tail1(O, code); + return IOFEOF; + } + return IOFEMPTY; + } + byte2: + if ((c3 = iof_get(I)) < 0) + { + set_tail2(state, c1, c2); + if (state->flush) + { + code = ((c1<<24)|(c2<<16))/85/85; + base85_encode_tail2(O, code); + return IOFEOF; + } + return IOFEMPTY; + } + byte3: + if ((c4 = iof_get(I)) < 0) + { + set_tail3(state, c1, c2, c3); + if (state->flush) + { + code = ((c1<<24)|(c2<<16)|(c3<<8))/85; + base85_encode_tail3(O, code); + return IOFEOF; + } + return IOFEMPTY; + } + code = (c1<<24)|(c2<<16)|(c3<<8)|c4; + if (code == 0) + { + iof_set(O, 'z'); + continue; + } + base85_encode_word(O, code); + } + return IOFFULL; +} + +iof_status base85_encode_ln (iof *I, iof *O, size_t line, size_t maxline) +{ + register int c1, c2, c3, c4; + register unsigned int code; + while(iof_ensure(O, 6)) + { + if ((c1 = iof_get(I)) < 0) + return IOFEOF; + if ((c2 = iof_get(I)) < 0) + { + code = (c1<<24)/85/85/85; + put_nl(O, line, maxline, 2); + base85_encode_tail1(O, code); + return IOFEOF; + } + if ((c3 = iof_get(I)) < 0) + { + code = ((c1<<24)|(c2<<16))/85/85; + put_nl(O, line, maxline, 3); + base85_encode_tail2(O, code); + return IOFEOF; + } + if ((c4 = iof_get(I)) < 0) + { + code = ((c1<<24)|(c2<<16)|(c3<<8))/85; + put_nl(O, line, maxline, 4); + base85_encode_tail3(O, code); + return IOFEOF; + } + code = (c1<<24)|(c2<<16)|(c3<<8)|c4; + if (code == 0) + { + put_nl(O, line, maxline, 1); + iof_set(O, 'z'); + continue; + } + put_nl(O, line, maxline, 5); + base85_encode_word(O, code); + } + return IOFFULL; +} + +iof_status base85_encode_state_ln (iof *I, iof *O, basexx_state *state) +{ + register int c1, c2, c3, c4; + register unsigned int code; + if (!(iof_ensure(O, 6))) + return IOFFULL; + switch(state->left) + { + case 0: goto byte0; + case 1: get_tail1(state, c1); goto byte1; + case 2: get_tail2(state, c1, c2); goto byte2; + case 3: get_tail3(state, c1, c2, c3); goto byte3; + } + while(iof_ensure(O, 6)) + { + byte0: + if ((c1 = iof_get(I)) < 0) + return (state->flush ? IOFEOF : IOFEMPTY); + byte1: + if ((c2 = iof_get(I)) < 0) + { + set_tail1(state, c1); + if (state->flush) + { + code = (c1<<24)/85/85/85; + put_nl(O, state->line, state->maxline, 2); + base85_encode_tail1(O, code); + return IOFEOF; + } + return IOFEMPTY; + } + byte2: + if ((c3 = iof_get(I)) < 0) + { + set_tail2(state, c1, c2); + if (state->flush) + { + code = ((c1<<24)|(c2<<16))/85/85; + put_nl(O, state->line, state->maxline, 3); + base85_encode_tail2(O, code); + return IOFEOF; + } + return IOFEMPTY; + } + byte3: + if ((c4 = iof_get(I)) < 0) + { + set_tail3(state, c1, c2, c3); + if (state->flush) + { + code = ((c1<<24)|(c2<<16)|(c3<<8))/85; + put_nl(O, state->line, state->maxline, 4); + base85_encode_tail3(O, code); + return IOFEOF; + } + return IOFEMPTY; + } + code = (c1<<24)|(c2<<16)|(c3<<8)|c4; + if (code == 0) + { + put_nl(O, state->line, state->maxline, 1); + iof_set(O, 'z'); + continue; + } + put_nl(O, state->line, state->maxline, 5); + base85_encode_word(O, code); + } + return IOFFULL; +} + +#define base85_code(c1, c2, c3, c4, c5) ((((c1*85+c2)*85+c3)*85+c4)*85+c5) + +iof_status base85_decode (iof *I, iof *O) +{ + register int c1, c2, c3, c4, c5; + register unsigned int code; + while (iof_ensure(O, 4)) + { + do { c1 = iof_get(I); } while (ignored(c1)); + if (base85_eof(c1)) + return IOFEOF; + switch (c1) + { + case 'z': + iof_set4(O, '\0', '\0', '\0', '\0'); + continue; + case 'y': + iof_set4(O, ' ', ' ', ' ', ' '); + continue; + } + do { c2 = iof_get(I); } while (ignored(c2)); + if (base85_eof(c2)) + return IOFERR; + do { c3 = iof_get(I); } while (ignored(c3)); + if (base85_eof(c3)) + { + if ((c1 = base85_value(c1)) < 0 || (c2 = base85_value(c2)) < 0) + return IOFERR; + code = base85_code(c1, c2, 84, 84, 84); /* padding with 'u' (117); 117-33 = 84 */ + iof_set(O, code>>24); + return IOFEOF; + } + do { c4 = iof_get(I); } while (ignored(c4)); + if (base85_eof(c4)) + { + if ((c1 = base85_value(c1)) < 0 || (c2 = base85_value(c2)) < 0 || (c3 = base85_value(c3)) < 0) + return IOFERR; + code = base85_code(c1, c2, c3, 84, 84); + iof_set2(O, code>>24, (code>>16)&255); + return IOFEOF; + } + do { c5 = iof_get(I); } while (ignored(c5)); + if (base85_eof(c5)) + { + if ((c1 = base85_value(c1)) < 0 || (c2 = base85_value(c2)) < 0 || + (c3 = base85_value(c3)) < 0 || (c4 = base85_value(c4)) < 0) + return IOFERR; + code = base85_code(c1, c2, c3, c4, 84); + iof_set3(O, code>>24, (code>>16)&255, (code>>8)&255); + return IOFEOF; + } + if ((c1 = base85_value(c1)) < 0 || (c2 = base85_value(c2)) < 0 || (c3 = base85_value(c3)) < 0 || + (c4 = base85_value(c4)) < 0 || (c5 = base85_value(c5)) < 0) + return IOFERR; + code = base85_code(c1, c2, c3, c4, c5); + iof_set4(O, code>>24, (code>>16)&255, (code>>8)&255, code&255); + } + return IOFFULL; +} + +iof_status base85_decode_state (iof *I, iof *O, basexx_state *state) +{ + register int c1, c2, c3, c4, c5; + register int d1, d2, d3, d4, d5; + register unsigned int code; + if (!(iof_ensure(O, 4))) + return IOFFULL; + switch(state->left) + { + case 0: goto byte0; + case 1: get_tail1(state, c1); goto byte1; + case 2: get_tail2(state, c1, c2); goto byte2; + case 3: get_tail3(state, c1, c2, c3); goto byte3; + case 4: get_tail4(state, c1, c2, c3, c4); goto byte4; + } + while (iof_ensure(O, 4)) + { + byte0: + do { c1 = iof_get(I); } while (ignored(c1)); + if (base85_eof(c1)) + return (state->flush ? IOFEOF : IOFEMPTY); + switch (c1) + { + case 'z': + iof_set4(O, '\0', '\0', '\0', '\0'); + continue; + case 'y': + iof_set4(O, ' ', ' ', ' ', ' '); + continue; + } + byte1: + do { c2 = iof_get(I); } while (ignored(c2)); + if (base85_eof(c2)) + { + set_tail1(state, c1); + return (state->flush ? IOFERR : IOFEMPTY); /* if state->flush then error; tail must have at least two bytes */ + } + byte2: + do { c3 = iof_get(I); } while (ignored(c3)); + if (base85_eof(c3)) + { + set_tail2(state, c1, c2); + if (state->flush) + { + if ((c1 = base85_value(c1)) < 0 || (c2 = base85_value(c2)) < 0) + return IOFERR; + code = base85_code(c1, c2, 84, 84, 84); + iof_set(O, code>>24); + return IOFEOF; + } + return IOFEMPTY; + } + byte3: + do { c4 = iof_get(I); } while (ignored(c4)); + if (base85_eof(c4)) + { + set_tail3(state, c1, c2, c3); + if (state->flush) + { + if ((c1 = base85_value(c1)) < 0 || (c2 = base85_value(c2)) < 0 || (c3 = base85_value(c3)) < 0) + return IOFERR; + code = base85_code(c1, c2, c3, 84, 84); + iof_set2(O, code>>24, (code>>16)&255); + return IOFEOF; + } + return IOFEMPTY; + } + byte4: + do { c5 = iof_get(I); } while (ignored(c5)); + if (base85_eof(c5)) + { + set_tail4(state, c1, c2, c3, c4); + if (state->flush) + { + if ((c1 = base85_value(c1)) < 0 || (c2 = base85_value(c2)) < 0 || + (c3 = base85_value(c3)) < 0 || (c4 = base85_value(c4)) < 0) + return IOFERR; + code = base85_code(c1, c2, c3, c4, 84); + iof_set3(O, code>>24, (code>>16)&255, (code>>8)&255); + return IOFEOF; + } + return IOFEMPTY; + } + if ((d1 = base85_value(c1)) < 0 || (d2 = base85_value(c2)) < 0 || (d3 = base85_value(c3)) < 0 || + (d4 = base85_value(c4)) < 0 || (d5 = base85_value(c5)) < 0) + { + set_tail5(state, c1, c2, c3, c4, c5); + return IOFERR; + } + code = base85_code(d1, d2, d3, d4, d5); + iof_set4(O, code>>24, (code>>16)&255, (code>>8)&255, code&255); + } + return IOFFULL; +} + +/* postscript run length */ + +void runlength_state_init (runlength_state *state) +{ + state->run = -1; + state->flush = 0; + state->c1 = 0; + state->c2 = 0; + state->pos = NULL; +} + +iof_status runlength_encode (iof *I, iof *O) +{ + register int c1, c2, run = -1; + uint8_t *pos; + c1 = 0, c2 = 0; /* avoid warning */ + while (iof_ensure(O, 1+128+1)) + { /* ensured space for single length byte, up to 128 bytes to be copied, possible eod marker */ + pos = O->pos++; + switch (run) + { + case -1: /* initial state; get first byte */ + if ((c1 = iof_get(I)) < 0) + return (*pos = 128, IOFEOF); + run = 0; + FALLTHRU // fall through + case 0: /* `repeat' state; get another byte and compare */ + if ((c2 = iof_get(I)) < 0) + return (*pos = 0, iof_set2(O, c1, 128), IOFEOF); + run = (c1 == c2 ? 257-2 : 0); + break; + } + if (run < 128) + { /* single length byte, up to 128 bytes to be copied, possible eod marker */ + iof_set(O, c1); + for (c1 = c2, c2 = iof_char(I); c1 != c2 && run < 127; c1 = c2, c2 = iof_next(I)) + { + if (c2 < 0) /* O->pos must not change until next call to calling encoder!!! */ + return (*pos = (uint8_t)run+1, iof_set2(O, c1, 128), IOFEOF); + iof_set(O, c1); + ++run; + } + } + else // if run > 128 + { + for (c2 = iof_get(I); c1 == c2 && run > 129; c2 = iof_get(I)) + --run; + if (c2 < 0) + return (*pos = (uint8_t)run, iof_set2(O, c1, 128), IOFEOF); + iof_set(O, c1); + } + *pos = (uint8_t)run; + c1 = c2; + run = 0; + } + return IOFFULL; +} + +iof_status runlength_encode_state (iof *I, iof *O, runlength_state *state) +{ + while (iof_ensure(O, 3)) /* single length byte, the byte to be repeated and eod */ + { + state->pos = O->pos++; + switch (state->run) + { + case -1: /* initial state; get first byte */ + if ((state->c1 = iof_get(I)) < 0) + return (state->flush ? (*state->pos = 128, IOFEOF) : IOFEMPTY); + state->run = 0; + FALLTHRU // fall through + case 0: /* `repeat' state; get another byte and compare */ + if ((state->c2 = iof_get(I)) < 0) + return (state->flush ? (*state->pos = 0, iof_set2(O, state->c1, 128), IOFEOF) : IOFEMPTY); + state->run = (state->c1 == state->c2 ? 257-2 : 0); + break; + } + if (state->run < 128) + { /* ensure space for single length byte, up to 128 bytes to be copied, plus possible eod marker, minus those already copied */ + if (!iof_ensure(O, 1+128+1-state->run)) + return IOFFULL; + iof_set(O, state->c1); + for (state->c1 = state->c2, state->c2 = iof_char(I); + state->c1 != state->c2 && state->run < 127; + state->c1 = state->c2, state->c2 = iof_next(I)) + { + if (state->c2 < 0) /* O->pos must not change until next call to calling encoder!!! */ + return (state->flush ? (*state->pos = (uint8_t)state->run+1, iof_set2(O, state->c1, 128), IOFEOF) : IOFEMPTY); + iof_set(O, state->c1); + ++state->run; + } + } + else // if run > 128 + { + for (state->c2 = iof_get(I); state->c1 == state->c2 && state->run > 129; state->c2 = iof_get(I)) + --state->run; + if (state->c2 < 0) + return (state->flush ? (*state->pos = (uint8_t)state->run, iof_set2(O, state->c1, 128), IOFEOF) : IOFEMPTY); + iof_set(O, state->c1); + } + *state->pos = (uint8_t)state->run; + state->c1 = state->c2; + state->run = 0; + } + return IOFFULL; +} + +iof_status runlength_decode (iof *I, iof *O) +{ + register int c, run = -1; + while (1) + { + if (run == -1) /* initial state */ + { + if ((run = iof_get(I)) < 0) + { + run = -1; /* don't assume IOFEOF == -1 */ + return IOFEOF; + } + } + if (run < 128) + { /* copy (run + 1) following bytes */ + while (run > -1) + { + if (iof_ensure(O, 1)) + { + if ((c = iof_get(I)) < 0) + return IOFERR; + iof_set(O, c); + --run; + continue; + } + return IOFFULL; + } + } + else if (run > 128) + { /* replicate the following byte (257 - run) times */ + if ((c = iof_get(I)) < 0) /* cf. state-wise version; don't change input position until we got this byte */ + return IOFERR; + while (run < 257) + { + if (iof_ensure(O, 1)) + { + iof_set(O, c); + ++run; + continue; + } + return IOFFULL; + } + run = -1; + } + else // c == 128 + return IOFEOF; + } + // return IOFFULL; +} + +iof_status runlength_decode_state (iof *I, iof *O, runlength_state *state) +{ + register int c; + while (1) + { + if (state->run == -1) /* initial state */ + { + if ((state->run = iof_char(I)) < 0) + { + state->run = -1; /* don't assume IOFEOF == -1 */ + return (state->flush ? IOFEOF : IOFEMPTY); + } + ++I->pos; + } + if (state->run < 128) + { /* copy (state->run + 1) following bytes */ + while (state->run > -1) + { + if (iof_ensure(O, 1)) + { + if ((c = iof_char(I)) < 0) + return (state->flush ? IOFERR : IOFEMPTY); + ++I->pos; + iof_set(O, c); + --state->run; + continue; + } + return IOFFULL; + } + } + else if (state->run > 128) + { /* replicate the following byte (257 - state->run) times */ + if ((c = iof_char(I)) < 0) + return (state->flush ? IOFERR : IOFEMPTY); + ++I->pos; + while (state->run < 257) + { + if (iof_ensure(O, 1)) + { + iof_set(O, c); + ++state->run; + continue; + } + return IOFFULL; + } + state->run = -1; + } + else // c == 128 + return IOFEOF; + } + // return IOFFULL; +} + +/* filters */ + +// base16 decoder function + +static size_t base16_decoder (iof *F, iof_mode mode) +{ + basexx_state *state; + iof_status status; + size_t tail; + + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + state = iof_filter_state(basexx_state *, F); + do { + status = base16_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "base16", status); + case IOFCLOSE: + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// base16 encoder function + +static size_t base16_encoder (iof *F, iof_mode mode) +{ + basexx_state *state; + iof_status status; + + state = iof_filter_state(basexx_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = base16_encode_state_ln(F, F->next, state); + return iof_encoder_retval(F, "base16", status); + case IOFCLOSE: + if (!state->flush) + base16_encoder(F, IOFFLUSH); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// base64 decoder function + +static size_t base64_decoder (iof *F, iof_mode mode) +{ + basexx_state *state; + iof_status status; + size_t tail; + + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + state = iof_filter_state(basexx_state *, F); + do { + status = base64_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "base64", status); + case IOFCLOSE: + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// base64 encoder function + +static size_t base64_encoder (iof *F, iof_mode mode) +{ + basexx_state *state; + iof_status status; + + state = iof_filter_state(basexx_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = base64_encode_state_ln(F, F->next, state); + return iof_encoder_retval(F, "base64", status); + case IOFCLOSE: + if (!state->flush) + base64_encoder(F, IOFFLUSH); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// base85 decoder function + +static size_t base85_decoder (iof *F, iof_mode mode) +{ + basexx_state *state; + iof_status status; + size_t tail; + + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + state = iof_filter_state(basexx_state *, F); + do { + status = base85_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "base85", status); + case IOFCLOSE: + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// base85 encoder function + +static size_t base85_encoder (iof *F, iof_mode mode) +{ + basexx_state *state; + iof_status status; + + state = iof_filter_state(basexx_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = base85_encode_state_ln(F, F->next, state); + return iof_encoder_retval(F, "base85", status); + case IOFCLOSE: + if (!state->flush) + base85_encoder(F, IOFFLUSH); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// runlength decoder function + +static size_t runlength_decoder (iof *F, iof_mode mode) +{ + runlength_state *state; + iof_status status; + size_t tail; + + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + state = iof_filter_state(runlength_state *, F); + do { + status = runlength_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "runlength", status); + case IOFCLOSE: + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// runlength encoder function + +static size_t runlength_encoder (iof *F, iof_mode mode) +{ + runlength_state *state; + iof_status status; + + state = iof_filter_state(runlength_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = runlength_encode_state(F, F->next, state); + return iof_encoder_retval(F, "runlength", status); + case IOFCLOSE: + if (!state->flush) + runlength_encoder(F, IOFFLUSH); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// + +int iof_filter_basexx_encoder_ln (iof *F, size_t line, size_t maxline) +{ + basexx_state *state; + if (maxline > 8 && line < maxline) + { + state = iof_filter_state(basexx_state *, F); + state->line = line; + state->maxline = maxline; + return 1; + } + return 0; +} + +/* base 16 */ + +iof * iof_filter_base16_decoder (iof *N) +{ + iof *I; + basexx_state_pointer P; + I = iof_filter_reader(base16_decoder, sizeof(basexx_state), &P.voidstate); + iof_setup_next(I, N); + basexx_state_init(P.basexxstate); + P.basexxstate->flush = 1; // means N is supposed to be continuous input + return I; +} + +iof * iof_filter_base16_encoder (iof *N) +{ + iof *O; + basexx_state_pointer P; + O = iof_filter_writer(base16_encoder, sizeof(basexx_state), &P.voidstate); + iof_setup_next(O, N); + basexx_state_init(P.basexxstate); + return O; +} + +/* base 64 */ + +iof * iof_filter_base64_decoder (iof *N) +{ + iof *I; + basexx_state_pointer P; + I = iof_filter_reader(base64_decoder, sizeof(basexx_state), &P.voidstate); + iof_setup_next(I, N); + basexx_state_init(P.basexxstate); + P.basexxstate->flush = 1; + return I; +} + +iof * iof_filter_base64_encoder (iof *N) +{ + iof *O; + basexx_state_pointer P; + O = iof_filter_writer(base64_encoder, sizeof(basexx_state), &P.voidstate); + iof_setup_next(O, N); + basexx_state_init(P.basexxstate); + return O; +} + +/* base 85 */ + +iof * iof_filter_base85_decoder (iof *N) +{ + iof *I; + basexx_state_pointer P; + I = iof_filter_reader(base85_decoder, sizeof(basexx_state), &P.voidstate); + iof_setup_next(I, N); + basexx_state_init(P.basexxstate); + P.basexxstate->flush = 1; + return I; +} + +iof * iof_filter_base85_encoder (iof *N) +{ + iof *O; + basexx_state_pointer P; + O = iof_filter_writer(base85_encoder, sizeof(basexx_state), &P.voidstate); + iof_setup_next(O, N); + basexx_state_init(P.basexxstate); + return O; +} + +/* runlength stream filter */ + +iof * iof_filter_runlength_decoder (iof *N) +{ + iof *I; + basexx_state_pointer P; + I = iof_filter_reader(runlength_decoder, sizeof(runlength_state), &P.voidstate); + iof_setup_next(I, N); + runlength_state_init(P.runlengthstate); + P.runlengthstate->flush = 1; + return I; +} + +iof * iof_filter_runlength_encoder (iof *N) +{ + iof *O; + basexx_state_pointer P; + O = iof_filter_writer(runlength_encoder, sizeof(runlength_state), &P.voidstate); + iof_setup_next(O, N); + runlength_state_init(P.runlengthstate); + return O; +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilbasexx.h b/Build/source/libs/pplib/pplib-src/src/util/utilbasexx.h new file mode 100644 index 00000000000..81891b549fb --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilbasexx.h @@ -0,0 +1,111 @@ + +/* base encodings */ + +#ifndef UTIL_BASEXX_H +#define UTIL_BASEXX_H + +#include "utiliof.h" + +/* base codecs state */ + +typedef struct basexx_state basexx_state; + +#define BASEXX_MAXLINE 80 +#define BASEXX_PDF + +void basexx_state_init_ln (basexx_state *state, size_t line, size_t maxline); +#define basexx_state_init(state) basexx_state_init_ln(state, 0, BASEXX_MAXLINE) + +/* base16 */ + +int base16_getc (iof *I); +int base16_uc_putc (iof *I, int c); +int base16_lc_putc (iof *I, int c); +#define base16_putc base16_uc_putc + +iof_status base16_encoded_uc (const void *data, size_t size, iof *O); +iof_status base16_encoded_lc (const void *data, size_t size, iof *O); +iof_status base16_encoded_uc_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline); +iof_status base16_encoded_lc_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline); + +iof_status base16_encode_uc (iof *I, iof *O); +iof_status base16_encode_lc (iof *I, iof *O); +iof_status base16_encode_uc_ln (iof *I, iof *O, size_t line, size_t maxline); +iof_status base16_encode_lc_ln (iof *I, iof *O, size_t line, size_t maxline); +iof_status base16_decode (iof *I, iof *O); + +#define base16_encoded base16_encoded_uc +#define base16_encoded_ln base16_encoded_uc_ln +#define base16_encode base16_encode_uc +#define base16_encode_ln base16_encode_uc_ln + +iof_status base16_encode_state_uc (iof *I, iof *O, basexx_state *state); +iof_status base16_encode_state_lc (iof *I, iof *O, basexx_state *state); +iof_status base16_encode_state_uc_ln (iof *I, iof *O, basexx_state *state); +iof_status base16_encode_state_lc_ln (iof *I, iof *O, basexx_state *state); +iof_status base16_decode_state (iof *I, iof *O, basexx_state *state); + +#define base16_encode_state base16_encode_state_uc +#define base16_encode_state_ln base16_encode_state_uc_ln + +/* base64 */ + +extern const char base64_alphabet[]; +extern const int base64_lookup[]; + +iof_status base64_encoded (const void *data, size_t size, iof *O); +iof_status base64_encoded_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline); + +iof_status base64_encode (iof *I, iof *O); +iof_status base64_encode_ln (iof *I, iof *O, size_t line, size_t maxline); +iof_status base64_decode (iof *I, iof *O); + +iof_status base64_encode_state (iof *I, iof *O, basexx_state *state); +iof_status base64_encode_state_ln (iof *I, iof *O, basexx_state *state); +iof_status base64_decode_state (iof *I, iof *O, basexx_state *state); + +/* base85 */ + +extern const char base85_alphabet[]; +extern const int base85_lookup[]; + +iof_status base85_encoded (const void *data, size_t size, iof *O); +iof_status base85_encoded_ln (const void *data, size_t size, iof *O, size_t line, size_t maxline); + +iof_status base85_encode (iof *I, iof *O); +iof_status base85_encode_ln (iof *I, iof *O, size_t line, size_t maxline); +iof_status base85_decode (iof *I, iof *O); + +iof_status base85_encode_state (iof *I, iof *O, basexx_state *state); +iof_status base85_encode_state_ln (iof *I, iof *O, basexx_state *state); +iof_status base85_decode_state (iof *I, iof *O, basexx_state *state); + +/* run length */ + +typedef struct runlength_state runlength_state; + +void runlength_state_init (runlength_state *state); + +iof_status runlength_encode (iof *I, iof *O); +iof_status runlength_encode_state (iof *I, iof *O, runlength_state *state); + +iof_status runlength_decode (iof *I, iof *O); +iof_status runlength_decode_state (iof *I, iof *O, runlength_state *state); + +/* filters */ + +int iof_filter_basexx_encoder_ln (iof *N, size_t line, size_t maxline); + +iof * iof_filter_base16_decoder (iof *N); +iof * iof_filter_base16_encoder (iof *N); + +iof * iof_filter_base64_decoder (iof *N); +iof * iof_filter_base64_encoder (iof *N); + +iof * iof_filter_base85_decoder (iof *N); +iof * iof_filter_base85_encoder (iof *N); + +iof * iof_filter_runlength_decoder (iof *N); +iof * iof_filter_runlength_encoder (iof *N); + +#endif diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilcrypt.c b/Build/source/libs/pplib/pplib-src/src/util/utilcrypt.c new file mode 100644 index 00000000000..2c77e42a4c7 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilcrypt.c @@ -0,0 +1,1190 @@ + +#include "utilmem.h" +#include "utilcrypt.h" +#include "utilcryptdef.h" +#include "utilmd5.h" + +/* rc4 */ + +/* +Initializer arguments: +- state - crypt state +- map - a space for rc4 bytes map; may be left NULL in which case will be allocated +- vkey - crypt key; may be left NULL iff map is provided and properly initialized +- keylength - the length of crypt key (from 5 to 16 bytes) +*/ + +rc4_state * rc4_state_initialize (rc4_state *state, rc4_map *map, const void *vkey, size_t keylength) +{ + int i, j; + uint8_t tmp; + const uint8_t *key; + key = (const uint8_t *)vkey; + if (keylength == 0 || keylength > 256) + return NULL; + state->flags = 0; + if (map != NULL) + { + state->map = map; + } + else + { + state->map = (rc4_map *)util_malloc(sizeof(rc4_map)); + state->flags |= RC4_STATE_ALLOC; + } + + if (key != NULL) + { + for (i = 0; i < 256; ++i) + state->smap[i] = (uint8_t)i; + for (i = 0, j = 0; i < 256; ++i) + { + j = (j + state->smap[i] + key[i % keylength]) & 255; + tmp = state->smap[i]; + state->smap[i] = state->smap[j]; + state->smap[j] = tmp; + } + } + state->i = 0; + state->j = 0; + state->flush = 0; /* caller is responsible to override if necessary */ + return state; +} + +void rc4_map_save (rc4_state *state, rc4_map *map) +{ + memcpy(map, state->map, sizeof(rc4_map)); +} + +void rc4_map_restore (rc4_state *state, rc4_map *map) +{ + memcpy(state->map, map, sizeof(rc4_map)); + //state->flags = 0; + //state->flush = 0; + state->i = 0; + state->j = 0; +} + +static uint8_t rc4_next_random_byte (rc4_state *state) +{ + uint8_t tmp; + state->i = (state->i + 1) & 255; + state->j = (state->j + state->smap[state->i]) & 255; + tmp = state->smap[state->i]; + state->smap[state->i] = state->smap[state->j]; + state->smap[state->j] = tmp; + return state->smap[(state->smap[state->i] + state->smap[state->j]) & 255]; +} + +iof_status rc4_crypt_state (iof *I, iof *O, rc4_state *state) +{ + uint8_t r; + int c; + while (iof_ensure(O, 1)) + { + if ((c = iof_get(I)) < 0) + return c == IOFERR ? IOFERR : (state->flush ? IOFEOF : IOFEMPTY); + r = rc4_next_random_byte(state); + //r = r ^ ((uint8_t)c); + //iof_set(O, r); + iof_set(O, r ^ ((uint8_t)c)); + } + return IOFFULL; +} + +iof_status rc4_crypt (iof *I, iof *O, const void *key, size_t keylength) +{ + int ret; + rc4_state state; + rc4_map map; + if (rc4_state_initialize(&state, &map, key, keylength) == NULL) + return IOFERR; + state.flush = 1; + ret = rc4_crypt_state(I, O, &state); + rc4_state_close(&state); + return ret; +} + +/* +Variants that operates on c-strings can worn inplace, so output and input can be the same address. +Variant that takes rc4_state pointer expects the state properly initialized. Keep in mind +the crypt procedure modifies rc4 bytes map. All returns the size of encrypted/decrypted +data, which is the same as input data length for rc4. +*/ + +size_t rc4_crypt_data (const void *input, size_t length, void *output, const void *key, size_t keylength) +{ + rc4_state state; + rc4_map map; + if (rc4_state_initialize(&state, &map, key, keylength) == NULL) + return 0; + return rc4_crypt_state_data(&state, input, length, output); + // no need to call rc4_state_close() +} + +size_t rc4_crypt_state_data (rc4_state *state, const void *input, size_t length, void *output) +{ /* state assumed to be initialized and with the proper state of smap */ + const uint8_t *inp; + uint8_t r, *out; + size_t size; + inp = (const uint8_t *)input; + out = (uint8_t *)output; + for (size = 0; size < length; ++size, ++inp, ++out) + { + r = rc4_next_random_byte(state); + *out = r ^ *inp; + } + return length; +} + +void rc4_state_close (rc4_state *state) +{ + if (state->smap != NULL && (state->flags & RC4_STATE_ALLOC)) + { + util_free(state->smap); + state->smap = NULL; + } +} + +/* aes; parts of code excerpted from https://github.com/kokke/tiny-AES128-C */ + +static const uint8_t sbox[256] = { + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 }; + +static const uint8_t rsbox[256] = +{ 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d }; + +/* +The round constant word array, rcon[i], contains the values given by +x to th e power (i-1) being powers of x (x is denoted as {02}) in the field GF(2^8) +Note that i starts at 1, not 0). +*/ + +static const uint8_t rcon[255] = { + 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, + 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, + 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, + 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, + 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, + 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, + 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, + 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, + 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, + 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, + 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, + 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, + 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, + 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, + 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, + 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb }; + +/* block copying */ + +#define aes_copy_block(output, input) memcpy(output, input, 16) + +static void aes_copy_cbc (uint8_t *data, const uint8_t *input) +{ + uint8_t i; + for (i = 0; i < 16; ++i) + data[i] ^= input[i]; +} + +static void aes_copy_xor (uint8_t *data, const uint8_t *input, const uint8_t *iv) +{ + uint8_t i; + for (i = 0; i < 16; ++i) + data[i] = input[i] ^ iv[i]; +} + +/* key expansion */ + +#define AES_COLUMNS 4 // constant in aes + +static void key_expansion (aes_state *state, const uint8_t *key) +{ + uint32_t i, j; + uint8_t t[4], temp; + uint8_t *keydata, keywords, columns; + + keywords = (uint8_t)(state->keylength >> 2); + keydata = (uint8_t *)state->keyblock; + + /* the first round key is the key itself */ + for(i = 0; i < keywords; ++i) + { + keydata[(i * 4) + 0] = key[(i * 4) + 0]; + keydata[(i * 4) + 1] = key[(i * 4) + 1]; + keydata[(i * 4) + 2] = key[(i * 4) + 2]; + keydata[(i * 4) + 3] = key[(i * 4) + 3]; + } + + /* others derived from the first */ + for(columns = AES_COLUMNS * (state->rounds + 1); i < columns; ++i) + { + for(j = 0; j < 4; ++j) + t[j] = keydata[(i - 1) * 4 + j]; + if (i % keywords == 0) + { + /* rotate the 4 bytes in a word to the left once; [a0,a1,a2,a3] becomes [a1,a2,a3,a0] */ + temp = t[0]; + t[0] = t[1]; + t[1] = t[2]; + t[2] = t[3]; + t[3] = temp; + + /* take a four-byte input word and apply the S-box to each of the four bytes to produce an output word */ + t[0] = sbox[t[0]]; + t[1] = sbox[t[1]]; + t[2] = sbox[t[2]]; + t[3] = sbox[t[3]]; + + t[0] = t[0] ^ rcon[i / keywords]; + } + else if (keywords > 6 && i % keywords == 4) + { + t[0] = sbox[t[0]]; + t[1] = sbox[t[1]]; + t[2] = sbox[t[2]]; + t[3] = sbox[t[3]]; + } + keydata[i * 4 + 0] = keydata[(i - keywords) * 4 + 0] ^ t[0]; + keydata[i * 4 + 1] = keydata[(i - keywords) * 4 + 1] ^ t[1]; + keydata[i * 4 + 2] = keydata[(i - keywords) * 4 + 2] ^ t[2]; + keydata[i * 4 + 3] = keydata[(i - keywords) * 4 + 3] ^ t[3]; + } + +} + +/* +An original implementation uses no private buffers except a keyblock. We need private buffers to +keep a CBC vector between calls and to be able to read input data not necessarily in 16-bytes blocks. +Encrypter would actually require only one such buffer, as CBC vector is applied on input data before +the actual cipher procedure. And CBC for the next chunk is simply the output from the previous. +Decrypter, however, applies the cipher first, then applies CBC to the output with a buffered init +vector, and the vector for the next call is the row input before cipher. Hence we need two 16-bytes +buffers for decrypter. +*/ + +/* +aes_state * aes_state_initialize_ecb (aes_state *State, uint8_t *keyblock, const uint8_t *key) +{ + state->flags = 0; + + state->flags |= AES_ECB_MODE; + + if (keyblock == NULL) + { + keyblock = util_malloc(sizeof(aes_keyblock)); + state->flags |= AES_STATE_ALLOC; + } + state->keyblock = keyblock; + key_expansion(state, key); + state->flush = 0; + return state; +} +*/ + +void aes_pdf_mode (aes_state *state) +{ + state->flags |= AES_INLINE_IV; + state->flags &= ~AES_NULL_PADDING; +} + +/* +Initialize arguments: +- state - crypt state +- keyblock - a space for aes key expansion; can be left NULL in which case will be allocated +- key - crypt key; can be left NULL iff keyblock is given and properly initialized +- keylength - the length of the key (16 or 32 bytes) +- iv - 16-bytes CBC initialization vector; + - if left NULL for encoder, one is generated and stored as state->iv + - can also be left NULL for decorer, but then AES_INLINE_IV must be set, as this informs decoder to take + an initialization vector from the beginning of the encrypted stream + +At the first approach, an initialization vector was copied to state block during initialization and encoders +assumed that the state block is the current initialization vector. This simplifies encrypting procedure, +as the output from every 16-bytes chunk encryption is an initialization vector for the next chunk. However, +it makes api usage cumbersome, as the user has to know that iv may need to be copied to state block +before each call. +*/ + +static int aes_key_length (aes_state *state, size_t keylength) +{ + state->keylength = keylength; + switch (keylength) + { + case 16: + state->rounds = 10; + break; + case 24: + state->rounds = 12; + break; + case 32: + state->rounds = 14; + break; + default: + return 0; + } + return 1; +} + +aes_state * aes_encode_initialize (aes_state *state, aes_keyblock *keyblock, const void *key, size_t keylength, const void *iv) +{ + state->flags = 0; + if (!aes_key_length(state, keylength)) + return NULL; + if (iv != NULL) + aes_copy_block(state->iv, iv); + else + aes_generate_iv(state->iv); + state->flags |= AES_HAS_IV; + + if (keyblock == NULL) + { + keyblock = (aes_keyblock *)util_malloc(sizeof(aes_keyblock)); + state->flags |= AES_STATE_ALLOC; + } + state->keyblock = keyblock; + if (key != NULL) /* if NULL we assume keyblock is given and already expanded */ + key_expansion(state, (const uint8_t *)key); + state->flush = 0; + return state; +} + +aes_state * aes_decode_initialize (aes_state *state, aes_keyblock *keyblock, const void *key, size_t keylength, const void *iv) +{ + state->flags = 0; + if (!aes_key_length(state, keylength)) + return NULL; + if (iv != NULL) + { + aes_copy_block(state->iv, iv); + state->flags |= AES_HAS_IV; + } + /* else if AES_INLINE_IV flag is set will be read from input */ + + if (keyblock == NULL) + { + keyblock = (aes_keyblock *)util_malloc(sizeof(aes_keyblock)); + state->flags |= AES_STATE_ALLOC; + } + state->keyblock = keyblock; + if (key != NULL) /* otherwise keyblock is assumed present and properly initialized */ + key_expansion(state, (const uint8_t *)key); + state->flush = 0; + return state; +} + +void aes_state_close (aes_state *state) +{ + if (state->keyblock != NULL && (state->flags & AES_STATE_ALLOC)) + util_free(state->keyblock); +} + +/* add round key */ + +static void aes_round_key (aes_block block, aes_block keyblock) +{ + uint8_t i, j; + for(i = 0; i < 4; ++i) + for(j = 0; j < 4; ++j) + block[i][j] ^= keyblock[i][j]; +} + +#define aes_add_key(block, keyblock, round) aes_round_key(block, (*keyblock)[round]) + +/* substitution */ + +static void aes_encode_sub (aes_block block) +{ + uint8_t i, j, v; + for(i = 0; i < 4; ++i) + for(j = 0; j < 4; ++j) + v = block[i][j], block[i][j] = sbox[v]; +} + +/* rows shift; the row index is the shift offset, the first order is not shifted */ + +static void aes_encode_shift (aes_block block) +{ + uint8_t tmp; + + /* 1st row rotated once */ + tmp = block[0][1]; + block[0][1] = block[1][1]; + block[1][1] = block[2][1]; + block[2][1] = block[3][1]; + block[3][1] = tmp; + + /* 2nd row rotated twice */ + tmp = block[0][2]; + block[0][2] = block[2][2]; + block[2][2] = tmp; + tmp = block[1][2]; + block[1][2] = block[3][2]; + block[3][2] = tmp; + + /* 3rd row rotated 3 times */ + tmp = block[0][3]; + block[0][3] = block[3][3]; + block[3][3] = block[2][3]; + block[2][3] = block[1][3]; + block[1][3] = tmp; +} + +static uint8_t xtime (uint8_t x) +{ + return ((x << 1) ^ (((x >> 7) & 1) * 0x1b)); +} + +/* mix columns */ + +static void aes_encode_mix (aes_block block) +{ + uint8_t i, tmp, tm, t; + + for(i = 0; i < 4; ++i) + { + t = block[i][0]; + tmp = block[i][0] ^ block[i][1] ^ block[i][2] ^ block[i][3] ; + tm = block[i][0] ^ block[i][1]; tm = xtime(tm); block[i][0] ^= tm ^ tmp; + tm = block[i][1] ^ block[i][2]; tm = xtime(tm); block[i][1] ^= tm ^ tmp; + tm = block[i][2] ^ block[i][3]; tm = xtime(tm); block[i][2] ^= tm ^ tmp; + tm = block[i][3] ^ t ; tm = xtime(tm); block[i][3] ^= tm ^ tmp; + } +} + +/* multiply is used to multiply numbers in the field GF(2^8) */ + +#define multiply(x, y) \ + ( ((y & 1) * x) ^ \ + ((y>>1 & 1) * xtime(x)) ^ \ + ((y>>2 & 1) * xtime(xtime(x))) ^ \ + ((y>>3 & 1) * xtime(xtime(xtime(x)))) ^ \ + ((y>>4 & 1) * xtime(xtime(xtime(xtime(x)))))) \ + +/* mix columns */ + +static void aes_decode_mix (aes_block block) +{ + int i; + uint8_t a, b, c, d; + + for(i = 0; i < 4; ++i) + { + a = block[i][0]; + b = block[i][1]; + c = block[i][2]; + d = block[i][3]; + block[i][0] = multiply(a, 0x0e) ^ multiply(b, 0x0b) ^ multiply(c, 0x0d) ^ multiply(d, 0x09); + block[i][1] = multiply(a, 0x09) ^ multiply(b, 0x0e) ^ multiply(c, 0x0b) ^ multiply(d, 0x0d); + block[i][2] = multiply(a, 0x0d) ^ multiply(b, 0x09) ^ multiply(c, 0x0e) ^ multiply(d, 0x0b); + block[i][3] = multiply(a, 0x0b) ^ multiply(b, 0x0d) ^ multiply(c, 0x09) ^ multiply(d, 0x0e); + } +} + +/* inverse substitution */ + +static void aes_decode_sub (aes_block block) +{ + uint8_t i, j, v; + for(i = 0; i < 4; ++i) + for(j = 0; j < 4; ++j) + v = block[i][j], block[i][j] = rsbox[v]; +} + +/* inverse shift rows */ + +static void aes_decode_shift (aes_block block) +{ + uint8_t tmp; + + /* 1st row rotated once right */ + tmp = block[3][1]; + block[3][1] = block[2][1]; + block[2][1] = block[1][1]; + block[1][1] = block[0][1]; + block[0][1] = tmp; + + /* 2st row rotated twice right */ + tmp = block[0][2]; + block[0][2] = block[2][2]; + block[2][2] = tmp; + tmp = block[1][2]; + block[1][2] = block[3][2]; + block[3][2] = tmp; + + /* 3rd row rotated 3 times right */ + tmp = block[0][3]; + block[0][3] = block[1][3]; + block[1][3] = block[2][3]; + block[2][3] = block[3][3]; + block[3][3] = tmp; +} + +/* aes block encoder */ + +static void aes_encode_cipher (aes_state *state) +{ + uint8_t round; + aes_add_key(state->block, state->keyblock, 0); + for (round = 1; round < state->rounds; ++round) + { + aes_encode_sub(state->block); + aes_encode_shift(state->block); + aes_encode_mix(state->block); + aes_add_key(state->block, state->keyblock, round); + } + aes_encode_sub(state->block); + aes_encode_shift(state->block); + aes_add_key(state->block, state->keyblock, state->rounds); +} + +/* aes block decoder */ + +static void aes_decode_cipher (aes_state *state) +{ + uint8_t round; + aes_add_key(state->block, state->keyblock, state->rounds); + for(round = state->rounds - 1; round > 0; --round) + { + aes_decode_shift(state->block); + aes_decode_sub(state->block); + aes_add_key(state->block, state->keyblock, round); + aes_decode_mix(state->block); + } + aes_decode_shift(state->block); + aes_decode_sub(state->block); + aes_add_key(state->block, state->keyblock, 0); +} + +/* tail block padding; RFC 2898, PKCS #5: Password-Based Cryptography Specification Version 2.0; pdf spec p. 119 */ + +#define aes_padding(state) ((state->flags & AES_NULL_PADDING) == 0) + +static void aes_put_padding (aes_state *state, uint8_t length) +{ + uint8_t pad; + pad = (aes_padding(state)) ? 16 - length : 0; + for (; length < 16; ++length) + state->data[length] = state->iv[length] ^ pad; +} + +static int aes_remove_padding (aes_state *state, uint8_t *data, uint8_t *length) +{ + uint8_t pad; + *length = 16; /* block length 16 means leave intact */ + if (aes_padding(state)) + { + pad = data[16 - 1]; + if (pad > 16) + return IOFERR; + for ( ; *length > 16 - pad; --(*length)) + if (data[*length - 1] != pad) + return IOFERR; + } + else + { + for ( ; *length > 0; --(*length)) + if (data[*length - 1] != '\0') + break; + } + return IOFEOF; +} + +/* aes codec */ + +/* make the cipher on input xor-ed with iv, save the output as a new iv, write the output */ +#define aes_encode_output(state, output) \ + (aes_encode_cipher(state), aes_copy_block(state->iv, state->data), aes_copy_block(output, state->data), output += 16) + +iof_status aes_encode_state (iof *I, iof *O, aes_state *state) +{ + int c; + + if (!(state->flags & AES_HAS_IV)) // weird + return IOFERR; + if ((state->flags & AES_INLINE_IV) && !(state->flags & AES_CONTINUE)) + { /* write iv at the beginning of encrypted data */ + if (!iof_ensure(O, 16)) + return IOFFULL; + aes_copy_block(O->pos, state->iv); + O->pos += 16; + state->flags |= AES_CONTINUE; + } + while (iof_ensure(O, 16)) + { + while (state->buffered < 16) + { + if ((c = iof_get(I)) != IOFEOF) + { /* get input byte XORed with iv */ + state->data[state->buffered] = state->iv[state->buffered] ^ ((uint8_t)c); + ++state->buffered; + } + else + { + if (state->flush) + { + if (state->buffered > 0 || aes_padding(state)) + { /* pad the last input chunk; for input divisable by 16, add 16 bytes 0x0f */ + aes_put_padding(state, state->buffered); + state->buffered = 16; + aes_encode_output(state, O->pos); + } + return IOFEOF; + } + else + return IOFEMPTY; + } + } + aes_encode_output(state, O->pos); + state->buffered = 0; + } + return IOFFULL; +} + +/* write iv to the output, save the raw input just buffered as iv for the next chunk, make the cipher, write out xoring with iv */ +#define aes_decode_output(state, output) \ + (aes_copy_block(output, state->iv), aes_copy_block(state->iv, state->data), aes_decode_cipher(state), aes_copy_cbc(output, state->data), output += 16) + +iof_status aes_decode_state (iof *I, iof *O, aes_state *state) +{ + int c, ret; + uint8_t lastlength; + + if ((state->flags & AES_INLINE_IV) && !(state->flags & AES_CONTINUE)) + { + while (state->buffered < 16) + { + if ((c = iof_get(I)) != IOFEOF) + state->iv[state->buffered++] = (uint8_t)c; + else + return state->flush ? IOFERR : IOFEMPTY; + } + state->flags |= AES_CONTINUE|AES_HAS_IV; + state->buffered = 0; + } + while (iof_ensure(O, 16)) + { + while (state->buffered < 16) + { + if ((c = iof_get(I)) != IOFEOF) + state->data[state->buffered++] = (uint8_t)c; + else + return state->flush ? IOFERR : IOFEMPTY; + } + aes_decode_output(state, O->pos); + if (state->flush) + { /* we have to check for EOF here, to remove eventual padding */ + if ((c = iof_get(I)) < 0) + { /* end of input at 16-bytes boundary; remove padding and quit */ + ret = aes_remove_padding(state, O->pos - 16, &lastlength); + O->pos -= 16 - lastlength; + return ret; + } + else + { /* beginning of the next block */ + state->buffered = 1; + state->data[0] = (uint8_t)c; + } + } + else + state->buffered = 0; + } + return IOFFULL; +} + +/* variants that works on c-strings; can work inplace (output==input) except encoder in pdf flavour */ + +/* +Codecs operating on c-string can generally work inplace (output==input), except encoder with AES_INLINE_IV flag set, +which outputs 16 bytes of initialization vector at the beginning of encrypted data. All return the size of encrypted/decrypted +data. Encoders output is the original length padded to a complete 16 bytes (plus eventual 16 bytes of initialization +vector, if AES_INLINE_IV is used). Default padding is unambiguously removed during decryption. AES_NULL_PADDING flag +forces using (ambiguous) NULL-byte padding, only if input length module 16 is greater then zero. + +An input data is supposed to be a complete data to be encrypted or decrypted. It is possible, however, to use those +codecs for scaterred data chunks by manipulating AES_INLINE_IV, AES_NULL_PADDING, AES_CONTINUE flags and data length. +Caller may assume that c-string codecs do not modify state flags. + +Encoder could actually be optimized by writing an initialization vector to a state block once. After every chunk encryption, +the output is the initialization vector for the next chunk. Since we use c-string codec variants on short strings, +the gain is neglectable in comparison with the weight of the aes crypt procedure. +*/ + +size_t aes_encode_data (const void *input, size_t length, void *output, const void *key, size_t keylength, const void *iv, int flags) +{ + aes_state state; + aes_keyblock keyblock; + + if (aes_encode_initialize(&state, &keyblock, key, keylength, iv) == NULL) + return 0; + state.flags |= flags; + return aes_encode_state_data(&state, input, length, output); + // aes_state_close(&state); +} + +size_t aes_encode_state_data (aes_state *state, const void *input, size_t length, void *output) +{ + const uint8_t *inp; + uint8_t *out, tail, t; + size_t size; + + inp = (const uint8_t *)input; + out = (uint8_t *)output; + + if (!(state->flags & AES_HAS_IV)) + return 0; + if ((state->flags & AES_INLINE_IV) && !(state->flags & AES_CONTINUE)) + { + aes_copy_block(out, state->iv); + out += 16; + } + // state->flags |= AES_CONTINUE; // do not modify state flags + + for (size = 0; size + 16 <= length; size += 16) + { + aes_copy_xor(state->data, inp, state->iv); + aes_encode_output(state, out); + inp += 16; + } + + if ((tail = (length % 16)) > 0 || aes_padding(state)) + { + for (t = 0; t < tail; ++t) + state->data[t] = inp[t] ^ state->iv[t]; + aes_put_padding(state, tail); + aes_encode_output(state, out); + size += 16; + } + if (state->flags & AES_INLINE_IV) + size += 16; /* iv written at the beginning of encoded data */ + + return size; +} + +size_t aes_decode_data (const void *input, size_t length, void *output, const void *key, size_t keylength, const void *iv, int flags) +{ + aes_state state; + aes_keyblock keyblock; + + if (aes_decode_initialize(&state, &keyblock, key, keylength, iv) == NULL) + return 0; + state.flags |= flags; + return aes_decode_state_data(&state, input, length, output); + // aes_state_close(&state); +} + +size_t aes_decode_state_data (aes_state *state, const void *input, size_t length, void *output) +{ + const uint8_t *inp; + uint8_t *out, lastlength; + size_t size; + + inp = (const uint8_t *)input; + out = (uint8_t *)output; + + if ((state->flags & AES_INLINE_IV) && !(state->flags & AES_CONTINUE)) + { + aes_copy_block(state->iv, inp); + // state->flags |= AES_HAS_IV; // do not modify state flags + inp += 16; + length = length >= 16 ? length - 16 : 0; + } + else if (!(state->flags & AES_HAS_IV)) + return 0; + // state->flags |= AES_CONTINUE; // do not modify state flags + for (size = 0; size + 16 <= length; size += 16) + { + aes_copy_block(state->data, inp); + aes_decode_output(state, out); + inp += 16; + } + + if (size >= 16) + { + aes_remove_padding(state, out - 16, &lastlength); + size = size - 16 + lastlength; + } + + return size; +} + +/* +pseudo-random bytes chain exceprted from eexec; not expected to have strong cryptographic properties +we only expect that it is (reasonably) unique and different for each call (not only function call, but also +a program call). A current trick with mangling pointer value gives satisfactory results, generally different +for every function call and a programm call. Note that the pseudo-input bytes starts from some inner address +bits, as they vary better; without that, the first byte tends to be "lazy". +*/ + +void random_bytes (uint8_t *output, size_t size) +{ + size_t i; + uint8_t p; + static uint16_t k = 55665; + for (i = 0; i < size; ++i) + { + p = ((uint8_t *)(&output))[(i + 2) % sizeof(uint8_t *)] ^ (uint8_t)size; // pseudo input byte ;) + k = (((p + k) * 52845 + 22719) & 65535); // xor-ed with pseudo-random sequence (kept between calls) + output[i] = p ^ (k >> 8); + } +} + +void aes_generate_iv (uint8_t output[16]) +{ + random_bytes(output, 16); +} + +/* filters */ + +// rc4 decoder function + +static size_t rc4_decoder (iof *F, iof_mode mode) +{ + rc4_state *state; + iof_status status; + size_t tail; + + state = iof_filter_state(rc4_state *, F); + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + do { + status = rc4_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "rc4", status); + case IOFCLOSE: + rc4_state_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// rc4 encoder function + +static size_t rc4_encoder (iof *F, iof_mode mode) +{ + rc4_state *state; + iof_status status; + + state = iof_filter_state(rc4_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = rc4_encode_state(F, F->next, state); + return iof_encoder_retval(F, "rc4", status); + case IOFCLOSE: + if (!state->flush) + rc4_encoder(F, IOFFLUSH); + rc4_state_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// aes decoder function + +static size_t aes_decoder (iof *F, iof_mode mode) +{ + aes_state *state; + iof_status status; + size_t tail; + + state = iof_filter_state(aes_state *, F); + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + do { + status = aes_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "aes", status); + case IOFCLOSE: + aes_state_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// aes encoder function + +static size_t aes_encoder (iof *F, iof_mode mode) +{ + aes_state *state; + iof_status status; + + state = iof_filter_state(aes_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = aes_encode_state(F, F->next, state); + return iof_encoder_retval(F, "aes", status); + case IOFCLOSE: + if (!state->flush) + aes_encoder(F, IOFFLUSH); + aes_state_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +iof * iof_filter_rc4_decoder (iof *N, const void *key, size_t keylength) +{ + iof *I; + crypt_state_pointer P; + + I = iof_filter_reader(rc4_decoder, sizeof(rc4_state), &P.voidstate); + iof_setup_next(I, N); + if (rc4_state_init(P.rc4state, key, keylength) == NULL) + { + iof_discard(I); + return NULL; + } + P.rc4state->flush = 1; + return I; +} + +iof * iof_filter_rc4_encoder (iof *N, const void *key, size_t keylength) +{ + iof *O; + crypt_state_pointer P; + + O = iof_filter_writer(rc4_encoder, sizeof(rc4_state), &P.voidstate); + iof_setup_next(O, N); + if (rc4_state_init(P.rc4state, key, keylength) == NULL) + { + iof_discard(O); + return NULL; + } + // P.rc4state->flush = 1; + return O; +} + +/* aes crypt filters */ + +iof * iof_filter_aes_decoder (iof *N, const void *key, size_t keylength) +{ + iof *I; + crypt_state_pointer P; + + I = iof_filter_reader(aes_decoder, sizeof(aes_state), &P.voidstate); + iof_setup_next(I, N); + if (aes_decode_init(P.aesstate, key, keylength) == NULL) + { + iof_discard(I); + return NULL; + } + aes_pdf_mode(P.aesstate); + P.aesstate->flush = 1; + return I; +} + +iof * iof_filter_aes_encoder (iof *N, const void *key, size_t keylength) +{ + iof *O; + crypt_state_pointer P; + + O = iof_filter_writer(aes_encoder, sizeof(aes_state), &P.voidstate); + iof_setup_next(O, N); + if (aes_encode_init(P.aesstate, key, keylength) == NULL) + { + iof_discard(O); + return NULL; + } + aes_pdf_mode(P.aesstate); + // P.aesstate->flush = 1; + return O; +} + +/* test */ + +/* +static void show (void *p, size_t size, uint8_t round, uint8_t sym) +{ + uint8_t i; + printf("%c%c:", round, sym); + for (i = 0; i < size; ++i) + printf("%02x", ((uint8_t *)p)[i]); + printf("\n"); +} + +void aes_test (void) +{ + const uint8_t key[] = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; + const uint8_t iv[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; + const uint8_t inp[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 }; + const uint8_t out[] = { + 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + 0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b, 0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16, + 0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09, 0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 }; + + uint8_t input[64], output[64]; + size_t inpsize, outsize; + int flags = AES_NULL_PADDING; + + //////////////////////////////////////////////////////////////////////////// + +//#define ENCODETO output +#define ENCODETO input // inplace + + inpsize = 64; + memcpy(input, inp, inpsize); + show(input, inpsize, '>', '>'); + outsize = aes_encode_data(input, inpsize, ENCODETO, key, 16, iv, flags); + show(ENCODETO, outsize, '<', '<'); + if (outsize == inpsize && memcmp(ENCODETO, out, outsize) == 0) + printf("ENCODER SUCCESS\n"); + else + printf("ENCODER FAILURE\n"); + + //////////////////////////////////////////////////////////////////////////// + +//#define DECODETO input +#define DECODETO output // in place + + outsize = 64; + memcpy(output, out, outsize); + show(output, outsize, '<', '<'); + inpsize = aes_decode_data(output, outsize, DECODETO, key, 16, iv, flags); + show(DECODETO, inpsize, '>', '>'); + if (inpsize == outsize && memcmp(DECODETO, inp, inpsize) == 0) + printf("DECODER SUCCESS\n"); + else + printf("DECODER FAILURE\n"); +} +*/ + +/* +Some example vectors + +================================ AES ECB 128-bit encryption mode ================================ + +Encryption key: 2b7e151628aed2a6abf7158809cf4f3c + +Test vector Cipher text +6bc1bee22e409f96e93d7e117393172a 3ad77bb40d7a3660a89ecaf32466ef97 +ae2d8a571e03ac9c9eb76fac45af8e51 f5d3d58503b9699de785895a96fdbaaf +30c81c46a35ce411e5fbc1191a0a52ef 43b1cd7f598ece23881b00e3ed030688 +f69f2445df4f9b17ad2b417be66c3710 7b0c785e27e8ad3f8223207104725dd4 + + +================================ AES ECB 192-bit encryption mode ================================ + +Encryption key: 8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b + +Test vector Cipher text +6bc1bee22e409f96e93d7e117393172a bd334f1d6e45f25ff712a214571fa5cc +ae2d8a571e03ac9c9eb76fac45af8e51 974104846d0ad3ad7734ecb3ecee4eef +30c81c46a35ce411e5fbc1191a0a52ef ef7afd2270e2e60adce0ba2face6444e +f69f2445df4f9b17ad2b417be66c3710 9a4b41ba738d6c72fb16691603c18e0e + + +================================ AES ECB 256-bit encryption mode ================================ + +Encryption key: 603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4 + +Test vector Cipher text +6bc1bee22e409f96e93d7e117393172a f3eed1bdb5d2a03c064b5a7e3db181f8 +ae2d8a571e03ac9c9eb76fac45af8e51 591ccb10d410ed26dc5ba74a31362870 +30c81c46a35ce411e5fbc1191a0a52ef b6ed21b99ca6f4f9f153e7b1beafed1d +f69f2445df4f9b17ad2b417be66c3710 23304b7a39f9f3ff067d8d8f9e24ecc7 + +================================ AES CBC 128-bit encryption mode ================================ + +Encryption key: 2b7e151628aed2a6abf7158809cf4f3c + +Initialization vector Test vector Cipher text +000102030405060708090A0B0C0D0E0F 6bc1bee22e409f96e93d7e117393172a 7649abac8119b246cee98e9b12e9197d +7649ABAC8119B246CEE98E9B12E9197D ae2d8a571e03ac9c9eb76fac45af8e51 5086cb9b507219ee95db113a917678b2 +5086CB9B507219EE95DB113A917678B2 30c81c46a35ce411e5fbc1191a0a52ef 73bed6b8e3c1743b7116e69e22229516 +73BED6B8E3C1743B7116E69E22229516 f69f2445df4f9b17ad2b417be66c3710 3ff1caa1681fac09120eca307586e1a7 + +================================ AES CBC 192-bit encryption mode ================================ + +Encryption key: 8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b + +Initialization vector Test vector Cipher text +000102030405060708090A0B0C0D0E0F 6bc1bee22e409f96e93d7e117393172a 4f021db243bc633d7178183a9fa071e8 +4F021DB243BC633D7178183A9FA071E8 ae2d8a571e03ac9c9eb76fac45af8e51 b4d9ada9ad7dedf4e5e738763f69145a +B4D9ADA9AD7DEDF4E5E738763F69145A 30c81c46a35ce411e5fbc1191a0a52ef 571b242012fb7ae07fa9baac3df102e0 +571B242012FB7AE07FA9BAAC3DF102E0 f69f2445df4f9b17ad2b417be66c3710 08b0e27988598881d920a9e64f5615cd + +================================ AES CBC 256-bit encryption mode ================================ + +Encryption key: 603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4 + +Initialization vector Test vector Cipher text +000102030405060708090A0B0C0D0E0F 6bc1bee22e409f96e93d7e117393172a f58c4c04d6e5f1ba779eabfb5f7bfbd6 +F58C4C04D6E5F1BA779EABFB5F7BFBD6 ae2d8a571e03ac9c9eb76fac45af8e51 9cfc4e967edb808d679f777bc6702c7d +9CFC4E967EDB808D679F777BC6702C7D 30c81c46a35ce411e5fbc1191a0a52ef 39f23369a9d9bacfa530e26304231461 +39F23369A9D9BACFA530E26304231461 f69f2445df4f9b17ad2b417be66c3710 b2eb05e2c39be9fcda6c19078c6a9d1b +*/
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilcrypt.h b/Build/source/libs/pplib/pplib-src/src/util/utilcrypt.h new file mode 100644 index 00000000000..e5bf53cc5ce --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilcrypt.h @@ -0,0 +1,90 @@ +#ifndef UTIL_CRYPT_H +#define UTIL_CRYPT_H + +#include <stdint.h> +#include <stddef.h> +#include "utiliof.h" + +#ifndef UTIL_CRYPT_TIME +# define UTIL_CRYPT_TIME 0 +#endif + +/* RC4 */ + +typedef uint8_t rc4_map[256]; + +typedef struct rc4_state rc4_state; + +#define RC4_STATE_ALLOC (1<<0) + +UTILAPI rc4_state * rc4_state_initialize (rc4_state *state, rc4_map *map, const void *vkey, size_t keylength); +#define rc4_state_init(state, vkey, keylength) rc4_state_initialize(state, NULL, vkey, keylength) +UTILAPI void rc4_map_save (rc4_state *state, rc4_map *map); +UTILAPI void rc4_map_restore (rc4_state *state, rc4_map *map); + +/* Codecs operating on iof */ + +UTILAPI iof_status rc4_crypt_state (iof *I, iof *O, rc4_state *state); +#define rc4_encode_state(I, O, state) rc4_crypt_state(I, O, state) +#define rc4_decode_state(I, O, state) rc4_crypt_state(I, O, state) + +UTILAPI iof_status rc4_crypt (iof *I, iof *O, const void *key, size_t length); +#define rc4_encode(I, O) rc4_crypt(I, O, key, length) +#define rc4_decode(I, O) rc4_crypt(I, O, key, length) + +UTILAPI size_t rc4_crypt_data (const void *input, size_t length, void *output, const void *key, size_t keylength); +UTILAPI size_t rc4_crypt_state_data (rc4_state *state, const void *input, size_t length, void *output); +#define rc4_encode_data(input, length, output, key, keylength) rc4_crypt_data(input, length, output, key, keylength) +#define rc4_decode_data(input, length, output, key, keylength) rc4_crypt_data(input, length, output, key, keylength) +#define rc4_encode_state_data(state, input, length, output) rc4_crypt_state_data(state, input, length, output) +#define rc4_decode_state_data(state, input, length, output) rc4_crypt_state_data(state, input, length, output) + +UTILAPI void rc4_state_close (rc4_state *state); + +/* AES */ + +typedef uint8_t aes_block[4][4]; +typedef aes_block aes_keyblock[15]; // aes128 - 10+1, aes192 - 12+1, aes256 - 14+1 + +typedef struct aes_state aes_state; + +#define AES_STATE_ALLOC (1<<0) +//#define AES_ECB_MODE (1<<2) +#define AES_HAS_IV (1<<3) +#define AES_INLINE_IV (1<<4) +#define AES_CONTINUE (1<<5) +#define AES_NULL_PADDING (1<<6) + +UTILAPI void aes_pdf_mode (aes_state *state); +//UTILAPI aes_state * aes_state_initialize_ecb (aes_state *State, uint8_t *roundkey, const uint8_t *key); +UTILAPI aes_state * aes_encode_initialize (aes_state *state, aes_keyblock *keyblock, const void *key, size_t keylength, const void *iv); +UTILAPI aes_state * aes_decode_initialize (aes_state *state, aes_keyblock *keyblock, const void *key, size_t keylength, const void *iv); +#define aes_encode_init(state, key, keylength) aes_encode_initialize(state, NULL, key, keylength, NULL) +#define aes_decode_init(state, key, keylength) aes_decode_initialize(state, NULL, key, keylength, NULL) + +UTILAPI void aes_state_close (aes_state *state); + +/* Codecs operating on iof */ + +UTILAPI iof_status aes_encode_state (iof *I, iof *O, aes_state *state); +UTILAPI iof_status aes_decode_state (iof *I, iof *O, aes_state *state); + +UTILAPI size_t aes_encode_data (const void *input, size_t length, void *output, const void *key, size_t keylength, const void *iv, int flags); +UTILAPI size_t aes_encode_state_data (aes_state *state, const void *input, size_t length, void *output); +UTILAPI size_t aes_decode_data (const void *input, size_t length, void *output, const void *key, size_t keylength, const void *iv, int flags); +UTILAPI size_t aes_decode_state_data (aes_state *state, const void *input, size_t length, void *output); + +/* random bytes generator */ + +UTILAPI void random_bytes (uint8_t *output, size_t size); +UTILAPI void aes_generate_iv (uint8_t output[16]); + +/* filters */ + +iof * iof_filter_rc4_decoder (iof *N, const void *key, size_t keylength); +iof * iof_filter_rc4_encoder (iof *N, const void *key, size_t keylength); + +iof * iof_filter_aes_decoder (iof *N, const void *key, size_t keylength); +iof * iof_filter_aes_encoder (iof *N, const void *key, size_t keylength); + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilcryptdef.h b/Build/source/libs/pplib/pplib-src/src/util/utilcryptdef.h new file mode 100644 index 00000000000..d43ea2e5b53 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilcryptdef.h @@ -0,0 +1,32 @@ + +#ifndef UTIL_CRYPTDEF_H +#define UTIL_CRYPTDEF_H + +struct rc4_state { + union { + rc4_map *map; + uint8_t *smap; + }; + int i, j; + int flush; + int flags; +}; + +struct aes_state { + size_t keylength; + int rounds; + //int keywords; + union { + aes_block block; + uint8_t data[16]; + }; + aes_keyblock *keyblock; + uint8_t iv[16]; + uint8_t buffered; + int flush; + int flags; +}; + +typedef union { rc4_state *rc4state; aes_state *aesstate; void *voidstate; } crypt_state_pointer; // to avoid 'dereferencing type-puned ...' warnings + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utildecl.h b/Build/source/libs/pplib/pplib-src/src/util/utildecl.h new file mode 100644 index 00000000000..b11e5b88432 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utildecl.h @@ -0,0 +1,28 @@ + +#ifndef UTIL_DECL_H +#define UTIL_DECL_H + +/* +UTILDLL - when building .dll +UTILEXE - when building .exe to import symbols from .dll +*/ + +#if defined (_WIN32) || defined(_WIN64) +# ifdef UTILDLL +# define UTILAPI __declspec(dllexport) +# define UTILDEF __declspec(dllexport) +# else +# ifdef UTILEXE +# define UTILAPI __declspec(dllimport) +# define UTILDEF +# else +# define UTILAPI +# define UTILDEF +# endif +# endif +#else +# define UTILAPI +# define UTILDEF +#endif + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilflate.c b/Build/source/libs/pplib/pplib-src/src/util/utilflate.c new file mode 100644 index 00000000000..27e44d409a1 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilflate.c @@ -0,0 +1,322 @@ + +#include <zlib.h> + +#include "utilmem.h" +#include "utillog.h" +#include "utilflate.h" + +/* flate codec */ + +/* +Flate codec example provided at http://www.zlib.net/zpipe.c (http://www.zlib.net/zlib_how.html) uses the following scheme: +- provide input data buffer +- keep providing output until codec function uses it + +For encoder: + + z->zalloc = z->zfree = z->zopaque = NULL; + deflateInit(z, compression_level); + do { + z->next_in = <input buffer> + z->avail_in = <input buffer bytes> + do { + z->next_out = <output buffer> + z->avail_out = <output buffer bytes> + deflate(z, flush); + // write obtained output from deflate + } while (z->avail_out == 0); + assert(z->avail_in == 0); + } while (flush != Z_FINISH); + deflateEnd(z); + +'z' is an internal codec state of type z_stream, 'flush' is either Z_NO_FLUSH or Z_FINISH at the end of data. +deflate() ensures to consume the entire input if there are no obstackles to write an output. The inner loop +provides an output space as long as it is used by deflate(). When deflate() wrote everything it could, +it leaves z->avail_out > 0, which breaks the inner loop. At this point z->avail_in should also be zero. +The example documentation claims that the return codes from deflate() doesn't really need to be checked, +as checking z->avail_out for zero is enough. + +The scheme for decoder is pretty similar, but with substantial differences: +- the end of stream is automatically found by decoder, so using Z_FINISH flag to indicate an end of stream + is not necessary, but if provided, it MUST be given only if the EOF marker actually occurs in the input chunk, + and subsequent calls to inflate() must consequently use Z_FINISH +- calling inflate() as long as it uses the output buffer provided still works for decoder, but inflate() + does not ensure to consume the entire input, as it will read until end of stream marker +- the return code from inflate() must be checked to ensure the proper reaction on invalid data stream and + end of stream signals +- initialization must set an input buffer to NULL or to some existing chunk (the later helps zlib to perform + better on inflate(), but inflate() does the research on the first call anyway) + + z->zalloc = z->zfree = z->zopaque = NULL; + z->next_in = NULL, z->avail_in = 0; + inflateInit(z); + do { + z->next_in = <input buffer> + z->avail_in = <input buffer bytes> + do { + z->next_out = <output buffer> + z->avail_out = <output buffer bytes> + status = inflate(z, flush); + // check return status + // write obtained output from inflate + } while (z->avail_out == 0); + } while (status != Z_STREAM_END); + inflateEnd(z); + +Our wrapper generally follows "prepare input, keep pomping output" scheme, but we need to support handler function +breaks on IOFEMPTY and IOFFULL. For a consistent come back from those on subsequent calls to the handler function, +we use 3 states: +- FLATE_IN - get input, when got something then goto FALTE_OUT +- FLATE_OUT - set z_stream buffers and keep writing output until enything to write, then goto FLATE_IN or FLATE_DONE +- FLATE_DONE - we are done, no return from that state +Distinction of FLATE_IN and FLATE_OUT states guarantees that we will not get more input until zlib consumes the stuff +from the previous feed, possibly interrupted by IOFFULL return on filling the output buffer. This distinction is not +critical, but makes the filter running according to the scheme described above. Note that we set zlib input buffer +(z->next_in, z->avail_in) at the beginning of FLATE_OUT state. Also note that we always update our buffers according +to updated avail_in / avail_out values, just after a call to inflate() / deflate(). So no matter what have happens +between handler calls, zlib input buffer is in sync with ours. +*/ + +struct flate_state { + z_stream z; + int flush; + int status; + int level; /* encoder compression level -1..9 */ +}; + +typedef union { flate_state *flatestate; void *voidstate; } flate_state_pointer; // to avoid 'dereferencing type-puned ...' warnings + +enum { + FLATE_IN, + FLATE_OUT, + FLATE_DONE +}; + +flate_state * flate_decoder_init (flate_state *state) +{ /* initialize zlib */ + z_stream *z = &state->z; + z->zalloc = Z_NULL; + z->zfree = Z_NULL; + z->opaque = Z_NULL; + z->avail_in = 0; /* must be initialized before inflateInit() */ + z->next_in = Z_NULL; /* ditto */ + if (inflateInit(z) != Z_OK) + return NULL; + state->status = FLATE_IN; + return state; +} + +flate_state * flate_encoder_init (flate_state *state) +{ + z_stream *z = &state->z; + z->zalloc = Z_NULL; + z->zfree = Z_NULL; + z->opaque = Z_NULL; + z->avail_in = 0; + z->next_in = Z_NULL; + state->level = Z_DEFAULT_COMPRESSION; // will probably be moved upward + if (deflateInit(z, state->level) != Z_OK) + return NULL; + state->status = FLATE_IN; + return state; +} + +static const char * zmess (int zstatus) +{ + switch (zstatus) + { + case Z_OK: return "ok"; + case Z_STREAM_END: return "end of stream"; + case Z_BUF_ERROR: return "buffer error"; + case Z_STREAM_ERROR: return "stream error"; + case Z_NEED_DICT: return "need dict"; + case Z_DATA_ERROR: return "data error"; + case Z_MEM_ERROR: return "memory error"; + case Z_VERSION_ERROR: return "version error"; + case Z_ERRNO: return "io error"; + default: + break; + } + return "unknown error"; +} + +iof_status flate_decode_state (iof *I, iof *O, flate_state *state) +{ + z_stream *z; + int zstatus = Z_OK; + z = &state->z; + while (state->status != FLATE_DONE) + { + if (state->status == FLATE_IN) + { + if (!iof_readable(I)) + return state->flush ? IOFERR : IOFEMPTY; + state->status = FLATE_OUT; + } + z->next_in = (Bytef *)I->pos; + z->avail_in = (uInt)iof_left(I); + do { + if (!iof_writable(O)) + return IOFFULL; + z->next_out = (Bytef *)O->pos; + z->avail_out = (uInt)iof_left(O); + zstatus = inflate(z, Z_NO_FLUSH); + I->pos += iof_left(I) - z->avail_in; + O->pos += iof_left(O) - z->avail_out; + switch (zstatus) + { + case Z_OK: + case Z_STREAM_END: + break; + default: + loggerf("flate decoder %s (%d)", zmess(zstatus), zstatus); + return IOFERR; + } + } while (z->avail_out == 0); + state->status = zstatus == Z_STREAM_END ? FLATE_DONE : FLATE_IN; + } + return IOFEOF; +} + +iof_status flate_encode_state (iof *I, iof *O, flate_state *state) +{ + z_stream *z; + int zstatus; + z = &state->z; + while (state->status != FLATE_DONE) + { + if (state->status == FLATE_IN) + { + if (!iof_readable(I)) + if (!state->flush) + return IOFEMPTY; + state->status = FLATE_OUT; + } + z->next_in = (Bytef *)I->pos; + z->avail_in = (uInt)iof_left(I); + do { + if (!iof_writable(O)) + return IOFFULL; + z->next_out = (Bytef *)O->pos; + z->avail_out = (uInt)iof_left(O); + zstatus = deflate(z, state->flush ? Z_FINISH : Z_NO_FLUSH); + I->pos += iof_left(I) - z->avail_in; + O->pos += iof_left(O) - z->avail_out; + switch (zstatus) + { + case Z_OK: + case Z_STREAM_END: + break; + default: + loggerf("flate encoder %s (%d)", zmess(zstatus), zstatus); + return IOFERR; + } + } while (z->avail_out == 0); + state->status = state->flush ? FLATE_DONE : FLATE_IN; + } + return IOFEOF; +} + + +void flate_decoder_close (flate_state *state) +{ + inflateEnd(&state->z); +} + +void flate_encoder_close (flate_state *state) +{ + deflateEnd(&state->z); +} + +/* filter */ + +// flate decoder function + +static size_t flate_decoder (iof *F, iof_mode mode) +{ + flate_state *state; + iof_status status; + size_t tail; + + state = iof_filter_state(flate_state *, F); + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + do { + status = flate_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "flate", status); + case IOFCLOSE: + flate_decoder_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// flate encoder function + +static size_t flate_encoder (iof *F, iof_mode mode) +{ + flate_state *state; + iof_status status; + + state = iof_filter_state(flate_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = flate_encode_state(F, F->next, state); + return iof_encoder_retval(F, "flate", status); + case IOFCLOSE: + if (!state->flush) + flate_encoder(F, IOFFLUSH); + flate_encoder_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +iof * iof_filter_flate_decoder (iof *N) +{ + iof *I; + flate_state_pointer P; + I = iof_filter_reader(flate_decoder, sizeof(flate_state), &P.voidstate); + iof_setup_next(I, N); + if (flate_decoder_init(P.flatestate) == NULL) + { + iof_discard(I); + return NULL; + } + P.flatestate->flush = 1; + return I; +} + +iof * iof_filter_flate_encoder (iof *N) +{ + iof *O; + flate_state_pointer P; + O = iof_filter_writer(flate_encoder, sizeof(flate_state), &P.voidstate); + iof_setup_next(O, N); + if (flate_encoder_init(P.flatestate) == NULL) + { + iof_discard(O); + return NULL; + } + return O; +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilflate.h b/Build/source/libs/pplib/pplib-src/src/util/utilflate.h new file mode 100644 index 00000000000..09bdd666112 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilflate.h @@ -0,0 +1,21 @@ +#ifndef UTIL_FLATE_H +#define UTIL_FLATE_H + +#include "utiliof.h" + +typedef struct flate_state flate_state; + +flate_state * flate_decoder_init (flate_state *state); +flate_state * flate_encoder_init (flate_state *state); + +iof_status flate_decode_state (iof *I, iof *O, flate_state *state); +iof_status flate_encode_state (iof *I, iof *O, flate_state *state); + +void flate_decoder_close (flate_state *state); +void flate_encoder_close (flate_state *state); + +iof * iof_filter_flate_decoder (iof *N); +iof * iof_filter_flate_encoder (iof *N); + + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilfpred.c b/Build/source/libs/pplib/pplib-src/src/util/utilfpred.c new file mode 100644 index 00000000000..9203c5e0742 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilfpred.c @@ -0,0 +1,778 @@ +/* predictor filters; common for flate and lzw */ + +#include "utilmem.h" +#include "utillog.h" +#include "utilfpred.h" + +/* +Here we implement predictor filters used with flate and lzw compressions in PDF streams. The main idea of data prediction +is to compute and output the differences between data records instead of those records. Adjacent pixels in images are usually +similar, so differences between pixel values tends to be zero. And both Flate and LZW performs better when the input +is rather smooth. Although a preliminary use of predictors is related to bitmap data, The actual need for predictor filter +came from the fact that xref streams may also be predicted (usually with PNG up-predictor). + +PDF specification allows to use several predictor algorithms, specified by /Predictor key in /DecodeParms dictionary: + + 1 - no predictor (default) + 2 - TIFF horizontal predictor + 10 - PNG none predictor + 11 - PNG sub predictor + 12 - PNG up predictor + 13 - PNG average predictor + 14 - PNG paeth predictor + +All PNG predictors works on bytes, regardless the image color-depth. While encoding, every input data byte is decreased +by the appropriate byte of the previous pixel. Even if the pixel does not fit a full byte, PNG predictors use an artificial +pixel size rounded up to a full byte. PNG predictors utilizes previous (left) pixel, pixel above and previous to above +pixel. In case of PNG, the type of the predictor is written on a dedicated byte at the beginning of every scanline. It +means all predictor functions must maintain and information about left, above and left-above pixels. + +Despite the same differencing idea, TIFF predictors are different. The prediction process bases on pixel components, +which are not necessarily bytes (component of a pixel is added/substracted from a relevant component of a previous +pixel). In TIFF predictor 2, only the previous (the left) pixel is taken into account, there is no need to keep +an information about other surrounding pixels. Also there is no expicit algorithm marker in data; the same prediction +method is applied to all input rows. + +Not surprisingly, predictor encoders and decoders are pretty similar. Encoders take some input value and the previous +input value (or 0 at the beginning of the scanline) and output a difference between them. Decoders takes an input value, +previously decoded value (or zero) and outputs their sum. When encoding, the result is cast to the proper unsigned integer, +when decoding, modulo 256 (or appropriate) is used, which makes encoding and decoding looseless. + +Some extra bits trickery is involved in TIFF predictor function, when components don't fit bytes boundary. In that case, +an input is treated as a bits stream. Every input byte is "buffered" in a larger integer, as its lower bits (from right). +Every output value is taken from its higher (left) bits. In a special case of bits-per-component equal 1, we buffer all +pixel bits and use XOR to compute bits difference between pixels. I've excerpted that trick from poppler, but I'm not +really sure if it works any better, especially when the number of components per pixel is 1. In that case we do a hard +bit-by-bit work anyway. + +In PNG prediction, we record every pixel byte (in decoded form) in state->rowsave. At the end of a scanline +we copy state->rowsave to state->rowup, so that in the next scanline we can access up-pixel byte. +Left pixel byte is accessed as state->rowsave (the byte recently stored or virtual left edge byte \0). +Up-left pixel byte is accessed via state->rowup, but with state->pixelsize offset (same as left byte, possibly \0 +at the left edge of the row). Both state->rowup and state->rowsave has a safe span of pixelsize bytes on the left, +that are permanently \0. +*/ + +#define predictor_component_t uint16_t +#define predictor_pixel1b_t uint32_t + +#define MAX_COMPONENTS 8 + +struct predictor_state { + int default_predictor; /* default predictor indicator */ + int current_predictor; /* current predictor, possibly taken from algorithm marker in PNG data */ + int rowsamples; /* number of pixels in a scanline (/DecodeParms << /Columns ... >>) */ + int compbits; /* number of bits per component (/DecodeParms << /BitsPerComponent ... >>) */ + int components; /* number of components (/DecodeParms << /Colors ... >>) */ + uint8_t *buffer; /* temporary private buffer area */ + uint8_t *rowin; /* an input row buffer position */ + int rowsize; /* size of a current scanline in bytes (rounded up) */ + int rowend; /* an input buffer end position */ + int rowindex; /* an output buffer position */ + union { + struct { /* used by PNG predictor codecs */ + uint8_t *rowup, *rowsave; /* previous scanline buffers */ + int predictorbyte; /* flag indicating that algorithm byte is read/written */ + int pixelsize; /* number of bytes per pixel (rounded up) */ + }; + struct { /* used by TIFF predictor codecs */ + predictor_component_t compbuffer[MAX_COMPONENTS]; + union { + predictor_component_t *prevcomp; /* an array of left pixel components, typically eq ->compbuffer */ + predictor_pixel1b_t *prevpixel; /* left pixel value stored on a single integer (for 1bit color-depth) */ + }; + int compin, compout; /* bit stream buffers */ + int bitsin, bitsout; /* bit stream counters */ + int sampleindex; /* pixel counter */ + int compindex; /* component counter */ + int pixbufsize; /* size of pixel buffer in bytes */ + }; + }; + int flush; + int status; +}; + +typedef union { predictor_state *predictorstate; void *voidstate; } predictor_state_pointer; // to avoid 'dereferencing type-puned ...' warnings + +enum { + STATUS_LAST = 0, + STATUS_CONTINUE = 1 // any value different then IOFEOF, IOFERR, ... which are < 0 +}; + +/* +Predictor type identifiers (pdf spec 76). lpdf doesn't hire the codec if predictor is 1. Predictor 15 indicates +that the type of PNG prediction algorithm may change in subsequent lines. We always check algorithm marker anyway. +*/ + +enum predictor_code { + NONE_PREDICTOR = 1, + TIFF_PREDICTOR = 2, + PNG_NONE_PREDICTOR = 10, + PNG_SUB_PREDICTOR = 11, + PNG_UP_PREDICTOR = 12, + PNG_AVERAGE_PREDICTOR = 13, + PNG_PAETH_PREDICTOR = 14, + PNG_OPTIMUM_PREDICTOR = 15 +}; + +predictor_state * predictor_decoder_init (predictor_state *state, int predictor, int rowsamples, int components, int compbits) +{ + int rowsize, pixelsize; +#define storage_pos(b, p, size) ((b = p), (p += size)) + uint8_t *buffer, *p; + size_t buffersize; + + pixelsize = (components * compbits + 7) >> 3; // to bytes, rounded up + rowsize = (rowsamples * components * compbits + 7) >> 3; + + state->default_predictor = state->current_predictor = predictor; + state->rowsamples = rowsamples; + state->components = components; + state->compbits = compbits; + + if (predictor == TIFF_PREDICTOR) + { /* tiff predictor */ + size_t compbuf, pixbuf; + compbuf = components * sizeof(predictor_component_t); + pixbuf = 1 * sizeof(predictor_pixel1b_t); + state->pixbufsize = (int)(compbuf > pixbuf ? compbuf : pixbuf); + buffersize = rowsize * sizeof(uint8_t); + buffer = (uint8_t *)util_calloc(buffersize, 1); + if ((size_t)state->pixbufsize > sizeof(state->compbuffer)) // components > MAX_COMPONENTS + state->prevcomp = (predictor_component_t *)util_calloc(state->pixbufsize, 1); + else + state->prevcomp = state->compbuffer; + // &state->prevcomp == &state->prevpixel + state->sampleindex = state->compindex = 0; + state->bitsin = state->bitsout = 0; + state->compin = state->compout = 0; + } + else + { /* png predictors */ + buffersize = (3 * rowsize + 2 * pixelsize + 1) * sizeof(uint8_t); + p = buffer = (uint8_t *)util_calloc(buffersize, 1); + storage_pos(state->rowin, p, 1 + rowsize); // one extra byte for prediction algorithm tag + p += pixelsize; // pixelsize extra bytes for virtual left pixel at the edge, eg. rowup[-1] (permanently \0) + storage_pos(state->rowup, p, rowsize); // actual row byte + p += pixelsize; // ditto + storage_pos(state->rowsave, p, rowsize); + state->pixelsize = pixelsize; + state->predictorbyte = 0; + } + state->buffer = buffer; + state->rowsize = rowsize; + state->rowindex = 0; + state->rowend = 0; + state->status = STATUS_CONTINUE; + return state; +} + +predictor_state * predictor_encoder_init (predictor_state *state, int predictor, int rowsamples, int components, int compbits) +{ + return predictor_decoder_init(state, predictor, rowsamples, components, compbits); +} + +void predictor_decoder_close (predictor_state *state) +{ + util_free(state->buffer); + if (state->default_predictor == TIFF_PREDICTOR && state->prevcomp != NULL && state->prevcomp != state->compbuffer) + util_free(state->prevcomp); +} + +void predictor_encoder_close (predictor_state *state) +{ + predictor_decoder_close(state); +} + +/* +All predoctor codecs first read the entire data row into a buffer. This is not crucial for the process, +but allows to separate read/write states. In particular, there is one place in which codec functions +may return on EOD. +*/ + +#define start_row(state) (state->rowindex = 0, state->rowin = state->buffer) + +static int read_scanline (predictor_state *state, iof *I, int size) +{ + int rowtail, left; + while ((rowtail = size - state->rowend) > 0) + { + left = (int)iof_left(I); + if (left >= rowtail) + { + memcpy(state->buffer + state->rowend, I->pos, (size_t)rowtail); + state->rowend += rowtail; + I->pos += rowtail; + start_row(state); + break; + } + else + { + if ((rowtail = left) > 0) + { + memcpy(state->buffer + state->rowend, I->pos, (size_t)rowtail); + state->rowend += rowtail; + I->pos += rowtail; + } + if (iof_input(I) == 0) + { + if (state->rowend == 0) // no scanline to process, no more input + return state->flush ? IOFEOF : IOFEMPTY; + /* If we are here, there is an incomplete scanline in buffer: + - if there is a chance for more (state->flush == 0), than wait for more + - otherwise encode/decode the last incomplete line? + pdf spec p. 76 says that "A row occupies a whole number of bytes", + so this situation should be considered abnormal (not found so far). + */ + if (!state->flush) + return IOFEMPTY; + loggerf("incomplete scanline in predictor filter"); + //return IOFERR; + state->status = STATUS_LAST; + state->rowsize -= size - state->rowend; + start_row(state); + break; + } + } + } + return STATUS_CONTINUE; +} + +#define read_row(state, I, size, status) if ((status = read_scanline(state, I, size)) != STATUS_CONTINUE) return status + +#define ensure_output_bytes(O, n) if (!iof_ensure(O, n)) return IOFFULL + +#define tobyte(c) ((uint8_t)(c)) +#define tocomp(c) ((uint16_t)(c)) + +#define row_byte(state) (state->rowin[state->rowindex]) + +/* png predictor macros; on bytes */ + +#define up_pixel_byte(state) (state->rowup[state->rowindex]) +#define upleft_pixel_byte(state) (state->rowup[state->rowindex - state->pixelsize]) +#define left_pixel_byte(state) (state->rowsave[state->rowindex - state->pixelsize]) +#define save_pixel_byte(state, c) (state->rowsave[state->rowindex] = (uint8_t)(c)) + +/* tiff predictor macros; on components */ + +#define left_pixel_component(state) (state->prevcomp[state->compindex]) // tiff predictor with 2, 4, 8, 16 components +#define left_pixel_value(state) (state->prevpixel[0]) // tiff predictor with 1bit components + +/* assignment in conditional +#define save_pixel_component(state, c) ((void)\ + ((state->prevcomp[state->compindex] = (predictor_component_t)(c)), \ + ++state->compindex, (state->compindex < state->components || (state->compindex = 0)))) +*/ +#define save_pixel_component(state, c) \ + do { state->prevcomp[state->compindex] = (predictor_component_t)(c); if (++state->compindex >= state->components) state->compindex = 0; } while (0) + +#define save_pixel_value(state, c) (state->prevpixel[0] = (predictor_pixel1b_t)(c)) + +/* Once the codec function is done with the scanline, we set imaginary left pixel data to zero, and reset row counters to +zero in order to allow buffering another input scanline. */ + +#define reset_row(state) state->rowend = 0 + +#define reset_png_row(state) (memcpy(state->rowup, state->rowsave, state->rowsize), state->predictorbyte = 0, reset_row(state)) + +#define reset_tiff_row(state) \ + memset(state->prevcomp, 0, state->pixbufsize), \ + state->bitsin = state->bitsout = 0, \ + state->compin = state->compout = 0, \ + reset_row(state), \ + state->sampleindex = state->compindex = 0 + +/* PNG paeth predictor function; http://www.libpng.org/pub/png/book/chapter09.html +Compute the base value p := left + up - upleft, then choose that byte the closest +(of the smallest absolute difference) to the base value. Left byte has a precedence. */ + + +static int paeth (predictor_state *state) +{ + int p, p1, p2, p3; + p = left_pixel_byte(state) + up_pixel_byte(state) - upleft_pixel_byte(state); + p1 = p >= left_pixel_byte(state) ? (p - left_pixel_byte(state)) : (left_pixel_byte(state) - p); + p2 = p >= up_pixel_byte(state) ? (p - up_pixel_byte(state)) : (up_pixel_byte(state) - p); + p3 = p >= upleft_pixel_byte(state) ? (p - upleft_pixel_byte(state)) : (upleft_pixel_byte(state) - p); + return (p1 <= p2 && p1 <= p3) ? left_pixel_byte(state) : (p2 <= p3 ? up_pixel_byte(state) : upleft_pixel_byte(state)); +} + +/* predictor decoder */ + +iof_status predictor_decode_state (iof *I, iof *O, predictor_state *state) +{ + int status, c, d, outbytes; + while (state->status == STATUS_CONTINUE) + { + if (state->default_predictor >= 10) // PNG predictor? + { + read_row(state, I, state->rowsize + 1, status); + if (state->predictorbyte == 0) + { // we could actually check state->rowin <> state->buffer, but we need this flag for encoder anyway + state->current_predictor = row_byte(state) + 10; + state->predictorbyte = 1; + ++state->rowin; + } + } + else + { + read_row(state, I, state->rowsize, status); + } + switch (state->current_predictor) + { + case NONE_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + iof_set(O, c); + } + reset_row(state); + break; + case TIFF_PREDICTOR: + switch (state->compbits) + { + case 1: + outbytes = (state->components + 7) >> 3; + for ( ; state->sampleindex < state->rowsamples; ++state->sampleindex) + { + ensure_output_bytes(O, outbytes); + while (state->bitsin < state->components) + { + state->compin = (state->compin << 8) | row_byte(state); + state->bitsin += 8; + ++state->rowindex; + } + state->bitsin -= state->components; + d = state->compin >> state->bitsin; + state->compin &= (1 << state->bitsin) - 1; + c = d ^ left_pixel_value(state); + save_pixel_value(state, c); + state->compout = (state->compout << state->components) | c; + state->bitsout += state->components; + while (state->bitsout >= 8) + { + state->bitsout -= 8; + iof_set(O, state->compout >> state->bitsout); + state->compout &= (1 << state->bitsout) - 1; + } + } + if (state->bitsout > 0) + { + ensure_output_bytes(O, 1); + iof_set(O, state->compin << (8 - state->bitsout)); + } + break; + case 2: case 4: + for ( ; state->sampleindex < state->rowsamples; ++state->sampleindex) + { + for ( ; state->compindex < state->components; ) // state->compindex is ++ed by save_pixel_component() + { + ensure_output_bytes(O, 1); + if (state->bitsin < state->compbits) + { + state->compin = (state->compin << 8) | row_byte(state); + state->bitsin += 8; + ++state->rowindex; + } + state->bitsin -= state->compbits; + d = state->compin >> state->bitsin; + state->compin &= (1 << state->bitsin) - 1; + c = (d + left_pixel_component(state)) & 0xff; + save_pixel_component(state, c); + state->compout = (state->compout << state->compbits) | c; + state->bitsout += state->compbits; + if (state->bitsout >= 8) + { + state->bitsout -= 8; + iof_set(O, state->compout >> state->bitsout); + state->compout &= (1 << state->bitsout) - 1; + } + } + } + if (state->bitsout > 0) + { + ensure_output_bytes(O, 1); + iof_set(O, state->compin << (8 - state->bitsout)); + } + break; + case 8: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = (row_byte(state) + left_pixel_component(state)) & 0xff; + save_pixel_component(state, c); + iof_set(O, c); + } + break; + case 16: + for ( ; state->rowindex < state->rowsize - 1; ++state->rowindex) + { + ensure_output_bytes(O, 2); + d = row_byte(state) << 8; + ++state->rowindex; + d |= row_byte(state); + c = (d + left_pixel_component(state)) & 0xffff; + save_pixel_component(state, c); + iof_set2(O, c >> 8, c & 0xff); + } + break; + default: + return IOFERR; + } + reset_tiff_row(state); + break; + case PNG_NONE_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + save_pixel_byte(state, c); // next row may need it + iof_set(O, c); + } + reset_png_row(state); + break; + case PNG_SUB_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = (row_byte(state) + left_pixel_byte(state)) & 0xff; + save_pixel_byte(state, c); + iof_set(O, c); + } + reset_png_row(state); + break; + case PNG_UP_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = (row_byte(state) + up_pixel_byte(state)) & 0xff; + save_pixel_byte(state, c); + iof_set(O, c); + } + reset_png_row(state); + break; + case PNG_AVERAGE_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = (row_byte(state) + ((up_pixel_byte(state) + left_pixel_byte(state)) / 2)) & 0xff; + save_pixel_byte(state, c); + iof_set(O, c); + } + reset_png_row(state); + break; + case PNG_PAETH_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = (row_byte(state) + paeth(state)) & 0xff; + save_pixel_byte(state, c); + iof_set(O, c); + } + reset_png_row(state); + break; + //case PNG_OPTIMUM_PREDICTOR: // valid as default_redictor, but not as algorithm identifier byte + default: + return IOFERR; + } + } + return state->status == STATUS_LAST ? IOFERR : IOFEOF; +} + +/* predictor encoder */ + +iof_status predictor_encode_state (iof *I, iof *O, predictor_state *state) +{ + int status, c, d, outbytes; + while (state->status == STATUS_CONTINUE) + { + read_row(state, I, state->rowsize, status); + if (state->current_predictor >= 10 && state->predictorbyte == 0) + { + ensure_output_bytes(O, 1); + iof_set(O, state->current_predictor - 10); + state->predictorbyte = 1; + } + switch (state->current_predictor) + { + case NONE_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + iof_set(O, c); + } + reset_row(state); + break; + case TIFF_PREDICTOR: + switch (state->compbits) + { + case 1: + outbytes = (state->components + 7) >> 3; + for ( ; state->sampleindex < state->rowsamples; ++state->sampleindex) + { + ensure_output_bytes(O, outbytes); + while (state->bitsin < state->components) + { + state->compin = (state->compin << 8) | row_byte(state); + state->bitsin += 8; + ++state->rowindex; + } + state->bitsin -= state->components; + c = state->compin >> state->bitsin; + state->compin &= (1 << state->bitsin) - 1; + d = c ^ left_pixel_value(state); + save_pixel_value(state, c); + state->compout = (state->compout << state->components) | d; + state->bitsout += state->components; + while (state->bitsout >= 8) + { + state->bitsout -= 8; + iof_set(O, state->compout >> state->bitsout); + state->compout &= (1 << state->bitsout) - 1; + } + } + if (state->bitsout > 0) + { + ensure_output_bytes(O, 1); + iof_set(O, state->compin << (8 - state->bitsout)); + } + break; + case 2: case 4: + for ( ; state->sampleindex < state->rowsamples; ++state->sampleindex) + { + for ( ; state->compindex < state->components; ) + { + ensure_output_bytes(O, 1); + if (state->bitsin < state->compbits) + { + state->compin = (state->compin << 8) | row_byte(state); + state->bitsin += 8; + ++state->rowindex; + } + state->bitsin -= state->compbits; + c = state->compin >> state->bitsin; + state->compin &= (1 << state->bitsin) - 1; + d = tocomp(c - left_pixel_component(state)); + save_pixel_component(state, c); + state->compout = (state->compout << state->compbits) | d; + state->bitsout += state->compbits; + if (state->bitsout >= 8) + { + state->bitsout -= 8; + iof_set(O, state->compout >> state->bitsout); + state->compout &= (1 << state->bitsout) - 1; + } + } + } + if (state->bitsout > 0) + { + ensure_output_bytes(O, 1); + iof_set(O, state->compin << (8 - state->bitsout)); + } + break; + case 8: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + d = tobyte(c - left_pixel_component(state)); + save_pixel_component(state, c); + iof_set(O, d); + } + break; + case 16: + for ( ; state->rowindex < state->rowsize - 1; ++state->rowindex) + { + ensure_output_bytes(O, 2); + c = row_byte(state) << 8; + ++state->rowindex; + c |= row_byte(state); + d = tocomp(c - left_pixel_component(state)); + save_pixel_component(state, c); + iof_set2(O, d >> 8, d & 0xff); + } + break; + default: + return IOFERR; + } + reset_tiff_row(state); + break; + case PNG_NONE_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + save_pixel_byte(state, c); // next row may need it + iof_set(O, c); + } + reset_png_row(state); + break; + case PNG_SUB_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + d = tobyte(c - left_pixel_byte(state)); + save_pixel_byte(state, c); + iof_set(O, d); + } + reset_png_row(state); + break; + case PNG_OPTIMUM_PREDICTOR: // not worthy to perform optimization + case PNG_UP_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + d = tobyte(c - up_pixel_byte(state)); + save_pixel_byte(state, c); + iof_set(O, d); + } + reset_png_row(state); + break; + case PNG_AVERAGE_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + d = tobyte(c - ((up_pixel_byte(state) + left_pixel_byte(state)) >> 1)); + save_pixel_byte(state, c); + iof_set(O, d); + } + reset_png_row(state); + break; + case PNG_PAETH_PREDICTOR: + for ( ; state->rowindex < state->rowsize; ++state->rowindex) + { + ensure_output_bytes(O, 1); + c = row_byte(state); + d = tobyte(c - paeth(state)); + save_pixel_byte(state, c); + iof_set(O, d); + } + reset_png_row(state); + break; + default: + return IOFERR; + } + } + return state->status == STATUS_LAST ? IOFERR : IOFEOF; +} + +iof_status predictor_decode (iof *I, iof *O, int predictor, int rowsamples, int components, int compbits) +{ + predictor_state state; + int ret; + predictor_decoder_init(&state, predictor, rowsamples, components, compbits); + state.flush = 1; + ret = predictor_decode_state(I, O, &state); + predictor_decoder_close(&state); + return ret; +} + +iof_status predictor_encode (iof *I, iof *O, int predictor, int rowsamples, int components, int compbits) +{ + predictor_state state; + int ret; + predictor_encoder_init(&state, predictor, rowsamples, components, compbits); + state.flush = 1; + ret = predictor_encode_state(I, O, &state); + predictor_encoder_close(&state); + return ret; +} + +/* filters */ + +// predictor decoder function + +static size_t predictor_decoder (iof *F, iof_mode mode) +{ + predictor_state *state; + iof_status status; + size_t tail; + + state = iof_filter_state(predictor_state *, F); + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + do { + status = predictor_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "predictor", status); + case IOFCLOSE: + predictor_decoder_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// predictor encoder function + +static size_t predictor_encoder (iof *F, iof_mode mode) +{ + predictor_state *state; + iof_status status; + + state = iof_filter_state(predictor_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = predictor_encode_state(F, F->next, state); + return iof_encoder_retval(F, "predictor", status); + case IOFCLOSE: + if (!state->flush) + predictor_encoder(F, IOFFLUSH); + predictor_encoder_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +iof * iof_filter_predictor_decoder (iof *N, int predictor, int rowsamples, int components, int compbits) +{ + iof *I; + predictor_state_pointer P; + I = iof_filter_reader(predictor_decoder, sizeof(predictor_state), &P.voidstate); + iof_setup_next(I, N); + if (predictor_decoder_init(P.predictorstate, predictor, rowsamples, components, compbits) == NULL) + { + iof_discard(I); + return NULL; + } + P.predictorstate->flush = 1; + return I; +} + +iof * iof_filter_predictor_encoder (iof *N, int predictor, int rowsamples, int components, int compbits) +{ + iof *O; + predictor_state_pointer P; + O = iof_filter_writer(predictor_encoder, sizeof(predictor_state), &P.voidstate); + iof_setup_next(O, N); + if (predictor_encoder_init(P.predictorstate, predictor, rowsamples, components, compbits) == NULL) + { + iof_discard(O); + return NULL; + } + return O; +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilfpred.h b/Build/source/libs/pplib/pplib-src/src/util/utilfpred.h new file mode 100644 index 00000000000..6ae2f893586 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilfpred.h @@ -0,0 +1,23 @@ +#ifndef UTIL_FILTER_PREDICTOR_H +#define UTIL_FILTER_PREDICTOR_H + +#include "utiliof.h" + +typedef struct predictor_state predictor_state; + +predictor_state * predictor_decoder_init (predictor_state *state, int predictor, int rowsamples, int components, int compbits); +predictor_state * predictor_encoder_init (predictor_state *state, int predictor, int rowsamples, int components, int compbits); + +void predictor_decoder_close (predictor_state *state); +void predictor_encoder_close (predictor_state *state); + +iof_status predictor_decode_state (iof *I, iof *O, predictor_state *state); +iof_status predictor_encode_state (iof *I, iof *O, predictor_state *state); + +iof_status predictor_decode (iof *I, iof *O, int predictor, int rowsamples, int components, int compbits); +iof_status predictor_encode (iof *I, iof *O, int predictor, int rowsamples, int components, int compbits); + +iof * iof_filter_predictor_decoder (iof *N, int predictor, int rowsamples, int components, int compbits); +iof * iof_filter_predictor_encoder (iof *N, int predictor, int rowsamples, int components, int compbits); + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utiliof.c b/Build/source/libs/pplib/pplib-src/src/util/utiliof.c new file mode 100644 index 00000000000..41d6fba38f1 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utiliof.c @@ -0,0 +1,2993 @@ +/* input/output stream */ + +#include <stdlib.h> +#include <string.h> +#include <stdarg.h> + +#include "utilmem.h" +#include "utillog.h" +#include "utiliof.h" + +/* commons */ + +void * iof_copy_data (const void *data, size_t size) +{ + return memcpy(util_malloc(size), data, size); +} + +uint8_t * iof_copy_file_data (const char *filename, size_t *psize) +{ + FILE *file; + size_t size; + uint8_t *data; + if ((file = fopen(filename, "rb")) == NULL) + return NULL; + fseek(file, 0, SEEK_END); + size = (size_t)ftell(file); + data = (uint8_t *)util_malloc(size); + fseek(file, 0, SEEK_SET); + if ((*psize = fread(data, 1, size, file)) != size) + { + util_free(data); + data = NULL; + } + fclose(file); + return data; +} + +uint8_t * iof_copy_file_handle_data (FILE *file, size_t *psize) +{ + size_t size; + uint8_t *data; + //long offset = ftell(file); // keep offset intact? + fseek(file, 0, SEEK_END); + size = (size_t)ftell(file); + data = (uint8_t *)util_malloc(size); + fseek(file, 0, SEEK_SET); + if ((*psize = fread(data, 1, size, file)) != size) + { + util_free(data); + data = NULL; + } + //fseek(file, offset, SEEK_SET) + return data; +} + +FILE * iof_get_file (iof *F) +{ + if (F->flags & IOF_FILE) + return iof_file_get_file(F->iofile); + if (F->flags & IOF_FILE_HANDLE) + return F->file; + return NULL; +} + +const char * iof_status_kind (iof_status status) +{ + switch (status) + { + case IOFEOF: + return "IOFEOF"; + case IOFERR: + return "IOFERR"; + case IOFEMPTY: + return "IOFEMPTY"; + case IOFFULL: + return "IOFFULL"; + default: + break; + } + return "(unknown)"; +} + +/* shared pseudofile */ + +#define IOF_FILE_DEFAULTS 0 + +iof_file * iof_file_new (FILE *file) +{ + iof_file *iofile = (iof_file *)util_malloc(sizeof(iof_file)); + iof_file_set_fh(iofile, file); + iofile->offset = NULL; + iofile->size = 0; + iofile->name = NULL; + iofile->refcount = 0; + iofile->flags = IOF_FILE_DEFAULTS|IOF_ALLOC; + return iofile; +} + +iof_file * iof_file_init (iof_file *iofile, FILE *file) +{ + iof_file_set_fh(iofile, file); + iofile->offset = NULL; + iofile->size = 0; + iofile->name = NULL; + iofile->refcount = 0; + iofile->flags = IOF_FILE_DEFAULTS; + return iofile; +} + +iof_file * iof_file_rdata (const void *data, size_t size) +{ + iof_file *iofile = (iof_file *)util_malloc(sizeof(iof_file)); + iofile->rbuf = iofile->rpos = (const uint8_t *)data; + iofile->rend = iofile->rbuf + size; + iofile->offset = NULL; + iofile->size = 0; + iofile->name = NULL; + iofile->refcount = 0; + iofile->flags = IOF_FILE_DEFAULTS|IOF_ALLOC|IOF_DATA; + return iofile; +} + +iof_file * iof_file_rdata_init (iof_file *iofile, const void *data, size_t size) +{ + iofile->rbuf = iofile->rpos = (const uint8_t *)data; + iofile->rend = iofile->rbuf + size; + iofile->offset = NULL; + iofile->size = 0; // lets keep it consequently set to zero (only for user disposition) + iofile->name = NULL; + iofile->refcount = 0; + iofile->flags = IOF_FILE_DEFAULTS|IOF_DATA; + return iofile; +} + +iof_file * iof_file_wdata (void *data, size_t size) +{ + return iof_file_rdata((const void *)data, size); +} + +iof_file * iof_file_wdata_init (iof_file *iofile, void *data, size_t size) +{ + return iof_file_rdata_init(iofile, (const void *)data, size); +} + +/* typical uses so far */ + +iof_file * iof_file_reader_from_file_handle (iof_file *iofile, const char *filename, FILE *file, int preload, int closefile) +{ + uint8_t *data; + size_t size; + + if (preload) + { + if ((data = iof_copy_file_handle_data(file, &size)) == NULL) + { + if (closefile) // callers expect close also on failure + fclose(file); + return NULL; + } + if (iofile == NULL) + iofile = iof_file_rdata(data, size); + else + iof_file_rdata_init(iofile, data, size); + iofile->flags |= IOF_BUFFER_ALLOC; + if (closefile) + fclose(file); + } + else + { + if (iofile == NULL) + iofile = iof_file_new(file); + else + iof_file_init(iofile, file); + if (closefile) + iofile->flags |= IOF_CLOSE_FILE; + } + if (filename != NULL) + iof_file_set_name(iofile, filename); + return iofile; +} + +iof_file * iof_file_reader_from_file (iof_file *iofile, const char *filename, int preload) +{ + FILE *file; + if ((file = fopen(filename, "rb")) == NULL) + return NULL; + return iof_file_reader_from_file_handle(iofile, filename, file, preload, 1); // takes care to fclose() on failure +} + +iof_file * iof_file_reader_from_data (iof_file *iofile, const void *data, size_t size, int preload, int freedata) +{ + void *newdata; + if (data == NULL) + return NULL; + if (preload) + { + newdata = iof_copy_data(data, size); + if (iofile == NULL) + iofile = iof_file_rdata(newdata, size); + else + iof_file_rdata_init(iofile, newdata, size); + iofile->flags |= IOF_BUFFER_ALLOC; + //if (freedata) // hardly makes sense... we can't free const void * + // util_free((void *)data); + } + else + { + if (iofile == NULL) + iofile = iof_file_rdata(data, size); + else + iof_file_rdata_init(iofile, data, size); + if (freedata) + iofile->flags |= IOF_BUFFER_ALLOC; + } + return iofile; +} + +/* +iof_file * iof_file_writer_from_file (iof_file *iofile, const char *filename) +{ + FILE *file; + if ((file = fopen(filename, "wb")) == NULL) + return NULL; + if (iofile == NULL) + iofile = iof_file_new(file); + else + iof_file_init(iofile, file); + iofile->flags |= IOF_CLOSE_FILE; + iof_file_set_name(iofile, filename); + return iofile; +} +*/ + +/* +Because of limited number of FILE* handles available, we may need to close/reopen a file handle +when accessing it. In applications so far (fonts, images) we typically need the entire source +to parse the file on object creation and to rewrite or reload the data on dump. All iof_file API +functions assume that iofile has FILE* opened. Reopening it on every access (ftell, fseek, read/write) +makes no sense. So if the caller invalidates iofile by closing and NULLing its file handle, +it is also responsible to reopen when necessary. +*/ + +int iof_file_reclose_input (iof_file *iofile) +{ + FILE *file; + if (iofile->flags & IOF_DATA) + return 0; + if ((file = iof_file_get_fh(iofile)) == NULL) + return 0; + fclose(file); + iof_file_set_fh(iofile, NULL); + iofile->flags &= ~IOF_RECLOSE_FILE; + iofile->flags |= IOF_REOPEN_FILE; + return 1; +} + +int iof_file_reopen_input (iof_file *iofile) +{ // returns true if iofile readable + FILE *file; + const char *filename; + if (iofile->flags & IOF_DATA) + return 1; + if ((file = iof_file_get_fh(iofile)) != NULL) + return 1; // if present, assumed readable + if ((filename = iofile->name) == NULL || (file = fopen(filename, "rb")) == NULL) + return 0; + iof_file_set_fh(iofile, file); + iofile->flags &= ~IOF_REOPEN_FILE; + iofile->flags |= IOF_RECLOSE_FILE; + return 1; +} + +/* freeing iof_file */ + +void iof_file_free (iof_file *iofile) +{ + FILE *file; + if (iofile->flags & IOF_DATA) + { + if (iofile->flags & IOF_BUFFER_ALLOC) + { + iofile->flags &= ~IOF_BUFFER_ALLOC; + if (iofile->buf != NULL) + { + util_free(iofile->buf); + iofile->buf = iofile->pos = iofile->end = NULL; + } + } + } + else if ((file = iof_file_get_fh(iofile)) != NULL) + { + if (iofile->flags & IOF_CLOSE_FILE) + fclose(file); + iof_file_set_fh(iofile, NULL); + } + iof_file_set_name(iofile, NULL); + if (iofile->flags & IOF_ALLOC) + util_free(iofile); +} + +/* +An attempt to close iofile input keeping things safe. In bindings we sometimes we need to force +closing the file handle, otherwise it is closed when garbage collector graciously calls destroyer. +Eg. we are done with an object representing pdf/image/font, but we can't move/replace it, as the +host language keeps the garbage that keeps a file handle. When we call fclose(), we also have to +set the handle to NULL. In many places we assume, that if the iofile wraps FILE *, than the handle +is operable (no NULL checks). To close the handle keeping iofile alive safe, we can silently convert +it dummy IOF_DATA buffer. +*/ + +void iof_file_close_input (iof_file *iofile) +{ + FILE *file; + if (iofile->flags & IOF_DATA) + { + if (iofile->flags & IOF_BUFFER_ALLOC) + { + iofile->flags &= ~IOF_BUFFER_ALLOC; + if (iofile->buf != NULL) + { + util_free(iofile->buf); + //iofile->buf = iofile->pos = iofile->end = NULL; + } + } + } + else if ((file = iof_file_get_fh(iofile)) != NULL) + { + iof_file_set_fh(iofile, NULL); + fclose(file); + } + iof_file_set_name(iofile, NULL); + /* now make it a dummy string iofile */ + iofile->buf = iofile->pos = iofile->end = NULL; + iofile->flags |= IOF_DATA; +} + +/* set filename for reopen */ + +void iof_file_set_name (iof_file *iofile, const char *name) +{ + if (iofile->name != NULL) + util_free(iofile->name); + if (name != NULL) + iofile->name = iof_copy_data(name, strlen(name) + 1); + else + iofile->name = NULL; +} + +/* seek */ + +int iof_file_seek (iof_file *iofile, long offset, int whence) +{ + if (iofile->flags & IOF_DATA) + { + switch (whence) + { + case SEEK_SET: + if (offset >= 0 && iofile->buf + offset <= iofile->end) + { + iofile->pos = iofile->buf + offset; + return 0; + } + return -1; + case SEEK_CUR: + if ((offset >= 0 && iofile->pos + offset <= iofile->end) || (offset < 0 && iofile->pos + offset >= iofile->buf)) + { + iofile->pos += offset; + return 0; + } + return -1; + case SEEK_END: + if (offset <= 0 && iofile->end + offset >= iofile->buf) + { + iofile->pos = iofile->end + offset; + return 0; + } + return -1; + } + return -1; + } + return fseek(iof_file_get_fh(iofile), offset, whence); +} + +/* */ + +long iof_file_tell (iof_file *iofile) +{ + return (iofile->flags & IOF_DATA) ? (long)(iofile->pos - iofile->buf) : ftell(iof_file_get_fh(iofile)); +} + +size_t iof_file_size (iof_file *iofile) +{ + long pos, size; + FILE *file; + if (iofile->flags & IOF_DATA) + return (size_t)iof_space(iofile); + file = iof_file_get_fh(iofile); + pos = ftell(file); + fseek(file, 0, SEEK_END); + size = ftell(file); + fseek(file, pos, SEEK_SET); + return size; +} + +int iof_file_eof (iof_file *iofile) +{ + if (iofile->flags & IOF_DATA) + return iofile->pos == iofile->end ? -1 : 0; + return feof(iof_file_get_fh(iofile)); +} + +int iof_file_flush (iof_file *iofile) +{ + if (iofile->flags & IOF_DATA) + return 0; + return fflush(iof_file_get_fh(iofile)); +} + +size_t iof_file_read (void *ptr, size_t size, size_t items, iof_file *iofile) +{ + if (iofile->flags & IOF_DATA) + { + size_t bytes = size * items; + if (bytes > (size_t)iof_left(iofile)) + bytes = (size_t)iof_left(iofile); + memcpy(ptr, iofile->pos, bytes); + iofile->pos += bytes; + return bytes / size; // number of elements read + } + return fread(ptr, size, items, iof_file_get_fh(iofile)); +} + +static size_t iof_file_data_resizeto (iof_file *iofile, size_t space) +{ + uint8_t *newbuf; + size_t size; + size = iof_size(iofile); + if (iofile->flags & IOF_BUFFER_ALLOC) + { + newbuf = (uint8_t *)util_realloc(iofile->buf, space); + } + else + { + newbuf = (uint8_t *)util_malloc(space); + if (size > 0) + memcpy(newbuf, iofile->buf, size); + iofile->flags |= IOF_BUFFER_ALLOC; + } + iofile->buf = newbuf; + iofile->pos = newbuf + size; + iofile->end = newbuf + space; + return space - size; +} + +#define iof_file_data_resize(iofile) iof_file_data_resizeto(iofile, iof_space(iofile) << 1) + +size_t iof_file_write (const void *ptr, size_t size, size_t items, iof_file *iofile) +{ + if (iofile->flags & IOF_DATA) + { + size_t space, sizesofar, bytes; + bytes = size * items; + if (bytes > (size_t)iof_left(iofile)) + { + if ((space = iof_space(iofile)) == 0) // allow iofile->buf/end initially NULL + space = BUFSIZ; + for (sizesofar = iof_size(iofile), space <<= 1; sizesofar + bytes > space; space <<= 1) + ; + if (iof_file_data_resizeto(iofile, space) == 0) + return 0; + } + memcpy(iofile->pos, ptr, bytes); + iofile->pos += bytes; + return bytes / size; + } + return fwrite(ptr, size, items, iof_file_get_fh(iofile)); +} + +size_t iof_file_ensure (iof_file *iofile, size_t bytes) +{ + if (iofile->flags & IOF_DATA) + { + size_t space, sizesofar, left; + left = (size_t)iof_left(iofile); + if (bytes > left) + { + if ((space = iof_space(iofile)) == 0) // allow iofile->buf/end initially NULL + space = BUFSIZ; + for (sizesofar = iof_size(iofile), space <<= 1; sizesofar + bytes > space; space <<= 1); + return iof_file_data_resizeto(iofile, space); + } + return left; + } + return 0; +} + +int iof_file_getc (iof_file *iofile) +{ + if (iofile->flags & IOF_DATA) + return iofile->pos < iofile->end ? *iofile->pos++ : IOFEOF; + return fgetc(iof_file_get_fh(iofile)); +} + +int iof_file_putc (iof_file *iofile, int c) +{ + if (iofile->flags & IOF_DATA) + { + if (iofile->pos >= iofile->end) + if (iof_file_data_resize(iofile) == 0) + return IOFEOF; + *iofile->pos++ = (uint8_t)c; + return c; + } + return fputc(c, iof_file_get_fh(iofile)); +} + +static int iof_file_sync (iof_file *iofile, size_t *offset) +{ + if (iofile->offset != offset) + { + if (iofile->offset != NULL) + *iofile->offset = iof_file_tell(iofile); + iofile->offset = offset; + if (offset) // let offset be NULL + return iof_file_seek(iofile, (long)*offset, SEEK_SET); + } + return 0; +} + +//#define iof_file_unsync(iofile, poffset) (void)((iofile)->offset == poffset && (((iofile)->offset = NULL), 0)) +#define iof_file_unsync(iofile, poffset) ((void)poffset, (iofile)->offset = NULL) + +/* iof seek */ + +#define iof_reader_reset(I) ((I)->pos = (I)->end = (I)->buf) +#define iof_reader_reseek_file(I, offset, whence) (fseek((I)->file, offset, whence) == 0 ? (iof_reader_reset(I), 0) : -1) +#define iof_reader_reseek_iofile(I, offset, whence) (iof_file_seek((I)->iofile, offset, whence) == 0 ? (iof_reader_reset(I), 0) : -1) + +#define iof_writer_reset(O) ((O)->pos = (O)->buf) +#define iof_writer_reseek_file(O, offset, whence) (iof_flush(O), (fseek((O)->file, offset, whence) == 0 ? (iof_writer_reset(O), 0) : -1)) +#define iof_writer_reseek_iofile(O, offset, whence) (iof_flush(O), (iof_file_seek((O)->iofile, offset, whence) == 0 ? (iof_writer_reset(O), 0) : -1)) + +static int iof_reader_seek_data (iof *I, long offset, int whence) +{ + switch (whence) + { + case SEEK_SET: + if (offset >= 0 && I->buf + offset <= I->end) + { + I->pos = I->buf + offset; + return 0; + } + return -1; + case SEEK_CUR: + if ((offset >= 0 && I->pos + offset <= I->end) || (offset < 0 && I->pos + offset >= I->buf)) + { + I->pos += offset; + return 0; + } + return -1; + case SEEK_END: + if (offset <= 0 && I->end + offset >= I->buf) + { + I->pos = I->end + offset; + return 0; + } + return -1; + } + return -1; +} + +static int iof_reader_seek_iofile (iof *I, long offset, int whence) +{ + long fileoffset; + switch (whence) + { + case SEEK_SET: + fileoffset = iof_file_tell(I->iofile); + if (offset <= fileoffset && offset >= fileoffset - iof_space(I)) + { + I->pos = I->end - (fileoffset - offset); + return 0; + } + return iof_reader_reseek_iofile(I, offset, SEEK_SET); + case SEEK_CUR: + if ((offset >= 0 && I->pos + offset <= I->end) || (offset < 0 && I->pos + offset >= I->buf)) + { + I->pos += offset; + return 0; + } + return iof_reader_reseek_iofile(I, offset, SEEK_CUR); + case SEEK_END: + return iof_reader_reseek_iofile(I, offset, SEEK_END); // can we do better? + } + return -1; +} + +static int iof_reader_seek_file (iof *I, long offset, int whence) +{ + long fileoffset; + switch (whence) + { + case SEEK_SET: + fileoffset = ftell(I->file); + if (offset <= fileoffset && offset >= fileoffset - iof_space(I)) + { + I->pos = I->end - (fileoffset - offset); + return 0; + } + return iof_reader_reseek_file(I, offset, SEEK_SET); + case SEEK_CUR: + if ((offset >= 0 && I->pos + offset <= I->end) || (offset < 0 && I->pos + offset >= I->buf)) + { + I->pos += offset; + return 0; + } + return iof_reader_reseek_file(I, offset, SEEK_CUR); + case SEEK_END: + return iof_reader_reseek_file(I, offset, SEEK_END); // can we do better? + } + return -1; +} + +int iof_reader_seek (iof *I, long offset, int whence) +{ + I->flags &= ~IOF_STOPPED; + if (I->flags & IOF_FILE) + return iof_reader_seek_iofile(I, offset, whence); + if (I->flags & IOF_FILE_HANDLE) + return iof_reader_seek_file(I, offset, whence); + if (I->flags & IOF_DATA) + return iof_reader_seek_data(I, offset, whence); + return -1; +} + +int iof_reader_reseek (iof *I, long offset, int whence) +{ + I->flags &= ~IOF_STOPPED; + if (I->flags & IOF_FILE) + return iof_reader_reseek_iofile(I, offset, whence); + if (I->flags & IOF_FILE_HANDLE) + return iof_reader_reseek_file(I, offset, whence); + if (I->flags & IOF_DATA) + return iof_reader_seek_data(I, offset, whence); + return -1; +} + +static int iof_writer_seek_data (iof *O, long offset, int whence) +{ + /* + fseek() allows to seek after the end of file. Seeking does not increase the output file. + No byte is written before fwirte(). It seems to fill the gap with zeros. Until we really need that, + no seeking out of bounds for writers. + */ + O->flags &= ~IOF_STOPPED; + return iof_reader_seek_data(O, offset, whence); +} + +static int iof_writer_seek_iofile (iof *O, long offset, int whence) +{ + long fileoffset; + switch (whence) + { + case SEEK_SET: + fileoffset = iof_file_tell(O->iofile); + if (offset >= fileoffset && offset <= fileoffset + iof_space(O)) + { + O->pos = O->buf + (offset - fileoffset); + return 0; + } + return iof_writer_reseek_iofile(O, offset, SEEK_SET); + case SEEK_CUR: + if ((offset >=0 && O->pos + offset <= O->end) || (offset < 0 && O->pos + offset >= O->buf)) + { + O->pos += offset; + return 0; + } + return iof_writer_reseek_iofile(O, offset, SEEK_CUR); + case SEEK_END: + return iof_writer_reseek_iofile(O, offset, SEEK_END); + } + return -1; +} + +static int iof_writer_seek_file (iof *O, long offset, int whence) +{ + long fileoffset; + switch (whence) + { + case SEEK_SET: + fileoffset = ftell(O->file); + if (offset >= fileoffset && offset <= fileoffset + iof_space(O)) + { + O->pos = O->buf + (offset - fileoffset); + return 0; + } + return iof_writer_reseek_file(O, offset, SEEK_SET); + case SEEK_CUR: + if ((offset >=0 && O->pos + offset <= O->end) || (offset < 0 && O->pos + offset >= O->buf)) + { + O->pos += offset; + return 0; + } + return iof_writer_reseek_file(O, offset, SEEK_CUR); + case SEEK_END: + return iof_writer_reseek_file(O, offset, SEEK_END); + } + return -1; +} + +int iof_writer_seek (iof *I, long offset, int whence) +{ + I->flags &= ~IOF_STOPPED; + if (I->flags & IOF_FILE) + return iof_writer_seek_iofile(I, offset, whence); + if (I->flags & IOF_FILE_HANDLE) + return iof_writer_seek_file(I, offset, whence); + if (I->flags & IOF_DATA) + return iof_writer_seek_data(I, offset, whence); + return -1; +} + +int iof_writer_reseek (iof *I, long offset, int whence) +{ + I->flags &= ~IOF_STOPPED; + if (I->flags & IOF_FILE) + return iof_writer_reseek_iofile(I, offset, whence); + if (I->flags & IOF_FILE_HANDLE) + return iof_writer_reseek_file(I, offset, whence); + if (I->flags & IOF_DATA) + return iof_writer_seek_data(I, offset, whence); + return -1; +} + +int iof_seek (iof *F, long offset, int whence) +{ + return (F->flags & IOF_WRITER) ? iof_writer_seek(F, offset, whence) : iof_reader_seek(F, offset, whence); +} + +int iof_reseek (iof *F, long offset, int whence) +{ + return (F->flags & IOF_WRITER) ? iof_writer_reseek(F, offset, whence) : iof_reader_reseek(F, offset, whence); +} + +/* tell */ + +long iof_reader_tell (iof *I) +{ + if (I->flags & IOF_FILE) + return iof_file_tell(I->iofile) - (long)iof_left(I); + if (I->flags & IOF_FILE_HANDLE) + return ftell(I->file) - (long)iof_left(I); + //if (I->flags & IOF_DATA) + return (long)iof_size(I); +} + +long iof_writer_tell (iof *O) +{ + if (O->flags & IOF_FILE) + return iof_file_tell(O->iofile) + (long)iof_size(O); + if (O->flags & IOF_FILE_HANDLE) + return ftell(O->file) + (long)iof_size(O); + //if (I->flags & IOF_DATA) + return (long)iof_size(O); +} + +long iof_tell (iof *I) +{ + return (I->flags & IOF_WRITER) ? iof_writer_tell(I) : iof_reader_tell(I); +} + +size_t iof_fsize (iof *I) +{ + size_t pos, size; + if (I->flags & IOF_FILE) + return iof_file_size(I->iofile); + if (I->flags & IOF_FILE_HANDLE) + { + pos = (size_t)ftell(I->file); + fseek(I->file, 0, SEEK_END); + size = (size_t)ftell(I->file); + fseek(I->file, (long)pos, SEEK_SET); + return size; + } + //if (I->flags & IOF_DATA) + return (size_t)iof_space(I); +} + +/* save reader tail */ + +size_t iof_save_tail (iof *I) +{ + size_t size, left; + size = iof_size(I); + left = iof_left(I); + if (size >= left) + memcpy(I->buf, I->pos, left); + else + memmove(I->buf, I->pos, left); + return left; +} + +size_t iof_input_save_tail (iof *I, size_t back) +{ + size_t size; + I->flags |= IOF_TAIL; + I->pos -= back; + size = iof_input(I); + I->pos += back; + I->flags &= ~IOF_TAIL; + return size; // + back - back +} + +/* read from file */ + +/* iof free*/ + +static size_t file_read (iof *I); +static size_t file_load (iof *I); + +static size_t file_reader (iof *I, iof_mode mode) +{ + switch (mode) + { + case IOFREAD: + return file_read(I); + case IOFLOAD: + return file_load(I); + case IOFCLOSE: + iof_free(I); + return 0; + default: + return 0; + } +} + +iof * iof_setup_file_handle_reader (iof *I, void *buffer, size_t space, FILE *f) +{ + iof_setup_reader(I, buffer, space); + iof_setup_file(I, f); + I->more = file_reader; + return I; +} + +iof * iof_setup_file_reader (iof *I, void *buffer, size_t space, const char *filename) +{ + FILE *f; + if ((f = fopen(filename, "rb")) == NULL) + return NULL; + iof_setup_reader(I, buffer, space); + iof_setup_file(I, f); + I->flags |= IOF_CLOSE_FILE; + I->more = file_reader; + return I; +} + +/* write to file */ + +static size_t file_write (iof *O, int flush); + +static size_t file_writer (iof *O, iof_mode mode) +{ + switch (mode) + { + case IOFWRITE: + return file_write(O, 0); + case IOFFLUSH: + return file_write(O, 1); + case IOFCLOSE: + file_write(O, 1); + iof_free(O); + return 0; + default: + return 0; + } +} + +iof * iof_setup_file_handle_writer (iof *O, void *buffer, size_t space, FILE *f) +{ + iof_setup_writer(O, buffer, space); + iof_setup_file(O, f); + O->more = file_writer; + return O; +} + +iof * iof_setup_file_writer (iof *O, void *buffer, size_t space, const char *filename) +{ + FILE *f; + if ((f = fopen(filename, "wb")) == NULL) + return NULL; + iof_setup_writer(O, buffer, space); + iof_setup_file(O, f); + O->flags |= IOF_CLOSE_FILE; + O->more = file_writer; + return O; +} + +/* a dedicated handler for stdout/stderr */ + +static size_t stdout_writer (iof *O, iof_mode mode) +{ + switch(mode) + { + case IOFWRITE: + { + fwrite(O->buf, sizeof(uint8_t), iof_size(O), stdout); + O->pos = O->buf; + return O->space; + } + case IOFCLOSE: + case IOFFLUSH: + { + fwrite(O->buf, sizeof(uint8_t), iof_size(O), stdout); + fflush(stdout); + O->pos = O->buf; + return 0; + } + default: + break; + } + return 0; +} + +static size_t stderr_writer (iof *O, iof_mode mode) +{ + switch(mode) + { + case IOFWRITE: + { + fwrite(O->buf, sizeof(uint8_t), iof_size(O), stderr); + O->pos = O->buf; + return O->space; + } + case IOFCLOSE: + case IOFFLUSH: + { + fwrite(O->buf, sizeof(uint8_t), iof_size(O), stderr); + fflush(stderr); + O->pos = O->buf; + return 0; + } + default: + break; + } + return 0; +} + +static uint8_t iof_stdout_buffer[BUFSIZ]; +iof iof_stdout = IOF_WRITER_INIT(stdout_writer, NULL, iof_stdout_buffer, BUFSIZ, 0); + +static uint8_t iof_stderr_buffer[BUFSIZ]; +iof iof_stderr = IOF_WRITER_INIT(stderr_writer, NULL, iof_stderr_buffer, BUFSIZ, 0); + +/* read from somewhere */ + +iof * iof_reader (iof *I, void *link, iof_handler reader, const void *m, size_t bytes) +{ + I->space = 0; + I->link = link; + I->more = reader; + I->flags = 0; + I->refcount = 0; + if (m != NULL) + { + I->rbuf = I->rpos = (const uint8_t *)m; + I->rend = (const uint8_t *)m + bytes; + return I; + } + return NULL; +} + +iof * iof_string_reader (iof *I, const void *s, size_t bytes) +{ + I->space = 0; + I->link = NULL; + I->more = NULL; + I->flags = 0; // iof_string() sets IOF_DATA + I->refcount = 0; + if (s != NULL) + return iof_string(I, s, bytes); + return NULL; +} + +/* write somewhere */ + +iof * iof_writer (iof *O, void *link, iof_handler writer, void *m, size_t bytes) +{ + O->space = 0; + O->link = link; + O->more = writer; + O->flags = 0; + O->refcount = 0; + if (m != NULL && bytes > 0) + { + O->buf = O->pos = (uint8_t *)m; + O->end = (uint8_t *)m + bytes; + return O; + } + // return iof_null(O); + return NULL; +} + +/* write to growing bytes buffer */ + +static size_t iof_mem_handler (iof *O, iof_mode mode) +{ + switch(mode) + { + case IOFWRITE: + return iof_resize_buffer(O); + case IOFCLOSE: + iof_free(O); + return 0; + default: + return 0; + } +} + +iof * iof_setup_buffer (iof *O, void *buffer, size_t space) +{ + iof_setup_writer(O, buffer, space); + O->link = NULL; + O->flags |= IOF_DATA; + O->more = iof_mem_handler; + return O; +} + +iof * iof_setup_buffermin (iof *O, void *buffer, size_t space, size_t min) +{ + iof_setup_buffer(O, buffer, space); + if (space < min) // allocate min to avoid further rewriting + { + O->buf = O->pos = (uint8_t *)util_malloc(min); + O->flags |= IOF_BUFFER_ALLOC; + O->end = O->buf + min; + } + return O; +} + +iof * iof_buffer_create (size_t space) +{ + uint8_t *buffer; + iof *O; + O = (iof *)util_malloc(space); + buffer = (uint8_t *)(O + 1); + iof_setup_buffer(O, buffer, space); + O->flags |= IOF_ALLOC; + return O; +} + +/* set/get */ + +int iof_getc (iof *I) +{ + if (iof_readable(I)) + return *I->pos++; + return IOFEOF; +} + +int iof_putc (iof *O, int u) +{ + if (iof_writable(O)) + { + iof_set(O, u); + return (uint8_t)u; + } + return IOFFULL; +} + +size_t iof_skip (iof *I, size_t bytes) +{ + while (bytes) + { + if (iof_readable(I)) + ++I->pos; + else + break; + --bytes; + } + return bytes; +} + +/* from iof to iof */ + +iof_status iof_pass (iof *I, iof *O) +{ + size_t leftin, leftout; + if ((leftin = iof_left(I)) == 0) + leftin = iof_input(I); + while (leftin) + { + if ((leftout = iof_left(O)) == 0) + if ((leftout = iof_output(O)) == 0) + return IOFFULL; + while (leftin > leftout) + { + memcpy(O->pos, I->pos, leftout); + I->pos += leftout; + O->pos = O->end; /* eq. += leftout */ + leftin -= leftout; + if ((leftout = iof_output(O)) == 0) + return IOFFULL; + } + if (leftin) + { + memcpy(O->pos, I->pos, leftin); + I->pos = I->end; /* eq. += leftin */ + O->pos += leftin; + } + leftin = iof_input(I); + } + return IOFEOF; +} + +/* read n-bytes */ + +size_t iof_read (iof *I, void *to, size_t size) +{ + size_t leftin, done = 0; + char *s = (char *)to; + + if ((leftin = iof_left(I)) == 0) + if ((leftin = iof_input(I)) == 0) + return done; + while (size > leftin) + { + memcpy(s, I->pos, leftin * sizeof(uint8_t)); + size -= leftin; + done += leftin; + s += leftin; + I->pos = I->end; + if ((leftin = iof_input(I)) == 0) + return done; + } + if (size) + { + memcpy(s, I->pos, size * sizeof(uint8_t)); + I->pos += size; + done += size; + } + return done; +} + +/* rewrite FILE content (use fseek if needed) */ + +size_t iof_write_file_handle (iof *O, FILE *file) +{ + size_t leftout, size, readout; + if ((leftout = iof_left(O)) == 0) + if ((leftout = iof_output(O)) == 0) + return 0; + size = 0; + do { + readout = fread(O->pos, 1, leftout, file); + O->pos += readout; + size += readout; + } while(readout == leftout && (leftout = iof_output(O)) > 0); + return size; +} + +size_t iof_write_file (iof *O, const char *filename) +{ + FILE *file; + size_t size; + if ((file = fopen(filename, "rb")) == NULL) + return 0; + size = iof_write_file_handle(O, file); + fclose(file); + return size; +} + +size_t iof_write_iofile (iof *O, iof_file *iofile, int savepos) +{ + long offset; + size_t size; + FILE *file; + if (iofile->flags & IOF_DATA) + return iof_write(O, iofile->pos, (size_t)(iofile->end - iofile->pos)); + file = iof_file_get_fh(iofile); + if (savepos) + { + offset = ftell(file); + size = iof_write_file_handle(O, file); + fseek(file, offset, SEEK_SET); + return size; + } + return iof_write_file_handle(O, file); +} + +/* write n-bytes */ + +size_t iof_write (iof *O, const void *data, size_t size) +{ + size_t leftout, done = 0; + const char *s = (const char *)data; + if ((leftout = iof_left(O)) == 0) + if ((leftout = iof_output(O)) == 0) + return done; + while (size > leftout) + { + memcpy(O->pos, s, leftout * sizeof(uint8_t)); + size -= leftout; + done += leftout; + s += leftout; + O->pos = O->end; + if ((leftout = iof_output(O)) == 0) + return done; + } + if (size) + { + memcpy(O->pos, s, size * sizeof(uint8_t)); + O->pos += size; + done += size; + } + return done; +} + +/* write '\0'-terminated string */ + +iof_status iof_puts (iof *O, const void *data) +{ + const char *s = (const char *)data; + while (*s) + { + if (iof_writable(O)) + iof_set(O, *s++); + else + return IOFFULL; + } + return IOFEOF; // ? +} + +size_t iof_put_string (iof *O, const void *data) +{ + const char *p, *s = (const char *)data; + for (p = s; *p != '\0' && iof_writable(O); iof_set(O, *p++)); + return p - s; +} + +/* write byte n-times */ + +/* +iof_status iof_repc (iof *O, char c, size_t bytes) +{ + while (bytes) + { + if (iof_writable(O)) + iof_set(O, c); + else + return IOFFULL; + --bytes; + } + return IOFEOF; // ? +} +*/ + +size_t iof_repc (iof *O, char c, size_t bytes) +{ + size_t leftout, todo = bytes; + if ((leftout = iof_left(O)) == 0) + if ((leftout = iof_output(O)) == 0) + return 0; + while (bytes > leftout) + { + memset(O->pos, c, leftout); + bytes -= leftout; + O->pos = O->end; + if ((leftout = iof_output(O)) == 0) + return todo - bytes; + } + if (bytes) + { + memset(O->pos, c, bytes); + O->pos += bytes; + } + return todo; +} + +/* putfs */ + +#define IOF_FMT_SIZE 1024 + +size_t iof_putfs (iof *O, const char *format, ...) +{ + static char buffer[IOF_FMT_SIZE]; + va_list args; + va_start(args, format); + if (vsnprintf(buffer, IOF_FMT_SIZE, format, args) > 0) + { + va_end(args); + return iof_put_string(O, buffer); + } + else + { + va_end(args); + return iof_write(O, buffer, IOF_FMT_SIZE); + } +} + +/* integer from iof; return 1 on success, 0 otherwise */ + +int iof_get_int32 (iof *I, int32_t *number) +{ + int sign, c = iof_char(I); + iof_scan_sign(I, c, sign); + if (!base10_digit(c)) return 0; + iof_read_integer(I, c, *number); + if (sign) *number = -*number; + return 1; +} + +int iof_get_slong (iof *I, long *number) +{ + int sign, c = iof_char(I); + iof_scan_sign(I, c, sign); + if (!base10_digit(c)) return 0; + iof_read_integer(I, c, *number); + if (sign) *number = -*number; + return 1; +} + +int iof_get_int64 (iof *I, int64_t *number) +{ + int sign, c = iof_char(I); + iof_scan_sign(I, c, sign); + if (!base10_digit(c)) return 0; + iof_read_integer(I, c, *number); + if (sign) *number = -*number; + return 1; +} + +int iof_get_uint32 (iof *I, uint32_t *number) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_integer(I, c, *number); + return 1; +} + +int iof_get_ulong (iof *I, unsigned long *number) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_integer(I, c, *number); + return 1; +} + +int iof_get_usize (iof *I, size_t *number) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_integer(I, c, *number); + return 1; +} + +int iof_get_uint64 (iof *I, uint64_t *number) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_integer(I, c, *number); + return 1; +} + +int iof_get_int32_radix (iof *I, int32_t *number, int radix) +{ + int sign, c = iof_char(I); + iof_scan_sign(I, c, sign); + if (!base10_digit(c)) return 0; + iof_read_radix(I, c, *number, radix); + if (sign) *number = -*number; + return 1; + +} + +int iof_get_slong_radix (iof *I, long *number, int radix) +{ + int sign, c = iof_char(I); + iof_scan_sign(I, c, sign); + if (!base10_digit(c)) return 0; + iof_read_radix(I, c, *number, radix); + if (sign) *number = -*number; + return 1; +} + +int iof_get_int64_radix (iof *I, int64_t *number, int radix) +{ + int sign, c = iof_char(I); + iof_scan_sign(I, c, sign); + if (!base10_digit(c)) return 0; + iof_read_radix(I, c, *number, radix); + if (sign) *number = -*number; + return 1; +} + +int iof_get_uint32_radix (iof *I, uint32_t *number, int radix) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_radix(I, c, *number, radix); + return 1; +} + +int iof_get_ulong_radix (iof *I, unsigned long *number, int radix) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_radix(I, c, *number, radix); + return 1; +} + +int iof_get_usize_radix (iof *I, size_t *number, int radix) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_radix(I, c, *number, radix); + return 1; +} + +int iof_get_uint64_radix (iof *I, uint64_t *number, int radix) +{ + int c = iof_char(I); + if (!base10_digit(c)) return 0; + iof_read_radix(I, c, *number, radix); + return 1; +} + +/* get roman to uint16_t, cf. roman_to_uint16() from utilnumber.c*/ + +/* todo: some trick in place of this macro horror? */ + +#define roman1000(c) (c == 'M' || c == 'm') +#define roman500(c) (c == 'D' || c == 'd') +#define roman100(c) (c == 'C' || c == 'c') +#define roman50(c) (c == 'L' || c == 'l') +#define roman10(c) (c == 'X' || c == 'x') +#define roman5(c) (c == 'V' || c == 'v') +#define roman1(c) (c == 'I' || c == 'i') + +#define roman100s(I, c) \ + (roman100(c) ? (100 + ((c = iof_next(I), roman100(c)) ? (100 + ((c = iof_next(I), roman100(c)) ? (c = iof_next(I), 100) : 0)) : 0)) : 0) +#define roman10s(I, c) \ + (roman10(c) ? (10 + ((c = iof_next(I), roman10(c)) ? (10 + ((c = iof_next(I), roman10(c)) ? (c = iof_next(I), 10) : 0)) : 0)) : 0) +#define roman1s(I, c) \ + (roman1(c) ? (1 + ((c = iof_next(I), roman1(c)) ? (1 + ((c = iof_next(I), roman1(c)) ? (c = iof_next(I), 1) : 0)) : 0)) : 0) + +int iof_get_roman (iof *I, uint16_t *number) +{ + int c; + /* M */ + for (*number = 0, c = iof_char(I); roman1000(c); *number += 1000, c = iof_next(I)); + /* D C */ + if (roman500(c)) + { + c = iof_next(I); + *number += 500 + roman100s(I, c); + } + else if (roman100(c)) + { + c = iof_next(I); + if (roman1000(c)) + { + c = iof_next(I); + *number += 900; + } + else if (roman500(c)) + { + c = iof_next(I); + *number += 400; + } + else + *number += 100 + roman100s(I, c); + } + /* L X */ + if (roman50(c)) + { + c = iof_next(I); + *number += 50 + roman10s(I, c); + } + else if (roman10(c)) + { + c = iof_next(I); + if (roman100(c)) + { + c = iof_next(I); + *number += 90; + } + else if (roman50(c)) + { + c = iof_next(I); + *number += 40; + } + else + *number += 10 + roman10s(I, c); + } + /* V I */ + if (roman5(c)) + { + c = iof_next(I); + *number += 5 + roman1s(I, c); + } + else if (roman1(c)) + { + c = iof_next(I); + if (roman10(c)) + { + c = iof_next(I); + *number += 9; + } + else if (roman5(c)) + { + c = iof_next(I); + *number += 4; + } + else + *number += 1 + roman1s(I, c); + } + return 1; +} + +/* double from iof; return 1 on success */ + +int iof_get_double (iof *I, double *number) // cf. string_to_double() +{ + int sign, exponent10, c = iof_char(I); + iof_scan_sign(I, c, sign); + iof_scan_decimal(I, c, *number); + if (c == '.') + { + c = iof_next(I); + iof_scan_fraction(I, c, *number, exponent10); + } + else + exponent10 = 0; + if (c == 'e' || c == 'E') + { + c = iof_next(I); + iof_scan_exponent10(I, c, exponent10); + } + double_exp10(*number, exponent10); + if (sign) *number = -*number; + return 1; +} + +int iof_get_float (iof *I, float *number) // cf. string_to_float() +{ + int sign, exponent10, c = iof_char(I); + iof_scan_sign(I, c, sign); + iof_scan_decimal(I, c, *number); + if (c == '.') + { + c = iof_next(I); + iof_scan_fraction(I, c, *number, exponent10); + } + else + exponent10 = 0; + if (c == 'e' || c == 'E') + { + c = iof_next(I); + iof_scan_exponent10(I, c, exponent10); + } + float_exp10(*number, exponent10); + if (sign) *number = -*number; + return 1; +} + +int iof_conv_double (iof *I, double *number) // cf. convert_to_double() +{ + int sign, exponent10, c = iof_char(I); + iof_scan_sign(I, c, sign); + iof_scan_decimal(I, c, *number); + if (c == '.' || c == ',') + { + c = iof_next(I); + iof_scan_fraction(I, c, *number, exponent10); + if (exponent10 < 0) + double_negative_exp10(*number, exponent10); + } + if (sign) *number = -*number; + return 1; +} + +int iof_conv_float (iof *I, float *number) // cf. convert_to_float() +{ + int sign, exponent10, c = iof_char(I); + iof_scan_sign(I, c, sign); + iof_scan_decimal(I, c, *number); + if (c == '.' || c == ',') + { + c = iof_next(I); + iof_scan_fraction(I, c, *number, exponent10); + if (exponent10 < 0) + float_negative_exp10(*number, exponent10); + } + if (sign) *number = -*number; + return 1; +} + +/* integer to iof; return a number of written bytes */ + +size_t iof_put_int32 (iof *O, int32_t number) +{ + const char *s; + size_t size; + s = int32_to_string(number, &size); + return iof_write(O, s, size); +} + +size_t iof_put_slong (iof *O, long number) +{ + const char *s; + size_t size; + s = slong_to_string(number, &size); + return iof_write(O, s, size); +} + +size_t iof_put_int64 (iof *O, int64_t number) +{ + const char *s; + size_t size; + s = int64_to_string(number, &size); + return iof_write(O, s, size); +} + +size_t iof_put_uint32 (iof *O, uint32_t number) +{ + const char *s; + size_t size; + s = uint32_to_string(number, &size); + return iof_write(O, s, size); +} + +size_t iof_put_ulong (iof *O, unsigned long number) +{ + const char *s; + size_t size; + s = ulong_to_string(number, &size); + return iof_write(O, s, size); +} + +size_t iof_put_usize (iof *O, size_t number) +{ + const char *s; + size_t size; + s = usize_to_string(number, &size); + return iof_write(O, s, size); +} + +size_t iof_put_uint64 (iof *O, uint64_t number) +{ + const char *s; + size_t size; + s = uint64_to_string(number, &size); + return iof_write(O, s, size); +} + +size_t iof_put_int32_radix (iof *O, int32_t number, int radix, int uc) +{ + const char *s; + size_t size; + s = int32_to_radix(number, radix, uc, &size); + return iof_write(O, s, size); +} + +size_t iof_put_slong_radix (iof *O, long number, int radix, int uc) +{ + const char *s; + size_t size; + s = slong_to_radix(number, radix, uc, &size); + return iof_write(O, s, size); +} + +size_t iof_put_int64_radix (iof *O, int64_t number, int radix, int uc) +{ + const char *s; + size_t size; + s = int64_to_radix(number, radix, uc, &size); + return iof_write(O, s, size); +} + +size_t iof_put_uint32_radix (iof *O, uint32_t number, int radix, int uc) +{ + const char *s; + size_t size; + s = uint32_to_radix(number, radix, uc, &size); + return iof_write(O, s, size); +} + +size_t iof_put_ulong_radix (iof *O, unsigned long number, int radix, int uc) +{ + const char *s; + size_t size; + s = ulong_to_radix(number, radix, uc, &size); + return iof_write(O, s, size); +} + +size_t iof_put_usize_radix (iof *O, size_t number, int radix, int uc) +{ + const char *s; + size_t size; + s = usize_to_radix(number, radix, uc, &size); + return iof_write(O, s, size); +} + +size_t iof_put_uint64_radix (iof *O, uint64_t number, int radix, int uc) +{ + const char *s; + size_t size; + s = uint64_to_radix(number, radix, uc, &size); + return iof_write(O, s, size); +} + +/* roman numerals */ + +size_t iof_put_roman (iof *O, uint16_t number, int uc) +{ + const char *s; + size_t size; + s = uint16_to_roman(number, uc, &size); + return iof_write(O, s, size); +} + +/* double/float to iof; return the number of written bytes */ + +size_t iof_put_double (iof *O, double number, int digits) +{ + const char *s; + size_t size; + s = double_to_string(number, digits, &size); + return iof_write(O, s, size); +} + +size_t iof_put_float (iof *O, float number, int digits) +{ + const char *s; + size_t size; + s = float_to_string(number, digits, &size); + return iof_write(O, s, size); +} + +/* iof to binary integer; pretty common */ + +int iof_get_be_uint2 (iof *I, uint32_t *pnumber) +{ + int c1, c2; + if ((c1 = iof_get(I)) < 0 || (c2 = iof_get(I)) < 0) + return 0; + *pnumber = (c1<<8)|c2; + return 1; +} + +int iof_get_be_uint3 (iof *I, uint32_t *pnumber) +{ + int c1, c2, c3; + if ((c1 = iof_get(I)) < 0 || (c2 = iof_get(I)) < 0 || (c3 = iof_get(I)) < 0) + return 0; + *pnumber = (c1<<16)|(c2<<8)|c3; + return 1; +} + +int iof_get_be_uint4 (iof *I, uint32_t *pnumber) +{ + int c1, c2, c3, c4; + if ((c1 = iof_get(I)) < 0 || (c2 = iof_get(I)) < 0 || (c3 = iof_get(I)) < 0 || (c4 = iof_get(I)) < 0) + return 0; + *pnumber = (c1<<24)|(c2<<16)|(c3<<8)|c4; + return 1; +} + +int iof_get_le_uint2 (iof *I, uint32_t *pnumber) +{ + int c1, c2; + if ((c1 = iof_get(I)) < 0 || (c2 = iof_get(I)) < 0) + return 0; + *pnumber = (c2<<8)|c1; + return 1; +} + +int iof_get_le_uint3 (iof *I, uint32_t *pnumber) +{ + int c1, c2, c3; + if ((c1 = iof_get(I)) < 0 || (c2 = iof_get(I)) < 0 || (c3 = iof_get(I)) < 0) + return 0; + *pnumber = (c3<<16)|(c2<<8)|c1; + return 1; +} + +int iof_get_le_uint4 (iof *I, uint32_t *pnumber) +{ + int c1, c2, c3, c4; + if ((c1 = iof_get(I)) < 0 || (c2 = iof_get(I)) < 0 || (c3 = iof_get(I)) < 0 || (c4 = iof_get(I)) < 0) + return 0; + *pnumber = (c4<<24)|(c3<<16)|(c2<<8)|c1; + return 1; +} + +/* iof input data */ + +uint8_t * iof_file_input_data (iof_file *iofile, size_t *psize, int *isnew) +{ + uint8_t *data; + if (iofile->flags & IOF_DATA) + { + data = iofile->buf; + *psize = iofile->end - iofile->buf; + *isnew = 0; + return data; + } + if (iof_file_reopen(iofile)) + { + data = iof_copy_file_handle_data(iof_file_get_fh(iofile), psize); + *isnew = 1; + iof_file_reclose(iofile); + return data; + } + return NULL; +} + +/* +uint8_t * iof_file_reader_data (iof_file *iofile, size_t *size) +{ + uint8_t *data; + if (!(iofile->flags & IOF_DATA) || iofile->pos == NULL || (*size = (size_t)iof_left(iofile)) == 0) + return NULL; + if (iofile->flags & IOF_BUFFER_ALLOC) + { + data = iofile->buf; // iofile->pos; // returned must be freeable, makes sense when ->buf == ->pos + iofile->flags &= ~IOF_BUFFER_ALLOC; + iofile->buf = iofile->pos = iofile->end = NULL; + return data; + } + data = (uint8_t *)util_malloc(*size); + memcpy(data, iofile->buf, *size); + return data; +} + +uint8_t * iof_file_writer_data (iof_file *iofile, size_t *size) +{ + uint8_t *data; + if (!(iofile->flags & IOF_DATA) || iofile->buf == NULL || (*size = (size_t)iof_size(iofile)) == 0) + return NULL; + if (iofile->flags & IOF_BUFFER_ALLOC) + { + iofile->flags &= ~IOF_BUFFER_ALLOC; + data = iofile->buf; + iofile->buf = iofile->pos = iofile->end = NULL; + return data; + } + data = (uint8_t *)util_malloc(*size); + memcpy(data, iofile->buf, *size); + return data; +} +*/ + +uint8_t * iof_reader_data (iof *I, size_t *psize) +{ + uint8_t *data; + *psize = (size_t)iof_left(I); + if (I->flags & IOF_BUFFER_ALLOC) + { + data = I->buf; // actually I->pos, but we have to return something freeable + I->flags &= ~IOF_BUFFER_ALLOC; + I->buf = NULL; + } + else + { + data = util_malloc(*psize); + memcpy(data, I->pos, *psize); + } + iof_close(I); + return data; +} + + +uint8_t * iof_writer_data (iof *O, size_t *psize) +{ + uint8_t *data; + *psize = (size_t)iof_size(O); + if (O->flags & IOF_BUFFER_ALLOC) + { + data = O->buf; + O->flags &= ~IOF_BUFFER_ALLOC; + O->buf = NULL; + } + else + { + data = util_malloc(*psize); + memcpy(data, O->buf, *psize); + } + iof_close(O); + return data; +} + +size_t iof_reader_to_file_handle (iof *I, FILE *file) +{ + size_t size; + for (size = 0; iof_readable(I); I->pos = I->end) + size += fwrite(I->buf, sizeof(uint8_t), iof_left(I), file); + return size; +} + +size_t iof_reader_to_file (iof *I, const char *filename) +{ + FILE *file; + size_t size; + if ((file = fopen(filename, "wb")) == NULL) + return 0; + for (size = 0; iof_readable(I); I->pos = I->end) + size += fwrite(I->buf, sizeof(uint8_t), iof_left(I), file); + fclose(file); + return size; +} + +/* debug */ + +size_t iof_data_to_file (const void *data, size_t size, const char *filename) +{ + FILE *fh; + if ((fh = fopen(filename, "wb")) == NULL) + return 0; + // size = fwrite(data, size, sizeof(uint8_t), fh); // WRONG, this always returns 1, as fwrite returns the number of elements successfully written out + size = fwrite(data, sizeof(uint8_t), size, fh); + fclose(fh); + return size; +} + +size_t iof_result_to_file_handle (iof *F, FILE *file) +{ + const void *data; + size_t size; + data = iof_result(F, size); + return iof_data_to_file_handle(data, size, file); +} + +size_t iof_result_to_file (iof *F, const char *filename) +{ + const void *data; + size_t size; + data = iof_result(F, size); + return iof_data_to_file(data, size, filename); +} + +void iof_debug (iof *I, const char *filename) +{ + FILE *file = fopen(filename, "wb"); + if (file != NULL) + { + fprintf(file, ">>> buf %p <<<\n", I->buf); + fwrite(I->buf, sizeof(uint8_t), iof_size(I), file); + fprintf(file, "\n>>> pos %p (%ld) <<<\n", I->pos, (long)iof_size(I)); + fwrite(I->pos, sizeof(uint8_t), iof_left(I), file); + fprintf(file, "\n>>> end %p (%ld) <<<\n", I->end, (long)iof_left(I)); + fwrite(I->end, sizeof(uint8_t), I->space - iof_space(I), file); + fprintf(file, "\n>>> end of buffer %p (%ld) <<<\n", I->buf + I->space, (long)(I->buf + I->space - I->end)); + fclose(file); + } +} + +/* common filters api */ + +/* sizes of filter states on x64 +size of iof_filter: 640 (no longer used; sizeof(iof) + sizeof larger state) +size of file_state: 16 +size of stream_state: 16 +size of flate_state: 104 +size of lzw_state: 56 +size of predictor_state: 104 +size of basexx_state: 48 +size of basexx_state: 48 +size of basexx_state: 48 +size of eexec_state: 40 +size of runlength_state: 24 +size of rc4_state: 24 +size of aes_state: 72 +size of img_state: 576 +size of img: 496 +*/ + +typedef struct iof_heap iof_heap; + +typedef struct { + iof_heap *heap; +} iof_heap_ghost; + + +struct iof_heap { + union { uint8_t *data; iof_heap_ghost *gdata; }; // union instead of casts (ARM) + union { uint8_t *pos; iof_heap_ghost *gpos; }; + size_t size, space; + iof_heap *next, *prev; + int refcount; + uint8_t dummy[4]; // pad to 8N bytes +}; + +/* +We use hidden heap pointer for every allocated buffer, so heap->data should be kept properly aligned. +Dummy 4-bytes pad doesn't really matter (the pad is there anyway), but iof_heap_take() must pad the +requested size. +*/ + +static iof_heap * iof_buffers_heap = NULL; +static iof_heap * iof_filters_heap = NULL; + +#define IOF_HEAP_FILTERS_COUNT 4 +#define IOF_BUFFER_SIZE 262144 // (1<<18) +#define IOF_FILTER_SIZE 1024 +// sizeof(iof_filter) on x64 is now 640, img_state 576, img 496, others 16-104 +#define IOF_BUFFER_HEAP_SIZE (IOF_HEAP_FILTERS_COUNT * (IOF_BUFFER_SIZE + sizeof(iof_heap_ghost))) +#define IOF_FILTER_HEAP_SIZE (IOF_HEAP_FILTERS_COUNT * (IOF_FILTER_SIZE + sizeof(iof_heap_ghost))) + +static iof_heap * iof_heap_new (size_t space) +{ + iof_heap *iofheap; + iofheap = (iof_heap *)util_malloc(sizeof(iof_heap) + space); + iofheap->gdata = iofheap->gpos = (iof_heap_ghost *)(iofheap + 1); + iofheap->size = iofheap->space = space; + iofheap->next = NULL; + iofheap->prev = NULL; + iofheap->refcount = 0; + return iofheap; +} + +#define iof_heap_free(iofheap) util_free(iofheap) + +void iof_filters_init (void) +{ + if (iof_buffers_heap == NULL) + iof_buffers_heap = iof_heap_new(IOF_BUFFER_HEAP_SIZE); + if (iof_filters_heap == NULL) + iof_filters_heap = iof_heap_new(IOF_FILTER_HEAP_SIZE); +} + +void iof_filters_free (void) +{ + iof_heap *heap, *next; + for (heap = iof_buffers_heap; heap != NULL; heap = next) + { + next = heap->next; + if (heap->refcount != 0) + loggerf("not closed iof filters left (%d)", heap->refcount); + if (next != NULL) + loggerf("iof filters heap left"); + iof_heap_free(heap); + } + iof_buffers_heap = NULL; + for (heap = iof_filters_heap; heap != NULL; heap = next) + { + next = heap->next; + if (heap->refcount != 0) + loggerf("not closed iof buffers left (%d)", heap->refcount); + if (next != NULL) + loggerf("iof buffers heap left"); + iof_heap_free(heap); + } + iof_filters_heap = NULL; +} + +#define iof_heap_get(hp, ghost, data, siz) \ + (ghost = (hp)->gpos, ghost->heap = (hp), data = (uint8_t *)(ghost + 1), (hp)->pos += siz, (hp)->size -= siz, ++(hp)->refcount) + +static void * iof_heap_take (iof_heap **pheap, size_t size) +{ + uint8_t *data; + iof_heap_ghost *ghost; + iof_heap *heap, *newheap, *next; + + heap = *pheap; + if (size & 7) + size += 8 - (size & 7); // pad to 8N bytes so that (heap->pos + size) remains properly aligned + size += sizeof(iof_heap_ghost); + if (heap->size >= size) + { /* take cheap mem from main heap */ + iof_heap_get(heap, ghost, data, size); + return data; + } + if (size <= (heap->space >> 1)) + { /* make new cheap heap, make it front */ + *pheap = newheap = iof_heap_new(heap->space); + newheap->next = heap; + heap->prev = newheap; + iof_heap_get(newheap, ghost, data, size); + return data; + } + /* size much larger than expected? should not happen. + make a single-item heap, keep the front heap intact. */ + newheap = iof_heap_new(size); + if ((next = heap->next) != NULL) + { + newheap->next = next; + next->prev = newheap; + } + heap->next = newheap; + newheap->prev = heap; + iof_heap_get(newheap, ghost, data, size); + return data; +} + +static void iof_heap_back (void *data) +{ + iof_heap_ghost *ghost; + iof_heap *heap, *next, *prev; + + ghost = ((iof_heap_ghost *)data) - 1; + heap = ghost->heap; + if (heap->refcount == 0) + loggerf("invalid use of iof heap, refcount < 0"); + if (--heap->refcount <= 0) + { + if ((prev = heap->prev) != NULL) + { /* free the heap */ + if ((next = heap->next) != NULL) + prev->next = next, next->prev = prev; + else + prev->next = NULL; + iof_heap_free(heap); + } + else + { /* this is the front heap, just reset */ + heap->pos = heap->data; + heap->size = heap->space; + } + } +} + +/**/ + +/* +void * iof_filter_new (size_t size) +{ + void *data; + iof_filters_init(); + data = iof_heap_take(&iof_filters_heap, size); + return memset(data, 0, size); +} +*/ + +iof * iof_filter_reader_new (iof_handler handler, size_t statesize, void **pstate) +{ + iof *F; + void *filter; + uint8_t *buffer; + size_t buffersize; + + iof_filters_init(); + filter = iof_heap_take(&iof_filters_heap, sizeof(iof) + statesize); + F = (iof *)memset(filter, 0, sizeof(iof) + statesize); + buffer = iof_heap_take(&iof_buffers_heap, IOF_BUFFER_SIZE); + buffersize = IOF_BUFFER_SIZE; + iof_setup_reader(F, buffer, buffersize); + F->flags |= IOF_HEAP|IOF_BUFFER_HEAP; + F->more = handler; + *pstate = (F + 1); + return F; +} + +iof * iof_filter_reader_with_buffer_new (iof_handler handler, size_t statesize, void **pstate, void *buffer, size_t buffersize) +{ // for filters that has own buffer (string, some image filters) + iof *F; + void *filter; + + iof_filters_init(); + filter = iof_heap_take(&iof_filters_heap, sizeof(iof) + statesize); + F = (iof *)memset(filter, 0, sizeof(iof) + statesize); + iof_setup_reader(F, buffer, buffersize); + F->flags |= IOF_HEAP; + F->more = handler; + *pstate = (F + 1); + return F; +} + +iof * iof_filter_writer_new (iof_handler handler, size_t statesize, void **pstate) +{ + iof *F; + void *filter; + uint8_t *buffer; + size_t buffersize; + + iof_filters_init(); + filter = iof_heap_take(&iof_filters_heap, sizeof(iof) + statesize); + F = (iof *)memset(filter, 0, sizeof(iof) + statesize); + buffer = iof_heap_take(&iof_buffers_heap, IOF_BUFFER_SIZE); + buffersize = IOF_BUFFER_SIZE; + iof_setup_writer(F, buffer, buffersize); + F->flags |= IOF_HEAP|IOF_BUFFER_HEAP; + F->more = handler; + *pstate = (F + 1); + return F; +} + +iof * iof_filter_writer_with_buffer_new (iof_handler handler, size_t statesize, void **pstate, void *buffer, size_t buffersize) +{ + iof *F; + void *filter; + + iof_filters_init(); + filter = iof_heap_take(&iof_filters_heap, sizeof(iof) + statesize); + F = (iof *)memset(filter, 0, sizeof(iof) + statesize); + iof_setup_writer(F, buffer, buffersize); + F->flags |= IOF_HEAP; + F->more = handler; + *pstate = (F + 1); + return F; +} + +/**/ + +#define iof_filter_free(F) iof_heap_back(F) +#define iof_filter_buffer_free(data) iof_heap_back(data) + +/* close */ + +#define iof_close_next(F) ((void)(iof_decref((F)->next), (F)->next = NULL, 0)) +/* when filter creation fails, we should take care to destroy the filter but leave ->next intact */ +#define iof_clear_next(F) ((void)(iof_unref((F)->next), (F)->next = NULL, 0)) + +#define iof_close_buffer(F) ((void)\ + ((F)->buf != NULL ? \ + ((F->flags & IOF_BUFFER_ALLOC) ? (util_free((F)->buf), (F)->buf = NULL, 0) : \ + ((F->flags & IOF_BUFFER_HEAP) ? (iof_filter_buffer_free((F)->buf), (F)->buf = NULL, 0) : ((F)->buf = NULL, 0))) : 0)) + +/* closing underlying file handle */ + +static void iof_close_file (iof *F) +{ + FILE *file; + //if (F->flags & IOF_FILE_HANDLE) + //{ + if ((file = F->file) != NULL) + { + if (F->flags & IOF_CLOSE_FILE) + fclose(F->file); + F->file = NULL; + } + //} +} + +/* a very special variant for reader filters initiated with iof_file_reopen(). It also calls + iof_file_reclose(), which takes an effect only if previously reopened, but better to keep + all this thin ice separated. Used in filters: iofile_reader, iofile_stream_reader, image + decoders. */ + +static void iof_close_iofile (iof *F) +{ + iof_file *iofile; + //if (F->flags & IOF_FILE) + //{ + if ((iofile = F->iofile) != NULL) + { + iof_file_unsync(iofile, NULL); + iof_file_reclose(iofile); // takes an effect iff prevoiusly reopened + iof_file_decref(iofile); + F->iofile = NULL; + } + //} +} + +void iof_free (iof *F) +{ + if (F->flags & IOF_FILE_HANDLE) + iof_close_file(F); + else if (F->flags & IOF_FILE) + iof_close_iofile(F); + else if (F->flags & IOF_NEXT) + iof_close_next(F); + iof_close_buffer(F); + if (F->flags & IOF_HEAP) + iof_filter_free(F); + else if (F->flags & IOF_ALLOC) + util_free(F); +} + +void iof_discard (iof *F) +{ // so far used only on failed filters creation; as iof_free() but don't dare to release ->next + if (F->flags & IOF_FILE_HANDLE) + iof_close_file(F); + else if (F->flags & IOF_FILE) + iof_close_iofile(F); + //else if (F->flags & IOF_NEXT) + // iof_close_next(F); + iof_close_buffer(F); + if (F->flags & IOF_HEAP) + iof_filter_free(F); + else if (F->flags & IOF_ALLOC) + util_free(F); +} + +/* resizing buffer */ + +size_t iof_resize_buffer_to (iof *O, size_t space) +{ + uint8_t *buf; + + if (O->flags & IOF_BUFFER_ALLOC) + { + buf = (uint8_t *)util_realloc(O->buf, space); + } + else + { + buf = (uint8_t *)util_malloc(space); + memcpy(buf, O->buf, iof_size(O)); + if (O->flags & IOF_BUFFER_HEAP) + { + iof_filter_buffer_free(O->buf); + O->flags &= ~IOF_BUFFER_HEAP; + } + O->flags |= IOF_BUFFER_ALLOC; + + } + O->pos = buf + iof_size(O); + O->end = buf + space; + O->buf = buf; + O->space = space; + return iof_left(O); +} + +/* */ + +size_t iof_decoder_retval (iof *I, const char *type, iof_status status) +{ + switch (status) + { + case IOFERR: + case IOFEMPTY: // should never happen as we set state.flush = 1 on decoders init + loggerf("%s decoder error (%d, %s)", type, status, iof_status_kind(status)); + I->flags |= IOF_STOPPED; + return 0; + case IOFEOF: // this is the last chunk, + I->flags |= IOF_STOPPED; // so stop it and fall + FALLTHRU // fall through + case IOFFULL: // prepare pointers to read from I->buf + I->end = I->pos; + I->pos = I->buf; + return I->end - I->buf; + } + loggerf("%s decoder bug, invalid retval %d", type, status); + return 0; +} + +size_t iof_encoder_retval (iof *O, const char *type, iof_status status) +{ + switch (status) + { + case IOFERR: + case IOFFULL: + loggerf("%s encoder error (%d, %s)", type, status, iof_status_kind(status)); + return 0; + case IOFEMPTY: + O->pos = O->buf; + O->end = O->buf + O->space; + return O->space; + case IOFEOF: + return 0; + } + loggerf("%s encoder bug, invalid retval %d", type, status); + return 0; +} + +/* file/stream state */ + +typedef struct { + size_t length; + size_t offset; +} file_state; + +#define file_state_init(state, off, len) ((state)->offset = off, (state)->length = len) + +typedef struct { + size_t length; + size_t offset; +} stream_state; + +#define stream_state_init(state, off, len) ((state)->offset = off, (state)->length = len) + +/* union type to avoid 'dereferencing type-punned .. ' warnings on (void **) case */ + +typedef union { file_state *filestate; stream_state *streamstate; void *voidstate; } fs_state_pointer; + +/**/ + +static size_t file_read (iof *I) +{ + size_t bytes, tail; + if (I->flags & IOF_STOPPED) + return 0; + tail = iof_tail(I); + if ((bytes = tail + fread(I->buf + tail, sizeof(uint8_t), I->space - tail, I->file)) < I->space) + I->flags |= IOF_STOPPED; + I->pos = I->buf; + I->end = I->buf + bytes; + return bytes; +} + +static size_t iofile_read (iof *I, size_t *poffset) +{ + size_t bytes, tail; + if (I->flags & IOF_STOPPED) + return 0; + iof_file_sync(I->iofile, poffset); + tail = iof_tail(I); + if ((bytes = tail + iof_file_read(I->buf + tail, sizeof(uint8_t), I->space - tail, I->iofile)) < I->space) + { + I->flags |= IOF_STOPPED; + iof_file_unsync(I->iofile, poffset); + } + I->pos = I->buf; + I->end = I->buf + bytes; + return bytes; +} + +static size_t file_load (iof *I) +{ + size_t bytes, left, tail; + if (I->flags & IOF_STOPPED) + return 0; + tail = iof_tail(I); + I->pos = I->buf + tail; + I->end = I->buf + I->space; /* don't assume its done when initializing the filter */ + left = I->space - tail; + do { + bytes = fread(I->pos, sizeof(uint8_t), left, I->file); + I->pos += bytes; + } while (bytes == left && (left = iof_resize_buffer(I)) > 0); + I->flags |= IOF_STOPPED; + return iof_loaded(I); +} + +static size_t iofile_load (iof *I, size_t *poffset) +{ + size_t bytes, left, tail; + if (I->flags & IOF_STOPPED) + return 0; + tail = iof_tail(I); + I->pos = I->buf + tail; + I->end = I->buf + I->space; /* don't assume its done when initializing the filter */ + left = I->space - tail; + iof_file_sync(I->iofile, poffset); + do { + bytes = iof_file_read(I->pos, sizeof(uint8_t), left, I->iofile); + I->pos += bytes; + } while (bytes == left && (left = iof_resize_buffer(I)) > 0); + I->flags |= IOF_STOPPED; + iof_file_unsync(I->iofile, poffset); + return iof_loaded(I); +} + +static size_t filter_file_reader (iof *I, iof_mode mode) +{ + switch (mode) + { + case IOFREAD: + return file_read(I); + case IOFLOAD: + return file_load(I); + case IOFCLOSE: + iof_free(I); + return 0; + default: + return 0; + } +} + +static size_t filter_iofile_reader (iof *I, iof_mode mode) +{ + file_state *state; + state = iof_filter_state(file_state *, I); + switch (mode) + { + case IOFREAD: + return iofile_read(I, &state->offset); + case IOFLOAD: + return iofile_load(I, &state->offset); + case IOFCLOSE: + iof_free(I); + return 0; + default: + return 0; + } +} + +static size_t file_write (iof *O, int flush) +{ + size_t bytes; + if ((bytes = iof_size(O)) > 0) + if (bytes != fwrite(O->buf, sizeof(uint8_t), bytes, O->file)) + return 0; + if (flush) + fflush(O->file); + O->end = O->buf + O->space; // remains intact actually + O->pos = O->buf; + return O->space; +} + +static size_t iofile_write (iof *O, size_t *poffset, int flush) +{ + size_t bytes; + iof_file_sync(O->iofile, poffset); + if ((bytes = iof_size(O)) > 0) + { + if (bytes != iof_file_write(O->buf, sizeof(uint8_t), bytes, O->iofile)) + { + iof_file_unsync(O->iofile, poffset); + return 0; + } + } + if (flush) + iof_file_flush(O->iofile); + O->end = O->buf + O->space; // remains intact actually + O->pos = O->buf; + return O->space; +} + +static size_t filter_file_writer (iof *O, iof_mode mode) +{ + switch (mode) + { + case IOFWRITE: + return file_write(O, 0); + case IOFFLUSH: + return file_write(O, 1); + case IOFCLOSE: + file_write(O, 1); + iof_free(O); + return 0; + default: + return 0; + } +} + +static size_t filter_iofile_writer (iof *O, iof_mode mode) +{ + file_state *state; + state = iof_filter_state(file_state *, O); + switch (mode) + { + case IOFWRITE: + return iofile_write(O, &state->offset, 0); + case IOFFLUSH: + return iofile_write(O, &state->offset, 1); + case IOFCLOSE: + iofile_write(O, &state->offset, 1); + iof_free(O); + return 0; + default: + return 0; + } +} + +/* filter from FILE* */ + +iof * iof_filter_file_handle_reader (FILE *file) +{ + iof *I; + fs_state_pointer P; + if (file == NULL) + return NULL; + I = iof_filter_reader(filter_file_reader, sizeof(file_state), &P.voidstate); + iof_setup_file(I, file); + file_state_init(P.filestate, 0, 0); + return I; +} + +iof * iof_filter_file_handle_writer (FILE *file) +{ + iof *O; + fs_state_pointer P; + if (file == NULL) + return NULL; + O = iof_filter_writer(filter_file_writer, sizeof(file_state), &P.voidstate); + iof_setup_file(O, file); + file_state_init(P.filestate, 0, 0); + return O; +} + +/* filter from iof_file * */ + +iof * iof_filter_iofile_reader (iof_file *iofile, size_t offset) +{ + iof *I; + fs_state_pointer P; + if (!iof_file_reopen(iofile)) + return NULL; + I = iof_filter_reader(filter_iofile_reader, sizeof(file_state), &P.voidstate); + iof_setup_iofile(I, iofile); + file_state_init(P.filestate, offset, 0); + return I; +} + +iof * iof_filter_iofile_writer (iof_file *iofile, size_t offset) +{ + iof *O; + fs_state_pointer P; + O = iof_filter_writer(filter_iofile_writer, sizeof(file_state), &P.voidstate); + iof_setup_iofile(O, iofile); + file_state_init(P.filestate, offset, 0); + return O; +} + +/* filter from filename */ + +iof * iof_filter_file_reader (const char *filename) +{ + iof *I; + fs_state_pointer P; + FILE *file; + if ((file = fopen(filename, "rb")) == NULL) + return NULL; + I = iof_filter_reader(filter_file_reader, sizeof(file_state), &P.voidstate); + iof_setup_file(I, file); + file_state_init(P.filestate, 0, 0); + I->flags |= IOF_CLOSE_FILE; + return I; +} + +iof * iof_filter_file_writer (const char *filename) +{ + iof *O; + fs_state_pointer P; + FILE *file; + if ((file = fopen(filename, "wb")) == NULL) + return NULL; + O = iof_filter_writer(filter_file_writer, sizeof(file_state), &P.voidstate); + iof_setup_file(O, file); + file_state_init(P.filestate, 0, 0); + O->flags |= IOF_CLOSE_FILE; + return O; +} + +/* from string */ + +static size_t dummy_handler (iof *I, iof_mode mode) +{ + switch (mode) + { + case IOFCLOSE: + iof_free(I); + return 0; + default: + return 0; + } +} + +iof * iof_filter_string_reader (const void *s, size_t length) +{ + iof *I; + void *dummy; + I = iof_filter_reader_with_buffer(dummy_handler, 0, &dummy, NULL, 0); + I->rbuf = I->rpos = (const uint8_t *)s; + I->rend = (const uint8_t *)s + length; + // I->space = length; + return I; +} + +iof * iof_filter_string_writer (const void *s, size_t length) +{ + iof *O; + void *dummy; + O = iof_filter_reader_with_buffer(dummy_handler, 0, &dummy, NULL, 0); + O->rbuf = O->rpos = (const uint8_t *)s; + O->rend = (const uint8_t *)s + length; + // O->space = length; + return O; +} + +iof * iof_filter_buffer_writer (size_t size) +{ // cmp iof_buffer_create() + iof *O; + fs_state_pointer dummy; + uint8_t *buffer; + if (size > IOF_BUFFER_SIZE) + { + buffer = (uint8_t *)util_malloc(size); + O = iof_filter_writer_with_buffer(iof_mem_handler, 0, &dummy.voidstate, buffer, size); + O->flags |= IOF_BUFFER_ALLOC; + return O; + } + return iof_filter_writer(iof_mem_handler, 0, &dummy.voidstate); +} + +/* stream */ + +static size_t file_stream_read (iof *I, size_t *plength) +{ + size_t bytes, tail; + if (I->flags & IOF_STOPPED || *plength == 0) + return 0; + tail = iof_tail(I); + if (I->space - tail >= *plength) + { + bytes = tail + fread(I->buf + tail, sizeof(uint8_t), *plength, I->file); + I->flags |= IOF_STOPPED; + *plength = 0; + } + else + { + bytes = tail + fread(I->buf + tail, sizeof(uint8_t), I->space - tail, I->file); + *plength -= bytes - tail; + } + I->pos = I->buf; + I->end = I->buf + bytes; + return bytes; +} + +static size_t iofile_stream_read (iof *I, size_t *plength, size_t *poffset) +{ + size_t bytes, tail; + if (I->flags & IOF_STOPPED || *plength == 0) + return 0; + tail = iof_tail(I); + iof_file_sync(I->iofile, poffset); + if (I->space - tail >= *plength) + { + bytes = tail + iof_file_read(I->buf + tail, sizeof(uint8_t), *plength, I->iofile); + iof_file_unsync(I->iofile, poffset); + I->flags |= IOF_STOPPED; + *plength = 0; + } + else + { + bytes = tail + iof_file_read(I->buf + tail, sizeof(uint8_t), I->space - tail, I->iofile); + *plength -= bytes - tail; + } + I->pos = I->buf; + I->end = I->buf + bytes; + return bytes; +} + +static size_t file_stream_load (iof *I, size_t *plength) +{ + size_t bytes, tail; + if (I->flags & IOF_STOPPED || *plength == 0) + return 0; + tail = iof_tail(I); + if (I->space - tail < *plength) + if (iof_resize_buffer_to(I, tail + *plength) == 0) + return 0; + bytes = tail + fread(I->buf + tail, sizeof(uint8_t), *plength, I->file); + I->flags |= IOF_STOPPED; + *plength = 0; + I->pos = I->buf; + I->end = I->buf + bytes; + return bytes; +} + +static size_t iofile_stream_load (iof *I, size_t *plength, size_t *poffset) +{ + size_t bytes, tail; + if (I->flags & IOF_STOPPED || *plength == 0) + return 0; + iof_file_sync(I->iofile, poffset); + tail = iof_tail(I); + if (I->space - tail < *plength) + if (iof_resize_buffer_to(I, tail + *plength) == 0) + return 0; + bytes = tail + iof_file_read(I->buf + tail, sizeof(uint8_t), *plength, I->iofile); + iof_file_unsync(I->iofile, poffset); + I->flags |= IOF_STOPPED; + *plength = 0; + I->pos = I->buf; + I->end = I->buf + bytes; + return bytes; +} + +static size_t filter_file_stream_reader (iof *I, iof_mode mode) +{ + stream_state *state; + state = iof_filter_state(stream_state *, I); + switch(mode) + { + case IOFREAD: + return file_stream_read(I, &state->length); + case IOFLOAD: + return file_stream_load(I, &state->length); + case IOFCLOSE: + iof_free(I); + return 0; + default: + return 0; + } +} + +static size_t filter_iofile_stream_reader (iof *I, iof_mode mode) +{ + stream_state *state; + state = iof_filter_state(stream_state *, I); + switch(mode) + { + case IOFREAD: + return iofile_stream_read(I, &state->length, &state->offset); + case IOFLOAD: + return iofile_stream_load(I, &state->length, &state->offset); + case IOFCLOSE: + iof_free(I); + return 0; + default: + return 0; + } +} + +iof * iof_filter_stream_reader (FILE *file, size_t offset, size_t length) +{ + iof *I; + fs_state_pointer P; + I = iof_filter_reader(filter_file_stream_reader, sizeof(stream_state), &P.voidstate); + iof_setup_file(I, file); + stream_state_init(P.streamstate, offset, length); + fseek(file, (long)offset, SEEK_SET); // or perhaps it should be call in file_stream_read(), like iof_file_sync()? + return I; +} + +iof * iof_filter_stream_coreader (iof_file *iofile, size_t offset, size_t length) +{ + iof *I; + fs_state_pointer P; + if (!iof_file_reopen(iofile)) + return NULL; + I = iof_filter_reader(filter_iofile_stream_reader, sizeof(stream_state), &P.voidstate); + iof_setup_iofile(I, iofile); + stream_state_init(P.streamstate, offset, length); + return I; +} + +static size_t file_stream_write (iof *O, size_t *plength, int flush) +{ + size_t bytes; + if ((bytes = iof_size(O)) > 0) + { + if (bytes != fwrite(O->buf, sizeof(uint8_t), bytes, O->file)) + { + *plength += bytes; + return 0; + } + } + if (flush) + fflush(O->file); + *plength += bytes; + O->end = O->buf + O->space; // remains intact + O->pos = O->buf; + return O->space; +} + +static size_t iofile_stream_write (iof *O, size_t *plength, size_t *poffset, int flush) +{ + size_t bytes; + if ((bytes = iof_size(O)) > 0) + { + iof_file_sync(O->iofile, poffset); + if (bytes != iof_file_write(O->buf, sizeof(uint8_t), bytes, O->iofile)) + { + *plength += bytes; + iof_file_unsync(O->iofile, poffset); + return 0; + } + } + if (flush) + iof_file_flush(O->iofile); + *plength += bytes; + O->end = O->buf + O->space; // remains intact + O->pos = O->buf; + return O->space; +} + +static size_t filter_file_stream_writer (iof *O, iof_mode mode) +{ + stream_state *state; + state = iof_filter_state(stream_state *, O); + switch (mode) + { + case IOFWRITE: + return file_stream_write(O, &state->length, 0); + case IOFFLUSH: + return file_stream_write(O, &state->length, 1); + case IOFCLOSE: + file_stream_write(O, &state->length, 1); + iof_free(O); + return 0; + default: + return 0; + } +} + +static size_t filter_iofile_stream_writer (iof *O, iof_mode mode) +{ + stream_state *state; + state = iof_filter_state(stream_state *, O); + switch (mode) + { + case IOFWRITE: + return iofile_stream_write(O, &state->length, &state->offset, 0); + case IOFFLUSH: + return iofile_stream_write(O, &state->length, &state->offset, 1); + case IOFCLOSE: + iofile_stream_write(O, &state->length, &state->offset, 1); + iof_free(O); + return 0; + default: + return 0; + } +} + +iof * iof_filter_stream_writer (FILE *file) +{ + iof *O; + fs_state_pointer P; + O = iof_filter_writer(filter_file_stream_writer, sizeof(stream_state), &P.voidstate); + iof_setup_file(O, file); + stream_state_init(P.streamstate, 0, 0); + return O; +} + +iof * iof_filter_stream_cowriter (iof_file *iofile, size_t offset) +{ + iof *O; + fs_state_pointer P; + O = iof_filter_writer(filter_iofile_stream_writer, sizeof(stream_state), &P.voidstate); + iof_setup_iofile(O, iofile); + stream_state_init(P.streamstate, offset, 0); + return O; +} + +/* very specific for images; get input from already created strem filter, exchange the filter but keep the buffer */ + +FILE * iof_filter_file_reader_source (iof *I, size_t *poffset, size_t *plength) +{ + fs_state_pointer P; + if (I->more == filter_file_stream_reader) // I is the result of iof_filter_stream_reader() + { + P.streamstate = iof_filter_state(stream_state *, I); + *poffset = P.streamstate->offset; + *plength = P.streamstate->length; // might be 0 but it is ok for file readers + return I->file; + } + if (I->more == filter_file_reader) + { + P.filestate = iof_filter_state(file_state *, I); + *poffset = P.filestate->offset; + *plength = P.filestate->length; // might be 0 but it is ok for file readers + return I->file; + } + return NULL; +} + +iof_file * iof_filter_file_coreader_source (iof *I, size_t *poffset, size_t *plength) +{ + fs_state_pointer P; + if (I->more == filter_iofile_stream_reader) // I is the result of iof_filter_stream_coreader() + { + P.streamstate = iof_filter_state(stream_state *, I); + *poffset = P.streamstate->offset; + *plength = P.streamstate->length; + return I->iofile; + } + if (I->more == filter_iofile_reader) + { + P.filestate = iof_filter_state(file_state *, I); + *poffset = P.filestate->offset; + *plength = P.filestate->length; + return I->iofile; + } + return NULL; +} + +iof * iof_filter_reader_replacement (iof *P, iof_handler handler, size_t statesize, void **pstate) +{ // called after iof_filter_file_reader_source(), no need to check if F is filter from iof heap and if has buffer from iof heap + iof *F; + F = iof_filter_reader_with_buffer(handler, statesize, pstate, P->buf, P->space); + F->flags |= IOF_BUFFER_HEAP; + //iof_setup_reader(P, NULL, 0); + //P->flags &= ~IOF_BUFFER_HEAP; + iof_filter_free(P); + return F; +} + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Build/source/libs/pplib/pplib-src/src/util/utiliof.h b/Build/source/libs/pplib/pplib-src/src/util/utiliof.h new file mode 100644 index 00000000000..bad43a77374 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utiliof.h @@ -0,0 +1,673 @@ + +#ifndef UTIL_IOF_H +#define UTIL_IOF_H + +#include <stdio.h> // for FILE * +#include <errno.h> // for errno +#include <string.h> // for strerror() +#include <stdint.h> // for uintN_t + +#include "utildecl.h" +#include "utilnumber.h" + +/* handler call modes */ + +typedef enum { + IOFREAD = 0, /* read to buffer */ + IOFLOAD = 1, /* read all to buffer */ + IOFWRITE = 2, /* write buffer to the output */ + IOFFLUSH = 3, /* flush buffer to the output */ + IOFCLOSE = 4 /* (flush and) close */ +} iof_mode; + +/* return statuses */ + +typedef enum { + IOFEOF = -1, /* end of input */ + IOFEMPTY = -2, /* end of input buffer*/ + IOFFULL = -3, /* end of output buffer */ + IOFERR = -4 /* error */ +} iof_status; + +const char * iof_status_kind (iof_status status); + +/* iof_file */ + +typedef struct iof_file { + union { + FILE *iofh; // access via iof_file_get_fh / iof_file_set_fh (below) + union { + struct { uint8_t *buf, *pos, *end; }; + struct { const uint8_t *rbuf, *rpos, *rend; }; // to trick compiler warnings about cast discarding const + }; + }; + size_t *offset; + char *name; + size_t size; + int refcount; + int flags; +} iof_file; + +/* iof handler function */ + +typedef struct iof iof; +typedef size_t (*iof_handler) (iof *I, iof_mode mode); + +/* iof structure; keep 8N bytes */ + +#define IOF_MEMBERS \ + union { \ + struct { uint8_t *buf, *pos, *end; }; \ + struct { uint16_t *hbuf, *hpos, *hend; }; \ + struct { uint32_t *ibuf, *ipos, *iend; }; \ + struct { const uint8_t *rbuf, *rpos, *rend; }; \ + }; \ + size_t space; \ + iof_handler more; \ + union { void *link; iof *next; FILE *file; iof_file *iofile; }; \ + int flags; \ + int refcount + +/* + buf -- the beginning of buffer + pos -- the current position + end -- the end of buffer + space -- private space size, not always eq. (end - buf) + more -- handler function + next/file/iofile/link -- reader source or writer target + source -- source filter + flags -- private filter info + refcount -- refcount +*/ + +struct iof { + IOF_MEMBERS; +}; + +typedef void (*iof_dump_function) (const void *value, iof *O); + +/* flags */ + +#define IOF_ALLOC (1<<0) // iof is allocated +#define IOF_HEAP (1<<1) // iof taken from iof heap +#define IOF_BUFFER_ALLOC (1<<2) // buffer allocated +#define IOF_BUFFER_HEAP (1<<3) // buffer taken from iof heap + +#define IOF_SHORT (1<<4) // buffer uses 16bit integers +#define IOF_LONG (1<<5) // buffer uses 32bit integers + +#define IOF_TAIL (1<<6) // preserve reader tail +#define IOF_READER (1<<7) // is reader +#define IOF_WRITER (1<<8) // is writer + +#define IOF_DATA (1<<9) // binds some memory +#define IOF_FILE_HANDLE (1<<10) // links FILE * +#define IOF_FILE (1<<11) // links iof_file * +#define IOF_NEXT (1<<12) // links next iof * +#define IOF_CLOSE_FILE (1<<13) // close FILE * on free +#define IOF_REOPEN_FILE (1<<14) // close/reopen mode for iof_file +#define IOF_RECLOSE_FILE (1<<15) // ditto + +#define IOF_STOPPED (1<<16) // stopped + +// #define IOF_CUSTOM (1<<17) // first custom flag + +#define IOF_BUFSIZ (sizeof(iof) + BUFSIZ*sizeof(uint8_t)) + +/* +reading buffer -- all of buf, pos, end pointers are initialized to the beginning of the private buffer, + next call to a handler function moves the end pointer to bufer+space +writer -- buf and pos pointers initialized to the beginning of the buffer, end initialized to bufer+space + +Every call to handler returns size_t number of bytes +available (to write/read) or 0 if there is no more space. + +We usually align the data buffer just after the iof structure. +This is convenient, especially when a memory for the structure +and its buffer is to be allocated. In the case of growing output +buffers we used to check if the memory of the buffer is allocated +by the handler function using test (O->buf != (O+1)). We don't use +it any longer not to rely on little secrets. Now there is an explicit +IOF_BUFFER_ALLOC flag for that. IOF_ALLOC tells if the structure +itself is taken from malloc (not used so far). Assuming the buffer size +is way larger the sizeof(iof) +*/ + +/* initializers */ + +#define IOF_READER_INIT(handler, file, buffer, size, flags) \ + { {{ (uint8_t *)(buffer), (uint8_t *)(buffer), (uint8_t *)(buffer) }}, size, handler, { file }, (flags)|IOF_READER, 0 } + +#define IOF_WRITER_INIT(handler, file, buffer, size, flags) \ + { {{ (uint8_t *)(buffer), (uint8_t *)(buffer), (uint8_t *)(buffer) + size }}, size, handler, { file }, (flags)|IOF_WRITER, 0 } + +#define IOF_STRING_INIT(buffer, size) \ + { {{ (uint8_t *)(buffer), (uint8_t *)(buffer), (uint8_t *)(buffer) + size }}, size, NULL, { NULL }, 0|IOF_READER|IOF_DATA, 0 } + +#define IOF_STRING() IOF_STRING_INIT(0, 0) + +/* refcount */ + +#define iof_incref(I) (++(I)->refcount) +#define iof_decref(I) ((void)(--(I)->refcount <= 0 && iof_close(I))) +#define iof_unref(I) (--(I)->refcount) + +/* binding buffer of a given size */ + +#define iof_setup_reader(I, buffer, size) \ + ((I)->buf = (I)->pos = (I)->end = (uint8_t *)(buffer), \ + (I)->space = size, (I)->flags = 0|IOF_READER, (I)->refcount = 0) + +#define iof_setup_writer(O, buffer, size) \ + ((O)->buf = (O)->pos = (uint8_t *)(buffer), \ + (O)->end = (uint8_t *)(buffer) + size, \ + (O)->space = size, (O)->flags = 0|IOF_WRITER, (O)->refcount = 0) + +/* basics */ + +#define iof_space(I) ((I)->end - (I)->buf) +#define iof_left(I) ((I)->end - (I)->pos) +#define iof_size(I) ((I)->pos - (I)->buf) + +#define iof_input(I) ((I)->more ? (I)->more((I), IOFREAD) : 0lu) +#define iof_load(I) ((I)->more ? (I)->more((I), IOFLOAD) : 0lu) + +#define iof_output(O) ((O)->more ? (O)->more((O), IOFWRITE) : 0lu) +//#define iof_flush(O) ((O)->pos > (O)->buf && (O)->more ? (O)->more(O, IOFFLUSH) : 0lu) +// flush should be unconditional, because encoders emits EOD markers only on flush +#define iof_flush(O) ((O)->more ? (O)->more(O, IOFFLUSH) : 0lu) +#define iof_close(O) ((O)->more ? (O)->more(O, IOFCLOSE) : 0lu) + +#define iof_stop(F) ((void)(F->pos = F->end = F->buf, F->flags |= IOF_STOPPED)) + +/* +Rewriting reader tail to the beginning of new data portion; readers reacting on IOFREAD +mode must be aware of some not yet read data, but treat it necessary only if IOF_TAIL flag is set. +Parsers using iof input may protect not yet read data when there may be a need to put bytes +back to the stream. This is trivial when I->pos > I->buf, as we can make a move by --I->pos. +But when there is a need to put back more then one byte, we can protect the data tail, so that +realoder will rewrite it to the beginning of new data chunk. + + iof_tail(I) - internal, used by iof handlers at IOFREAD mode + iof_protect_tail(I) - used by parsers to ensure some bytes chunk in one piece + +*/ + +size_t iof_save_tail (iof *I); +#define iof_tail(I) (((I)->flags & IOF_TAIL) && (I)->pos < (I)->end ? iof_save_tail(I) : 0) + +size_t iof_input_save_tail (iof *I, size_t back); +#define iof_protect_tail(I, back, length) ((iof_left(I) >= (length) - (back)) ? 1 : (iof_input_save_tail(I, back) >= length - back)) + +//uint8_t * iof_tail_data (iof *I, size_t *ptail); +//#define iof_tail_free(data) util_free(data) + +/* panic */ + +// #define iof_panic(mess) return 0 +#ifndef iof_panic + #define iof_panic(mess) (fputs(mess, stderr), abort()) +#endif +//#define iof_memory_error() iof_panic(strerror(errno)) +#define iof_fwrite_error() iof_panic(strerror(errno)) + +/* generic helpers */ + +UTILAPI uint8_t * iof_copy_file_data (const char *filename, size_t *psize); +UTILAPI uint8_t * iof_copy_file_handle_data (FILE *file, size_t *psize); + +/* In the future we may need releasing file handle and restoring it from iofile->name, so access file handle via macros */ + +#define iof_file_get_fh(iofile) ((iofile)->iofh) +#define iof_file_set_fh(iofile, fh) ((iofile)->iofh = fh) +#define iof_file_get_file(iofile) (((iofile)->flags & IOF_DATA) ? NULL : iof_file_get_fh(iofile)) +FILE * iof_get_file (iof *F); + +/* basic iof_file interface */ + +iof_file * iof_file_new (FILE *file); +iof_file * iof_file_init (iof_file *iofile, FILE *file); + +iof_file * iof_file_rdata (const void *data, size_t size); +iof_file * iof_file_wdata (void *data, size_t size); + +iof_file * iof_file_rdata_init (iof_file *iofile, const void *data, size_t size); +iof_file * iof_file_wdata_init (iof_file *iofile, void *data, size_t size); + +iof_file * iof_file_reader_from_file_handle (iof_file *iofile, const char *filename, FILE *file, int preload, int closefile); +iof_file * iof_file_reader_from_file (iof_file *iofile, const char *filename, int preload); +iof_file * iof_file_reader_from_data (iof_file *iofile, const void *data, size_t size, int preload, int freedata); +//iof_file * iof_file_writer_from_file (iof_file *iofile, const char *filename); + +void * iof_copy_data (const void *data, size_t size); +#define iof_data_free(data) util_free(data) +#define iof_file_wdata_copy(data, size) iof_file_wdata(iof_copy_data(data, size), size) +#define iof_file_rdata_copy(data, size) iof_file_rdata(iof_copy_data(data, size), size) + +void iof_file_free (iof_file *iofile); + +#define iof_file_get_name(iofile) ((iofile)->name) +void iof_file_set_name (iof_file *iofile, const char *name); + +#define iof_file_incref(iofile) (++(iofile)->refcount) +#define iof_file_decref(iofile) ((void)(--(iofile)->refcount <= 0 && (iof_file_free(iofile), 0))) + +int iof_file_seek (iof_file *iofile, long offset, int whence); +long iof_file_tell (iof_file *iofile); +size_t iof_file_size (iof_file *iofile); +int iof_file_eof (iof_file *iofile); + +size_t iof_file_read (void *ptr, size_t size, size_t items, iof_file *iofile); +size_t iof_file_write (const void *ptr, size_t size, size_t items, iof_file *iofile); +size_t iof_file_ensure (iof_file *iofile, size_t bytes); +int iof_file_flush (iof_file *iofile); + +int iof_file_getc (iof_file *iofile); +int iof_file_putc (iof_file *iofile, int c); + +int iof_file_reclose_input (iof_file *iofile); +int iof_file_reopen_input (iof_file *iofile); + +#define iof_file_reopen(iofile) (((iofile)->flags & IOF_REOPEN_FILE) ? iof_file_reopen_input(iofile) : 1) +#define iof_file_reclose(iofile) (void)(((iofile)->flags & IOF_RECLOSE_FILE) ? iof_file_reclose_input(iofile) : 0) + +void iof_file_close_input (iof_file *iofile); + +/* wrappers of basic operations for iof */ + +int iof_reader_seek (iof *I, long offset, int whence); +int iof_reader_reseek (iof *I, long offset, int whence); +int iof_writer_seek (iof *I, long offset, int whence); +int iof_writer_reseek (iof *I, long offset, int whence); + +int iof_seek (iof *I, long offset, int whence); +int iof_reseek (iof *I, long offset, int whence); + +long iof_reader_tell (iof *I); +long iof_writer_tell (iof *I); +long iof_tell (iof *I); +size_t iof_fsize (iof *I); + +#define iof_setup_iofile(I, f) (iof_file_incref(f), (I)->iofile = f, (I)->flags |= IOF_FILE) +#define iof_setup_file(I, fh) ((I)->file = fh, (I)->flags |= IOF_FILE_HANDLE) +#define iof_setup_next(I, N) ((I)->next = N, iof_incref(N), (I)->flags |= IOF_NEXT) + +/* file handler reader and writer */ + +UTILAPI iof * iof_setup_file_handle_reader (iof *I, void *buffer, size_t space, FILE *f); +UTILAPI iof * iof_setup_file_handle_writer (iof *O, void *buffer, size_t space, FILE *f); + +/* file reader and writer */ + +UTILAPI iof * iof_setup_file_reader (iof *I, void *buffer, size_t space, const char *filename); +UTILAPI iof * iof_setup_file_writer (iof *O, void *buffer, size_t space, const char *filename); + +/* mem writer */ + +UTILAPI iof * iof_setup_buffer (iof *O, void *buffer, size_t space); +UTILAPI iof * iof_setup_buffermin (iof *O, void *buffer, size_t space, size_t min); + +UTILAPI iof * iof_buffer_create (size_t space); +#define iof_buffer_new() iof_buffer_create(BUFSIZ) + +/* custom handler */ + +UTILAPI iof * iof_reader (iof *I, void *link, iof_handler reader, const void *s, size_t bytes); +UTILAPI iof * iof_writer (iof *O, void *link, iof_handler writer, void *s, size_t bytes); + +/* stdout wrapper */ + +extern UTILAPI iof iof_stdout; +extern UTILAPI iof iof_stderr; + +/* simple string reader */ + +UTILAPI iof * iof_string_reader (iof *I, const void *s, size_t bytes); + +#define iof_string(I, s, bytes) \ + (((I)->rbuf = (I)->rpos = (const uint8_t *)s), ((I)->rend = (I)->rbuf + (bytes)), ((I)->flags |= IOF_DATA), (I)) + +/* dummies */ + +UTILAPI iof * iof_dummy (void *buffer, size_t space); +UTILAPI iof * iof_null (void *buffer, size_t space); + +/* checking available space */ + +#define iof_loadable(I) ((I)->pos < (I)->end || iof_load(I)) +#define iof_readable(I) ((I)->pos < (I)->end || iof_input(I)) +#define iof_writable(O) ((O)->pos < (O)->end || iof_output(O)) + +#define iof_hloadable iof_loadable +#define iof_iloadable iof_loadable + +#define iof_hreadable iof_readable +#define iof_ireadable iof_readable + +#define iof_hwritable iof_writable +#define iof_iwritable iof_writable + +/* ensure space to write several bytes (several means less then I->space) */ + +#define iof_ensure(O, n) ((O)->pos+(n)-1 < (O)->end || iof_output(O)) // iof_ensure(O, 1) eq iof_writable(O) +#define iof_hensure(O, n) ((O)->hpos+(n)-1 < (O)->hend || iof_output(O)) +#define iof_iensure(O, n) ((O)->ipos+(n)-1 < (O)->iend || iof_output(O)) + +/* reading */ + +UTILAPI int iof_getc (iof *I); +UTILAPI int iof_hgetc (iof *I); +UTILAPI int iof_igetc (iof *I); + +// UTILAPI int iof_cmp (iof *I, const char *s); +// UTILAPI int iof_cmpn (iof *I, const char *s, size_t bytes); + +UTILAPI iof_status iof_pass (iof *I, iof *O); +#define iof_hpass iof_pass +#define iof_ipass iof_pass + +/* readers helpers */ + +UTILAPI size_t iof_read (iof *I, void *s, size_t bytes); +UTILAPI size_t iof_hread (iof *I, void *s, size_t bytes); +UTILAPI size_t iof_iread (iof *I, void *s, size_t bytes); + +UTILAPI size_t iof_skip (iof *I, size_t bytes); +UTILAPI size_t iof_hskip (iof *I, size_t bytes); +UTILAPI size_t iof_iskip (iof *I, size_t bytes); + +/* get */ + +#define iof_pos(I) (*(I)->pos++) +#define iof_hpos(I) (*(I)->hpos++) +#define iof_ipos(I) (*(I)->ipos++) + +#define iof_get(I) (iof_readable(I) ? (int)(*(I)->pos++) : IOFEOF) +#define iof_hget(I) (iof_hreadable(I) ? (int)(*(I)->hpos++) : IOFEOF) +#define iof_iget(I) (iof_ireadable(I) ? (int)(*(I)->ipos++) : IOFEOF) + +#define iof_char(I) (iof_readable(I) ? (int)(*(I)->pos) : IOFEOF) +#define iof_hcurr(I) (iof_hreadable(I) ? (int)(*(I)->hpos) : IOFEOF) +#define iof_icurr(I) (iof_ireadable(I) ? (int)(*(I)->ipos) : IOFEOF) + +#define iof_next(I) (++(I)->pos, iof_char(I)) +#define iof_hnext(I) (++(I)->hpos, iof_hcurr(I)) +#define iof_inext(I) (++(I)->ipos, iof_icurr(I)) + +/* unget */ + +/* +If possible, we just move the position backward. If it is not possible to +move backward, we call iof_backup(I, c) that sets all pointers to the end of +a private backup space, then moves buf AND pos pointers backward and set c at +pos (==buf). We can backup characters as long as there is a private space. If +several calls to iof_backup() are followed by iof_get(), pos pointer +increases in normal way and so the use of another iof_unget() works just fine +by moving the position. Once we swallow all backup characters (when +pos==end), backup handler restores the previous pointers. + +Obviously we assume that the character provided to iof_unget() is always the +character just obtained from iof_get(). We CAN'T just overwrite the character +at a given position as the space we read may not be writable. + +When backup is in use, we can only get bytes until automatically restored. +*/ + +/* backup */ + +/* +#define iof_uses_backup(I) ((I)->more == iof_unget_handler) + +#define iof_save(I, B) \ + ((B)->buf = (I)->buf, (B)->pos = (I)->pos, (B)->end = (I)->end, (B)->space = (I)->space, \ + (B)->link = I->link, (B)->more = (I)->more, (B)->flags = (I)->flags) +#define iof_restore(B, I) iof_save(I, B) + +#define iof_unget(I, c) \ + ((void)(c == (uint8_t)c ? ((I)->pos > (I)->buf ? --(I)->pos : iof_backup(I, c)) : 0) +int iof_backup (iof *I, int c); +*/ + +/* writing */ + +UTILAPI size_t iof_write_file_handle (iof *O, FILE *file); +UTILAPI size_t iof_write_file (iof *O, const char *filename); +UTILAPI size_t iof_write_iofile (iof *O, iof_file *iofile, int savepos); + +UTILAPI int iof_putc (iof *O, int u); +UTILAPI int iof_hputc (iof *O, int u); +UTILAPI int iof_iputc (iof *O, int u); + +UTILAPI size_t iof_write (iof *O, const void *data, size_t size); +UTILAPI size_t iof_hwrite (iof *O, const void *data, size_t size); +UTILAPI size_t iof_iwrite (iof *O, const void *data, size_t size); + +UTILAPI iof_status iof_puts (iof *O, const void *data); +UTILAPI size_t iof_put_string (iof *O, const void *data); +UTILAPI size_t iof_putfs (iof *O, const char *format, ...); +UTILAPI size_t iof_repc (iof *O, char c, size_t bytes); + +#define iof_putl(O, s) iof_write(O, "" s, sizeof(s)-1) +//#define iof_putl iof_puts + +#define iof_set(O, c) (*(O)->pos++ = (uint8_t)(c)) +#define iof_set2(O, c1, c2) (iof_set(O, c1), iof_set(O, c2)) +#define iof_set3(O, c1, c2, c3) (iof_set(O, c1), iof_set(O, c2), iof_set(O, c3)) +#define iof_set4(O, c1, c2, c3, c4) (iof_set(O, c1), iof_set(O, c2), iof_set(O, c3), iof_set(O, c4)) +#define iof_set5(O, c1, c2, c3, c4, c5) (iof_set(O, c1), iof_set(O, c2), iof_set(O, c3), iof_set(O, c4), iof_set(O, c5)) + +#define iof_hset(O, c) (*(O)->hpos++ = (uint16_t)(c)) +#define iof_iset(O, c) (*(O)->ipos++ = (uint32_t)(c)) + +#define iof_put(O, c) ((void)iof_ensure(O, 1), iof_set(O, c)) +#define iof_put2(O, c1, c2) ((void)iof_ensure(O, 2), iof_set2(O, c1, c2)) +#define iof_put3(O, c1, c2, c3) ((void)iof_ensure(O, 3), iof_set3(O, c1, c2, c3)) +#define iof_put4(O, c1, c2, c3, c4) ((void)iof_ensure(O, 4), iof_set4(O, c1, c2, c3, c4)) +#define iof_put5(O, c1, c2, c3, c4, c5) ((void)iof_ensure(O, 5), iof_set5(O, c1, c2, c3, c4, c5)) + +#define iof_hput(O, c) ((void)iof_hensure(O, 1), iof_hset(O, c)) +#define iof_iput(O, c) ((void)iof_iensure(O, 1), iof_iset(O, c)) + +#define iof_put_uc_hex(O, c) iof_put2(O, base16_uc_digit1(c), base16_uc_digit2(c)) +#define iof_put_lc_hex(O, c) iof_put2(O, base16_lc_digit1(c), base16_lc_digit2(c)) +#define iof_set_uc_hex(O, c) iof_set2(O, base16_uc_digit1(c), base16_uc_digit2(c)) +#define iof_set_lc_hex(O, c) iof_set2(O, base16_lc_digit1(c), base16_lc_digit2(c)) +#define iof_put_hex iof_put_uc_hex +#define iof_set_hex iof_set_uc_hex + +/* number from iof; return 1 on success, 0 otherwise */ + +#define iof_scan_sign(I, c, sign) _scan_sign(c, sign, iof_next(I)) +#define iof_scan_integer(I, c, number) _scan_integer(c, number, iof_next(I)) +#define iof_scan_radix(I, c, number, radix) _scan_radix(c, number, radix, iof_next(I)) +#define iof_read_integer(I, c, number) _read_integer(c, number, iof_next(I)) +#define iof_read_radix(I, c, number, radix) _read_radix(c, number, radix, iof_next(I)) + +#define iof_scan_decimal(I, c, number) _scan_decimal(c, number, iof_next(I)) +#define iof_scan_fraction(I, c, number, exponent10) _scan_fraction(c, number, exponent10, iof_next(I)) +#define iof_scan_exponent10(I, c, exponent10) _scan_exponent10(c, exponent10, iof_next(I)) + +UTILAPI int iof_get_int32 (iof *I, int32_t *number); +UTILAPI int iof_get_slong (iof *I, long *number); +UTILAPI int iof_get_int64 (iof *I, int64_t *number); + +UTILAPI int iof_get_uint32 (iof *I, uint32_t *number); +UTILAPI int iof_get_ulong (iof *I, unsigned long *number); +UTILAPI int iof_get_usize (iof *I, size_t *number); +UTILAPI int iof_get_uint64 (iof *I, uint64_t *number); + +UTILAPI int iof_get_int32_radix (iof *I, int32_t *number, int radix); +UTILAPI int iof_get_slong_radix (iof *I, long *number, int radix); +UTILAPI int iof_get_int64_radix (iof *I, int64_t *number, int radix); + +UTILAPI int iof_get_uint32_radix (iof *I, uint32_t *number, int radix); +UTILAPI int iof_get_ulong_radix (iof *I, unsigned long *number, int radix); +UTILAPI int iof_get_usize_radix (iof *I, size_t *number, int radix); +UTILAPI int iof_get_uint64_radix (iof *I, uint64_t *number, int radix); + +#if defined(INTLW_IS_INT64) +# define iof_get_intlw(I, number) iof_get_int64(I, number) +# define iof_get_uintlw(I, number) iof_get_uint64(I, number) +# define iof_get_intlw_radix(I, number, radix) iof_get_int64_radix(I, number, radix) +# define iof_get_uintlw_radix(I, number, radix) iof_get_uint64_radix(I, number, radix) +#elif defined(INTLW_IS_LONG) +# define iof_get_intlw(I, number) iof_get_slong(I, number) +# define iof_get_uintlw(I, number) iof_get_ulong(I, number) +# define iof_get_intlw_radix(I, number, radix) iof_get_slong_radix(I, number, radix) +# define iof_get_uintlw_radix(I, number, radix) iof_get_ulong_radix(I, number, radix) +#endif + +UTILAPI int iof_get_roman (iof *I, uint16_t *number); + +UTILAPI int iof_get_double (iof *I, double *number); +UTILAPI int iof_get_float (iof *I, float *number); + +UTILAPI int iof_conv_double (iof *I, double *number); +UTILAPI int iof_conv_float (iof *I, float *number); + +/* number to iof; return a number of written bytes */ + +UTILAPI size_t iof_put_int32 (iof *O, int32_t number); +UTILAPI size_t iof_put_slong (iof *O, long number); +UTILAPI size_t iof_put_int64 (iof *O, int64_t number); + +UTILAPI size_t iof_put_uint32 (iof *O, uint32_t number); +UTILAPI size_t iof_put_ulong (iof *O, unsigned long number); +UTILAPI size_t iof_put_usize (iof *O, size_t number); +UTILAPI size_t iof_put_uint64 (iof *O, uint64_t number); + +UTILAPI size_t iof_put_int32_radix (iof *O, int32_t number, int radix, int uc); +UTILAPI size_t iof_put_slong_radix (iof *O, long number, int radix, int uc); +UTILAPI size_t iof_put_int64_radix (iof *O, int64_t number, int radix, int uc); + +UTILAPI size_t iof_put_uint32_radix (iof *O, uint32_t number, int radix, int uc); +UTILAPI size_t iof_put_ulong_radix (iof *O, unsigned long number, int radix, int uc); +UTILAPI size_t iof_put_usize_radix (iof *O, size_t number, int radix, int uc); +UTILAPI size_t iof_put_uint64_radix (iof *O, uint64_t number, int radix, int uc); + +#if defined(INTLW_IS_INT64) +# define iof_put_intlw(O, number) iof_put_int64(O, number) +# define iof_put_uintlw(O, number) iof_put_uint64(O, number) +# define iof_put_intlw_radix(O, number, radix, uc) iof_put_int64_radix(O, number, radix, uc) +# define iof_put_uintlw_radix(O, number, radix, uc) iof_put_uint64_radix(O, number, radix, uc) +#elif defined(INTLW_IS_LONG) +# define iof_put_intlw(O, number) iof_put_slong(O, number) +# define iof_put_uintlw(O, number) iof_put_ulong(O, number) +# define iof_put_intlw_radix(O, number, radix, uc) iof_put_slong_radix(O, number, radix, uc) +# define iof_put_uintlw_radix(O, number, radix, uc) iof_put_ulong_radix(O, number, radix, uc) +#endif + +UTILAPI size_t iof_put_roman (iof *O, uint16_t number, int uc); + +UTILAPI size_t iof_put_double(iof *O, double number, int digits); +UTILAPI size_t iof_put_float(iof *O, float number, int digits); + +/* common helpers for binary parsers */ + +UTILAPI int iof_get_be_uint2 (iof *I, uint32_t *pnumber); +UTILAPI int iof_get_be_uint3 (iof *I, uint32_t *pnumber); +UTILAPI int iof_get_be_uint4 (iof *I, uint32_t *pnumber); + +UTILAPI int iof_get_le_uint2 (iof *I, uint32_t *pnumber); +UTILAPI int iof_get_le_uint3 (iof *I, uint32_t *pnumber); +UTILAPI int iof_get_le_uint4 (iof *I, uint32_t *pnumber); + +// iof_set() and iof_put() suite casts arguments to uint8_t, so we don't need &0xff mask + +#define iof_set_be_uint1(O, u) iof_set(O, u) +#define iof_set_be_uint2(O, u) iof_set2(O, (u)>>8, u) +#define iof_set_be_uint3(O, u) iof_set3(O, (u)>>16, (u)>>8, u) +#define iof_set_be_uint4(O, u) iof_set4(O, (u)>>24, (u)>>16, (u)>>8, u) + +#define iof_set_le_uint1(O, u) iof_set(O, u) +#define iof_set_le_uint2(O, u) iof_set2(O, u, (u)>>8) +#define iof_set_le_uint3(O, u) iof_set3(O, u, (u)>>8, (u)>>16) +#define iof_set_le_uint4(O, u) iof_set4(O, u, (u)>>8, (u)>>16, (u)>>24) + +#define iof_put_be_uint1(O, u) iof_put(O, u) +#define iof_put_be_uint2(O, u) iof_put2(O, (u)>>8, u) +#define iof_put_be_uint3(O, u) iof_put3(O, (u)>>16, (u)>>8, u) +#define iof_put_be_uint4(O, u) iof_put4(O, (u)>>24, (u)>>16, (u)>>8, u) + +#define iof_put_le_uint1(O, u) iof_put(O, u) +#define iof_put_le_uint2(O, u) iof_put2(O, u, (u)>>8) +#define iof_put_le_uint3(O, u) iof_put3(O, u, (u)>>8, (u)>>16) +#define iof_put_le_uint4(O, u) iof_put4(O, u, (u)>>8, (u)>>16, (u)>>24) + +/* buffer results */ + +#define iof_reader_result(I, size) ((size = (size_t)iof_left(I)), (I)->pos) +#define iof_writer_result(I, size) ((size = (size_t)iof_size(I)), (I)->buf) +#define iof_result(I, size) (((I)->flags & IOF_READER) ? iof_reader_result(I, size) : iof_writer_result(I, size)) + +uint8_t * iof_file_input_data (iof_file *iofile, size_t *psize, int *isnew); +//uint8_t * iof_file_reader_data (iof_file *iofile, size_t *size); +//uint8_t * iof_file_writer_data (iof_file *iofile, size_t *size); + +uint8_t * iof_reader_data (iof *I, size_t *psize); +uint8_t * iof_writer_data (iof *O, size_t *psize); +size_t iof_reader_to_file_handle (iof *I, FILE *file); +size_t iof_reader_to_file (iof *I, const char *filename); + +#define iof_loaded(I) ((I)->end = (I)->pos, (I)->pos = (I)->buf, iof_left(I)) + +#define iof_data_to_file_handle(data, size, file) fwrite(data, sizeof(uint8_t), size, file) +UTILAPI size_t iof_data_to_file (const void *data, size_t size, const char *filename); + +UTILAPI size_t iof_result_to_file_handle (iof *F, FILE *file); +UTILAPI size_t iof_result_to_file (iof *F, const char *filename); +UTILAPI void iof_debug (iof *I, const char *filename); + +/* common filters allocator */ + +void iof_filters_init (void); +void iof_filters_free (void); + +iof * iof_filter_reader_new (iof_handler handler, size_t statesize, void **pstate); +#define iof_filter_reader(handler, statesize, pstate) iof_filter_reader_new(handler, statesize, (void **)(pstate)) +iof * iof_filter_reader_with_buffer_new (iof_handler handler, size_t statesize, void **pstate, void *buffer, size_t buffersize); +#define iof_filter_reader_with_buffer(handler, statesize, pstate, buffer, buffersize) iof_filter_reader_with_buffer_new(handler, statesize, (void **)(pstate), buffer, buffersize) +iof * iof_filter_writer_new (iof_handler handler, size_t statesize, void **pstate); +#define iof_filter_writer(handler, statesize, pstate) iof_filter_writer_new(handler, statesize, (void **)(pstate)) +iof * iof_filter_writer_with_buffer_new (iof_handler handler, size_t statesize, void **pstate, void *buffer, size_t buffersize); +#define iof_filter_writer_with_buffer(handler, statesize, pstate, buffer, buffersize) iof_filter_writer_with_buffer_new(handler, statesize, (void **)(pstate), buffer, buffersize) + +#define iof_filter_state(statetype, F) (statetype)((void *)((F) + 1)) + +void iof_free (iof *F); +void iof_discard (iof *F); + +size_t iof_resize_buffer_to (iof *O, size_t space); +#define iof_resize_buffer(O) iof_resize_buffer_to(O, (O)->space << 1) + +size_t iof_decoder_retval (iof *I, const char *type, iof_status status); +size_t iof_encoder_retval (iof *O, const char *type, iof_status status); + +/* filters */ + +iof * iof_filter_file_handle_reader (FILE *file); +iof * iof_filter_file_handle_writer (FILE *file); + +iof * iof_filter_iofile_reader (iof_file *iofile, size_t offset); +iof * iof_filter_iofile_writer (iof_file *iofile, size_t offset); + +iof * iof_filter_file_reader (const char *filename); +iof * iof_filter_file_writer (const char *filename); + +iof * iof_filter_string_reader (const void *s, size_t length); +iof * iof_filter_string_writer (const void *s, size_t length); + +iof * iof_filter_buffer_writer (size_t size); + +iof * iof_filter_stream_reader (FILE *file, size_t offset, size_t length); +iof * iof_filter_stream_coreader (iof_file *iofile, size_t offset, size_t length); + +iof * iof_filter_stream_writer (FILE *file); +iof * iof_filter_stream_cowriter (iof_file *iofile, size_t offset); + +FILE * iof_filter_file_reader_source (iof *I, size_t *poffset, size_t *plength); +iof_file * iof_filter_file_coreader_source (iof *I, size_t *poffset, size_t *plength); +iof * iof_filter_reader_replacement (iof *P, iof_handler handler, size_t statesize, void **pstate); +#define iof_filter_reader_replace(P, handler, statesize, pstate) iof_filter_reader_replacement(P, handler, statesize, (void **)(pstate)) + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utillog.c b/Build/source/libs/pplib/pplib-src/src/util/utillog.c new file mode 100644 index 00000000000..6d32514a7a7 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utillog.c @@ -0,0 +1,60 @@ + +#include <stdio.h> +#include <string.h> // strlen +#include <stdarg.h> +#include "utillog.h" + +#define LOGGER_BUFFER_SIZE 256 +#define LOGGER_PREFIX_SIZE 32 + +typedef struct { + logger_function callback; + void *context; + size_t pfxlen; +} logger_struct; + +static logger_struct logger = { 0, NULL, 0 }; + +static char logger_buffer[LOGGER_BUFFER_SIZE+LOGGER_PREFIX_SIZE]; + +void loggerf (const char *format, ...) +{ + va_list args; + int length; + + va_start(args, format); + length = vsnprintf(logger_buffer + logger.pfxlen, LOGGER_BUFFER_SIZE, format, args); + if (length > 0) + { + if (length > LOGGER_BUFFER_SIZE) + length = LOGGER_BUFFER_SIZE; + } + else + { + loggerf("logger encoding error '%s'", format); + length = (int)strlen(logger_buffer); + } + length += (int)logger.pfxlen; + if (logger.callback) + logger.callback(logger_buffer, logger.context); + else + printf("\n%s\n", logger_buffer); + va_end(args); +} + +void logger_callback (logger_function callback, void *context) +{ + logger.callback = callback; + logger.context = context; +} + +int logger_prefix (const char *prefix) +{ + size_t pfxlen; + pfxlen = strlen(prefix); + if (pfxlen > LOGGER_PREFIX_SIZE) + return 0; + memcpy(logger_buffer, prefix, pfxlen); + logger.pfxlen = pfxlen; + return 1; +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utillog.h b/Build/source/libs/pplib/pplib-src/src/util/utillog.h new file mode 100644 index 00000000000..c30e0ff0f57 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utillog.h @@ -0,0 +1,10 @@ + +#ifndef UTIL_LOG_H +#define UTIL_LOG_H + +typedef void (*logger_function) (const char *message, void *alien); +void loggerf (const char *format, ...); +void logger_callback (logger_function callback, void *context); +int logger_prefix (const char *prefix); + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utillzw.c b/Build/source/libs/pplib/pplib-src/src/util/utillzw.c new file mode 100644 index 00000000000..e5134e79473 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utillzw.c @@ -0,0 +1,705 @@ +/* lzw implementation for postscript/pdf filters +# Notes on LZW + +# Encoder + +Initially the table contains 256 entires for single bytes. Encoder consumes +input bytes trying to find the longest sequence stored so far in the table. +Once it finds a sequence that is not present in the table, it outputs the table +index of the longest sequence found (accumulated bytes except the last +consumed) and pushes the new sequence (accumulated bytes including the last +one) on the top of the table. The last taken byte is not yet written to the +output, it becomes the beginning of the new sequence to accumulate. Initially, +encoder outputs 9-bit codes. While the table grows, the number of bits for each +code increases up to 12. In example, after adding a table entry of index 511 it +is high time to switch to 10-bit bytes. /EarlyChange=true parameter in stream +dictionary (both postscript and pdf) informs to increase the number of bits one +code earlier then necessary. Looks pretty much like an early days bug that +became a specification :) I have never found a PDF having /EarlyChange key +specified anyway. + +Once the table becomes full (or when encoder decides it is worthy), +a clear-table marker (code 256) purges the table and restores codes length to +9. End-of-data marker (code 257) ends the stream. Conventionally, the beginning +of the stream starts with clear-table marker. + +Postscript allows to provide a /UnitLength which determines the bit length of +codes. The above description assumes UnitLength=8 (default). Allowed values are +from 3 to 8. Different UnitLength also affects markers; clear-table is then +2^UnitLength and end-of-data marker is 2^UnitLenth+1. + +Encoder outputs 9-12bit codes that are packed into bytes using high-bits-first +scheme (default) or low-bits-scheme. + +PDF spec p. 73 (PS spec p. 135 gives an mistaken output sequence and so +mistaken output bytes) + +Input character sequence (decimal) +45 45 45 45 45 65 45 45 45 66 + +Output 9bit codes (decimal) +256 45 258 258 65 259 66 257 + +Output 9bit codes (binary) +100000000 000101101 100000010 100000010 001000001 100000011 001000010 100000001 + +Output bytes (LowBitsFirst=false); eight high-order bits of code becomes +the first byte, remaining low-order bit of code becomes the high-order bit of the +next byte; +10000000 00001011 01100000 01010000 00100010 00001100 00001100 10000101 00000001 +-> 80 0B 60 50 22 0C 0C 85 01 + +Output bytes (binary, LowBitsFirst=true); eight low-order bits of code becomes +the first byte, remaining high-order bit of code becomes low-order bit of the +next byte; +00000000 01011011 00001000 00010100 00011000 01100100 10100000 10000000 10010000 +-> 00 5B 08 14 18 64 A0 80 90 + +# Decoder + +Decoder consumes input bytes transforming them to 9 to 12 bit codes. Initially +it starts with 9bit codes and the table of 258 fixed codes (same as encoder). +Basically, it interprets incoming codes as table indices (except 256 and 257 +markers) and it outputs byte sequences stored at given indices. It also +upbuilds the table and changes the number of bits of codes when necessary. The +key point on lzw is that both encoder and decoder builds the table +synchronously. + +However, decoder needs some "knowledge" about how encoder works to be able to +interpret a table index that it doesn't have so far. Look that the output from +encoder in the example above. The first output code is conventional clear-table +(256). Then comes a code 45. So far so good, decoder interprets code 45 as +a (fixed) entry of the table, emitting byte 45. The next code is 258, which is +should be interpreted as an index in the table. Oops, encoder doesn't have one +yet. If that occurs, it means that encoder was able to output the new entry +code just after adding it to a table. It means that + + sequence_before + next_byte == next_byte + sequence_after + +This may happen not only for sequences like 45 45 45, but also symmetric series +such as abcbabcba; abcb + a == a + bcba. Decoder must be aware of that and if +it gets a code one larger than the top table index, it should create one on-fly +by appending last entry sequence by the first by of the last entry. + +# UnitLength + +Postscript specification mentions about UnitLength parameter that can be used +in LZW decoder (not allowed in encoder), with possible values from 3 to 8. This +parameter determines the number of bits per code; form UnitLength + 1 to 12. It +also determines which codes are used for clear-table marker (2^UnitLength) and +end-of-data marker ((2^UnitLength)+1). Postscript specification says (page 134): + +"Initially, the code length is (UnitLength + 1) bits and the table contains only +entries for the (2^UnitLength + 2) fixed codes. As encoding proceeds, entries are +appended to the table, associating new codes with longer and longer input character +sequences. The encoding and decoding filters maintain identical copies of +this table." + +Later on page 136 Postscript specification says: + +"Data that has been LZW-encoded with a UnitLength less than 8 consists only of +codes in the range 0 to 2^UnitLength - 1; consequently, the LZWDecode filter produces +only codes in that range when read. UnitLength also affects the encoded +representation, as described above." + +UnitLength (Postscript only) and LowBitsFirst are used only by decoder. +EarlyChange should obviously be respected by both encoder and decoder. When +table index reaches current bit length boundary (511, 1023, ...) it must react +by increasing the number of bits of input code. But if the index reaches it +maximum value (when the table is full), decoder is NOT supposed to clear the +table. When the table is full, encoder must emit clear-table marker and it +emits this code using 12 bits and reinitialize code bits after that. It means +that, when the table is full, decoder should get one more 12-bit code (which +should be clear-table marker) and actually clear the table and reinitialize +code bits after that. + +# Clear-table vs last entry track (after tries and checks) + +It is also not quite clear what should actually happen when encoder gets a full +table and it is supposed to emit clear-table marker. When it gets full, it +means that it has just appended another entry to the table. And that happens +only the input sequence collected so far plus the last byte is not present in +the table. Encoder is supposed to output the table index of the present +sequence and set the recent byte as a starting index of the new sequence to be +collected. Even if it is time to clear the table, encoder is still supposed to +keep the track of the last table entry. Decoder, however, must drop the track of the +last code on clear-table. + +# Decoder table vs encoder table + +While decoding we need query lzw table by (subsequent) numeric codes and output +character sequences stored in the table. While encoding we need to query the +table on every input byte and fetch indices pointing to character sequences. +Note that we never need to query the entire table for the longest sequence +found so far. The encoder table do not need to access the longest character +sequence at one piece. It is enough to keep the track of the current table +index and the very next byte. We organize an encoder table into a search tree, +where every node contains its table index (value) and last byte (key). Except +initial tree content, every node is created on the base of the previous node +and it conceptually point the sequence represented by that nodo consists of the +previous node sequence plus the next byte. + +Every new node is a descendant of the node it has been derived from. Every node +has a map (a search subtree) indexed by suffix byte value, pointing to +descendants nodes. Every node also has binary tentackles (left/right fields) +necessary to search the map (except initials, every node lives in a map of some +ancestor node). The key point is that on every input byte we don't search the +entire tree, but only the map of the current node children. The map tree is +a simple binary tree with no balancing mechanism (not worthy to optimize an +ephemeric structure that may be upbuilt more often then queried). + +In our implementation, decoder table requires 4069 entries (topmost index 4095). +Encoder table, however, needs 4097 entries to handle the case when EarlyIndex +parameter is 0 (I have never a chance to test that in practise). The node of index +4096 might be added to a search tree, but its code is never emitted; the lookup +is purged just after adding that node. + +todo: +- support for LowBitsFirst encoding +*/ + +#include "utilmem.h" +#include "utillzw.h" + +/* filter state struct */ + +typedef struct lzw_entry { + union { + const char *rdata; // to be able to init with string literal + char *data; + }; + int size; +} lzw_entry; + +#define lzw_index short + +typedef struct lzw_node lzw_node; + +struct lzw_node { + lzw_index index; + unsigned char suffix; + lzw_node *left; + lzw_node *right; + lzw_node *map; +}; + +struct lzw_state { + union { + lzw_node *lookup; /* encoder table */ + lzw_entry *table; /* decoder table */ + }; + lzw_index index; /* table index */ + union { + lzw_node *lastnode; /* previous encoder table node */ + struct { + lzw_entry *lastentry; /* previous decoder table entry */ + int tailbytes; /* num of bytes of lastentry not yet written out */ + }; + }; + int basebits; /* /UnitLength parameter (8) */ + int codebits; /* current code bits */ + int lastbyte; /* previosly read byte */ + int tailbits; /* lastbyte bits not yet consumed */ + int flush; /* encoder */ + int flags; /* options */ +}; + +typedef union { lzw_state *lzwstate; void *voidstate; } lzw_state_pointer; // to avoid 'dereferencing type-puned ...' warnings + +#define LZW_INIT_STATE { { 0 }, 0, { 0 }, 0, 0, 0, 0, 0, 0 } + +/* macros */ + +#define LZW_MIN_BITS 3 +#define LZW_MAX_BITS 12 +#define LZW_TABLE_SIZE (1 << LZW_MAX_BITS) +#define LZW_LOOKUP_SIZE (LZW_TABLE_SIZE + 1) + +#define lzw_bit_range(bits) (bits >= LZW_MIN_BITS && bits <= LZW_BASE_BITS) +#define lzw_base_bits(flags) (flags & ((1 << 4) - 1)) // 4 low bits of flags is basebits (UnitLength) + +#define lzw_initial_codes(state) (1 << state->basebits) +#define lzw_clear_code(state) lzw_initial_codes(state) +#define lzw_eod_code(state) (lzw_initial_codes(state) + 1) +#define lzw_initial_index(state) (lzw_initial_codes(state) + 2) + +#define lzw_max_index(state) ((1 << state->codebits) - ((state->flags & LZW_EARLY_INDEX) ? 1 : 0)) +#define lzw_check_bits(state) ((void)(state->index == lzw_max_index(state) && state->codebits < LZW_MAX_BITS && ++state->codebits)) + +#define lzw_malloc util_malloc +#define lzw_free util_free + +/* decoder */ + +static struct lzw_entry lzw_initial_table[] = { + {{"\x00"}, 1}, {{"\x01"}, 1}, {{"\x02"}, 1}, {{"\x03"}, 1}, {{"\x04"}, 1}, {{"\x05"}, 1}, {{"\x06"}, 1}, {{"\x07"}, 1}, {{"\x08"}, 1}, {{"\x09"}, 1}, {{"\x0A"}, 1}, {{"\x0B"}, 1}, {{"\x0C"}, 1}, {{"\x0D"}, 1}, {{"\x0E"}, 1}, {{"\x0F"}, 1}, + {{"\x10"}, 1}, {{"\x11"}, 1}, {{"\x12"}, 1}, {{"\x13"}, 1}, {{"\x14"}, 1}, {{"\x15"}, 1}, {{"\x16"}, 1}, {{"\x17"}, 1}, {{"\x18"}, 1}, {{"\x19"}, 1}, {{"\x1A"}, 1}, {{"\x1B"}, 1}, {{"\x1C"}, 1}, {{"\x1D"}, 1}, {{"\x1E"}, 1}, {{"\x1F"}, 1}, + {{"\x20"}, 1}, {{"\x21"}, 1}, {{"\x22"}, 1}, {{"\x23"}, 1}, {{"\x24"}, 1}, {{"\x25"}, 1}, {{"\x26"}, 1}, {{"\x27"}, 1}, {{"\x28"}, 1}, {{"\x29"}, 1}, {{"\x2A"}, 1}, {{"\x2B"}, 1}, {{"\x2C"}, 1}, {{"\x2D"}, 1}, {{"\x2E"}, 1}, {{"\x2F"}, 1}, + {{"\x30"}, 1}, {{"\x31"}, 1}, {{"\x32"}, 1}, {{"\x33"}, 1}, {{"\x34"}, 1}, {{"\x35"}, 1}, {{"\x36"}, 1}, {{"\x37"}, 1}, {{"\x38"}, 1}, {{"\x39"}, 1}, {{"\x3A"}, 1}, {{"\x3B"}, 1}, {{"\x3C"}, 1}, {{"\x3D"}, 1}, {{"\x3E"}, 1}, {{"\x3F"}, 1}, + {{"\x40"}, 1}, {{"\x41"}, 1}, {{"\x42"}, 1}, {{"\x43"}, 1}, {{"\x44"}, 1}, {{"\x45"}, 1}, {{"\x46"}, 1}, {{"\x47"}, 1}, {{"\x48"}, 1}, {{"\x49"}, 1}, {{"\x4A"}, 1}, {{"\x4B"}, 1}, {{"\x4C"}, 1}, {{"\x4D"}, 1}, {{"\x4E"}, 1}, {{"\x4F"}, 1}, + {{"\x50"}, 1}, {{"\x51"}, 1}, {{"\x52"}, 1}, {{"\x53"}, 1}, {{"\x54"}, 1}, {{"\x55"}, 1}, {{"\x56"}, 1}, {{"\x57"}, 1}, {{"\x58"}, 1}, {{"\x59"}, 1}, {{"\x5A"}, 1}, {{"\x5B"}, 1}, {{"\x5C"}, 1}, {{"\x5D"}, 1}, {{"\x5E"}, 1}, {{"\x5F"}, 1}, + {{"\x60"}, 1}, {{"\x61"}, 1}, {{"\x62"}, 1}, {{"\x63"}, 1}, {{"\x64"}, 1}, {{"\x65"}, 1}, {{"\x66"}, 1}, {{"\x67"}, 1}, {{"\x68"}, 1}, {{"\x69"}, 1}, {{"\x6A"}, 1}, {{"\x6B"}, 1}, {{"\x6C"}, 1}, {{"\x6D"}, 1}, {{"\x6E"}, 1}, {{"\x6F"}, 1}, + {{"\x70"}, 1}, {{"\x71"}, 1}, {{"\x72"}, 1}, {{"\x73"}, 1}, {{"\x74"}, 1}, {{"\x75"}, 1}, {{"\x76"}, 1}, {{"\x77"}, 1}, {{"\x78"}, 1}, {{"\x79"}, 1}, {{"\x7A"}, 1}, {{"\x7B"}, 1}, {{"\x7C"}, 1}, {{"\x7D"}, 1}, {{"\x7E"}, 1}, {{"\x7F"}, 1}, + {{"\x80"}, 1}, {{"\x81"}, 1}, {{"\x82"}, 1}, {{"\x83"}, 1}, {{"\x84"}, 1}, {{"\x85"}, 1}, {{"\x86"}, 1}, {{"\x87"}, 1}, {{"\x88"}, 1}, {{"\x89"}, 1}, {{"\x8A"}, 1}, {{"\x8B"}, 1}, {{"\x8C"}, 1}, {{"\x8D"}, 1}, {{"\x8E"}, 1}, {{"\x8F"}, 1}, + {{"\x90"}, 1}, {{"\x91"}, 1}, {{"\x92"}, 1}, {{"\x93"}, 1}, {{"\x94"}, 1}, {{"\x95"}, 1}, {{"\x96"}, 1}, {{"\x97"}, 1}, {{"\x98"}, 1}, {{"\x99"}, 1}, {{"\x9A"}, 1}, {{"\x9B"}, 1}, {{"\x9C"}, 1}, {{"\x9D"}, 1}, {{"\x9E"}, 1}, {{"\x9F"}, 1}, + {{"\xA0"}, 1}, {{"\xA1"}, 1}, {{"\xA2"}, 1}, {{"\xA3"}, 1}, {{"\xA4"}, 1}, {{"\xA5"}, 1}, {{"\xA6"}, 1}, {{"\xA7"}, 1}, {{"\xA8"}, 1}, {{"\xA9"}, 1}, {{"\xAA"}, 1}, {{"\xAB"}, 1}, {{"\xAC"}, 1}, {{"\xAD"}, 1}, {{"\xAE"}, 1}, {{"\xAF"}, 1}, + {{"\xB0"}, 1}, {{"\xB1"}, 1}, {{"\xB2"}, 1}, {{"\xB3"}, 1}, {{"\xB4"}, 1}, {{"\xB5"}, 1}, {{"\xB6"}, 1}, {{"\xB7"}, 1}, {{"\xB8"}, 1}, {{"\xB9"}, 1}, {{"\xBA"}, 1}, {{"\xBB"}, 1}, {{"\xBC"}, 1}, {{"\xBD"}, 1}, {{"\xBE"}, 1}, {{"\xBF"}, 1}, + {{"\xC0"}, 1}, {{"\xC1"}, 1}, {{"\xC2"}, 1}, {{"\xC3"}, 1}, {{"\xC4"}, 1}, {{"\xC5"}, 1}, {{"\xC6"}, 1}, {{"\xC7"}, 1}, {{"\xC8"}, 1}, {{"\xC9"}, 1}, {{"\xCA"}, 1}, {{"\xCB"}, 1}, {{"\xCC"}, 1}, {{"\xCD"}, 1}, {{"\xCE"}, 1}, {{"\xCF"}, 1}, + {{"\xD0"}, 1}, {{"\xD1"}, 1}, {{"\xD2"}, 1}, {{"\xD3"}, 1}, {{"\xD4"}, 1}, {{"\xD5"}, 1}, {{"\xD6"}, 1}, {{"\xD7"}, 1}, {{"\xD8"}, 1}, {{"\xD9"}, 1}, {{"\xDA"}, 1}, {{"\xDB"}, 1}, {{"\xDC"}, 1}, {{"\xDD"}, 1}, {{"\xDE"}, 1}, {{"\xDF"}, 1}, + {{"\xE0"}, 1}, {{"\xE1"}, 1}, {{"\xE2"}, 1}, {{"\xE3"}, 1}, {{"\xE4"}, 1}, {{"\xE5"}, 1}, {{"\xE6"}, 1}, {{"\xE7"}, 1}, {{"\xE8"}, 1}, {{"\xE9"}, 1}, {{"\xEA"}, 1}, {{"\xEB"}, 1}, {{"\xEC"}, 1}, {{"\xED"}, 1}, {{"\xEE"}, 1}, {{"\xEF"}, 1}, + {{"\xF0"}, 1}, {{"\xF1"}, 1}, {{"\xF2"}, 1}, {{"\xF3"}, 1}, {{"\xF4"}, 1}, {{"\xF5"}, 1}, {{"\xF6"}, 1}, {{"\xF7"}, 1}, {{"\xF8"}, 1}, {{"\xF9"}, 1}, {{"\xFA"}, 1}, {{"\xFB"}, 1}, {{"\xFC"}, 1}, {{"\xFD"}, 1}, {{"\xFE"}, 1}, {{"\xFF"}, 1} +}; + +#define lzw_entry_at(state, index) (&state->table[index]) + +static lzw_state * lzw_decoder_init_table (lzw_state *state, lzw_entry *table, int flags) +{ + state->basebits = lzw_base_bits(flags); // first four bits or flags + if (!lzw_bit_range(state->basebits)) + return NULL; + state->flags = flags; + if ((state->table = table) == NULL) + { + state->table = (lzw_entry *)lzw_malloc(LZW_TABLE_SIZE * sizeof(lzw_entry)); + state->flags |= LZW_TABLE_ALLOC; + } + memcpy(state->table, lzw_initial_table, (size_t)lzw_initial_codes(state)*sizeof(lzw_entry)); + // memset(&state->table[lzw_initial_codes(state)], 0, 2*sizeof(lzw_entry)); // eod and clear entries never accessed + state->codebits = state->basebits + 1; + state->index = lzw_initial_index(state); + state->lastentry = NULL; + state->tailbytes = 0; + state->lastbyte = 0; + state->tailbits = 0; + return state; +} + +lzw_state * lzw_decoder_init (lzw_state *state, int flags) +{ + return lzw_decoder_init_table(state, NULL, flags); +} + +static void lzw_decoder_clear (lzw_state *state) +{ + lzw_entry *entry; + lzw_index initindex = lzw_initial_index(state); + while (state->index > initindex) + { + entry = lzw_entry_at(state, --state->index); + lzw_free(entry->data); + // entry->data = NULL; + // entry->size = 0; + } + state->lastentry = NULL; + state->tailbytes = 0; + state->codebits = state->basebits + 1; +} + +void lzw_decoder_close (lzw_state *state) +{ + lzw_decoder_clear(state); + if (state->flags & LZW_TABLE_ALLOC) + lzw_free(state->table); +} + +static int lzw_next_entry (lzw_state *state, lzw_entry *nextentry) +{ + lzw_entry *lastentry, *newentry; + if ((lastentry = state->lastentry) == NULL) + return 1; /* its ok */ + if (state->index == LZW_TABLE_SIZE) + return 0; /* invalid input; eod marker expected earlier */ + /* put the new entry on the top of the table */ + newentry = lzw_entry_at(state, state->index++); + /* its size is the last entrtyy size plus 1 */ + newentry->size = lastentry->size + 1; + /* its content is the content of the last entry, */ + newentry->data = (char *)lzw_malloc((size_t)newentry->size); + memcpy(newentry->data, lastentry->data, lastentry->size); + /* plus the first byte of the new entry (usually fixed code entry) */ + newentry->data[newentry->size - 1] = nextentry->data[0]; + return 1; +} + +#define lzw_write_bytes(O, state) ((state->tailbytes -= (int)iof_write(O, state->lastentry->data, (size_t)state->tailbytes)) == 0) + +iof_status lzw_decode_state (iof *I, iof *O, lzw_state *state) +{ + const lzw_index clear = lzw_clear_code(state), eod = lzw_eod_code(state); + lzw_index code; + lzw_entry *entry; + if (state->lastentry != NULL) + { /* write out the tail from the last call */ + if (state->tailbytes > 0 && !lzw_write_bytes(O, state)) + return IOFFULL; + /* do what we normally do at the end of the loop body below */ + lzw_check_bits(state); + } + // if (state->flags & LZW_LOW_BITS_FIRST) + // return IOFERR; + while (1) + { + /* get input code of length state->codebits */ + code = (state->lastbyte & ((1 << state->tailbits) - 1)) << (state->codebits - state->tailbits); + for (state->tailbits -= state->codebits; state->tailbits < 0; ) + { + get_code: + if ((state->lastbyte = iof_get(I)) < 0) + return state->flush ? IOFEOF : state->lastbyte; + state->tailbits += 8; + if (state->tailbits < 0) + { + code |= (state->lastbyte << (-state->tailbits)); + goto get_code; + } + else + { + code |= (state->lastbyte >> state->tailbits); + break; + } + } + /* interpret the code */ + if (code < state->index) + { /* single byte code or special marker */ + if (code == clear) + { + lzw_decoder_clear(state); + continue; + } + if (code == eod) + return IOFEOF; + entry = lzw_entry_at(state, code); + if (!lzw_next_entry(state, entry)) + return IOFERR; + } + else if (code == state->index) + { /* apparently encoder has emitted the code of the key just created (see notes) */ + if (!lzw_next_entry(state, state->lastentry)) + return IOFERR; + entry = lzw_entry_at(state, state->index - 1); + } + else + { /* invalid input code */ + return IOFERR; + } + /* record the entry found */ + state->lastentry = entry; + /* emit the sequence pointed by that entry */ + state->tailbytes = entry->size; + if (!lzw_write_bytes(O, state)) + return IOFFULL; + /* check and update code bits */ + lzw_check_bits(state); + } + return state->lastbyte; // never reached +} + +/* encoder */ + +#define lzw_node_at(state, index) (&state->lookup[index]) + +#define lzw_node_init(node, i, c) (node->index = i, node->suffix = c, node->left = NULL, node->right = NULL, node->map = NULL) + +static lzw_state * lzw_encoder_init_table (lzw_state *state, lzw_node *lookup, int flags) +{ + lzw_index index; + lzw_node *node; + state->basebits = lzw_base_bits(flags); // first four bits of flags is base bits of code (default 8) + if (!lzw_bit_range(state->basebits)) + return NULL; + state->flags = flags; + if ((state->lookup = lookup) == NULL) + { + state->lookup = lzw_malloc(LZW_LOOKUP_SIZE*sizeof(lzw_node)); + state->flags |= LZW_TABLE_ALLOC; + } + state->index = lzw_initial_index(state); + for (index = 0; index < lzw_initial_codes(state); ++index) + { + node = lzw_node_at(state, index); + lzw_node_init(node, index, (unsigned char)index); + } + state->codebits = state->basebits + 1; + state->lastnode = NULL; + state->lastbyte = 0; + state->tailbits = 0; + return state; +} + +lzw_state * lzw_encoder_init (lzw_state *state, int flags) +{ + return lzw_encoder_init_table(state, NULL, flags); +} + +void lzw_encoder_close (lzw_state *state) +{ + if (state->flags & LZW_TABLE_ALLOC) + lzw_free(state->lookup); +} + +static void lzw_encoder_clear (lzw_state *state) +{ + lzw_node *node; + lzw_index index; + /* clear fixed nodes */ + for (index = 0; index < lzw_initial_codes(state); ++index) + { + node = lzw_node_at(state, index); + lzw_node_init(node, index, (unsigned char)index); + } + /* reset table index */ + state->index = lzw_initial_index(state); + /* reset code bits */ + state->codebits = state->basebits + 1; +} + +static void lzw_put_code (iof *O, lzw_state *state, lzw_index code, int todobits) +{ + int leftbits, rightbits; + do + { + leftbits = 8 - state->tailbits; + rightbits = todobits - leftbits; + if (rightbits >= 0) + { + state->lastbyte |= (code >> rightbits); + iof_put(O, state->lastbyte); + code = code & ((1 << rightbits) - 1); + todobits -= leftbits; + state->lastbyte = 0; + state->tailbits = 0; + } + else + { + state->lastbyte |= (code << (-rightbits)); + state->tailbits += todobits; + return; + } + } while (1); +} + +static iof_status lzw_encode_last (iof *O, lzw_state *state) +{ + if (state->flush) + { + /* put the last code if any */ + if (state->lastnode != NULL) + lzw_put_code(O, state, state->lastnode->index, state->codebits); + /* put eod marker, */ + lzw_put_code(O, state, lzw_eod_code(state), state->codebits); + /* with tail bits set to 0 */ + if (state->tailbits > 0) + lzw_put_code(O, state, 0, 8 - state->tailbits); + return IOFEOF; + } + return IOFEMPTY; +} + +static lzw_node * lzw_node_push (lzw_state *state, unsigned char suffix) +{ + lzw_node *node; + node = lzw_node_at(state, state->index); + lzw_node_init(node, state->index, suffix); + ++state->index; + return node; +} + +static int lzw_next_node (lzw_state *state, unsigned char suffix) +{ + lzw_node *node; + if ((node = state->lastnode->map) == NULL) + { + state->lastnode->map = lzw_node_push(state, suffix); + return 0; + } + while (1) + { + if (suffix < node->suffix) + { + if (node->left == NULL) + { + node->left = lzw_node_push(state, suffix); + return 0; + } + node = node->left; + } + else if (suffix > node->suffix) + { + if (node->right == NULL) + { + node->right = lzw_node_push(state, suffix); + return 0; + } + node = node->right; + } + else + { + state->lastnode = node; + return 1; + } + } + return 0; // never reached +} + +iof_status lzw_encode_state (iof *I, iof *O, lzw_state *state) +{ + int byte; + if (state->lastnode == NULL) + { /* first call only; following convention, put clear-table marker */ + if (!iof_ensure(O, 2)) + return IOFFULL; + lzw_put_code(O, state, lzw_clear_code(state), state->codebits); + /* get the first input byte and initialize the current table entry */ + if ((byte = iof_get(I)) < 0) + return lzw_encode_last(O, state); + state->lastnode = lzw_node_at(state, byte); + } + while (iof_ensure(O, 2)) + { /* we need to write at most 2 bytes on each iteration */ + if ((byte = iof_get(I)) < 0) + return lzw_encode_last(O, state); + if (lzw_next_node(state, (unsigned char)byte) == 0) + { /* means that the key hasn't been found and the new entry has just been created */ + /* output the code pointing the longest sequence so far */ + lzw_put_code(O, state, state->lastnode->index, state->codebits); + /* update code bits */ + if (state->index == lzw_max_index(state) + 1) + { + if (state->codebits < LZW_MAX_BITS) + ++state->codebits; + else + { + /* put clear-table marker */ + lzw_put_code(O, state, lzw_clear_code(state), state->codebits); + /* reset the table */ + lzw_encoder_clear(state); + } + } + /* in any case, recent byte becomes the current table code */ + state->lastnode = lzw_node_at(state, byte); + } + /* otherwise no new entry is appended and state->lastnode points the longer sequence just found */ + } + return IOFFULL; +} + +/* single call codecs */ + +iof_status lzw_decode (iof *I, iof *O, int flags) +{ + lzw_state state = LZW_INIT_STATE; + lzw_entry table[LZW_TABLE_SIZE]; + int ret; + lzw_decoder_init_table(&state, table, flags); + state.flush = 1; + ret = lzw_decode_state(I, O, &state); + // iof_flush(O); // ? + lzw_decoder_close(&state); + return ret; +} + +iof_status lzw_encode (iof *I, iof *O, int flags) +{ + lzw_state state = LZW_INIT_STATE; + lzw_node lookup[LZW_LOOKUP_SIZE]; + int ret; + lzw_encoder_init_table(&state, lookup, flags); + state.flush = 1; + ret = lzw_encode_state(I, O, &state); + // iof_flush(O); // ? + lzw_encoder_close(&state); + return ret; +} + +/* filters */ + +// lzw decoder function + +static size_t lzw_decoder (iof *F, iof_mode mode) +{ + lzw_state *state; + iof_status status; + size_t tail; + + state = iof_filter_state(lzw_state *, F); + switch(mode) + { + case IOFLOAD: + case IOFREAD: + if (F->flags & IOF_STOPPED) + return 0; + tail = iof_tail(F); + F->pos = F->buf + tail; + F->end = F->buf + F->space; + do { + status = lzw_decode_state(F->next, F, state); + } while (mode == IOFLOAD && status == IOFFULL && iof_resize_buffer(F)); + return iof_decoder_retval(F, "lzw", status); + case IOFCLOSE: + lzw_decoder_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +// lzw encoder function + +static size_t lzw_encoder (iof *F, iof_mode mode) +{ + lzw_state *state; + iof_status status; + + state = iof_filter_state(lzw_state *, F); + switch (mode) + { + case IOFFLUSH: + state->flush = 1; + FALLTHRU // fall through + case IOFWRITE: + F->end = F->pos; + F->pos = F->buf; + status = lzw_encode_state(F, F->next, state); + return iof_encoder_retval(F, "lzw", status); + case IOFCLOSE: + if (!state->flush) + lzw_encoder(F, IOFFLUSH); + lzw_encoder_close(state); + iof_free(F); + return 0; + default: + break; + } + return 0; +} + +iof * iof_filter_lzw_decoder (iof *N, int flags) +{ + iof *I; + lzw_state_pointer P; + I = iof_filter_reader(lzw_decoder, sizeof(lzw_state), &P.voidstate); + iof_setup_next(I, N); + if (lzw_decoder_init(P.lzwstate, flags) == NULL) + { + iof_discard(I); + return NULL; + } + P.lzwstate->flush = 1; + return I; +} + +iof * iof_filter_lzw_encoder (iof *N, int flags) +{ + iof *O; + lzw_state_pointer P; + O = iof_filter_writer(lzw_encoder, sizeof(lzw_state), &P.voidstate); + iof_setup_next(O, N); + if (lzw_encoder_init(P.lzwstate, flags) == NULL) + { + iof_discard(O); + return NULL; + } + return O; +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utillzw.h b/Build/source/libs/pplib/pplib-src/src/util/utillzw.h new file mode 100644 index 00000000000..9e3a085d43c --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utillzw.h @@ -0,0 +1,30 @@ +#ifndef UTIL_LZW_H +#define UTIL_LZW_H + +#include "utiliof.h" + +typedef struct lzw_state lzw_state; + +#define LZW_BASE_BITS 8 +#define LZW_TABLE_ALLOC (1<<4) +#define LZW_EARLY_INDEX (1<<5) +//#define LZW_LOW_BITS_FIRST (1<<6) +#define LZW_DECODER_DEFAULTS (LZW_BASE_BITS|LZW_EARLY_INDEX|0) +#define LZW_ENCODER_DEFAULTS (LZW_BASE_BITS|LZW_EARLY_INDEX|0) + +lzw_state * lzw_decoder_init (lzw_state *state, int flags); +lzw_state * lzw_encoder_init (lzw_state *state, int flags); + +void lzw_decoder_close (lzw_state *state); +void lzw_encoder_close (lzw_state *state); + +iof_status lzw_encode_state (iof *I, iof *O, lzw_state *state); +iof_status lzw_decode_state (iof *I, iof *O, lzw_state *state); + +iof_status lzw_encode (iof *I, iof *O, int flags); +iof_status lzw_decode (iof *I, iof *O, int flags); + +iof * iof_filter_lzw_decoder (iof *N, int flags); +iof * iof_filter_lzw_encoder (iof *N, int flags); + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmd5.c b/Build/source/libs/pplib/pplib-src/src/util/utilmd5.c new file mode 100644 index 00000000000..871984229c2 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmd5.c @@ -0,0 +1,447 @@ + +/* md5 implementation excerpted from code by Peter Deutsch */ + +/* begin of md5.c */ + +/* + Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.c is L. Peter Deutsch + <ghost@aladdin.com>. Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order + either statically or dynamically; added missing #include <string.h> + in library. + 2002-03-11 lpd Corrected argument list for main(), and added int return + type, in test program and T value program. + 2002-02-21 lpd Added missing #include <stdio.h> in test program. + 2000-07-03 lpd Patched to eliminate warnings about "constant is + unsigned in ANSI C, signed in traditional"; made test program + self-checking. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5). + 1999-05-03 lpd Original version. + */ + +#include <string.h> // memcpy +#include <stdio.h> // FILE + +#include "utilmd5.h" + +#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */ +#ifdef ARCH_IS_BIG_ENDIAN +# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1) +#else +# define BYTE_ORDER 0 +#endif + +#define T_MASK ((uint32_t)~0) +#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87) +#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9) +#define T3 0x242070db +#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111) +#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050) +#define T6 0x4787c62a +#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec) +#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe) +#define T9 0x698098d8 +#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850) +#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e) +#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841) +#define T13 0x6b901122 +#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c) +#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71) +#define T16 0x49b40821 +#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d) +#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf) +#define T19 0x265e5a51 +#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855) +#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2) +#define T22 0x02441453 +#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e) +#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437) +#define T25 0x21e1cde6 +#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829) +#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278) +#define T28 0x455a14ed +#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa) +#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07) +#define T31 0x676f02d9 +#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375) +#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd) +#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e) +#define T35 0x6d9d6122 +#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3) +#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb) +#define T38 0x4bdecfa9 +#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f) +#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f) +#define T41 0x289b7ec6 +#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805) +#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a) +#define T44 0x04881d05 +#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6) +#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a) +#define T47 0x1fa27cf8 +#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a) +#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb) +#define T50 0x432aff97 +#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58) +#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6) +#define T53 0x655b59c3 +#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d) +#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82) +#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e) +#define T57 0x6fa87e4f +#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f) +#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb) +#define T60 0x4e0811a1 +#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d) +#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca) +#define T63 0x2ad7d2bb +#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e) + +static void md5_process (md5_state *state, const uint8_t *data /*[64]*/) +{ + uint32_t + a = state->words[0], b = state->words[1], + c = state->words[2], d = state->words[3]; + uint32_t t; +#if BYTE_ORDER > 0 + /* Define storage only for big-endian CPUs. */ + uint32_t X[16]; +#else + /* Define storage for little-endian or both types of CPUs. */ + uint32_t xbuf[16]; + const uint32_t *X; +#endif + + { +#if BYTE_ORDER == 0 + /* + * Determine dynamically whether this is a big-endian or + * little-endian machine, since we can use a more efficient + * algorithm on the latter. + */ + static const int w = 1; + + if (*((const uint8_t *)&w)) /* dynamic little-endian */ +#endif +#if BYTE_ORDER <= 0 /* little-endian */ + { + /* + * On little-endian machines, we can process properly aligned + * data without copying it. + */ + if (!((data - (const uint8_t *)0) & 3)) { + /* data are properly aligned */ + X = (const uint32_t *)((const void *)data); // avoid compiler warning + } else { + /* not aligned */ + memcpy(xbuf, data, 64); + X = xbuf; + } + } +#endif +#if BYTE_ORDER == 0 + else /* dynamic big-endian */ +#endif +#if BYTE_ORDER >= 0 /* big-endian */ + { + /* + * On big-endian machines, we must arrange the bytes in the + * right order. + */ + const uint8_t *xp = data; + int i; +# if BYTE_ORDER == 0 + X = xbuf; /* (dynamic only) */ +# else +# define xbuf X /* (static only) */ +# endif + for (i = 0; i < 16; ++i, xp += 4) + xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24); + } +#endif + } + +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + + /* Round 1. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */ +#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + F(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 7, T1); + SET(d, a, b, c, 1, 12, T2); + SET(c, d, a, b, 2, 17, T3); + SET(b, c, d, a, 3, 22, T4); + SET(a, b, c, d, 4, 7, T5); + SET(d, a, b, c, 5, 12, T6); + SET(c, d, a, b, 6, 17, T7); + SET(b, c, d, a, 7, 22, T8); + SET(a, b, c, d, 8, 7, T9); + SET(d, a, b, c, 9, 12, T10); + SET(c, d, a, b, 10, 17, T11); + SET(b, c, d, a, 11, 22, T12); + SET(a, b, c, d, 12, 7, T13); + SET(d, a, b, c, 13, 12, T14); + SET(c, d, a, b, 14, 17, T15); + SET(b, c, d, a, 15, 22, T16); +#undef SET + + /* Round 2. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */ +#define G(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + G(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 1, 5, T17); + SET(d, a, b, c, 6, 9, T18); + SET(c, d, a, b, 11, 14, T19); + SET(b, c, d, a, 0, 20, T20); + SET(a, b, c, d, 5, 5, T21); + SET(d, a, b, c, 10, 9, T22); + SET(c, d, a, b, 15, 14, T23); + SET(b, c, d, a, 4, 20, T24); + SET(a, b, c, d, 9, 5, T25); + SET(d, a, b, c, 14, 9, T26); + SET(c, d, a, b, 3, 14, T27); + SET(b, c, d, a, 8, 20, T28); + SET(a, b, c, d, 13, 5, T29); + SET(d, a, b, c, 2, 9, T30); + SET(c, d, a, b, 7, 14, T31); + SET(b, c, d, a, 12, 20, T32); +#undef SET + + /* Round 3. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */ +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + H(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 5, 4, T33); + SET(d, a, b, c, 8, 11, T34); + SET(c, d, a, b, 11, 16, T35); + SET(b, c, d, a, 14, 23, T36); + SET(a, b, c, d, 1, 4, T37); + SET(d, a, b, c, 4, 11, T38); + SET(c, d, a, b, 7, 16, T39); + SET(b, c, d, a, 10, 23, T40); + SET(a, b, c, d, 13, 4, T41); + SET(d, a, b, c, 0, 11, T42); + SET(c, d, a, b, 3, 16, T43); + SET(b, c, d, a, 6, 23, T44); + SET(a, b, c, d, 9, 4, T45); + SET(d, a, b, c, 12, 11, T46); + SET(c, d, a, b, 15, 16, T47); + SET(b, c, d, a, 2, 23, T48); +#undef SET + + /* Round 4. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */ +#define I(x, y, z) ((y) ^ ((x) | ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + I(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 6, T49); + SET(d, a, b, c, 7, 10, T50); + SET(c, d, a, b, 14, 15, T51); + SET(b, c, d, a, 5, 21, T52); + SET(a, b, c, d, 12, 6, T53); + SET(d, a, b, c, 3, 10, T54); + SET(c, d, a, b, 10, 15, T55); + SET(b, c, d, a, 1, 21, T56); + SET(a, b, c, d, 8, 6, T57); + SET(d, a, b, c, 15, 10, T58); + SET(c, d, a, b, 6, 15, T59); + SET(b, c, d, a, 13, 21, T60); + SET(a, b, c, d, 4, 6, T61); + SET(d, a, b, c, 11, 10, T62); + SET(c, d, a, b, 2, 15, T63); + SET(b, c, d, a, 9, 21, T64); +#undef SET + + /* Then perform the following additions. (That is increment each + of the four registers by the value it had before this block + was started.) */ + state->words[0] += a; + state->words[1] += b; + state->words[2] += c; + state->words[3] += d; +} + +/* api */ + +md5_state * md5_digest_init (md5_state *state) +{ + state->bitcount[0] = state->bitcount[1] = 0; + state->words[0] = 0x67452301; + state->words[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476; + state->words[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301; + state->words[3] = 0x10325476; + return state; +} + +void md5_digest_add (md5_state *state, const void *input, size_t size) +{ + const uint8_t *p = (const uint8_t *)input; + int nbytes = (int)size; // PJ + int left = nbytes; + int offset = (state->bitcount[0] >> 3) & 63; + uint32_t nbits = (uint32_t)(nbytes << 3); + + if (nbytes <= 0) + return; + + /* Update the message length. */ + state->bitcount[1] += nbytes >> 29; + state->bitcount[0] += nbits; + if (state->bitcount[0] < nbits) + state->bitcount[1]++; + + /* Process an initial partial block. */ + if (offset) { + int copy = (offset + nbytes > 64 ? 64 - offset : nbytes); + + memcpy(state->buffer + offset, p, copy); + if (offset + copy < 64) + return; + p += copy; + left -= copy; + md5_process(state, state->buffer); + } + + /* Process full blocks. */ + for (; left >= 64; p += 64, left -= 64) + md5_process(state, p); + + /* Process a final partial block. */ + if (left) + memcpy(state->buffer, p, left); +} + +#define md5_digest_byte(state, i) (uint8_t)(state->words[i >> 2] >> ((i & 3) << 3)) + +void md5_digest_get (md5_state *state, uint8_t digest[], int flags) +{ + static const uint8_t pad[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + uint8_t data[8]; + int i; + + /* Save the length before padding. */ + for (i = 0; i < 8; ++i) + data[i] = (uint8_t)(state->bitcount[i >> 2] >> ((i & 3) << 3)); + /* Pad to 56 bytes mod 64. */ + md5_digest_add(state, pad, ((55 - (state->bitcount[0] >> 3)) & 63) + 1); + /* Append the length. */ + md5_digest_add(state, data, 8); + + /* Output */ + if (flags & MD5_HEX) + { // expected digest buffer size MD5_STRING_LENGTH + uint8_t byte; + const char *alphabet; + alphabet = (flags & MD5_LCHEX) ? "0123456789abcdef" : "0123456789ABCDEF"; + for (i = 0; i < MD5_DIGEST_LENGTH; ++i) + { + byte = md5_digest_byte(state, i); + *digest++ = (uint8_t)alphabet[byte >> 4]; + *digest++ = (uint8_t)alphabet[byte & 15]; + } + *digest = 0; + } + else + { // expected digest buffer size MD5_DIGEST_LENGTH + for (i = 0; i < MD5_DIGEST_LENGTH; ++i) + *digest++ = md5_digest_byte(state, i); + } +} + +void md5_digest (const void *input, size_t length, uint8_t digest[], int flags) +{ + md5_state md5; + md5_digest_init(&md5); + md5_digest_add(&md5, input, length); + md5_digest_get(&md5, digest, flags); +} + +/* file checksum */ + +#define DIGEST_BUFFER_SIZE 4096 + +int md5_digest_add_file (md5_state *state, const char *filename) +{ + FILE *fh; + uint8_t buffer[DIGEST_BUFFER_SIZE]; + size_t read; + + if ((fh = fopen(filename, "rb")) == NULL) + return 0; + do { + read = fread(buffer, 1, DIGEST_BUFFER_SIZE, fh); + md5_digest_add(state, buffer, read); + } while (read == DIGEST_BUFFER_SIZE); + fclose(fh); + return 1; +} + +int md5_digest_file (const char *filename, uint8_t digest[], int flags) +{ + md5_state state; + + md5_digest_init(&state); + if (md5_digest_add_file(&state, filename)) + { + md5_digest_get(&state, digest, flags); + return 1; + } + return 0; +}
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmd5.h b/Build/source/libs/pplib/pplib-src/src/util/utilmd5.h new file mode 100644 index 00000000000..3964d59df21 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmd5.h @@ -0,0 +1,49 @@ + +/* sha2 implementation excerpted from code by Aaron D. Gifford */ + +#ifndef UTIL_MD5_H +#define UTIL_MD5_H + +#include <stdint.h> +#include <stddef.h> // for size_t +#include "utildecl.h" + +//#define md5_state md5_state_t + +typedef struct { + uint32_t bitcount[2]; + uint32_t words[4]; + uint8_t buffer[64]; +} md5_state; + +#define MD5_DIGEST_LENGTH 16 +#define MD5_STRING_LENGTH (MD5_DIGEST_LENGTH * 2 + 1) + +enum { + MD5_BYTES = 0, + MD5_UCHEX = (1<<0), + MD5_LCHEX = (1<<1) +}; + +#define MD5_DEFAULT MD5_BYTES +#define MD5_HEX (MD5_UCHEX|MD5_LCHEX) + +#ifdef __cplusplus +extern "C" +{ +#endif + +UTILAPI md5_state * md5_digest_init (md5_state *state); +UTILAPI void md5_digest_add (md5_state *state, const void *input, size_t size); +UTILAPI void md5_digest_get (md5_state *state, uint8_t digest[], int flags); + +UTILAPI void md5_digest (const void *input, size_t length, uint8_t digest[], int flags); + +UTILAPI int md5_digest_add_file (md5_state *state, const char *filename); +UTILAPI int md5_digest_file (const char *filename, uint8_t digest[], int flags); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmem.c b/Build/source/libs/pplib/pplib-src/src/util/utilmem.c new file mode 100644 index 00000000000..9a32247ab11 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmem.c @@ -0,0 +1,67 @@ + +#include <string.h> // for memcpy + +#include "utilmem.h" +#include "utillog.h" + +#ifndef util_memerr +# if defined(_WIN64) || defined(__MINGW32__) +# define util_memerr(size) { loggerf("ooops, not enough memory (%I64u)", ((unsigned long long)(size))); abort(); } +# else +# define util_memerr(size) { loggerf("ooops, not enough memory (%llu)", ((unsigned long long)(size))); abort(); } +# endif +#endif + +void * util_malloc (size_t size) +{ + void *m; + if ((m = malloc(size)) == NULL) + util_memerr(size); + return m; +} + +void * util_calloc (size_t num, size_t size) +{ + void *m; + if ((m = calloc(num, size)) == NULL) + util_memerr(size); + return m; +} + +void * util_realloc (void *m, size_t size) +{ + if ((m = realloc(m, size)) == NULL) + util_memerr(size); + return m; +} + +/* common array resizer + +data -- the beginning of array +unit -- sizeof array element +size -- current array size +extra -- requested extra size +space -- pointer to available space +allocated -- flag indicating if *data has been allocated (with malloc) + +*/ + +void util_resize (void **data, size_t unit, size_t size, size_t extra, size_t *space, int allocated) +{ + if (*space == 0) + *space = 4; // better keep *space non-zero to avoid it + do { *space <<= 1; } while (size + extra > *space); + + if (allocated) + { + *data = util_realloc(*data, *space * unit); + } + else + { + void *newdata = util_malloc(*space * unit); + if (*data != NULL) + memcpy(newdata, *data, size * unit); + *data = newdata; + } +} + diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmem.h b/Build/source/libs/pplib/pplib-src/src/util/utilmem.h new file mode 100644 index 00000000000..4cfcfaba2a7 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmem.h @@ -0,0 +1,16 @@ + +#ifndef UTIL_MEM_H +#define UTIL_MEM_H + +#include <stdlib.h> // for size_t and alloc functions +#include "utildecl.h" + +UTILAPI void * util_malloc (size_t size); +UTILAPI void * util_calloc (size_t num, size_t size); +UTILAPI void * util_realloc (void *m, size_t size); + +void util_resize (void **data, size_t unit, size_t size, size_t extra, size_t *space, int allocated); + +#define util_free free // not a call, might be used as identifier + +#endif diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmemallc.h b/Build/source/libs/pplib/pplib-src/src/util/utilmemallc.h new file mode 100644 index 00000000000..6d0ed2a06e9 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmemallc.h @@ -0,0 +1,569 @@ +/* +Allocators +========== + +Using allocators defined here makes sense if there is a need to take a space for rather large amount of rather small objects. +The basic idea is to take memory in reasonably large blocks and to distribute small chunks from those blocks. Once chunks are +no longer needed, one can free them all at once, or free taken chunks individually. + +We define 3 types of allocators: + +1. HEAP - the simplest one, provides variable length memory chunks from larger blocks and frees them all on explicit + request. There is no way to free individual objects, only the entire heap. The heap only grows, until freed. + +2. STOCK - provides variable length memory chunks from larger blocks, but allows to free individual objects as well as the + entire stock. The stock grows and shrinks, by leaps and bounds, depending on parameters given during initialization. + +3. POOL - provides fixed size memory chunks from larger blocks. It allows to free individual chunks as well as the entire pool. + In opposite to a stock, a pool also reuses a space reclaimed by freeing individual objects; before allocating a new block it + firsts recycles freed chunks, if any. + +In general, memory chunks provided by allocators are never reallocated. Allocators do nothing with provided chunks until freed. + +Allocators are represented as small structures (several pointers and integers). We assume that such structures are either static +variables, or members of larger structures. We don't bother to allocate a memory for them. Usage scheme is pretty similar for +all: + + init() - just inititalize, don't allocate anything yet + take() - take chunks + take() + take() + ... + free() - free the all at once + +For stocks and pools there is a possibility to give back individual chunks: + + init() - like above + take() - take chunks + take() + take() + back() - give chunks back when no longer needed + take() + back() + ... + free() - needed only if not all chunks have been given back + +All calls take a shell structure pointer as an argument. take() returns a void pointer, aligned according to used variant +(8, 16, 32, 64). back() takes a void pointer as the second argument. It must be the pointer previously returned by take(). + +back() can be called in any order and can obviously be plotted with take(). By default, after back()-ing all taken chunks, the +stock returns to its initial state (zero memory used). A special KEEP flag can be used during initialization to prevent +freeing the last (sole) allocated block. If KEEP option is used, the allocator always keeps a single segment for further +allocations. This is necessary only when there is a risk that just several take() calls will be immediatelly followed by the +same number of back() calls. KEEP flag prevents allocating and freeing rather large blocks just to serve several chunks. And +this is actually important only if there are no other blocks taken, that is, if there is only one, nearly empty block in use. +In other cases KEEP flag doesn't matter, but allocators takes care to always have a block for fast allocs. + +There is also clear() operation that frees all but the recent block. One can use it to free all chunks taken so far, but to +make the allocator ready for further allocs. If either KEEP flag is used or clear() is called, soner or later the user have to +call free() explicitly, to remove all the remaining memory kept by the allocator. There is no KEEP flag for heaps, as heaps +don't allow to free individual chunks. And so, the heap never needs to make a decision if the last sole block should be removed +or not. The user makes the decision by calling clear() vs free() respectively. + +Pop +=== + +A very last chunk taken can be quickly given back with + + pop(heap, taken, size) // for heap or stock + pop(pool, taken) // for pool + +taken must be the chunk returned by the very last take(), size must be the size requested. If the chunk has been taken from +the head block (more about blocks below), the block pointer returns to its previous position, as it was before the last take(). +If the chunk has been taken from the sole block beneatch the head, the entire sole block (containing just that single chunk) +is freed. The pop() operation is different than back(); the popped chunk doesn't cause freeing the head block when its refcount +gets zero. So pop() operation breaks the concept of stock that frees all the memory once all taken chunks are given back. +on the other hand, if for some reason the very last taken chunk is to be ignored, pop() is better, as it doesn't cause blocks +scattering. The popped chunk pointer will probably be returned by the very next call to take(). In case of heap, pop() is +the only way to discard the chunk, as there is no back() operation. + +Buffer interface +================ + +When heap or stock is used by parsers, the caller oftenly doesn't know how many space will be needed for a data (this doesn't +apply to pools, which returns constant size memory chunks). Here is an interface for the allocator-as-bufer case (same for +heap and stock): + + some(heap, atleast, &space); + ... + atleast <<= 1; + more(heap, taken, written, atleast, &space); + ... + done(heap, taken, written); + +some() operation provides a data pointer to at least a given bytes. The actual space provided for writing is set to the third +argument. The caller may write space-bytes. If more space is needed, more() operation takes care to provide a chunk for a given +amount of bytes and rewrites already written amount of bytes from a previous chunk to a new location. Same as with() some, the +requests for atleast bytes, and the actual provided chunk size is given as space (space >= atleast). + +The function takes the pointer to the chunk previously taken; the one returned by some() or more(). This argument must not be NULL. +If you don't want to copy a data, set written argument to zero. No matter if more() operation was used zero, one or multiple times, +all the cycle must end with done(). Calls triple - some(), more() and done() - must not be interrupted by any other api calls. +In particular, using take() or back() smells like a segfault. However, if there is a need discard the buffer being written +(eg. input data error), instead of done() one may use + + giveup(heap, taken) + +If done() has already been called, pop() is the only option to discard the chunk + + pop(heap, taken, written) + +some() operation usually doesn't change the state of the heap, unless the heap head block is NULL, or atleast parameter is too +large to fit the remaining block. more() usually changes the state, either by allocating a new head block, or by allocating +a sole block just beneath the head (blocks and blocks tiling mechanism are described below). If a sole block has been taken for +some large chunk subsequent calls to more() reallocate this sole block in place. It is assumed, that the size you request in subsequent +calls generally grows. It is ok to request space-bytes, then call done() with written value less then requested. But the drawback +is that if the chunk has already been allocated from a sole chunk, the space requested but not used is a waste. + +iof interface +============= + +iof is an independent interface for buffers written/read byte-by-byte. When used together with allocators, it provides +a convenient way to write byte data to the heap or stock, without a need for intermediate buffers. The buffer is setup with + + iof output, *O + O = buffer_init(heap, &output); // doesn't allocate anything + +or + + output = BUFFER_INIT(heap); // doesn't allocate anything + O = &output; + +iof keeps pointers to the beginning of the buffer, end of buffer, and current position. Once the position reaches the end, +the iof internal handler updates the buffer providing more space to write. When used in conjunction with heap or stock, +the space to write is the space provided by the heap or stock. To start the buffer session: + + O = buffer_some(heap, O, atleast) // ensure iof *O to have atleast bytes to be written + +Once you are done with writing some chunk + + buffer_done(heap, O) + +instead of buffer_done(), one may also use + + iof_flush(O) // calls buffer_done() and buffer_some() again + +which updates the underlying heap or stock, and makes the iof ready for a new chunk. iof itself does not allocate a memory, +so it doesn't need finalizer. iof_close(output) does nothing. To drop the buffer use: + + buffer_giveup(heap, O) // restore the from before buffer_some() + +More often then not, we need to specify a minimal space for buffer each time, eg. for memcpy() or so. The actual space left +can be checked with iof_left(O). The entire space of recent chunk is O->space (eq. O->end - O->buf). + +Identical interface for heap and stock. + +Blocks +====== + +Each alloctor structure keeps a pointer to a head block, initially NULL. Most of new chunks are taken from the head. Once the +space left in the head block is to small to provide a chunk of requested size, a new head is created and the previous one is +linked to the head (blocks form a linked list). A stock block is named a ream, a heap block is named a pyre, a pool block is +named pile (we need to distinguish structure names in code but in the description below they are all called blocks). Every +block knows a number of chunks taken from that block (refcont). A stock also keeps a number of freed chunks [actually only +for statistics; in most cases it doesn't need an extra space in struct ream, as thies structure member lays in the place +f padding bytes.] + +We change the head block only if the new block is allocated, but we never change the head backward. Once some block became +->prev, it will never became a head again. This ensures that the allocator have the head block that usually has a lot of space +for new allocs. This needs a special care when removing a block that is not a head block. We check if the next block to the one +being removed is the head. If it is, and if its refcount is zero (and no KEEP flag is used) the head is removed as well. + +The basis of pools is similar to stocks and heaps, but there are some significant differences. A pool servers memory chunks of +equal size, specified during initialization. This also means that the pool knows the boundaries of individual chunks (stock and +heap doesn't). A pool provides iterators over chunks in use (taken but not given back yet). A pool shell structure keeps +a pointer to a head block and a tail block (both may point a different block, the same block or NULL). This is necessary only +for iterators to let the user follow the chunks from the first or from the last taken. The extra cost of maintaining both +->head and ->tail is neglectable. + +Refcounting +=========== + +Heap refcounting: whenever a new chunk is taken, the block refcount is incremented. It is never decremented, but plays an +important role in block tiling algorithm (below). No KEEP flag is used here. All the blocks are removed on free(), all but +recent are removed on clear(). + +Stock refcounting: whenever a new chunk in taken from the block, the block refcount is incremented. Whenever the chunk is given +back, the refcount is decremented. When the refcount gets zero, the block is removed and freed. To remove the block from the +list (any block, not necessarily a head block), a stock needs 2-directional list; every block has ->next and ->prev links. The +head block of the stock is freed only if this is the last (sole) block and no KEEP flag was used during initialization. +Otherwise the block is just reset, becoming ready for further allocations - refcount gets zero, data space reset to an initial +state. + +Pool refcounting: pretty much like with stocks, except that any chunk given back can be recycled on further take(). + +Ghosts +====== + +Every allocated block starts with a private structure for next/prev links, data pointer, refcount. We call it a block ghost. +Except from heap, individual chunks also need a ghost (chunk ghost) so that we are able to know from which block the chunk +comes from once the chunk is given back by the user (heaps don't have back() operation so data chunks have no ghosts). We keep +ghosts possibly small. Chunk ghosts are of size natural for alignment variant (1, 2, 4 or 8 bytes). Block ghosts are somewhat +larger. Statistics show clearly that it is worthy to keep them as small as possible: +- chunk ghosts keep offset to the block ghost, not a pointer to it (we use the pointer only if it makes no difference + to the chunk size; 64-bit aligned variant on 64-bit machine, 32 and 64 variants on 32-bit machine) +- block ghosts uses a data pointer (not an offset) so that we are able to record any requested chunk size (size_t) and to avoid + long array indexing on every chunk request + +At some point we considered storing a sheel structure pointer in the block ghost, then back() operation wouldn't need an extra +argument. But stats showed that the size of the block ghost is the most significant factor in memory usage efficiency, so eliminating +this extra pointer pays off. Besides, this would make impossible to relocate the shell structure. We don't allocate a memory +for the shell, so we shouldn't make assumptions of shell structure address. + +Tiling algorithm +================ + +Tiling the block size refers to stocks and heaps that serves memory chunks of variable size. Both stock and heap performs best +when the average size of requested chunks is a way smaller that the configured block size. But both also put no limitations on +chunk sizes, so they need to cope with situation, where the requested size is quite large, eg. half of the block size or even +more than the block size. Here is the algorithm used for blocks tiling: + +1. When the requested chunk size fills in the recent block, just serve it from that block. This is the best and hopefully the + most common case. + +2. When the requested chunk size is larger that the space left in the recent block, the new block must be allocated. But there +are two ways: + + a) either replace the head block with the new block so that the list of blocks is + + ... <- prev <- head so far <- new head + + b) or insert the block just "below the head", keeping the head intact, + + ... <- prev <- new single head <- head + +The first is the regular case. It is used when the space left in the head so far is small (can be neglected), and the requested +size is relatively small (will fit the new block). If the space left in the head block is worthy to bother, or the requested +chunk size is rather large, the new chunk is served from a single block, allocated just for that chunk. The block is of the +size needed for that chunk. The block never becomes the head, no other chunks will be served from it (its refcount is +permanently 1, until freed). + +Much depends on what is considered 'small, neglectable block space' and 'rather large chunk size'. The later is easier to +imagine. When the requested size is larger than the block size used for a given allocator, then the size is definitelly +considered large. When it is smaller than the block size, but still large enough to occupy most of the block size (grabbing +quite some space for tiny chunks), it is also considered large. As the block size, what is considered 'large' can be spcified +during initialization. A setup that works fine for me is (large = block_size / 2). + +Making a decision what is the left block space we can neglect is quite not obvious. At first approach we used a constant value, +requested from the user during allocator initialization. But it is hard to select a good default. Now we compute this value +from block params, by dividing a complete space occupied so far in the block by the number of chunks served from that block +(the average size of chunks allocated from this block). We assume that the average chunk size (or smaller) is the space we can +neglect. The logic behind is the following: if the space left in the block is larger than the average, it makes sense not to +waste this space and keep it for further allocs. If the space left in the block is less than the average, there is only a little +chance we will get a request for suitable size, so we sacrifice that space and we start allocating from a new block. + +Statistics showed a caveat in average == treshold approach. Suppose we have a block that has the average chunk size 16, there +is 18 bytes left in the block (not neglectable), and the user request is 20 bytes. Allocating a single block for 20 bytes is +bad, because the block ghost is 24 bytes (more internal than allocated memory). Allocating many of such blocks gives bad results; +much more allocs than necessary, large waste. To avoid that, we help to neglect the remaining block space by checking if the +space left is smaller than the block ghost size, which is an inevitable cost anyway. + +Stats below shows clearly that we should rather focus on "how to avoid producing sole-chunk blocks" instead of "how to feel the +remaining space". + +Recycling +========= + +Recycling applies only to pools. When a chunk is given back, it is inserted into a list of items for recycling. Every pool +block keeps a head of that list. Once a chunk is given back, it is inserted as recycling head and the previous head is attached +to a new head. Since every chunk is associated with a ghost, we use ghosts to store a link (pointer or offset) to another item +for recycling. Note that the ghost always keeps either a link to the block it belongs to, or a link to another recyclable ghost +of the same block. This is used by iteratos to distinguish the chunk currently in use from the chunk that has already been +given back; if the link points the block, the chunk is in use. + +A pool block that has at least one recyclable chunk is called a squot. A pool shell structure keeps 2-directional list of +squots. Once a pool block becomes a squot, it is inserted to that list. Once its all recyclable items has been used, it is +removed from the squots list. In every moment, the pool has an access to a list of all squots, and therefore, to a list of all +recyclable items. + +Whenever there is a request for a new chunk, at first it is served from the head block, as this is the easiest and the cheapest way. +Once the recent block has no more place for new items, recycling list is used, starting from the head recyclable chunk of the head squot. +In practise this is always the most recently reclaimed chunk ghost. During further allocs, a pool will first utilize all recyclables +from all squots before allocating a new block. + +Stats +===== + +Some numbers. The test made on a stock8, block size 255 bytes, 10000 allocations, random chunk sizes from 1 to 32 bytes +(average 16). These are rather tight constraints because of 255 buffer limit. First approach: + + blocks: 903 - this is the actual number of malloc() calls + singles: 214, 23.70% of all blocks + waste: 20.16% - total memory that was allocated but not requested by the user + block ghosts 10.04%, plus single block ghosts 3.12% + chunk ghosts 4.55% + neglected block tails 2.45% + +After adding a test for left space that helps in 'neglect remainig space or make sole chunk block' decision: + + blocks: 723 - a way better + singles 0 + waste: 19.04% - slightly better + block ghosts 10.67% + chunk ghosts 4.61% + neglected block tails 3.76% + +The actual numbers vary depending on the buffer size, the average elements size and, of course, taken alignment variant. After +some parameters tuning, on various tests we get 5-19% total waste for stocks, 3-14% total waste for heaps. But the basic scheme +of statistics remains similar: we take relatively lots of space for blocks ghost (5-10% of total memory taken), some inevitable +space for chunk ghosts (varies, 4-13% on various tests), and a little waste of neglected block tails (2-4%). Quite +surprisingly, block ghosts are, in sum, oftenly more significant than individual chunk ghosts (for the test above over half of +all the waste!). The number of block ghosts (equals the number of blocks) mostly depends on block size vs chunk size relation. +But seemingly it is worthy to bother about the size of the block ghost and the number of blocks taken - the less the better. +The waste of ghosts of individual objects (stock and pool) is inevitable, and depends only on the number/size of objects taken. +We can't use smaller ghosts, we can't do better. Anyways, the least significant is the waste of neglected block tails. + +Pools stats are pretty similar, but more predictable because of known chunks size. A pool block ghost is somewhat larger +structure because it keeps ->nextsquot / ->prevsquot pointers among ->next / ->prev. On the other hand, it doesn't need +->unused counter, as for fixed-length chunks it can always be computed from the refcount and used data. Also somewhat larger +block ghost structure is compensated by the fact that the are no tail block waste and there is no 'neglect or not' problem. + +Alignment +========= + +Each allocator has 4 variants for 1, 2, 4, 8 bytes alignment respectively. Eg. stock32_take() always returns a pointer aligned +to 4 bytes, heap64_take() returns a pointer aligned to 8 bytes. You can ask for any data length, but in practise you'll always +obtain 1N, 2N, 4N or 8N. Alignment implies data padding unless the user requests for "aligned" sizes. In statistics the padding +is not considered a waste. + +Zeroing +======= + +All heap, stock and pool may return zeroed memory chunks, depending on initial flags: + + HEAP_ZERO + STOCK_ZERO + POOL_ZERO + +There are also take0() variants that simply return memset(take(), 0, size), regardless the flag. +*/ + +#ifndef UTIL_MEM_ALLC_C +#define UTIL_MEM_ALLC_C + +/* +Common internals for allocators suite. A selection or all of the following defines (from api headers) should already be there: + + UTIL_MEM_HEAP_H // utilmemheap.h + UTIL_MEM_STOCK_H // utilmemstock.h + UTIL_MEM_POOL_H // utilmempool.h + +*/ + +#include <string.h> // memset() +#include <stdio.h> // printf() + +#include "utilmem.h" + +//#if defined(DEBUG) && debug != 0 +#if 1 +# define ASSERT8(cond) ((void)((cond) || (printf("8bit allocator assertion, %s:%d: %s\n", __FILE__, __LINE__, #cond), 0))) +# define ASSERT16(cond) ((void)((cond) || (printf("16bit allocator assertion, %s:%d: %s\n", __FILE__, __LINE__, #cond), 0))) +# define ASSERT32(cond) ((void)((cond) || (printf("32bit allocator assertion, %s:%d: %s\n", __FILE__, __LINE__, #cond), 0))) +# define ASSERT64(cond) ((void)((cond) || (printf("64bit allocator assertion, %s:%d: %s\n", __FILE__, __LINE__, #cond), 0))) +#else +# define ASSERT8(cond) (void)0 +# define ASSERT16(cond) (void)0 +# define ASSERT32(cond) (void)0 +# define ASSERT64(cond) (void)0 +#endif + +#if defined(UTIL_MEM_STOCK_H) || defined(UTIL_MEM_POOL_H) +struct ghost8{ + uint8_t offset; +}; + +struct ghost16 { + uint16_t offset; +}; + +#ifdef BIT32 +struct ghost32 { + union { +#ifdef UTIL_MEM_STOCK_H + ream32 *ream; +#endif +#ifdef UTIL_MEM_POOL_H + pile32 *pile; + ghost32 *nextfree; +#endif + void *block; + }; +}; +#else +struct ghost32 { + uint32_t offset; +}; +#endif + +struct ghost64 { + union { +#ifdef UTIL_MEM_STOCK_H + ream64 *ream; +#endif +#ifdef UTIL_MEM_POOL_H + pile64 *pile; + ghost64 *nextfree; +#endif + void *block; + }; +#ifdef BIT32 + uint8_t dummy[4]; // force 8 +#endif +}; +#endif + +/* +All offsets related macro horror is here. Block is 4/8-bytes aligned (32/64 pointer size), ream->data is adjusted to 1/2/4/8-bytes accordingly. +Therefore all offsets we store and pointers we cast, should be properly aligned. In all cases, sizes and offsets refers to bytes. +We need data ghosts only to access the block. For 8 and 16 we use 8/16 bit offsets to keep the ghost smaller. For 32 and 64 we either use offset, +or a pointer to the ream. + +malloc() is obviously expected to return a pointer properly allowed for all standard c-types. For 64-bit we can safely expect at least 8-bytes aligned. +(at least, because long double may need 16 bytes on gcc64, or 8 bytes on msvc64, or weird on some exotics). On 32 bit machines pointers are 4 bytes +aligned, even long long is 4-bytes aligned. But double on 32bit machine is 8-bytes aligned on windows, 4 bytes aligned in linux (compiler option +-malign-double makes it 8-bytes aligned). Anyways, we cannot expect that on 32bit machine the result of malloc is always 8-bytes aligned. +This requires a very special treatment of 64-variant on 32bit machine: the first data ghost may need to be 4-bytes off. Should we ensure 4 bytes +more from malloc just in case? Hmm padding will be there anyway, as we adjust ream->data size to bytes boundaries. + +In both 32/64bit environments, the ghost keeps a pointer to the block. On 32bit machine, the first chunk ghost address may need to be +4, +as this is not ensured by malloc(). See struct ream64 {}. We have an extra test; the final ghost pointer will be properly aligned iff + + ((block & 7 == 0) && (sizeof(block64) & 7 == 0)) || ((block & 7 == 4) && (sizeof(block64) & 7 == 4) + +or in short + + ((block + 1) & 7) == 0 + +otherwise it needs 4 bytes offset. +*/ + +#define pointer_tointeger(p) ((size_t)(p)) // & not allowed on pointer + +#define pointer_aligned32(p) ((pointer_tointeger(p) & 3) == 0) +#define pointer_aligned64(p) ((pointer_tointeger(p) & 7) == 0) + +#define void_data(data) ((void *)(data)) +#define byte_data(data) ((uint8_t *)(data)) + +/* top of the block ghost */ + +#define block_top(block) (byte_data(block + 1)) + +/* where the data begins */ + +#define block_edge8(block) block_top(block) +#define block_edge16(block) block_top(block) +#define block_edge32(block) block_top(block) + +#ifdef BIT32 +# define ALIGN64ON32(block) (pointer_aligned64(block + 1) ? 0 : 4) +# define block_edge64(block) (block_top(block) + ALIGN64ON32(block)) +#else +# define block_edge64(block) block_top(block) +#endif + +#define block_left8(block, size) (size) +#define block_left16(block, size) (size) +#define block_left32(block, size) (size) +#ifdef BIT32 +# define block_left64(block, size) (size - ALIGN64ON32(block)) +#else +# define block_left64(block, size) (size) +#endif + +/* consumed block space; it is important to use edge() macros that involves ALIGN64ON32() */ + +#define block_used8(block) (block->data - block_edge8(block)) +#define block_used16(block) (block->data - block_edge16(block)) +#define block_used32(block) (block->data - block_edge32(block)) +#define block_used64(block) (block->data - block_edge64(block)) + +/* align requested size to keep ream->data / pyre->data always aligned. size is always size_t, no insane overflow checks */ + +#define align_size8(size) ((void)size) +#define align_size16(size) (size = aligned_size16(size)) +#define align_size32(size) (size = aligned_size32(size)) +#define align_size64(size) (size = aligned_size64(size)) + +/* +done() and pop() operations decrements block->left space by an aligned size; block->left -= alignedwritten. Lets have 8-bytes aligned +variant block. If we tell the user there is 15 bytes left (block->left == 15) and the user taked 12. Aligned is 16, we cannot substract. +We could eventually set block->left to 0, but then pop() operation would no be allowed. Hance, block->left must be aligned. The procedure +is different than for size (size_t), we cannot cross 0xff/0xffff,... bondaries. +*/ + +#define align_space8(space) ((void)space) +#define align_space16(space) (space = aligned_space16(space)) +#define align_space32(space) (space = aligned_space32(space)) +#define align_space64(space) (space = aligned_space64(space)) + +/* handling ghost structure (stock and pool) */ + +#if defined(UTIL_MEM_STOCK_H) || defined(UTIL_MEM_POOL_H) + +/* ghost offset from block top; not from bottom because we must not exceed offset limit */ + +#define ghost_offset(block, ghost) (byte_data(ghost) - block_top(block)) + +/* ghost <-> data */ + +#define ghost_data(ghost) ((void *)(ghost + 1)) + +/* cast from data to ghost structure goes via (void *) to shut up warnigns, alignment ok */ + +#define data_ghost8(data) (((ghost8 *)void_data(data)) - 1) +#define data_ghost16(data) (((ghost16 *)void_data(data)) - 1) +#define data_ghost32(data) (((ghost32 *)void_data(data)) - 1) +#define data_ghost64(data) (((ghost64 *)void_data(data)) - 1) + +/* ghost <-> block */ + +#define ghost_block8(ghost, block8) ((block8 *)void_data(byte_data(ghost) - ghost->offset - sizeof(block8))) +#define ghost_block16(ghost, block16) ((block16 *)void_data(byte_data(ghost) - ghost->offset - sizeof(block16))) +#ifdef BIT32 +# define ghost_block32(ghost, block32) (ghost->block) +#else +# define ghost_block32(ghost, block32) ((block32 *)void_data(byte_data(ghost) - ghost->offset - sizeof(block32))) +#endif +#define ghost_block64(ghost, block64) (ghost->block) + +/* ghost init */ + +#define ghost_next8(block, ghost) ((ghost = block->dataghost), (ghost->offset = (uint8_t)ghost_offset(block, ghost))) +#define ghost_next16(block, ghost) ((ghost = block->dataghost), (ghost->offset = (uint16_t)ghost_offset(block, ghost))) +#ifdef BIT32 +# define ghost_next32(bl0ck, ghost) ((ghost = bl0ck->dataghost), (ghost->block = bl0ck)) +#else +# define ghost_next32(block, ghost) ((ghost = block->dataghost), (ghost->offset = (uint32_t)ghost_offset(block, ghost))) +#endif +#define ghost_next64(bl0ck, ghost) ((ghost = bl0ck->dataghost), (ghost->block = bl0ck)) + +#endif + +/* average block chunk size */ + +#define average_block_chunk8(ream) (block_used8(ream) / ream->chunks) +#define average_block_chunk16(ream) (block_used16(ream) / ream->chunks) +#define average_block_chunk32(ream) (block_used32(ream) / ream->chunks) +#define average_block_chunk64(ream) (block_used64(ream) / ream->chunks) + +/* +neglect remaining block tail and start a new block or create a single block; a test for (block->chunks > 0) is a sanity; +if block->chunks is zero (block has a full space left), we shouldn't get there, except when alloc->large is larger then alloc->space +*/ + +#define take_new_block8(alloc, ghoststruct, block, size) \ + ((size < alloc->large) && (block->left <= sizeof(ghoststruct) || (block->chunks > 0 && block->left <= average_block_chunk8(block)))) +#define take_new_block16(alloc, ghoststruct, block, size) \ + ((size < alloc->large) && (block->left <= sizeof(ghoststruct) || (block->chunks > 0 && block->left <= average_block_chunk16(block)))) +#define take_new_block32(alloc, ghoststruct, block, size) \ + ((size < alloc->large) && (block->left <= sizeof(ghoststruct) || (block->chunks > 0 && block->left <= average_block_chunk32(block)))) +#define take_new_block64(alloc, ghoststruct, block, size) \ + ((size < alloc->large) && (block->left <= sizeof(ghoststruct) || (block->chunks > 0 && block->left <= average_block_chunk64(block)))) + +/* empty */ + +#define head_block_empty(alloc, block) (((block = alloc->head) == NULL) || (block->chunks == 0 && block->prev == NULL)) + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmemallh.h b/Build/source/libs/pplib/pplib-src/src/util/utilmemallh.h new file mode 100644 index 00000000000..a543d1acb06 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmemallh.h @@ -0,0 +1,36 @@ + +#ifndef UTIL_MEM_ALLH_H +#define UTIL_MEM_ALLH_H + +#include <stddef.h> // size_t +#include <stdint.h> + +#include "utildecl.h" + +typedef struct ghost8 ghost8; +typedef struct ghost16 ghost16; +typedef struct ghost32 ghost32; +typedef struct ghost64 ghost64; + +#define aligned_size8(size) (size) +#define aligned_size16(size) ((((size) + 1) >> 1) << 1) +#define aligned_size32(size) ((((size) + 3) >> 2) << 2) +#define aligned_size64(size) ((((size) + 7) >> 3) << 3) + +#define aligned_space8(size) (size) +#define aligned_space16(size) (((size) & 1) ? ((size) < 0xFFFF ? ((size) + 1) : ((size) - 1)) : (size)) +#define aligned_space32(size) (((size) & 3) ? ((size) < 0xFFFFFFFD ? ((size) - ((size) & 3) + 4) : (size) - ((size) & 3)) : (size)) +#define aligned_space64(size) (((size) & 7) ? ((size) < 0xFFFFFFFFFFFFFFF8ULL ? ((size) - ((size) & 7) + 8) : (size) - ((size) & 7)) : (size)) + +/* info stub */ + +typedef struct { + size_t blocks, singles; + size_t chunks, unused; + size_t used, singleused, left; + size_t ghosts, blockghosts, singleghosts; +} mem_info; + +#define MEM_INFO_INIT() = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmemheap.c b/Build/source/libs/pplib/pplib-src/src/util/utilmemheap.c new file mode 100644 index 00000000000..cce28e0ed07 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmemheap.c @@ -0,0 +1,1078 @@ + +#include "utilmemheap.h" +#include "utilmemallc.h" + +#define pyre_alloc8(heap, space) ((pyre8 *)((heap->flags & HEAP_ZERO) ? util_calloc(1, sizeof(pyre8) + space * sizeof(uint8_t)) : util_malloc(sizeof(pyre8) + space * sizeof(uint8_t)))) +#define pyre_alloc16(heap, space) ((pyre16 *)((heap->flags & HEAP_ZERO) ? util_calloc(1, sizeof(pyre16) + space * sizeof(uint8_t)) : util_malloc(sizeof(pyre16) + space * sizeof(uint8_t)))) +#define pyre_alloc32(heap, space) ((pyre32 *)((heap->flags & HEAP_ZERO) ? util_calloc(1, sizeof(pyre32) + space * sizeof(uint8_t)) : util_malloc(sizeof(pyre32) + space * sizeof(uint8_t)))) +#define pyre_alloc64(heap, space) ((pyre64 *)((heap->flags & HEAP_ZERO) ? util_calloc(1, sizeof(pyre64) + space * sizeof(uint8_t)) : util_malloc(sizeof(pyre64) + space * sizeof(uint8_t)))) + +#define pyre_free(pyre) util_free(pyre) + +/* block reset */ + +#define reset_heap_head8(heap, pyre, used) \ + ((used = block_used8(pyre)), (pyre->data -= used), ((heap->flags & HEAP_ZERO) ? (memset(pyre->data, 0, used), 0) : 0), (pyre->left += (uint8_t)used)) +#define reset_heap_head16(heap, pyre, used) \ + ((used = block_used16(pyre)), (pyre->data -= used), ((heap->flags & HEAP_ZERO) ? (memset(pyre->data, 0, used), 0) : 0), (pyre->left += (uint16_t)used)) +#define reset_heap_head32(heap, pyre, used) \ + ((used = block_used32(pyre)), (pyre->data -= used), ((heap->flags & HEAP_ZERO) ? (memset(pyre->data, 0, used), 0) : 0), (pyre->left += (uint32_t)used)) +#define reset_heap_head64(heap, pyre, used) \ + ((used = block_used64(pyre)), (pyre->data -= used), ((heap->flags & HEAP_ZERO) ? (memset(pyre->data, 0, used), 0) : 0), (pyre->left += (uint64_t)used)) + +/* init heap */ + +heap8 * heap8_init (heap8 *heap, uint8_t space, uint8_t large, uint8_t flags) +{ + align_space8(space); + if (large > space) large = space; + heap->head = NULL; + heap->space = space; + heap->large = large; + heap->flags = flags; + return heap; +} + +heap16 * heap16_init (heap16 *heap, uint16_t space, uint16_t large, uint8_t flags) +{ + align_space16(space); + if (large > space) large = space; + heap->head = NULL; + heap->space = space; + heap->large = large; + heap->flags = flags; + return heap; +} + +heap32 * heap32_init (heap32 *heap, uint32_t space, uint32_t large, uint8_t flags) +{ + align_space32(space); + if (large > space) large = space; + heap->head = NULL; + heap->space = space; + heap->large = large; + heap->flags = flags; + return heap; +} + +heap64 * heap64_init (heap64 *heap, uint64_t space, uint64_t large, uint8_t flags) +{ + align_space64(space); + if (large > space) large = space; + heap->head = NULL; + heap->space = space; + heap->large = large; + heap->flags = flags; + return heap; +} + +/* free heap */ + +void heap8_free (heap8 *heap) +{ + pyre8 *pyre, *prev; + pyre = heap->head; + heap->head = NULL; + while (pyre != NULL) + { + prev = pyre->prev; + pyre_free(pyre); + pyre = prev; + } +} + +void heap16_free (heap16 *heap) +{ + pyre16 *pyre, *prev; + pyre = heap->head; + heap->head = NULL; + while (pyre != NULL) + { + prev = pyre->prev; + pyre_free(pyre); + pyre = prev; + } +} + +void heap32_free (heap32 *heap) +{ + pyre32 *pyre, *prev; + pyre = heap->head; + heap->head = NULL; + while (pyre != NULL) + { + prev = pyre->prev; + pyre_free(pyre); + pyre = prev; + } +} + +void heap64_free (heap64 *heap) +{ + pyre64 *pyre, *prev; + pyre = heap->head; + heap->head = NULL; + while (pyre != NULL) + { + prev = pyre->prev; + pyre_free(pyre); + pyre = prev; + } +} + +/* clear heap */ + +void heap8_clear (heap8 *heap) +{ + pyre8 *pyre, *prev; + size_t used; + if ((pyre = heap->head) == NULL) + return; + prev = pyre->prev; + pyre->prev = NULL; + reset_heap_head8(heap, pyre, used); + for (; prev != NULL; prev = pyre) + { + pyre = prev->prev; + pyre_free(prev); + } +} + +void heap16_clear (heap16 *heap) +{ + pyre16 *pyre, *prev; + size_t used; + if ((pyre = heap->head) == NULL) + return; + prev = pyre->prev; + pyre->prev = NULL; + reset_heap_head16(heap, pyre, used); + for (; prev != NULL; prev = pyre) + { + pyre = prev->prev; + pyre_free(prev); + } +} + +void heap32_clear (heap32 *heap) +{ + pyre32 *pyre, *prev; + size_t used; + if ((pyre = heap->head) == NULL) + return; + prev = pyre->prev; + pyre->prev = NULL; + reset_heap_head32(heap, pyre, used); + for (; prev != NULL; prev = pyre) + { + pyre = prev->prev; + pyre_free(prev); + } +} + +void heap64_clear (heap64 *heap) +{ + pyre64 *pyre, *prev; + size_t used; + if ((pyre = heap->head) == NULL) + return; + prev = pyre->prev; + pyre->prev = NULL; + reset_heap_head64(heap, pyre, used); + for (; prev != NULL; prev = pyre) + { + pyre = prev->prev; + pyre_free(prev); + } +} + +/* heap head */ + +void heap8_head (heap8 *heap) +{ + pyre8 *pyre; + heap->head = pyre = pyre_alloc8(heap, heap->space); + pyre->prev = NULL; + pyre->data = block_edge8(pyre); + pyre->left = block_left8(pyre, heap->space); + pyre->chunks = 0; +} + +void heap16_head (heap16 *heap) +{ + pyre16 *pyre; + heap->head = pyre = pyre_alloc16(heap, heap->space); + pyre->prev = NULL; + pyre->data = block_edge16(pyre); + pyre->left = block_left16(pyre, heap->space); + pyre->chunks = 0; +} + +void heap32_head (heap32 *heap) +{ + pyre32 *pyre; + heap->head = pyre = pyre_alloc32(heap, heap->space); + pyre->prev = NULL; + pyre->data = block_edge32(pyre); + pyre->left = block_left32(pyre, heap->space); + pyre->chunks = 0; +} + +void heap64_head (heap64 *heap) +{ + pyre64 *pyre; + heap->head = pyre = pyre_alloc64(heap, heap->space); + pyre->prev = NULL; + pyre->data = block_edge64(pyre); + pyre->left = block_left64(pyre, heap->space); + pyre->chunks = 0; +} + +/* next heap head */ + +static pyre8 * heap8_new (heap8 *heap) +{ + pyre8 *pyre; + pyre = pyre_alloc8(heap, heap->space); + pyre->prev = heap->head; + heap->head = pyre; + pyre->data = block_edge8(pyre); + pyre->left = block_left8(pyre, heap->space); + pyre->chunks = 0; + return pyre; +} + +static pyre16 * heap16_new (heap16 *heap) +{ + pyre16 *pyre; + pyre = pyre_alloc16(heap, heap->space); + pyre->prev = heap->head; + heap->head = pyre; + pyre->data = block_edge16(pyre); + pyre->left = block_left16(pyre, heap->space); + pyre->chunks = 0; + return pyre; +} + +static pyre32 * heap32_new (heap32 *heap) +{ + pyre32 *pyre; + pyre = pyre_alloc32(heap, heap->space); + pyre->prev = heap->head; + heap->head = pyre; + pyre->data = block_edge32(pyre); + pyre->left = block_left32(pyre, heap->space); + pyre->chunks = 0; + return pyre; +} + +static pyre64 * heap64_new (heap64 *heap) +{ + pyre64 *pyre; + pyre = pyre_alloc64(heap, heap->space); + pyre->prev = heap->head; + heap->head = pyre; + pyre->data = block_edge64(pyre); + pyre->left = block_left64(pyre, heap->space); + pyre->chunks = 0; + return pyre; +} + +/* next heap sole */ + +static pyre8 * heap8_sole (heap8 *heap, size_t size) +{ + pyre8 *pyre, *head, *prev; + pyre = pyre_alloc8(heap, size); + head = heap->head; + prev = head->prev; + pyre->prev = prev; + head->prev = pyre; + pyre->data = block_edge8(pyre); + pyre->left = 0; // (uint8_t)size makes no sense, even with buffer api it will finally become 0 + return pyre; +} + +static pyre16 * heap16_sole (heap16 *heap, size_t size) +{ + pyre16 *pyre, *head, *prev; + pyre = pyre_alloc16(heap, size); + head = heap->head; + prev = head->prev; + pyre->prev = prev; + head->prev = pyre; + pyre->data = block_edge16(pyre); + pyre->left = 0; + return pyre; +} + +static pyre32 * heap32_sole (heap32 *heap, size_t size) +{ + pyre32 *pyre, *head, *prev; + pyre = pyre_alloc32(heap, size); + head = heap->head; + prev = head->prev; + pyre->prev = prev; + head->prev = pyre; + pyre->data = block_edge32(pyre); + pyre->left = 0; + return pyre; +} + +static pyre64 * heap64_sole (heap64 *heap, size_t size) +{ + pyre64 *pyre, *head, *prev; + pyre = pyre_alloc64(heap, size); + head = heap->head; + prev = head->prev; + pyre->prev = prev; + head->prev = pyre; + pyre->data = block_edge64(pyre); + pyre->left = 0; + return pyre; +} + +/* take from heap */ + +#define pyre_next8(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->left -= (uint8_t)size, ++pyre->chunks) +#define pyre_next16(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->left -= (uint16_t)size, ++pyre->chunks) +#define pyre_next32(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->left -= (uint32_t)size, ++pyre->chunks) +#define pyre_next64(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->left -= (uint64_t)size, ++pyre->chunks) + +// for sole blocks, block->left is permanently 0, we can't store size_t there +#define pyre_last8(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->chunks = 1) +#define pyre_last16(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->chunks = 1) +#define pyre_last32(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->chunks = 1) +#define pyre_last64(d, pyre, size) (d = pyre->data, pyre->data += size, pyre->chunks = 1) + +void * _heap8_take (heap8 *heap, size_t size) +{ + pyre8 *pyre; + void *data; + pyre = heap->head; + align_size8(size); + if (size <= pyre->left) + { + pyre_next8(data, pyre, size); + } + else if (take_new_block8(heap, pyre8, pyre, size)) + { + pyre = heap8_new(heap); + pyre_next8(data, pyre, size); + } + else + { + pyre = heap8_sole(heap, size); + pyre_last8(data, pyre, size); + } + return data; +} + +void * _heap16_take (heap16 *heap, size_t size) +{ + pyre16 *pyre; + void *data; + pyre = heap->head; + align_size16(size); + if (size <= pyre->left) + { + pyre_next16(data, pyre, size); + } + else if (take_new_block16(heap, pyre16, pyre, size)) + { + pyre = heap16_new(heap); + pyre_next16(data, pyre, size); + } + else + { + pyre = heap16_sole(heap, size); + pyre_last16(data, pyre, size); + } + return data; +} + +void * _heap32_take (heap32 *heap, size_t size) +{ + pyre32 *pyre; + void *data; + pyre = heap->head; + align_size32(size); + if (size <= pyre->left) + { + pyre_next32(data, pyre, size); + } + else if (take_new_block32(heap, pyre32, pyre, size)) + { + pyre = heap32_new(heap); + pyre_next32(data, pyre, size); + } + else + { + pyre = heap32_sole(heap, size); + pyre_last32(data, pyre, size); + } + return data; +} + +void * _heap64_take (heap64 *heap, size_t size) +{ + pyre64 *pyre; + void *data; + pyre = heap->head; + align_size64(size); + if (size <= pyre->left) + { + pyre_next64(data, pyre, size); + } + else if (take_new_block64(heap, pyre64, pyre, size)) + { + pyre = heap64_new(heap); + pyre_next64(data, pyre, size); + } + else + { + pyre = heap64_sole(heap, size); + pyre_last64(data, pyre, size); + } + return data; +} + +void * _heap8_take0 (heap8 *heap, size_t size) +{ + return memset(_heap8_take(heap, size), 0, size); +} + +void * _heap16_take0 (heap16 *heap, size_t size) +{ + return memset(_heap16_take(heap, size), 0, size); +} + +void * _heap32_take0 (heap32 *heap, size_t size) +{ + return memset(_heap32_take(heap, size), 0, size); +} + +void * _heap64_take0 (heap64 *heap, size_t size) +{ + return memset(_heap64_take(heap, size), 0, size); +} + +/* pop last heap chunk */ + +#define taken_from_head(taken, head) (byte_data(taken) == head->data) +#define taken_from_sole(taken, head, sole) ((sole = head->prev) != NULL && byte_data(taken) == sole->data) + +#define taken_prev_head(taken, head, size) (byte_data(taken) == head->data - size) +#define taken_prev_sole(taken, head, sole, size) ((sole = head->prev) != NULL && byte_data(taken) == sole->data - size) + +void heap8_pop (heap8 *heap, void *taken, size_t size) +{ + pyre8 *pyre, *head; + head = heap->head; + align_size8(size); + if (taken_prev_head(taken, head, size)) + { + + head->data -= size; + head->left += (uint8_t)size; + --head->chunks; + } + else if (taken_prev_sole(taken, head, pyre, size)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } + else + { + ASSERT8(0); + } +} + +void heap16_pop (heap16 *heap, void *taken, size_t size) +{ + pyre16 *pyre, *head; + head = heap->head; + align_size16(size); + if (taken_prev_head(taken, head, size)) + { + + head->data -= size; + head->left += (uint16_t)size; + --head->chunks; + } + else if (taken_prev_sole(taken, head, pyre, size)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } + else + { + ASSERT16(0); + } +} + +void heap32_pop (heap32 *heap, void *taken, size_t size) +{ + pyre32 *pyre, *head; + head = heap->head; + align_size32(size); + if (taken_prev_head(taken, head, size)) + { + + head->data -= size; + head->left += (uint32_t)size; + --head->chunks; + } + else if (taken_prev_sole(taken, head, pyre, size)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } + else + { + ASSERT32(0); + } +} + +void heap64_pop (heap64 *heap, void *taken, size_t size) +{ + pyre64 *pyre, *head; + head = heap->head; + align_size64(size); + if (taken_prev_head(taken, head, size)) + { + + head->data -= size; + head->left += (uint64_t)size; + --head->chunks; + } + else if (taken_prev_sole(taken, head, pyre, size)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } + else + { + ASSERT64(0); + } +} + +/* heap buffer */ + +void * _heap8_some (heap8 *heap, size_t size, size_t *pspace) +{ + pyre8 *pyre; + pyre = heap->head; + align_size8(size); + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block8(heap, pyre8, pyre, size)) + { + pyre = heap8_new(heap); + *pspace = pyre->left; + } + else + { + pyre = heap8_sole(heap, size); + *pspace = size; + } + return void_data(pyre->data); +} + +void * _heap16_some (heap16 *heap, size_t size, size_t *pspace) +{ + pyre16 *pyre; + pyre = heap->head; + align_size16(size); + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block16(heap, pyre16, pyre, size)) + { + pyre = heap16_new(heap); + *pspace = pyre->left; + } + else + { + pyre = heap16_sole(heap, size); + *pspace = size; + } + return void_data(pyre->data); +} + +void * _heap32_some (heap32 *heap, size_t size, size_t *pspace) +{ + pyre32 *pyre; + pyre = heap->head; + align_size32(size); + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block32(heap, pyre32, pyre, size)) + { + pyre = heap32_new(heap); + *pspace = pyre->left; + } + else + { + pyre = heap32_sole(heap, size); + *pspace = size; + } + return void_data(pyre->data); +} + +void * _heap64_some (heap64 *heap, size_t size, size_t *pspace) +{ + pyre64 *pyre; + pyre = heap->head; + align_size64(size); + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block64(heap, pyre64, pyre, size)) + { + pyre = heap64_new(heap); + *pspace = pyre->left; + } + else + { + pyre = heap64_sole(heap, size); + *pspace = size; + } + return void_data(pyre->data); +} + +void * heap8_more (heap8 *heap, void *taken, size_t written, size_t size, size_t *pspace) +{ + pyre8 *pyre, *prev; + pyre = heap->head; + align_size8(size); + if (taken_from_head(taken, pyre)) + { + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block8(heap, pyre8, pyre, size)) + { + pyre = heap8_new(heap); + memcpy(pyre->data, taken, written); + *pspace = pyre->left; + } + else + { + pyre = heap8_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + } + } + else if (taken_from_sole(taken, pyre, prev)) + { + pyre = heap8_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + pyre->prev = prev->prev; + pyre_free(prev); + } + else + { + ASSERT8(0); + *pspace = 0; + return NULL; + } + return void_data(pyre->data); +} + +void * heap16_more (heap16 *heap, void *taken, size_t written, size_t size, size_t *pspace) +{ + pyre16 *pyre, *prev; + pyre = heap->head; + align_size16(size); + if (taken_from_head(taken, pyre)) + { + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block16(heap, pyre16, pyre, size)) + { + pyre = heap16_new(heap); + memcpy(pyre->data, taken, written); + *pspace = pyre->left; + } + else + { + pyre = heap16_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + } + } + else if (taken_from_sole(taken, pyre, prev)) + { + pyre = heap16_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + pyre->prev = prev->prev; + pyre_free(prev); + } + else + { + ASSERT16(0); + *pspace = 0; + return NULL; + } + return void_data(pyre->data); +} + +void * heap32_more (heap32 *heap, void *taken, size_t written, size_t size, size_t *pspace) +{ + pyre32 *pyre, *prev; + pyre = heap->head; + align_size32(size); + if (taken_from_head(taken, pyre)) + { + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block32(heap, pyre32, pyre, size)) + { + pyre = heap32_new(heap); + memcpy(pyre->data, taken, written); + *pspace = pyre->left; + } + else + { + pyre = heap32_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + } + } + else if (taken_from_sole(taken, pyre, prev)) + { + pyre = heap32_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + pyre->prev = prev->prev; + pyre_free(prev); + } + else + { + ASSERT32(0); + *pspace = 0; + return NULL; + } + return void_data(pyre->data); +} + +void * heap64_more (heap64 *heap, void *taken, size_t written, size_t size, size_t *pspace) +{ + pyre64 *pyre, *prev; + pyre = heap->head; + align_size64(size); + if (taken_from_head(taken, pyre)) + { + if (size <= pyre->left) + { + *pspace = pyre->left; + } + else if (take_new_block64(heap, pyre64, pyre, size)) + { + pyre = heap64_new(heap); + memcpy(pyre->data, taken, written); + *pspace = pyre->left; + } + else + { + pyre = heap64_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + } + } + else if (taken_from_sole(taken, pyre, prev)) + { + pyre = heap64_sole(heap, size); + memcpy(pyre->data, taken, written); + *pspace = size; + pyre->prev = prev->prev; + pyre_free(prev); + } + else + { + ASSERT64(0); + *pspace = 0; + return NULL; + } + return void_data(pyre->data); +} + +void heap8_done (heap8 *heap, void *taken, size_t written) +{ + pyre8 *pyre; + pyre = heap->head; + align_size8(written); + if (taken_from_head(taken, pyre)) + { + pyre->data += written; + pyre->left -= (uint8_t)written; + ++pyre->chunks; + } + else if (taken_from_sole(taken, pyre, pyre)) + { + pyre->data += written; + pyre->chunks = 1; + } + else + { + ASSERT8(0); + } +} + +void heap16_done (heap16 *heap, void *taken, size_t written) +{ + pyre16 *pyre; + pyre = heap->head; + align_size16(written); + if (taken_from_head(taken, pyre)) + { + pyre->data += written; + pyre->left -= (uint16_t)written; + ++pyre->chunks; + } + else if (taken_from_sole(taken, pyre, pyre)) + { + pyre->data += written; + pyre->chunks = 1; + } + else + { + ASSERT16(0); + } +} + +void heap32_done (heap32 *heap, void *taken, size_t written) +{ + pyre32 *pyre; + pyre = heap->head; + align_size32(written); + if (taken_from_head(taken, pyre)) + { + pyre->data += written; + pyre->left -= (uint32_t)written; + ++pyre->chunks; + } + else if (taken_from_sole(taken, pyre, pyre)) + { + pyre->data += written; + pyre->chunks = 1; + } + else + { + ASSERT32(0); + } +} + +void heap64_done (heap64 *heap, void *taken, size_t written) +{ + pyre64 *pyre; + pyre = heap->head; + align_size64(written); + if (taken_from_head(taken, pyre)) + { + pyre->data += written; + pyre->left -= (uint64_t)written; + ++pyre->chunks; + } + else if (taken_from_sole(taken, pyre, pyre)) + { + pyre->data += written; + pyre->chunks = 1; + } + else + { + ASSERT64(0); + } +} + +/* giveup */ + +void heap8_giveup (heap8 *heap, void *taken) +{ + pyre8 *head, *pyre; + head = heap->head; + if (taken_from_sole(taken, head, pyre)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } +} + +void heap16_giveup (heap16 *heap, void *taken) +{ + pyre16 *head, *pyre; + head = heap->head; + if (taken_from_sole(taken, head, pyre)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } +} + +void heap32_giveup (heap32 *heap, void *taken) +{ + pyre32 *head, *pyre; + head = heap->head; + if (taken_from_sole(taken, head, pyre)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } +} + +void heap64_giveup (heap64 *heap, void *taken) +{ + pyre64 *head, *pyre; + head = heap->head; + if (taken_from_sole(taken, head, pyre)) + { + head->prev = pyre->prev; + pyre_free(pyre); + } +} + +/* heap empty */ + +int heap8_empty (heap8 *heap) +{ + pyre8 *pyre; + return head_block_empty(heap, pyre); +} + +int heap16_empty (heap16 *heap) +{ + pyre16 *pyre; + return head_block_empty(heap, pyre); +} + +int heap32_empty (heap32 *heap) +{ + pyre32 *pyre; + return head_block_empty(heap, pyre); +} + +int heap64_empty (heap64 *heap) +{ + pyre64 *pyre; + return head_block_empty(heap, pyre); +} + +/* heap stats */ + +void heap8_stats (heap8 *heap, mem_info *info, int append) +{ + pyre8 *pyre; + size_t used, chunks = 0, blocks = 0, singles = 0; + if (!append) + memset(info, 0, sizeof(mem_info)); + for (pyre = heap->head; pyre != NULL; pyre = pyre->prev) + { + ++blocks; + chunks += pyre->chunks; + used = block_used8(pyre); + info->used += used; + info->left += pyre->left; + if (pyre->chunks == 1 && pyre->left == 0) + { + ++singles; + info->singleused += used; + } + } + info->chunks += chunks; + info->blocks += blocks; + info->blockghosts += blocks * sizeof(pyre8); + info->singles += singles; + info->singleghosts += singles * sizeof(pyre8); +} + +void heap16_stats (heap16 *heap, mem_info *info, int append) +{ + pyre16 *pyre; + size_t used, chunks = 0, blocks = 0, singles = 0; + if (!append) + memset(info, 0, sizeof(mem_info)); + for (pyre = heap->head; pyre != NULL; pyre = pyre->prev) + { + ++blocks; + chunks += pyre->chunks; + used = block_used16(pyre); + info->used += used; + info->left += pyre->left; + if (pyre->chunks == 1 && pyre->left == 0) + { + ++singles; + info->singleused += used; + } + } + info->chunks += chunks; + info->blocks += blocks; + info->blockghosts += blocks * sizeof(pyre16); + info->singles += singles; + info->singleghosts += singles * sizeof(pyre16); +} + +void heap32_stats (heap32 *heap, mem_info *info, int append) +{ + pyre32 *pyre; + size_t used, chunks = 0, blocks = 0, singles = 0; + if (!append) + memset(info, 0, sizeof(mem_info)); + for (pyre = heap->head; pyre != NULL; pyre = pyre->prev) + { + ++blocks; + chunks += pyre->chunks; + used = block_used32(pyre); + info->used += used; + info->left += pyre->left; + if (pyre->chunks == 1 && pyre->left == 0) + { + ++singles; + info->singleused += used; + } + } + info->chunks += chunks; + info->blocks += blocks; + info->blockghosts += blocks * sizeof(pyre32); + info->singles += singles; + info->singleghosts += singles * sizeof(pyre32); +} + +void heap64_stats (heap64 *heap, mem_info *info, int append) +{ + pyre64 *pyre; + size_t used, chunks = 0, blocks = 0, singles = 0; + if (!append) + memset(info, 0, sizeof(mem_info)); + for (pyre = heap->head; pyre != NULL; pyre = pyre->prev) + { + ++blocks; + chunks += pyre->chunks; + used = block_used64(pyre); + info->used += used; + info->left += pyre->left; + if (pyre->chunks == 1 && pyre->left == 0) + { + ++singles; + info->singleused += used; + } + } + info->chunks += chunks; + info->blocks += blocks; + info->blockghosts += blocks * sizeof(pyre64); + info->singles += singles; + info->singleghosts += singles * sizeof(pyre64); +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmemheap.h b/Build/source/libs/pplib/pplib-src/src/util/utilmemheap.h new file mode 100644 index 00000000000..8776419c284 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmemheap.h @@ -0,0 +1,188 @@ + +#ifndef UTIL_MEM_HEAP_H +#define UTIL_MEM_HEAP_H + +#include "utilmemallh.h" + +typedef struct pyre8 pyre8; +typedef struct pyre16 pyre16; +typedef struct pyre32 pyre32; +typedef struct pyre64 pyre64; + +struct pyre8 { + pyre8 *prev; + uint8_t *data; + uint8_t left; + uint8_t chunks; +#ifdef BIT32 + uint8_t dummy[2]; // 10->12 +#else + uint8_t dummy[6]; // 18->24 +#endif +}; + +struct pyre16 { + pyre16 *prev; + uint8_t *data; + uint16_t left; + uint16_t chunks; +#ifdef BIT32 + //uint8_t dummy[0]; // 12->12 +#else + uint8_t dummy[4]; // 20->24 +#endif +}; + +struct pyre32 { + pyre32 *prev; + uint8_t *data; + uint32_t left; + uint32_t chunks; +#ifdef BIT32 + //uint8_t dummy[0]; // 16->16 +#else + //uint8_t dummy[0]; // 24->24 +#endif +}; + +struct pyre64 { + pyre64 *prev; + uint8_t *data; + uint64_t left; + uint64_t chunks; +#ifdef BIT32 + //uint8_t dummy[0]; // 24->24 +#else + //uint8_t dummy[0]; // 32->32 +#endif +}; + +/* heaps */ + +typedef struct heap8 heap8; +typedef struct heap16 heap16; +typedef struct heap32 heap32; +typedef struct heap64 heap64; + +struct heap8 { + pyre8 *head; + uint8_t space; + uint8_t large; + uint8_t flags; +}; + +struct heap16 { + pyre16 *head; + uint16_t space; + uint16_t large; + uint8_t flags; +}; + +struct heap32 { + pyre32 *head; + uint32_t space; + uint32_t large; + uint8_t flags; +}; + +struct heap64 { + pyre64 *head; + uint64_t space; + uint64_t large; + uint8_t flags; +}; + +#define HEAP_ZERO (1 << 0) +#define HEAP_DEFAULTS 0 + +#define HEAP8_INIT(space, large, flags) { NULL, aligned_space8(space), large, flags } +#define HEAP16_INIT(space, large, flags) { NULL, aligned_space16(space), large, flags } +#define HEAP32_INIT(space, large, flags) { NULL, aligned_space32(space), large, flags } +#define HEAP64_INIT(space, large, flags) { NULL, aligned_space64(space), large, flags } + +UTILAPI heap8 * heap8_init (heap8 *heap, uint8_t space, uint8_t large, uint8_t flags); +UTILAPI heap16 * heap16_init (heap16 *heap, uint16_t space, uint16_t large, uint8_t flags); +UTILAPI heap32 * heap32_init (heap32 *heap, uint32_t space, uint32_t large, uint8_t flags); +UTILAPI heap64 * heap64_init (heap64 *heap, uint64_t space, uint64_t large, uint8_t flags); + +UTILAPI void heap8_head (heap8 *heap); +UTILAPI void heap16_head (heap16 *heap); +UTILAPI void heap32_head (heap32 *heap); +UTILAPI void heap64_head (heap64 *heap); + +#define heap8_ensure_head(heap) ((void)((heap)->head != NULL || (heap8_head(heap), 0))) +#define heap16_ensure_head(heap) ((void)((heap)->head != NULL || (heap16_head(heap), 0))) +#define heap32_ensure_head(heap) ((void)((heap)->head != NULL || (heap32_head(heap), 0))) +#define heap64_ensure_head(heap) ((void)((heap)->head != NULL || (heap64_head(heap), 0))) + +UTILAPI void heap8_free (heap8 *heap); +UTILAPI void heap16_free (heap16 *heap); +UTILAPI void heap32_free (heap32 *heap); +UTILAPI void heap64_free (heap64 *heap); + +UTILAPI void heap8_clear (heap8 *heap); +UTILAPI void heap16_clear (heap16 *heap); +UTILAPI void heap32_clear (heap32 *heap); +UTILAPI void heap64_clear (heap64 *heap); + +UTILAPI void * _heap8_take (heap8 *heap, size_t size); +UTILAPI void * _heap16_take (heap16 *heap, size_t size); +UTILAPI void * _heap32_take (heap32 *heap, size_t size); +UTILAPI void * _heap64_take (heap64 *heap, size_t size); + +UTILAPI void * _heap8_take0 (heap8 *heap, size_t size); +UTILAPI void * _heap16_take0 (heap16 *heap, size_t size); +UTILAPI void * _heap32_take0 (heap32 *heap, size_t size); +UTILAPI void * _heap64_take0 (heap64 *heap, size_t size); + +#define heap8_take(heap, size) (heap8_ensure_head(heap), _heap8_take(heap, size)) +#define heap16_take(heap, size) (heap16_ensure_head(heap), _heap16_take(heap, size)) +#define heap32_take(heap, size) (heap32_ensure_head(heap), _heap32_take(heap, size)) +#define heap64_take(heap, size) (heap64_ensure_head(heap), _heap64_take(heap, size)) + +#define heap8_take0(heap, size) (heap8_ensure_head(heap), _heap8_take0(heap, size)) +#define heap16_take0(heap, size) (heap16_ensure_head(heap), _heap16_take0(heap, size)) +#define heap32_take0(heap, size) (heap32_ensure_head(heap), _heap32_take0(heap, size)) +#define heap64_take0(heap, size) (heap64_ensure_head(heap), _heap64_take0(heap, size)) + +UTILAPI void heap8_pop (heap8 *heap, void *taken, size_t size); +UTILAPI void heap16_pop (heap16 *heap, void *taken, size_t size); +UTILAPI void heap32_pop (heap32 *heap, void *taken, size_t size); +UTILAPI void heap64_pop (heap64 *heap, void *taken, size_t size); + +UTILAPI void * _heap8_some (heap8 *heap, size_t size, size_t *pspace); +UTILAPI void * _heap16_some (heap16 *heap, size_t size, size_t *pspace); +UTILAPI void * _heap32_some (heap32 *heap, size_t size, size_t *pspace); +UTILAPI void * _heap64_some (heap64 *heap, size_t size, size_t *pspace); + +#define heap8_some(heap, size, pspace) (heap8_ensure_head(heap), _heap8_some(heap, size, pspace)) +#define heap16_some(heap, size, pspace) (heap16_ensure_head(heap), _heap16_some(heap, size, pspace)) +#define heap32_some(heap, size, pspace) (heap32_ensure_head(heap), _heap32_some(heap, size, pspace)) +#define heap64_some(heap, size, pspace) (heap64_ensure_head(heap), _heap64_some(heap, size, pspace)) + +UTILAPI void * heap8_more (heap8 *heap, void *taken, size_t written, size_t size, size_t *pspace); +UTILAPI void * heap16_more (heap16 *heap, void *taken, size_t written, size_t size, size_t *pspace); +UTILAPI void * heap32_more (heap32 *heap, void *taken, size_t written, size_t size, size_t *pspace); +UTILAPI void * heap64_more (heap64 *heap, void *taken, size_t written, size_t size, size_t *pspace); + +UTILAPI void heap8_done (heap8 *heap, void *taken, size_t written); +UTILAPI void heap16_done (heap16 *heap, void *taken, size_t written); +UTILAPI void heap32_done (heap32 *heap, void *taken, size_t written); +UTILAPI void heap64_done (heap64 *heap, void *taken, size_t written); + +UTILAPI void heap8_giveup (heap8 *heap, void *taken); +UTILAPI void heap16_giveup (heap16 *heap, void *taken); +UTILAPI void heap32_giveup (heap32 *heap, void *taken); +UTILAPI void heap64_giveup (heap64 *heap, void *taken); + +UTILAPI int heap8_empty (heap8 *heap); +UTILAPI int heap16_empty (heap16 *heap); +UTILAPI int heap32_empty (heap32 *heap); +UTILAPI int heap64_empty (heap64 *heap); + +UTILAPI void heap8_stats (heap8 *heap, mem_info *info, int append); +UTILAPI void heap16_stats (heap16 *heap, mem_info *info, int append); +UTILAPI void heap32_stats (heap32 *heap, mem_info *info, int append); +UTILAPI void heap64_stats (heap64 *heap, mem_info *info, int append); + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmemheapiof.c b/Build/source/libs/pplib/pplib-src/src/util/utilmemheapiof.c new file mode 100644 index 00000000000..cd9609da877 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmemheapiof.c @@ -0,0 +1,142 @@ + +#include "utilmemheapiof.h" + +// this is identical to stock iof suite, keep in sync + +size_t heap8_writer (iof *O, iof_mode mode) +{ + heap8 *heap; + size_t written; + heap = (heap8 *)O->link; + switch (mode) + { + case IOFFLUSH: + heap8_buffer_done(heap, O); + O->buf = _heap8_some(heap, 0, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + break; + case IOFWRITE: + written = (size_t)iof_size(O); + O->buf = heap8_more(heap, O->buf, written, written << 1, &O->space); + O->pos = O->buf + written; + O->end = O->buf + O->space; + return O->space - written; + case IOFCLOSE: + default: + break; + } + return 0; +} + +size_t heap16_writer (iof *O, iof_mode mode) +{ + heap16 *heap; + size_t written; + heap = (heap16 *)O->link; + switch (mode) + { + case IOFFLUSH: + heap16_buffer_done(heap, O); + O->buf = _heap16_some(heap, 0, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + break; + case IOFWRITE: + written = (size_t)iof_size(O); + O->buf = heap16_more(heap, O->buf, written, written << 1, &O->space); + O->pos = O->buf + written; + O->end = O->buf + O->space; + return O->space - written; + case IOFCLOSE: + default: + break; + } + return 0; +} + +size_t heap32_writer (iof *O, iof_mode mode) +{ + heap32 *heap; + size_t written; + heap = (heap32 *)O->link; + switch (mode) + { + case IOFFLUSH: + heap32_buffer_done(heap, O); + O->buf = _heap32_some(heap, 0, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + break; + case IOFWRITE: + written = (size_t)iof_size(O); + O->buf = heap32_more(heap, O->buf, written, written << 1, &O->space); + O->pos = O->buf + written; + O->end = O->buf + O->space; + return O->space - written; + case IOFCLOSE: + default: + break; + } + return 0; +} + +size_t heap64_writer (iof *O, iof_mode mode) +{ + heap64 *heap; + size_t written; + heap = (heap64 *)O->link; + switch (mode) + { + case IOFFLUSH: + heap64_buffer_done(heap, O); + O->buf = _heap64_some(heap, 0, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + break; + case IOFWRITE: + written = (size_t)iof_size(O); + O->buf = heap64_more(heap, O->buf, written, written << 1, &O->space); + O->pos = O->buf + written; + O->end = O->buf + O->space; + return O->space - written; + case IOFCLOSE: + default: + break; + } + return 0; +} + +/* buffer for some */ + +iof * _heap8_buffer_some (heap8 *heap, iof *O, size_t atleast) +{ + O->buf = _heap8_some(heap, atleast, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + return O; +} + +iof * _heap16_buffer_some (heap16 *heap, iof *O, size_t atleast) +{ + O->buf = _heap16_some(heap, atleast, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + return O; +} + +iof * _heap32_buffer_some (heap32 *heap, iof *O, size_t atleast) +{ + O->buf = _heap32_some(heap, atleast, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + return O; +} + +iof * _heap64_buffer_some (heap64 *heap, iof *O, size_t atleast) +{ + O->buf = _heap64_some(heap, atleast, &O->space); + O->pos = O->buf; + O->end = O->buf + O->space; + return O; +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmemheapiof.h b/Build/source/libs/pplib/pplib-src/src/util/utilmemheapiof.h new file mode 100644 index 00000000000..1f3da7efb2f --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmemheapiof.h @@ -0,0 +1,43 @@ + +#ifndef UTIL_MEM_HEAP_IOF_H +#define UTIL_MEM_HEAP_IOF_H + +#include "utilmemheap.h" +#include "utiliof.h" + +UTILAPI size_t heap8_writer (iof *O, iof_mode mode); +UTILAPI size_t heap16_writer (iof *O, iof_mode mode); +UTILAPI size_t heap32_writer (iof *O, iof_mode mode); +UTILAPI size_t heap64_writer (iof *O, iof_mode mode); + +#define HEAP8_BUFFER_INIT(heap) IOF_WRITER_INIT(heap8_writer, (void *)(heap), NULL, 0, 0) +#define HEAP16_BUFFER_INIT(heap) IOF_WRITER_INIT(heap16_writer, (void *)(heap), NULL, 0, 0) +#define HEAP32_BUFFER_INIT(heap) IOF_WRITER_INIT(heap32_writer, (void *)(heap), NULL, 0, 0) +#define HEAP64_BUFFER_INIT(heap) IOF_WRITER_INIT(heap64_writer, (void *)(heap), NULL, 0, 0) + +#define heap8_buffer_init(heap, O) iof_writer(O, (void *)(heap), heap8_writer, NULL, 0) +#define heap16_buffer_init(heap, O) iof_writer(O, (void *)(heap), heap16_writer, NULL, 0) +#define heap32_buffer_init(heap, O) iof_writer(O, (void *)(heap), heap32_writer, NULL, 0) +#define heap64_buffer_init(heap, O) iof_writer(O, (void *)(heap), heap64_writer, NULL, 0) + +UTILAPI iof * _heap8_buffer_some (heap8 *heap, iof *O, size_t atleast); +UTILAPI iof * _heap16_buffer_some (heap16 *heap, iof *O, size_t atleast); +UTILAPI iof * _heap32_buffer_some (heap32 *heap, iof *O, size_t atleast); +UTILAPI iof * _heap64_buffer_some (heap64 *heap, iof *O, size_t atleast); + +#define heap8_buffer_some(heap, O, atleast) (heap8_ensure_head(heap), _heap8_buffer_some(heap, O, atleast)) +#define heap16_buffer_some(heap, O, atleast) (heap16_ensure_head(heap), _heap16_buffer_some(heap, O, atleast)) +#define heap32_buffer_some(heap, O, atleast) (heap32_ensure_head(heap), _heap32_buffer_some(heap, O, atleast)) +#define heap64_buffer_some(heap, O, atleast) (heap64_ensure_head(heap), _heap64_buffer_some(heap, O, atleast)) + +#define heap8_buffer_done(heap, O) heap8_done(heap, (O)->buf, (size_t)iof_size(O)) +#define heap16_buffer_done(heap, O) heap16_done(heap, (O)->buf, (size_t)iof_size(O)) +#define heap32_buffer_done(heap, O) heap32_done(heap, (O)->buf, (size_t)iof_size(O)) +#define heap64_buffer_done(heap, O) heap64_done(heap, (O)->buf, (size_t)iof_size(O)) + +#define heap8_buffer_giveup(heap, O) heap8_giveup(heap, (O)->buf) +#define heap16_buffer_giveup(heap, O) heap16_giveup(heap, (O)->buf) +#define heap32_buffer_giveup(heap, O) heap32_giveup(heap, (O)->buf) +#define heap64_buffer_giveup(heap, O) heap64_giveup(heap, (O)->buf) + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmeminfo.c b/Build/source/libs/pplib/pplib-src/src/util/utilmeminfo.c new file mode 100644 index 00000000000..d3f61d5cac9 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmeminfo.c @@ -0,0 +1,38 @@ +/* print stats; common for heap, stock and pool */ + +#include <stdio.h> + +#include "utilmeminfo.h" + +#define UINT(i) ((unsigned long)(i)) + +void show_mem_info (mem_info *info) +{ + size_t totalwaste, totalmem, averagechunk, singlemem; + double ftotalwaste, fblockwaste, fghostwaste, ftailwaste, fsinglewaste; + double funused, fsingles, fsinglemem, fsingleeff; + + totalwaste = info->ghosts + info->blockghosts + info->left; + totalmem = info->used + totalwaste; + + ftotalwaste = totalmem > 0 ? totalwaste * 100.0 / totalmem : 0; + fblockwaste = totalmem > 0 ? (info->blockghosts - info->singleghosts) * 100.0 / totalmem : 0; + fsinglewaste = totalmem > 0 ? info->singleghosts * 100.0 / totalmem : 0; + fghostwaste = totalmem > 0 ? info->ghosts * 100.0 / totalmem : 0; + ftailwaste = totalmem > 0 ? info->left * 100.0 / totalmem : 0; + + averagechunk = info->chunks > 0 ? info->used / info->chunks : 0; + funused = info->chunks > 0 ? info->unused * 100.0 / info->chunks : 0.0; + + fsingles = info->blocks > 0 ? info->singles * 100.0 / info->blocks : 0; + fsinglemem = info->used > 0 ? info->singleused * 100.0 / info->used : 0; + singlemem = info->singleused + info->singleghosts; + fsingleeff = singlemem > 0 ? info->singleused * 100.0 / singlemem : 0; + + printf("total: %lu + %lu = %lu\n", UINT(info->used), UINT(totalwaste), UINT(totalmem)); + printf("chunks: %lu of average size %lu, unused %lu[%.2f%%]\n", UINT(info->chunks), UINT(averagechunk), UINT(info->unused), funused); + printf("blocks: %lu, singles %lu[%.2f%%], %.2f%% of allocs, efficiency %.2f%%\n", + UINT(info->blocks), UINT(info->singles), fsingles, fsinglemem, fsingleeff); + printf("waste: %lu[%0.2f%%], block ghosts %0.2f%%, single ghosts %.2f%%, chunk ghosts %0.2f%%, tails %0.2f%%\n\n", + UINT(totalwaste), ftotalwaste, fblockwaste, fsinglewaste, fghostwaste, ftailwaste); +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilmeminfo.h b/Build/source/libs/pplib/pplib-src/src/util/utilmeminfo.h new file mode 100644 index 00000000000..cfa0fd670ac --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilmeminfo.h @@ -0,0 +1,9 @@ + +#ifndef UTIL_MEM_INFO_H +#define UTIL_MEM_INFO_H + +#include "utilmemallh.h" + +UTILAPI void show_mem_info (mem_info *info); + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilnumber.c b/Build/source/libs/pplib/pplib-src/src/util/utilnumber.c new file mode 100644 index 00000000000..4352c26fbd0 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilnumber.c @@ -0,0 +1,1177 @@ + +#include <math.h> /* for log10() and floor() */ +#include <stdio.h> /* for printf() */ + +#include "utilnumber.h" + +// todo: lookups can be chars +// change lookup arrays to some __name to discourage accessing them directly; they always should be accessed via macros; base16_value() base16_digit() +// + +const int base10_lookup[] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +}; + +const int base16_lookup[] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1, + -1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +}; + +const int base26_lookup[] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15, + 16,17,18,19,20,21,22,23,24,25,26,-1,-1,-1,-1,-1, + -1, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15, + 16,17,18,19,20,21,22,23,24,25,26,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +}; + +const int base36_lookup[] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1, + -1,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24, + 25,26,27,28,29,30,31,32,33,34,35,-1,-1,-1,-1,-1, + -1,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24, + 25,26,27,28,29,30,31,32,33,34,35,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +}; + +/* common buffer for quick conversions (unsafe) */ + +char util_number_buffer[NUMBER_BUFFER_SIZE] = { 0 }; + +/* integer from string; return a pointer to character next to the last digit */ + +#define string_scan_sign(s, c, sign) _scan_sign(c, sign, *++s) +#define string_scan_integer(s, c, number) _scan_integer(c, number, *++s) +#define string_scan_radix(s, c, number, radix) _scan_radix(c, number, radix, *++s) +#define string_read_integer(s, c, number) _read_integer(c, number, *++s) +#define string_read_radix(s, c, number, radix) _read_radix(c, number, radix, *++s) + +const char * string_to_int32 (const char *s, int32_t *number) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_integer(s, c, *number); + if (sign) *number = -*number; + return s; +} + +const char * string_to_slong (const char *s, long *number) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_integer(s, c, *number); + if (sign) *number = -*number; + return s; +} + +const char * string_to_int64 (const char *s, int64_t *number) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_integer(s, c, *number); + if (sign) *number = -*number; + return s; +} + +const char * string_to_uint32 (const char *s, uint32_t *number) +{ + int c = *s; + string_scan_integer(s, c, *number); + return s; +} + +const char * string_to_ulong (const char *s, unsigned long *number) +{ + int c = *s; + string_scan_integer(s, c, *number); + return s; +} + +const char * string_to_usize (const char *s, size_t *number) +{ + int c = *s; + string_scan_integer(s, c, *number); + return s; +} + +const char * string_to_uint64 (const char *s, uint64_t *number) +{ + int c = *s; + string_scan_integer(s, c, *number); + return s; +} + +const char * radix_to_int32 (const char *s, int32_t *number, int radix) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_radix(s, c, *number, radix); + if (sign) *number = -*number; + return s; +} + +const char * radix_to_slong (const char *s, long *number, int radix) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_radix(s, c, *number, radix); + if (sign) *number = -*number; + return s; +} + +const char * radix_to_int64 (const char *s, int64_t *number, int radix) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_radix(s, c, *number, radix); + if (sign) *number = -*number; + return s; +} + +const char * radix_to_uint32 (const char *s, uint32_t *number, int radix) +{ + int c = *s; + string_scan_radix(s, c, *number, radix); + return s; +} + +const char * radix_to_ulong (const char *s, unsigned long *number, int radix) +{ + int c = *s; + string_scan_radix(s, c, *number, radix); + return s; +} + +const char * radix_to_usize (const char *s, size_t *number, int radix) +{ + int c = *s; + string_scan_radix(s, c, *number, radix); + return s; +} + +const char * radix_to_uint64 (const char *s, uint64_t *number, int radix) +{ + int c = *s; + string_scan_radix(s, c, *number, radix); + return s; +} + +/* roman to uint16_t */ + +#define roman1000(c) (c == 'M' || c == 'm') +#define roman500(c) (c == 'D' || c == 'd') +#define roman100(c) (c == 'C' || c == 'c') +#define roman50(c) (c == 'L' || c == 'l') +#define roman10(c) (c == 'X' || c == 'x') +#define roman5(c) (c == 'V' || c == 'v') +#define roman1(c) (c == 'I' || c == 'i') + +#define roman100s(p) (roman100(*p) ? (100 + ((++p, roman100(*p)) ? (100 + ((++p, roman100(*p)) ? (++p, 100) : 0)) : 0)) : 0) +#define roman10s(p) (roman10(*p) ? (10 + ((++p, roman10(*p)) ? (10 + ((++p, roman10(*p)) ? (++p, 10) : 0)) : 0)) : 0) +#define roman1s(p) (roman1(*p) ? (1 + ((++p, roman1(*p)) ? (1 + ((++p, roman1(*p)) ? (++p, 1) : 0)) : 0)) : 0) + +const char * roman_to_uint16 (const char *s, uint16_t *number) +{ + const char *p; + /* M */ + for (*number = 0, p = s; roman1000(*p); *number += 1000, ++p); + /* D C */ + if (roman500(*p)) + { + ++p; + *number += 500 + roman100s(p); + } + else if (roman100(*p)) + { + ++p; + if (roman1000(*p)) + { + ++p; + *number += 900; + } + else if (roman500(*p)) + { + ++p; + *number += 400; + } + else + *number += 100 + roman100s(p); + } + /* L X */ + if (roman50(*p)) + { + ++p; + *number += 50 + roman10s(p); + } + else if (roman10(*p)) + { + ++p; + if (roman100(*p)) + { + ++p; + *number += 90; + } + else if (roman50(*p)) + { + ++p; + *number += 40; + } + else + *number += 10 + roman10s(p); + } + /* V I */ + if (roman5(*p)) + { + ++p; + *number += 5 + roman1s(p); + } + else if (roman1(*p)) + { + ++p; + if (roman10(*p)) + { + ++p; + *number += 9; + } + else if (roman5(*p)) + { + ++p; + *number += 4; + } + else + *number += 1 + roman1s(p); + } + return p; +} + +/* integer to string; return a pointer to null-terminated static const string */ + +#define end_of_integer_buffer(integer_buffer) (integer_buffer + MAX_INTEGER_DIGITS - 1) + +#define number_printrev_signed(p, number, quotient) \ + do { \ + quotient = number; number /= 10; \ + *--p = base10_palindrome[9 + (quotient - number*10)]; \ + } while (number); \ + if (quotient < 0) *--p = '-' + +#define number_printrev_unsigned(p, number, quotient) \ + do { \ + quotient = number; number /= 10; \ + *--p = (char)(quotient - integer_multiplied10(number)) + '0'; \ + } while (number) + +#define SINTTYPE_AS_STRING(inttype, number, ibuf, psize) \ + char *p, *e; \ + inttype quotient; \ + e = p = end_of_integer_buffer(ibuf); *p = '\0'; \ + number_printrev_signed(p, number, quotient); \ + *psize = (size_t)(e - p) + +#define UINTTYPE_AS_STRING(inttype, number, ibuf, psize) \ + char *p, *e; \ + inttype quotient; \ + e = p = end_of_integer_buffer(ibuf); *p = '\0'; \ + number_printrev_unsigned(p, number, quotient); \ + *psize = (size_t)(e - p) + +char * int32_as_string (int32_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + SINTTYPE_AS_STRING(int32_t, number, ibuf, psize); + return p; +} + +char * slong_as_string (long number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + SINTTYPE_AS_STRING(long, number, ibuf, psize); + return p; +} + +char * int64_as_string (int64_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + SINTTYPE_AS_STRING(int64_t, number, ibuf, psize); + return p; +} + +char * uint32_as_string (uint32_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_STRING(uint32_t, number, ibuf, psize); + return p; +} + +char * ulong_as_string (unsigned long number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_STRING(unsigned long, number, ibuf, psize); + return p; +} + +char * usize_as_string (size_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_STRING(size_t, number, ibuf, psize); + return p; +} + +char * uint64_as_string (uint64_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_STRING(uint64_t, number, ibuf, psize); + return p; +} + +/* radix variant */ + +#define number_printrev_signed_radix_uc(p, number, radix, quotient) \ + do { \ + quotient = number; number /= radix; \ + *--p = base36_uc_palindrome[MAX_RADIX - 1 + (quotient - number*radix)]; \ + } while (number) + +#define number_printrev_signed_radix_lc(p, number, radix, quotient) \ + do { \ + quotient = number; number /= radix; \ + *--p = base36_lc_palindrome[MAX_RADIX - 1 + (quotient - number*radix)]; \ + } while (number) + +#define number_printrev_signed_radix(p, number, radix, quotient, uc) \ + do { \ + if (uc) { number_printrev_signed_radix_uc(p, number, radix, quotient); } \ + else { number_printrev_signed_radix_lc(p, number, radix, quotient); } \ + if (quotient < 0) *--p = '-'; \ + } while (0) + +#define number_printrev_unsigned_radix_uc(p, number, radix, quotient) \ + do { \ + quotient = number; number /= radix; \ + *--p = base36_uc_alphabet[quotient % radix]; \ + } while (number) + +#define number_printrev_unsigned_radix_lc(p, number, radix, quotient) \ + do { \ + quotient = number; number /= radix; \ + *--p = base36_lc_alphabet[quotient % radix]; \ + } while (number) + +#define number_printrev_unsigned_radix(p, number, radix, quotient, uc) \ + do { \ + if (uc) { number_printrev_unsigned_radix_uc(p, number, radix, quotient); } \ + else { number_printrev_unsigned_radix_lc(p, number, radix, quotient); } \ + } while (0) + +#define SINTTYPE_AS_RADIX(inttype, number, radix, uc, ibuf, psize) \ + char *p, *e; \ + inttype quotient; \ + e = p = end_of_integer_buffer(ibuf); *p = '\0'; \ + number_printrev_signed_radix(p, number, radix, quotient, uc); \ + *psize = (size_t)(e - p) + +#define UINTTYPE_AS_RADIX(inttype, number, radix, uc, ibuf, psize) \ + char *p, *e; \ + inttype quotient; \ + e = p = end_of_integer_buffer(ibuf); *p = '\0'; \ + number_printrev_unsigned_radix(p, number, radix, quotient, uc); \ + *psize = (size_t)(e - p) + +char * int32_as_radix (int32_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + SINTTYPE_AS_RADIX(int32_t, number, radix, uc, ibuf, psize); + return p; +} + +char * slong_as_radix (long number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + SINTTYPE_AS_RADIX(long, number, radix, uc, ibuf, psize); + return p; +} + +/* +char * ssize_as_radix (ssize_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + SINTTYPE_AS_RADIX(ssize_t, number, radix, uc, ibuf, psize); + return p; +} +*/ + +char * int64_as_radix (int64_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + SINTTYPE_AS_RADIX(int64_t, number, radix, uc, ibuf, psize); + return p; +} + +char * uint32_as_radix (uint32_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_RADIX(uint32_t, number, radix, uc, ibuf, psize); + return p; +} + +char * ulong_as_radix (unsigned long number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_RADIX(unsigned long, number, radix, uc, ibuf, psize); + return p; +} + +char * usize_as_radix (size_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_RADIX(size_t, number, radix, uc, ibuf, psize); + return p; +} + +char * uint64_as_radix (uint64_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_RADIX(uint64_t, number, radix, uc, ibuf, psize); + return p; +} + +/* aaa, aab, aac, ...; unsigned only. 0 gives empty string */ + +#define string_scan_alpha(s, c, number, radix) \ + for (number = 0, c = *s; (c = base26_value(c)) > 0; number = number * radix + c, c = *++s) + +const char * alpha_to_uint32 (const char *s, uint32_t *number) +{ + int c; + string_scan_alpha(s, c, *number, 26); + return s; +} + +const char * alpha_to_ulong (const char *s, unsigned long *number) +{ + int c; + string_scan_alpha(s, c, *number, 26); + return s; +} + +const char * alpha_to_usize (const char *s, size_t *number) +{ + int c; + string_scan_alpha(s, c, *number, 26); + return s; +} + +const char * alpha_to_uint64 (const char *s, uint64_t *number) +{ + int c; + string_scan_alpha(s, c, *number, 26); + return s; +} + +#define number_printrev_unsigned_alpha_uc(p, number, radix, quotient) \ + while (number > 0) { \ + quotient = --number; number /= radix; \ + *--p = base26_uc_alphabet[quotient % radix]; \ + } + +#define number_printrev_unsigned_alpha_lc(p, number, radix, quotient) \ + while (number > 0) { \ + quotient = --number; number /= radix; \ + *--p = base26_lc_alphabet[quotient % radix]; \ + } + +#define UINTTYPE_AS_ALPHA(inttype, number, uc, ibuf, psize) \ + char *p, *e; \ + inttype quotient; \ + e = p = end_of_integer_buffer(ibuf); *p = '\0'; \ + if (uc) { number_printrev_unsigned_alpha_uc(p, number, 26, quotient); } \ + else { number_printrev_unsigned_alpha_lc(p, number, 26, quotient); } \ + *psize = (size_t)(e - p) + +char * uint32_as_alpha (uint32_t number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_ALPHA(uint32_t, number, uc, ibuf, psize); + return p; +} + +char * ulong_as_alpha (unsigned long number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_ALPHA(unsigned long, number, uc, ibuf, psize); + return p; +} + +char * usize_as_alpha (size_t number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_ALPHA(size_t, number, uc, ibuf, psize); + return p; +} + +char * uint64_as_alpha (uint64_t number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize) +{ + UINTTYPE_AS_ALPHA(uint64_t, number, uc, ibuf, psize); + return p; +} + +/* a variant of alphabetic, a, b, c, ..., z, aa, bb, cc, ..., zz (eg. pdf page labelling) + watch out: unsafe for large numbers; for buffer size N we can handle max. N * 26. */ + +#define string_scan_alphan(s, c, number, radix) \ + do { \ + number = 0; \ + if ((c = (uint16_t)base26_value(*s)) > 0) { \ + number = c; \ + while (c == (uint16_t)base26_value(*++s)) number += radix; \ + } \ + } while (0) + +const char * alphan_to_uint16 (const char *s, uint16_t *number) +{ + uint16_t c; + string_scan_alphan(s, c, *number, 26); + return s; +} + +#define number_print_alphan_uc(p, e, c, number, radix) \ + for (c = (--number) % radix, number -= c; ; number -= radix) { \ + *p++ = base26_uc_alphabet[c]; \ + if (number == 0 || p >= e) break; \ + } + +#define number_print_alphan_lc(p, e, c, number, radix) \ + for (c = (--number) % radix, number -= c; ; number -= radix) { \ + *p++ = base26_lc_alphabet[c]; \ + if (number == 0 || p >= e) break; \ + } + +#define UINTTYPE_AS_ALPHAN(inttype, number, uc, ibuf, size, psize) \ + char *p, *e; \ + uint8_t c; \ + p = ibuf; \ + e = p + size; \ + if (number > 0) { \ + if (uc) { number_print_alphan_uc(p, e, c, number, 26); } \ + else { number_print_alphan_lc(p, e, c, number, 26); } \ + } \ + *p = '\0'; \ + *psize = (size_t)(p - ibuf) + +char * uint16_as_alphan (uint16_t number, int uc, char ibuf[], size_t size, size_t *psize) +{ + UINTTYPE_AS_ALPHAN(uint16_t, number, uc, ibuf, size, psize); + return ibuf; +} + +/* roman numeral */ + +/* large roman numerals? http://mathforum.org/library/drmath/view/57569.html */ + +#define base_roman_uc_alphabet "MDCLXVI" +#define base_roman_lc_alphabet "mdclxvi" + +char * uint16_as_roman (uint16_t number, int uc, char ibuf[MAX_ROMAN_DIGITS], size_t *psize) +{ + static const uint32_t base_roman_values[] = { 1000, 500, 100, 50, 10, 5, 1 }; + const char *alphabet; + char *p; + uint32_t k, j, v, u, n; + + n = (uint32_t)number; // uint16_t used to limit leding 'M' + alphabet = uc ? base_roman_uc_alphabet : base_roman_lc_alphabet; + for (p = ibuf, j = 0, v = base_roman_values[0]; n > 0; ) + { + if (n >= v) + { + *p++ = alphabet[j]; + n -= v; + continue; + } + if (j & 1) + k = j + 1; + else + k = j + 2; + u = base_roman_values[k]; + if (n + u >= v) + { + *p++ = alphabet[k]; + n += u; + } + else + v = base_roman_values[++j]; + } + *p = '\0'; + *psize = (size_t)(p - ibuf); + return ibuf; +} + +/* IEEE-754 */ + +#define BINARY_MODF 1 + +#define NOT_A_NUMBER_STRING "NaN" +#define INFINITY_STRING "INF" +#define SIGNED_INFINITY 1 +#define SIGNED_ZERO 0 +#define SIGNED_NOT_A_NUMBER 0 +#define RADIX_CHAR '.' + +/* double/float to decimal */ + +typedef struct ieee_double { + union { + double number; + uint64_t bits; + }; + uint64_t fraction; + int exponent, sign; +} ieee_double; + +typedef struct ieee_float { + union { + float number; + uint32_t bits; + }; + uint32_t fraction; + int exponent, sign; +} ieee_float; + +#define IEEE_DOUBLE_BIAS 1023 +#define IEEE_DOUBLE_MIN_EXPONENT -1023 +#define IEEE_DOUBLE_MAX_EXPONENT (0x7ff - IEEE_DOUBLE_BIAS) + +#define IEEE_FLOAT_BIAS 127 +#define IEEE_FLOAT_MIN_EXPONENT -127 +#define IEEE_FLOAT_MAX_EXPONENT (0xff - IEEE_FLOAT_BIAS) + +#define ieee_double_fraction(i) (i & 0x000fffffffffffffull) +#define ieee_double_exponent(i) ((0x7ff & (i >> 52)) - IEEE_DOUBLE_BIAS) +#define ieee_double_init(ieee_number, number) \ + ieee_number.number = number, \ + ieee_number.fraction = ieee_double_fraction(ieee_number.bits), \ + ieee_number.exponent = ieee_double_exponent(ieee_number.bits) + +#define ieee_float_fraction(i) (i & 0x007fffff) +#define ieee_float_exponent(i) ((0xff & (i >> 23)) - IEEE_FLOAT_BIAS) +#define ieee_float_init(ieee_number, number) \ + ieee_number.number = number, \ + ieee_number.fraction = ieee_float_fraction(ieee_number.bits), \ + ieee_number.exponent = ieee_float_exponent(ieee_number.bits) + +/* special cases */ + +#define ieee_double_is_zero(ieee_number) (ieee_number.number == 0) // || ieee_double_too_small(ieee_number) ? +#define ieee_double_too_small(ieee_number) (ieee_number.exponent == 0 && ieee_number.fraction != 0) // denormalized, implicit fracion bit not set + +#define ieee_float_is_zero(ieee_number) (ieee_number.number == 0) // || ieee_float_too_small(ieee_number) ? +#define ieee_float_too_small(ieee_number) (ieee_number.exponent == 0 && ieee_number.fraction != 0) + +#define ieee_double_zero_string(ieee_number) (SIGNED_ZERO && ieee_number.sign ? "-0" : "0") +#define ieee_double_infinity_string(ieee_number) (SIGNED_INFINITY && ieee_number.sign ? "-" INFINITY_STRING : INFINITY_STRING) + +#define ieee_float_zero_string ieee_double_zero_string +#define ieee_float_infinity_string ieee_double_infinity_string + +#define ieee_double_special_case(ieee_number) (ieee_number.exponent == IEEE_DOUBLE_MAX_EXPONENT) +#define ieee_double_special_string(ieee_number) (ieee_number.fraction ? NOT_A_NUMBER_STRING : ieee_double_infinity_string(ieee_number)) + +#define ieee_float_special_case(ieee_number) (ieee_number.exponent == IEEE_FLOAT_MAX_EXPONENT) +#define ieee_float_special_string(ieee_number) (ieee_number.fraction ? NOT_A_NUMBER_STRING : ieee_float_infinity_string(ieee_number)) + +#if 0 + +const double double_binary_power10[] = +{ + 1.0e1, 1.0e2, 1.0e4, 1.0e8, 1.0e16, 1.0e32, 1.0e64, 1.0e128, 1.0e256 +}; + +const float float_binary_power10[] = +{ + 1.0e1, 1.0e2, 1.0e4, 1.0e8, 1.0e16, 1.0e32 +}; + +const double double_binary_negpower10[] = +{ + 1.0e-1, 1.0e-2, 1.0e-4, 1.0e-8, 1.0e-16, 1.0e-32 +}; + +const float float_binary_negpower10[] = +{ + 1.0e-1, 1.0e-2, 1.0e-4, 1.0e-8, 1.0e-16, 1.0e-32 +}; + +#else + +const double double_decimal_power10[] = { + 1.0e0, 1.0e1, 1.0e2, 1.0e3, 1.0e4, 1.0e5, 1.0e6, 1.0e7, 1.0e8, 1.0e9, + 1.0e10, 1.0e11, 1.0e12, 1.0e13, 1.0e14, 1.0e15, 1.0e16, 1.0e17, 1.0e18, 1.0e19, + 1.0e20, 1.0e21, 1.0e22, 1.0e23, 1.0e24, 1.0e25, 1.0e26, 1.0e27, 1.0e28, 1.0e29, + 1.0e30, 1.0e31, 1.0e32, 1.0e33, 1.0e34, 1.0e35, 1.0e36, 1.0e37, 1.0e38, 1.0e39, + 1.0e40, 1.0e41, 1.0e42, 1.0e43, 1.0e44, 1.0e45, 1.0e46, 1.0e47, 1.0e48, 1.0e49, + 1.0e50, 1.0e51, 1.0e52, 1.0e53, 1.0e54, 1.0e55, 1.0e56, 1.0e57, 1.0e58, 1.0e59, + 1.0e60, 1.0e61, 1.0e62, 1.0e63, 1.0e64, 1.0e65, 1.0e66, 1.0e67, 1.0e68, 1.0e69, + 1.0e70, 1.0e71, 1.0e72, 1.0e73, 1.0e74, 1.0e75, 1.0e76, 1.0e77, 1.0e78, 1.0e79, + 1.0e80, 1.0e81, 1.0e82, 1.0e83, 1.0e84, 1.0e85, 1.0e86, 1.0e87, 1.0e88, 1.0e89, + 1.0e90, 1.0e91, 1.0e92, 1.0e93, 1.0e94, 1.0e95, 1.0e96, 1.0e97, 1.0e98, 1.0e99, + 1.0e100, 1.0e101, 1.0e102, 1.0e103, 1.0e104, 1.0e105, 1.0e106, 1.0e107, 1.0e108, 1.0e109, + 1.0e110, 1.0e111, 1.0e112, 1.0e113, 1.0e114, 1.0e115, 1.0e116, 1.0e117, 1.0e118, 1.0e119, + 1.0e120, 1.0e121, 1.0e122, 1.0e123, 1.0e124, 1.0e125, 1.0e126, 1.0e127, 1.0e128, 1.0e129, + 1.0e130, 1.0e131, 1.0e132, 1.0e133, 1.0e134, 1.0e135, 1.0e136, 1.0e137, 1.0e138, 1.0e139, + 1.0e140, 1.0e141, 1.0e142, 1.0e143, 1.0e144, 1.0e145, 1.0e146, 1.0e147, 1.0e148, 1.0e149, + 1.0e150, 1.0e151, 1.0e152, 1.0e153, 1.0e154, 1.0e155, 1.0e156, 1.0e157, 1.0e158, 1.0e159, + 1.0e160, 1.0e161, 1.0e162, 1.0e163, 1.0e164, 1.0e165, 1.0e166, 1.0e167, 1.0e168, 1.0e169, + 1.0e170, 1.0e171, 1.0e172, 1.0e173, 1.0e174, 1.0e175, 1.0e176, 1.0e177, 1.0e178, 1.0e179, + 1.0e180, 1.0e181, 1.0e182, 1.0e183, 1.0e184, 1.0e185, 1.0e186, 1.0e187, 1.0e188, 1.0e189, + 1.0e190, 1.0e191, 1.0e192, 1.0e193, 1.0e194, 1.0e195, 1.0e196, 1.0e197, 1.0e198, 1.0e199, + 1.0e200, 1.0e201, 1.0e202, 1.0e203, 1.0e204, 1.0e205, 1.0e206, 1.0e207, 1.0e208, 1.0e209, + 1.0e210, 1.0e211, 1.0e212, 1.0e213, 1.0e214, 1.0e215, 1.0e216, 1.0e217, 1.0e218, 1.0e219, + 1.0e220, 1.0e221, 1.0e222, 1.0e223, 1.0e224, 1.0e225, 1.0e226, 1.0e227, 1.0e228, 1.0e229, + 1.0e230, 1.0e231, 1.0e232, 1.0e233, 1.0e234, 1.0e235, 1.0e236, 1.0e237, 1.0e238, 1.0e239, + 1.0e240, 1.0e241, 1.0e242, 1.0e243, 1.0e244, 1.0e245, 1.0e246, 1.0e247, 1.0e248, 1.0e249, + 1.0e250, 1.0e251, 1.0e252, 1.0e253, 1.0e254, 1.0e255, 1.0e256, 1.0e257, 1.0e258, 1.0e259, + 1.0e260, 1.0e261, 1.0e262, 1.0e263, 1.0e264, 1.0e265, 1.0e266, 1.0e267, 1.0e268, 1.0e269, + 1.0e270, 1.0e271, 1.0e272, 1.0e273, 1.0e274, 1.0e275, 1.0e276, 1.0e277, 1.0e278, 1.0e279, + 1.0e280, 1.0e281, 1.0e282, 1.0e283, 1.0e284, 1.0e285, 1.0e286, 1.0e287, 1.0e288, 1.0e289, + 1.0e290, 1.0e291, 1.0e292, 1.0e293, 1.0e294, 1.0e295, 1.0e296, 1.0e297, 1.0e298, 1.0e299, + 1.0e300, 1.0e301, 1.0e302, 1.0e303, 1.0e304, 1.0e305, 1.0e306, 1.0e307, 1.0e308 +}; + +const float float_decimal_power10[] = { + 1.0e0f, 1.0e1f, 1.0e2f, 1.0e3f, 1.0e4f, 1.0e5f, 1.0e6f, 1.0e7f, 1.0e8f, 1.0e9f, + 1.0e10f, 1.0e11f, 1.0e12f, 1.0e13f, 1.0e14f, 1.0e15f, 1.0e16f, 1.0e17f, 1.0e18f, 1.0e19f, + 1.0e20f, 1.0e21f, 1.0e22f, 1.0e23f, 1.0e24f, 1.0e25f, 1.0e26f, 1.0e27f, 1.0e28f, 1.0e29f, + 1.0e30f, 1.0e31f, 1.0e32f, 1.0e33f, 1.0e34f, 1.0e35f, 1.0e36f, 1.0e37f, 1.0e38f +}; + +const double double_decimal_negpower10[] = { + 1.0e0, 1.0e-1, 1.0e-2, 1.0e-3, 1.0e-4, 1.0e-5, 1.0e-6, 1.0e-7, 1.0e-8, 1.0e-9, + 1.0e-10, 1.0e-11, 1.0e-12, 1.0e-13, 1.0e-14, 1.0e-15, 1.0e-16, 1.0e-17, 1.0e-18, 1.0e-19, + 1.0e-20, 1.0e-21, 1.0e-22, 1.0e-23, 1.0e-24, 1.0e-25, 1.0e-26, 1.0e-27, 1.0e-28, 1.0e-29, + 1.0e-30, 1.0e-31, 1.0e-32, 1.0e-33, 1.0e-34, 1.0e-35, 1.0e-36, 1.0e-37, 1.0e-38, 1.0e-39, + 1.0e-40, 1.0e-41, 1.0e-42, 1.0e-43, 1.0e-44, 1.0e-45, 1.0e-46, 1.0e-47, 1.0e-48, 1.0e-49, + 1.0e-50, 1.0e-51, 1.0e-52, 1.0e-53, 1.0e-54, 1.0e-55, 1.0e-56, 1.0e-57, 1.0e-58, 1.0e-59, + 1.0e-60, 1.0e-61, 1.0e-62, 1.0e-63, 1.0e-64, 1.0e-65, 1.0e-66, 1.0e-67, 1.0e-68, 1.0e-69, + 1.0e-70, 1.0e-71, 1.0e-72, 1.0e-73, 1.0e-74, 1.0e-75, 1.0e-76, 1.0e-77, 1.0e-78, 1.0e-79, + 1.0e-80, 1.0e-81, 1.0e-82, 1.0e-83, 1.0e-84, 1.0e-85, 1.0e-86, 1.0e-87, 1.0e-88, 1.0e-89, + 1.0e-90, 1.0e-91, 1.0e-92, 1.0e-93, 1.0e-94, 1.0e-95, 1.0e-96, 1.0e-97, 1.0e-98, 1.0e-99, + 1.0e-100, 1.0e-101, 1.0e-102, 1.0e-103, 1.0e-104, 1.0e-105, 1.0e-106, 1.0e-107, 1.0e-108, 1.0e-109, + 1.0e-110, 1.0e-111, 1.0e-112, 1.0e-113, 1.0e-114, 1.0e-115, 1.0e-116, 1.0e-117, 1.0e-118, 1.0e-119, + 1.0e-120, 1.0e-121, 1.0e-122, 1.0e-123, 1.0e-124, 1.0e-125, 1.0e-126, 1.0e-127, 1.0e-128, 1.0e-129, + 1.0e-130, 1.0e-131, 1.0e-132, 1.0e-133, 1.0e-134, 1.0e-135, 1.0e-136, 1.0e-137, 1.0e-138, 1.0e-139, + 1.0e-140, 1.0e-141, 1.0e-142, 1.0e-143, 1.0e-144, 1.0e-145, 1.0e-146, 1.0e-147, 1.0e-148, 1.0e-149, + 1.0e-150, 1.0e-151, 1.0e-152, 1.0e-153, 1.0e-154, 1.0e-155, 1.0e-156, 1.0e-157, 1.0e-158, 1.0e-159, + 1.0e-160, 1.0e-161, 1.0e-162, 1.0e-163, 1.0e-164, 1.0e-165, 1.0e-166, 1.0e-167, 1.0e-168, 1.0e-169, + 1.0e-170, 1.0e-171, 1.0e-172, 1.0e-173, 1.0e-174, 1.0e-175, 1.0e-176, 1.0e-177, 1.0e-178, 1.0e-179, + 1.0e-180, 1.0e-181, 1.0e-182, 1.0e-183, 1.0e-184, 1.0e-185, 1.0e-186, 1.0e-187, 1.0e-188, 1.0e-189, + 1.0e-190, 1.0e-191, 1.0e-192, 1.0e-193, 1.0e-194, 1.0e-195, 1.0e-196, 1.0e-197, 1.0e-198, 1.0e-199, + 1.0e-200, 1.0e-201, 1.0e-202, 1.0e-203, 1.0e-204, 1.0e-205, 1.0e-206, 1.0e-207, 1.0e-208, 1.0e-209, + 1.0e-210, 1.0e-211, 1.0e-212, 1.0e-213, 1.0e-214, 1.0e-215, 1.0e-216, 1.0e-217, 1.0e-218, 1.0e-219, + 1.0e-220, 1.0e-221, 1.0e-222, 1.0e-223, 1.0e-224, 1.0e-225, 1.0e-226, 1.0e-227, 1.0e-228, 1.0e-229, + 1.0e-230, 1.0e-231, 1.0e-232, 1.0e-233, 1.0e-234, 1.0e-235, 1.0e-236, 1.0e-237, 1.0e-238, 1.0e-239, + 1.0e-240, 1.0e-241, 1.0e-242, 1.0e-243, 1.0e-244, 1.0e-245, 1.0e-246, 1.0e-247, 1.0e-248, 1.0e-249, + 1.0e-250, 1.0e-251, 1.0e-252, 1.0e-253, 1.0e-254, 1.0e-255, 1.0e-256, 1.0e-257, 1.0e-258, 1.0e-259, + 1.0e-260, 1.0e-261, 1.0e-262, 1.0e-263, 1.0e-264, 1.0e-265, 1.0e-266, 1.0e-267, 1.0e-268, 1.0e-269, + 1.0e-270, 1.0e-271, 1.0e-272, 1.0e-273, 1.0e-274, 1.0e-275, 1.0e-276, 1.0e-277, 1.0e-278, 1.0e-279, + 1.0e-280, 1.0e-281, 1.0e-282, 1.0e-283, 1.0e-284, 1.0e-285, 1.0e-286, 1.0e-287, 1.0e-288, 1.0e-289, + 1.0e-290, 1.0e-291, 1.0e-292, 1.0e-293, 1.0e-294, 1.0e-295, 1.0e-296, 1.0e-297, 1.0e-298, 1.0e-299, + 1.0e-300, 1.0e-301, 1.0e-302, 1.0e-303, 1.0e-304, 1.0e-305, 1.0e-306, 1.0e-307, 1.0e-308 +}; + +const float float_decimal_negpower10[] = { + 1.0e0f, 1.0e-1f, 1.0e-2f, 1.0e-3f, 1.0e-4f, 1.0e-5f, 1.0e-6f, 1.0e-7f, 1.0e-8f, 1.0e-9f, + 1.0e-10f, 1.0e-11f, 1.0e-12f, 1.0e-13f, 1.0e-14f, 1.0e-15f, 1.0e-16f, 1.0e-17f, 1.0e-18f, 1.0e-19f, + 1.0e-20f, 1.0e-21f, 1.0e-22f, 1.0e-23f, 1.0e-24f, 1.0e-25f, 1.0e-26f, 1.0e-27f, 1.0e-28f, 1.0e-29f, + 1.0e-30f, 1.0e-31f, 1.0e-32f, 1.0e-33f, 1.0e-34f, 1.0e-35f, 1.0e-36f, 1.0e-37f, 1.0e-38f +}; + +#endif + +/* scale number by floor(log10(number)) + 1 so that the result is in range [0.1, 1) */ + +#define ieee_double_exponent10(ieee_number) ((int)floor(log10(ieee_number.number)) + 1) +#define ieee_float_exponent10(ieee_number) ((int)floorf(log10f(ieee_number.number)) + 1) // floorf, log10f ? + +#define ieee_double_exp10(ieee_number, exponent10) \ + exponent10 = ieee_double_exponent10(ieee_number); \ + if (exponent10 > 0) { \ + double_negative_exp10(ieee_number.number, -exponent10); \ + ieee_number.fraction = ieee_double_fraction(ieee_number.bits); \ + ieee_number.exponent = ieee_double_exponent(ieee_number.bits); \ + } else if (exponent10 < 0) { \ + double_positive_exp10(ieee_number.number, -exponent10); \ + ieee_number.fraction = ieee_double_fraction(ieee_number.bits); \ + ieee_number.exponent = ieee_double_exponent(ieee_number.bits); \ + } + +#define ieee_float_exp10(ieee_number, exponent10) \ + exponent10 = ieee_float_exponent10(ieee_number); \ + if (exponent10 > 0) { \ + float_negative_exp10(ieee_number.number, -exponent10); \ + ieee_number.fraction = ieee_float_fraction(ieee_number.bits); \ + ieee_number.exponent = ieee_float_exponent(ieee_number.bits); \ + } else if (exponent10 < 0) { \ + float_positive_exp10(ieee_number.number, -exponent10); \ + ieee_number.fraction = ieee_float_fraction(ieee_number.bits); \ + ieee_number.exponent = ieee_float_exponent(ieee_number.bits); \ + } + +#if BINARY_MODF + +/* unhide implicit bit 53, produce 56-bit denormalised fraction (binary exponent already in range [-4, -1]) */ + +#define ieee_double_denormalize(ieee_number) \ + (ieee_number.exponent == IEEE_DOUBLE_MIN_EXPONENT ? (++ieee_number.exponent, 0) : (ieee_number.fraction |= (1ull<<52))), \ + ieee_number.fraction <<= (ieee_number.exponent + 4) + +/* unhide implicit bit 24, produce 27-bit denormalized fraction (binary exponent already in range [-4, -1]) */ + +#define ieee_float_denormalize(ieee_number) \ + (ieee_number.exponent == IEEE_FLOAT_MIN_EXPONENT ? (++ieee_number.exponent, 0) : (ieee_number.fraction |= (1<<23))), \ + ieee_number.fraction <<= (ieee_number.exponent + 4) + +/* turn off significant bits over 56 (integer part), multiply by 10, return new integer part (subsequent decimal digit) */ + +#define ieee_double_binary_fraction(ieee_number) \ + (ieee_number.fraction &= ((1ull<<56) - 1), \ + ieee_number.fraction = (ieee_number.fraction << 1) + (ieee_number.fraction << 3), \ + ieee_number.fraction >> 56) + +/* turn off significant bits over 27 (integer part), multiply by 10, return the integer part (subsequent decimal digit) */ + +#define ieee_float_binary_fraction(ieee_number) \ + (ieee_number.fraction &= ((1<<27) - 1), \ + ieee_number.fraction = (ieee_number.fraction << 1) + (ieee_number.fraction << 3), \ + ieee_number.fraction >> 27) + +#define ieee_double_decimal(ieee_number, exponent10, digits, p) \ + ieee_number_decimal(ieee_double_binary_fraction, ieee_number, exponent10, digits, p) +#define ieee_float_decimal(ieee_number, exponent10, digits, p) \ + ieee_number_decimal(ieee_float_binary_fraction, ieee_number, exponent10, digits, p) + +#else + +/* generic method */ + +#define ieee_double_decimal_fraction(ieee_number, i) (ieee_number.number = modf(10*ieee_number.number, &i), i) +#define ieee_float_decimal_fraction(ieee_number, i) (ieee_number.number = (float)modf(10*ieee_number.number, &i), i) // ??? + +#define ieee_double_decimal(ieee_number, exponent10, digits, p) \ + ieee_number_decimal(ieee_double_decimal_fraction, ieee_number, exponent10, digits, p) +#define ieee_float_decimal(ieee_number, exponent10, digits, p) \ + ieee_number_decimal(ieee_float_decimal_fraction, ieee_number, exponent10, digits, p) + +#endif + +#define ieee_number_decimal(method, ieee_number, exponent10, digits, p) \ + ieee_double_denormalize(ieee_number); \ + if (ieee_number.sign) *p++ = '-'; \ + if (exponent10 <= 0) \ + for (*p++ = '0', *p++ = RADIX_CHAR; exponent10 && digits; *p++ = '0', ++exponent10, --digits); \ + else \ + { \ + do { *p++ = '0' + (char)method(ieee_number); } while (--exponent10); \ + *p++ = RADIX_CHAR; \ + } \ + for ( ; digits && ieee_number.fraction; --digits) \ + *p++ = '0' + (char)method(ieee_number) + +/* rounding to nearest integer */ + +#if BINARY_MODF +/* check if the mantissa has the most significant bit set, means >= 0.5 */ +# define ieee_double_half(ieee_number) (ieee_number.fraction & (1ull<<55)) +# define ieee_float_half(ieee_number) (ieee_number.fraction & (1<<26)) +#else +# define ieee_double_half(ieee_number) (ieee_number.number >= 0.5) +# define ieee_float_half(ieee_number) (ieee_number.number >= 0.5) +#endif + +/* rounding to nearest integer */ + +#define buffer_ceil(s, p, sign) \ + { \ + while (*--p == '9'); \ + if (*p != RADIX_CHAR) ++*p++; \ + else { \ + char *q; \ + for (q = p - 1; ; --q) { \ + if (*q < '9') { ++*q; break; } \ + *q = '0'; \ + if (q == s) \ + *--s = '1'; \ + else if (sign && q - 1 == s) \ + *s = '1', *--s = '-'; \ + } \ + } \ + } + +#define buffer_remove_trailing_zeros(s, p, sign) \ + { \ + while (*--p == '0'); \ + if (*p != RADIX_CHAR) \ + ++p; \ + else if (!SIGNED_ZERO && sign && p - 2 == s && *(p - 1) == '0') \ + p -= 2, *p++ = '0'; \ + } + +// if digits parameter was initially less then exponent10, then exponent10 > 0 and ieee_double_half(ieee_number) is irrelevant +#define ieee_double_round(ieee_number, exponent10, s, p) \ + if (exponent10 == 0 && ieee_double_half(ieee_number)) \ + { buffer_ceil(s, p, ieee_number.sign); } \ + else \ + { buffer_remove_trailing_zeros(s, p, ieee_number.sign); } + +#define ieee_float_round(ieee_number, exponent10, s, p) \ + if (exponent10 == 0 && ieee_float_half(ieee_number)) \ + { buffer_ceil(s, p, ieee_number.sign); } \ + else \ + { buffer_remove_trailing_zeros(s, p, ieee_number.sign); } + +/* double to decimal */ + +#define ieee_copy_special_string(nbuf, special, p, _p) \ + for (p = nbuf, _p = special; ; ++p, ++_p) { \ + if ((*p = *_p) == '\0') break; \ + } + +#define ieee_copy_special_string_re(nbuf, special, p, _p, r, e) \ + for (p = nbuf, _p = special; ; ++p, ++_p) { \ + if ((*p = *_p) == '\0') { \ + if (r != NULL) *r = NULL; \ + if (e != NULL) *e = p; \ + break; \ + } \ + } + +char * double_as_string (double number, int digits, char nbuf[MAX_NUMBER_DIGITS], size_t *psize) +{ + ieee_double ieee_number; + int exponent10; + char *s, *p; const char *_p; + s = p = nbuf + 1; // for sign/rounding + ieee_double_init(ieee_number, number); + if ((ieee_number.sign = ieee_number.bits >> 63) != 0) + ieee_number.number = -ieee_number.number; + if (ieee_double_is_zero(ieee_number)) // to avoid crash on log10(number) + { + ieee_copy_special_string(nbuf, ieee_double_zero_string(ieee_number), p, _p); + *psize = (size_t)(p - nbuf); + return nbuf; + } + if (ieee_double_special_case(ieee_number)) + { + ieee_copy_special_string(nbuf, ieee_double_special_string(ieee_number), p, _p); + *psize = (size_t)(p - nbuf); + return nbuf; + } + ieee_double_exp10(ieee_number, exponent10); + ieee_double_decimal(ieee_number, exponent10, digits, p); + ieee_double_round(ieee_number, exponent10, s, p); + *p = '\0'; + *psize = (size_t)(p - s); + return s; +} + +/* float to decimal */ + +char * float_as_string (float number, int digits, char nbuf[MAX_NUMBER_DIGITS], size_t *psize) +{ + ieee_float ieee_number; + int exponent10; + char *s, *p; const char *_p; + s = p = nbuf + 1; // for sign/rounding + ieee_float_init(ieee_number, number); + if ((ieee_number.sign = ieee_number.bits >> 31) != 0) + ieee_number.number = -ieee_number.number; + if (ieee_float_is_zero(ieee_number)) + { + ieee_copy_special_string(nbuf, ieee_float_zero_string(ieee_number), p, _p); + *psize = (size_t)(p - nbuf); + return nbuf; + } + if (ieee_float_special_case(ieee_number)) + { + ieee_copy_special_string(nbuf, ieee_float_special_string(ieee_number), p, _p); + *psize = (size_t)(p - nbuf); + return nbuf; + } + ieee_float_exp10(ieee_number, exponent10); + ieee_float_decimal(ieee_number, exponent10, digits, p); + ieee_float_round(ieee_number, exponent10, s, p); + *p = '\0'; + *psize = (size_t)(p - s); + return s; +} + +/* decimal string to double/float */ + +#define string_scan_decimal(s, c, number) _scan_decimal(c, number, *++s) +#define string_scan_fraction(s, c, number, exponent10) _scan_fraction(c, number, exponent10, *++s) +#define string_scan_exponent10(s, c, exponent10) _scan_exponent10(c, exponent10, *++s) + +const char * string_to_double (const char *s, double *number) +{ + int sign, exponent10, c = *s; + string_scan_sign(s, c, sign); + string_scan_decimal(s, c, *number); + if (c == '.') + { + c = *++s; + string_scan_fraction(s, c, *number, exponent10); + } + else + exponent10 = 0; + if (c == 'e' || c == 'E') + { + c = *++s; + string_scan_exponent10(s, c, exponent10); + } + double_exp10(*number, exponent10); + if (sign) *number = -*number; + return s; +} + +const char * string_to_float (const char *s, float *number) +{ + int sign, exponent10, c = *s; + string_scan_sign(s, c, sign); + string_scan_decimal(s, c, *number); + if (c == '.') + { + c = *++s; + string_scan_fraction(s, c, *number, exponent10); + } + else + exponent10 = 0; + if (c == 'e' || c == 'E') + { + c = *++s; + string_scan_exponent10(s, c, exponent10); + } + float_exp10(*number, exponent10); + if (sign) *number = -*number; + return s; +} + +/* conventional form */ + +const char * convert_to_double (const char *s, double *number) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_decimal(s, c, *number); + if (c == '.' || c == ',') + { + int exponent10; + c = *++s; + string_scan_fraction(s, c, *number, exponent10); + if (exponent10 < 0) + double_negative_exp10(*number, exponent10); + } + if (sign) *number = -*number; + return s; +} + +const char * convert_to_float (const char *s, float *number) +{ + int sign, c = *s; + string_scan_sign(s, c, sign); + string_scan_decimal(s, c, *number); + if (c == '.' || c == ',') + { + int exponent10; + c = *++s; + string_scan_fraction(s, c, *number, exponent10); + if (exponent10 < 0) + float_negative_exp10(*number, exponent10); + } + if (sign) *number = -*number; + return s; +} + +/* pretty common stuff */ + +size_t bytes_to_hex_lc (const void *input, size_t size, unsigned char *output) +{ + size_t i; + const unsigned char *p; + for (i = 0, p = (const unsigned char *)input; i < size; ++i, ++p) + { + *output++ = base16_lc_digit1(*p); + *output++ = base16_lc_digit2(*p); + } + *output = '\0'; + return 2*size + 1; +} + +size_t bytes_to_hex_uc (const void *input, size_t size, unsigned char *output) +{ + size_t i; + const unsigned char *p; + for (i = 0, p = (const unsigned char *)input; i < size; ++i, ++p) + { + *output++ = base16_uc_digit1(*p); + *output++ = base16_uc_digit2(*p); + } + *output = '\0'; + return 2*size + 1; +} + +size_t hex_to_bytes (const void *input, size_t size, unsigned char *output) +{ + size_t i; + int c1, c2; + const unsigned char *p; + for (i = 1, p = (const unsigned char *)input; i < size; i += 2) + { + c1 = base16_value(*p); + ++p; + c2 = base16_value(*p); + ++p; + if (c1 >= 0 && c2 >= 0) + *output++ = (unsigned char)((c1<<4)|c2); + else + break; + } + return i >> 1; +} + +void print_as_hex (const void *input, size_t bytes) +{ + const unsigned char *p; + for (p = (const unsigned char *)input; bytes > 0; --bytes, ++p) + printf("%02x", *p); +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilnumber.h b/Build/source/libs/pplib/pplib-src/src/util/utilnumber.h new file mode 100644 index 00000000000..735432b8dd8 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilnumber.h @@ -0,0 +1,428 @@ +#ifndef UTIL_NUMBER_H +#define UTIL_NUMBER_H + +#include <stddef.h> // for size_t + +#include "utilplat.h" +#include "utildecl.h" + +#if defined(__cplusplus) && defined(_MSC_VER) +// int*_t types are in standard in msvc++ +#else +# include <stdint.h> +#endif + +/* 'long' isn't long for msvc64/mingw64, we need a type for machine word */ + +#if defined(_WIN64) || defined(__MINGW32__) +# define INT64F "%I64d" +# define UINT64F "%I64u" +#else +# define INT64F "%lld" +# define UINT64F "%llu" +#endif + +#if defined(MSVC64) +# define INTLW_IS_INT64 +# define intlw_t int64_t +# define uintlw_t uint64_t +# define INTLW(N) N##I64 +# define UINTLW(N) N##UI64 +# define INTLWF INT64F +# define UINTLWF UINT64F +#elif defined(__MINGW64__) +# define INTLW_IS_INT64 +# define intlw_t int64_t +# define uintlw_t uint64_t +# define INTLW(N) N##LL +# define UINTLW(N) N##ULL +# define INTLWF INT64F +# define UINTLWF UINT64F +#else // 32bit or sane 64bit (LP64) +# define INTLW_IS_LONG +# define intlw_t long +# define uintlw_t unsigned long +# define INTLW(N) N##L +# define UINTLW(N) N##UL +# define INTLWF "%ld" +# define UINTLWF "%lu" +#endif + +// ssize_t is missing in MSVC, but defining it is risky; some environments (eg. python) typedefs ssize_t on its own way.. +// #if defined(MSVC64) +// # define ssize_t int32_t +// #else +// # if defined(MSVC32) +// # define ssize_t int64_t +// # endif +// #endif + +/* basic constants */ + +#define MAX_RADIX 36 +#define MAX_INTEGER_DIGITS 65 /* 64-bit number in binary form plus '\0' */ +#define MAX_ROMAN_DIGITS 128 /* to handle romannumeral of short int (up to 65 leading 'M') */ +#define MAX_NUMBER_DIGITS 512 +#define NUMBER_BUFFER_SIZE MAX_NUMBER_DIGITS + +#define base36_uc_alphabet "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +#define base36_lc_alphabet "0123456789abcdefghijklmnopqrstuvwxyz" + +#define base26_uc_alphabet "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +#define base26_lc_alphabet "abcdefghijklmnopqrstuvwxyz" +extern const int base26_lookup[]; + +#define base36_lc_palindrome "zyxwvutsrqponmlkjihgfedcba9876543210123456789abcdefghijklmnopqrstuvwxyz" +#define base36_uc_palindrome "ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +extern const int base36_lookup[]; + +#define base10_palindrome "9876543210123456789" +#define base10_alphabet "0123456789" +extern const int base10_lookup[]; + +#define base16_uc_alphabet "0123456789ABCDEF" +#define base16_lc_alphabet "0123456789abcdef" +extern const int base16_lookup[]; + +#define base16_uc_digit1(c) base16_uc_alphabet[(c)>>4] +#define base16_uc_digit2(c) base16_uc_alphabet[(c)&15] +#define base16_lc_digit1(c) base16_lc_alphabet[(c)>>4] +#define base16_lc_digit2(c) base16_lc_alphabet[(c)&15] + +#define base8_digit(c) ((unsigned)(c - '0') <= (unsigned)('7' - '0')) +#define base8_value(c) (base8_digit(c) ? (c) - '0' : -1) + +#define base10_digit(c) ((unsigned)(c - '0') <= (unsigned)('9' - '0')) +#define base10_value(c) (base10_lookup[(uint8_t)(c)]) + +#define base16_digit(c) (base16_lookup[(uint8_t)(c)] >= 0) +#define base16_value(c) (base16_lookup[(uint8_t)(c)]) + +#define base26_digit(c) (base26_lookup[(uint8_t)(c)] >= 0) +#define base26_value(c) (base26_lookup[(uint8_t)(c)]) + +#define base36_digit(c) (base36_lookup[(uint8_t)(c)] >= 0) +#define base36_value(c) (base36_lookup[(uint8_t)(c)]) + +//#define base_digit(c, radix) ((unsigned)(base36_lookup[c]) < (unsigned)(radix)) +//#define base_value(c, radix) (base_digit(c, radix) ? base36_lookup[c] : -1) + +UTILDEF extern char util_number_buffer[NUMBER_BUFFER_SIZE]; + +/* integer from string; return a pointer to character next to the last digit */ + +UTILAPI const char * string_to_int32 (const char *s, int32_t *number); +UTILAPI const char * string_to_slong (const char *s, long *number); +UTILAPI const char * string_to_int64 (const char *s, int64_t *number); + +UTILAPI const char * string_to_uint32 (const char *s, uint32_t *number); +UTILAPI const char * string_to_ulong (const char *s, unsigned long *number); +UTILAPI const char * string_to_usize (const char *s, size_t *number); +UTILAPI const char * string_to_uint64 (const char *s, uint64_t *number); + +UTILAPI const char * radix_to_int32 (const char *s, int32_t *number, int radix); +UTILAPI const char * radix_to_slong (const char *s, long *number, int radix); +UTILAPI const char * radix_to_int64 (const char *s, int64_t *number, int radix); + +UTILAPI const char * radix_to_uint32 (const char *s, uint32_t *number, int radix); +UTILAPI const char * radix_to_ulong (const char *s, unsigned long *number, int radix); +UTILAPI const char * radix_to_usize (const char *s, size_t *number, int radix); +UTILAPI const char * radix_to_uint64 (const char *s, uint64_t *number, int radix); + +UTILAPI const char * alpha_to_uint32 (const char *s, uint32_t *number); +UTILAPI const char * alpha_to_ulong (const char *s, unsigned long *number); +UTILAPI const char * alpha_to_usize (const char *s, size_t *number); +UTILAPI const char * alpha_to_uint64 (const char *s, uint64_t *number); + +/* integer to string */ + +UTILAPI char * int32_as_string (int32_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * slong_as_string (long number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * int64_as_string (int64_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); + +#define int32_to_string(number, psize) int32_as_string(number, util_number_buffer, psize) +#define slong_to_string(number, psize) slong_as_string(number, util_number_buffer, psize) +#define int64_to_string(number, psize) int64_as_string(number, util_number_buffer, psize) + +UTILAPI char * uint32_as_string (uint32_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * ulong_as_string (unsigned long number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * usize_as_string (size_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * uint64_as_string (uint64_t number, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); + +#define uint32_to_string(number, psize) uint32_as_string(number, util_number_buffer, psize) +#define ulong_to_string(number, psize) ulong_as_string(number, util_number_buffer, psize) +#define usize_to_string(number, psize) usize_as_string(number, util_number_buffer, psize) +#define uint64_to_string(number, psize) uint64_as_string(number, util_number_buffer, psize) + +UTILAPI char * int32_as_radix (int32_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * slong_as_radix (long number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * int64_as_radix (int64_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); + +#define int32_to_radix(number, radix, uc, psize) int32_as_radix(number, radix, uc, util_number_buffer, psize) +#define slong_to_radix(number, radix, uc, psize) slong_as_radix(number, radix, uc, util_number_buffer, psize) +#define int64_to_radix(number, radix, uc, psize) int64_as_radix(number, radix, uc, util_number_buffer, psize) + +UTILAPI char * uint32_as_radix (uint32_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * ulong_as_radix (unsigned long number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * usize_as_radix (size_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * uint64_as_radix (uint64_t number, int radix, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); + +#define uint32_to_radix(number, radix, uc, psize) uint32_as_radix(number, radix, uc, util_number_buffer, psize) +#define ulong_to_radix(number, radix, uc, psize) ulong_as_radix(number, radix, uc, util_number_buffer, psize) +#define usize_to_radix(number, radix, uc, psize) usize_as_radix(number, radix, uc, util_number_buffer, psize) +#define uint64_to_radix(number, radix, uc, psize) uint64_as_radix(number, radix, uc, util_number_buffer, psize) + +UTILAPI char * uint32_as_alpha (uint32_t number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * ulong_as_alpha (unsigned long number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * usize_as_alpha (size_t number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); +UTILAPI char * uint64_as_alpha (uint64_t number, int uc, char ibuf[MAX_INTEGER_DIGITS], size_t *psize); + +#define uint32_to_alpha(number, uc, psize) uint32_as_alpha(number, uc, util_number_buffer, psize) +#define ulong_to_alpha(number, uc, psize) ulong_as_alpha(number, uc, util_number_buffer, psize) +#define usize_to_alpha(number, uc, psize) usize_as_alpha(number, uc, util_number_buffer, psize) +#define uint64_to_alpha(number, uc, psize) uint64_as_alpha(number, uc, util_number_buffer, psize) + +#if defined(INTLW_IS_INT64) + +# define string_to_intlw(s, number) string_to_int64(s, number) +# define string_to_uintlw(s, number) string_to_uint64(s, number) + +# define radix_to_intlw(s, number, radix) radix_to_int64(s, number, radix) +# define radix_to_uintlw(s, number, radix) radix_to_uint64(s, number, radix) + +# define alpha_to_uintlw(s, number) alpha_to_uint64(s, number) + +# define intlw_as_string(number, ibuf, psize) int64_as_string(number, ibuf, psize) +# define uintlw_as_string(number, ibuf, psize) uint64_as_string(number, ibuf, psize) + +# define intlw_to_string(number, psize) int64_to_string(number, psize) +# define uintlw_to_string(number, psize) uint64_to_string(number, psize) + +# define intlw_as_radix(number, radix, uc, ibuf, psize) int64_as_radix(number, radix, uc, ibuf, psize) +# define uintlw_as_radix(number, radix, uc, ibuf, psize) uint64_as_radix(number, radix, uc, ibuf, psize) + +# define intlw_to_radix(number, radix, uc, psize) int64_to_radix(number, radix, uc, psize) +# define uintlw_to_radix(number, radix, uc, psize) uint64_to_radix(number, radix, uc, psize) + +# define uintlw_as_alpha(number, uc, ibuf, psize) uint64_as_alpha(number, uc, ibuf, psize) +# define uintlw_to_alpha(number, uc, psize) uint64_to_alpha(number, uc, ibuf, psize) + +#elif defined(INTLW_IS_LONG) + +# define string_to_intlw(s, number) string_to_slong(s, number) +# define string_to_uintlw(s, number) string_to_ulong(s, number) + +# define radix_to_intlw(s, number, radix) radix_to_slong(s, number, radix) +# define radix_to_uintlw(s, number, radix) radix_to_ulong(s, number, radix) + +# define alpha_to_uintlw(s, number) alpha_to_ulong(s, number) + +# define intlw_as_string(number, ibuf, psize) slong_as_string(number, ibuf, psize) +# define uintlw_as_string(number, ibuf, psize) ulong_as_string(number, ibuf, psize) + +# define intlw_to_string(number, psize) slong_to_string(number, psize) +# define uintlw_to_string(number, psize) ulong_to_string(number, psize) + +# define intlw_as_radix(number, radix, uc, ibuf, psize) slong_as_radix(number, radix, uc, ibuf, psize) +# define uintlw_as_radix(number, radix, uc, ibuf, psize) ulong_as_radix(number, radix, uc, ibuf, psize) + +# define intlw_to_radix(number, radix, uc, psize) slong_to_radix(number, radix, uc, psize) +# define uintlw_to_radix(number, radix, uc, psize) ulong_to_radix(number, radix, uc, psize) + +# define uintlw_as_alpha(number, uc, ibuf, psize) ulong_as_alpha(number, uc, ibuf, psize) +# define uintlw_to_alpha(number, uc, psize) ulong_to_alpha(number, uc, ibuf, psize) + +#endif + +/* a..z, aa..zz, aaa..zzz (limited to uint16_t, valid for N <= buffer_size * 26) */ + +UTILAPI const char * alphan_to_uint16 (const char *s, uint16_t *number); +UTILAPI char * uint16_as_alphan (uint16_t number, int uc, char ibuf[], size_t size, size_t *psize); +#define uint16_to_alphan(number, uc, psize) uint16_as_alphan(number, uc, util_number_buffer, NUMBER_BUFFER_SIZE, psize) + +/* roman numeral (limited to uint16_t) */ + +UTILAPI const char * roman_to_uint16 (const char *s, uint16_t *number); +UTILAPI char * uint16_as_roman (uint16_t number, int uc, char ibuf[MAX_ROMAN_DIGITS], size_t *psize); +#define uint16_to_roman(number, uc, psize) uint16_as_roman(number, uc, util_number_buffer, psize) + +/* double/float to string */ + +UTILAPI char * double_as_string (double number, int digits, char nbuf[MAX_NUMBER_DIGITS], size_t *psize); +#define double_to_string(number, digits, psize) double_as_string(number, digits, util_number_buffer, psize) + +UTILAPI char * float_as_string (float number, int digits, char nbuf[MAX_NUMBER_DIGITS], size_t *psize); +#define float_to_string(number, digits, psize) float_as_string(number, digits, util_number_buffer, psize) + +/* string to double/float */ + +UTILAPI const char * string_to_double (const char *s, double *number); +UTILAPI const char * string_to_float (const char *s, float *number); + +/* convenience form accepting comma among a dot, with not exp notation (eg. pdf) */ + +UTILAPI const char * convert_to_double (const char *s, double *number); +UTILAPI const char * convert_to_float (const char *s, float *number); + +/* binary data parsers helpers */ + +#if 0 // masking gives more overactive warnings +#define get_number_byte1(n) ((n) & 0x000000ffu) +#define get_number_byte2(n) (((n) & 0x0000ff00u) >> 8) +#define get_number_byte3(n) (((n) & 0x00ff0000u) >> 16) +#define get_number_byte4(n) (((n) & 0xff000000u) >> 24) +#define get_number_byte5(n) (((n) & 0x000000ff00000000ull) >> 32) +#define get_number_byte6(n) (((n) & 0x0000ff0000000000ull) >> 40) +#define get_number_byte7(n) (((n) & 0x00ff000000000000ull) >> 48) +#define get_number_byte8(n) (((n) & 0xff00000000000000ull) >> 56) +#else +#define get_number_byte1(n) ((n) & 0xff) +#define get_number_byte2(n) (((n) >> 8) & 0xff) +#define get_number_byte3(n) (((n) >> 16) & 0xff) +#define get_number_byte4(n) (((n) >> 24) & 0xff) +#define get_number_byte5(n) (((n) >> 32) & 0xff) +#define get_number_byte6(n) (((n) >> 40) & 0xff) +#define get_number_byte7(n) (((n) >> 48) & 0xff) +#define get_number_byte8(n) (((n) >> 56) & 0xff) +#endif + +#define get_number_bytes_be1(n, b) (b[0] = (uint8_t)get_number_byte1(n)) +#define get_number_bytes_be2(n, b) (b[0] = (uint8_t)get_number_byte2(n), b[1] = (uint8_t)get_number_byte1(n)) +#define get_number_bytes_be3(n, b) (b[0] = (uint8_t)get_number_byte3(n), b[1] = (uint8_t)get_number_byte2(n), b[2] = (uint8_t)get_number_byte1(n)) +#define get_number_bytes_be4(n, b) (b[0] = (uint8_t)get_number_byte4(n), b[1] = (uint8_t)get_number_byte3(n), b[2] = (uint8_t)get_number_byte2(n), b[3] = (uint8_t)get_number_byte1(n)) + +#define get_number_bytes_be5(n, b) (b[0] = (uint8_t)get_number_byte5(n), b[1] = (uint8_t)get_number_byte4(n), b[2] = (uint8_t)get_number_byte3(n), b[3] = (uint8_t)get_number_byte2(n), \ + b[4] = (uint8_t)get_number_byte1(n)) +#define get_number_bytes_be6(n, b) (b[0] = (uint8_t)get_number_byte6(n), b[1] = (uint8_t)get_number_byte5(n), b[2] = (uint8_t)get_number_byte4(n), b[3] = (uint8_t)get_number_byte3(n), \ + b[4] = (uint8_t)get_number_byte2(n), b[5] = (uint8_t)get_number_byte1(n)) +#define get_number_bytes_be7(n, b) (b[0] = (uint8_t)get_number_byte7(n), b[1] = (uint8_t)get_number_byte6(n), b[2] = (uint8_t)get_number_byte5(n), b[3] = (uint8_t)get_number_byte4(n), \ + b[4] = (uint8_t)get_number_byte3(n), b[5] = (uint8_t)get_number_byte2(n), b[6] = (uint8_t)get_number_byte1(n)) +#define get_number_bytes_be8(n, b) (b[0] = (uint8_t)get_number_byte8(n), b[1] = (uint8_t)get_number_byte7(n), b[2] = (uint8_t)get_number_byte6(n), b[3] = (uint8_t)get_number_byte5(n), \ + b[4] = (uint8_t)get_number_byte4(n), b[5] = (uint8_t)get_number_byte3(n), b[6] = (uint8_t)get_number_byte2(n), b[7] = (uint8_t)get_number_byte1(n)) + +#define read_uint16be_as(s, int_type) ((int_type)((s[0]<<8)|s[1])) +#define read_uint32be_as(s, int_type) ((int_type)((s[0]<<24)|(s[1]<<16)|(s[2]<<8)|s[3])) + +#define read_uint16le_as(s, int_type) ((int_type)((s[1]<<8)|s[0])) +#define read_uint32le_as(s, int_type) ((int_type)((s[3]<<24)|(s[2]<<16)|(s[1]<<8)|s[0])) + +#define read_uint16_native(s) (*((uint16_t *)(s))) +#define read_uint32_native(s) (*((uint32_t *)(s))) +#define read_int16_native(s) (*((int16_t *)(s))) +#define read_int32_native(s) (*((int32_t *)(s))) + +#define scan_uint16be_as(s, int_type) (s += 2, (int_type)((s[-2]<<8)|s[-1])) +#define scan_uint32be_as(s, int_type) (s += 4, (int_type)((s[-4]<<24)|(s[-3]<<16)|(s[-2]<<8)|s[-1])) + +#define scan_uint16le_as(s, int_type) (s += 2, (int_type)((s[-1]<<8)|s[-2])) +#define scan_uint32le_as(s, int_type) (s += 4, (int_type)((s[-1]<<24)|(s[-2]<<16)|(s[-3]<<8)|s[-4])) + +#define scan_uint16_native(s) (s += 2, read_uint16_native(s-2)) +#define scan_uint32_native(s) (s += 4, read_uint32_native(s-4)) +#define scan_int16_native(s) (s += 2, read_int16_native(s-2)) +#define scan_int32_native(s) (s += 4, read_int32_native(s-4)) + +#define read_fixed16_16_as(s, float_type) (((float_type)read_uint32be_as(s, signed int))/(1<<16)) +#define read_fixed2_14_as(s, float_type) (((float_type)read_uint16be_as(s, signed short))/(1<<14)) + +#define scan_fixed16_16_as(s, float_type) (((float_type)scan_uint32be_as(s, signed int))/(1<<16)) +#define scan_fixed2_14_as(s, float_type) (((float_type)scan_uint16be_as(s, signed short))/(1<<14)) + +/* internal procedures */ + +#define _scan_sign(c, sign, next) \ + do { if (c == '-') { sign = 1; c = next; } else if (c == '+') { sign = 0; c = next; } else sign = 0; } while (0) + +#define integer_multiplied10(number) (((number) << 1) + ((number) << 3)) + +#define _scan_integer(c, number, next) \ + for (number = 0; base10_digit(c); number = integer_multiplied10(number) + (c - '0'), c = next) +#define _scan_radix(c, number, radix, next) \ + for (number = 0; (c = base36_value(c)) >= 0 && c < radix; number = number * radix + c, c = next) + +#define _read_integer(c, number, next) \ + for (number = c - '0', c = next; base10_digit(c); number = integer_multiplied10(number) + (c - '0'), c = next) +#define _read_radix(c, number, radix, next) \ + for (number = c - '0', c = next; (c = base36_value(c)) >= 0 && c < radix; number = number * radix + c, c = next) + +/* rationals */ + +#define _scan_decimal(c, number, next) \ + for (number = 0; base10_digit(c); number = number*10 + (c - '0'), c = next) +#define _scan_fraction(c, number, exponent10, next) \ + for (exponent10 = 0; base10_digit(c); --exponent10, number = number*10 + (c - '0'), c = next) + +#define _scan_exponent10(c, exponent10, next) \ + do { \ + int eexponent10, eexpsign; \ + _scan_sign(c, eexpsign, next); \ + _scan_integer(c, eexponent10, next); \ + if (eexpsign) \ + exponent10 -= eexponent10; \ + else \ + exponent10 += eexponent10; \ + } while(0) + +#if 0 + +// kept just for sentiment ;) + +extern const double double_binary_power10[]; +extern const float float_binary_power10[]; +extern const double double_binary_negpower10[]; +extern const float float_binary_negpower10[]; + +#define double_negative_exp10(number, exponent) \ +{ const double *bp10; int e = ((exponent) < 511 ? 511 : -(exponent)); \ + for (bp10 = double_binary_negpower10; e > 0; e >>= 1, ++bp10) \ + if (e & 1) number *= *bp10; } + +#define float_negative_exp10(number, exponent) \ +{ const float *bp10; int e = ((exponent) < 64 ? 64 : -(exponent)); \ + for (bp10 = float_binary_negpower10; e > 0; e >>= 1, ++bp10) \ + if (e & 1) number *= *bp10; } + +#define double_positive_exp10(number, exponent) \ +{ const double *bp10; int e = ((exponent) > 511 ? 511 : (exponent)); \ + for (bp10 = double_binary_power10; e > 0; e >>= 1, ++bp10) \ + if (e & 1) number *= *bp10; } + +#define float_positive_exp10(number, exponent) \ +{ const float *bp10; int e = ((exponent) > 64 ? 64 : (exponent)); \ + for (bp10 = double_binary_power10; e > 0; e >>= 1, ++bp10) \ + if (e & 1) number *= *bp10; } + +#define double_exp10(number, exponent) \ + if ((exponent) < 0) double_negative_exp10(number, exponent) else if ((exponent) > 0) double_positive_exp10(number, exponent) + +#define float_exp10(number, exponent) \ + if ((exponent) < 0) float_negative_exp10(number, exponent) else if ((exponent) > 0) float_positive_exp10(number, exponent) + +#else + +extern const double double_decimal_power10[]; +extern const float float_decimal_power10[]; +extern const double double_decimal_negpower10[]; +extern const float float_decimal_negpower10[]; + +#define double_negative_exp10(number, exponent) ((number) *= double_decimal_negpower10[(exponent) < -308 ? 308 : -(exponent)]) +#define double_positive_exp10(number, exponent) ((number) *= double_decimal_power10[(exponent) > 308 ? 308 : (exponent)]) + +#define float_negative_exp10(number, exponent) ((number) *= float_decimal_negpower10[(exponent) < -38 ? 38 : -(exponent)]) +#define float_positive_exp10(number, exponent) ((number) *= float_decimal_power10[(exponent) > 38 ? 38 : (exponent)]) + +#define double_exp10(number, exponent) ((void)(((exponent) < 0 && double_negative_exp10(number, exponent)) || (((exponent) > 0 && double_positive_exp10(number, exponent))))) +#define float_exp10(number, exponent) ((void)(((exponent) < 0 && float_negative_exp10(number, exponent)) || (((exponent) > 0 && float_positive_exp10(number, exponent))))) + +#endif + +/* pretty common stuff */ + +#define bytes_to_hex(input, size, output) bytes_to_hex_lc(input, size, output) +UTILAPI size_t bytes_to_hex_lc (const void *input, size_t size, uint8_t *output); +UTILAPI size_t bytes_to_hex_uc (const void *input, size_t size, uint8_t *output); +UTILAPI size_t hex_to_bytes (const void *input, size_t size, uint8_t *output); +UTILAPI void print_as_hex (const void *input, size_t bytes); + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilplat.h b/Build/source/libs/pplib/pplib-src/src/util/utilplat.h new file mode 100644 index 00000000000..8838f702b92 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilplat.h @@ -0,0 +1,31 @@ + +#ifndef UTIL_PLAT_H +#define UTIL_PLAT_H + +#if defined(_WIN32) || defined(WIN32) +# ifdef _MSC_VER +# if defined(_M_64) || defined(_WIN64) +# define MSVC64 +# else +# define MSVC32 +# endif +# else +# if defined(__MINGW64__) +# define MINGW64 +# else +# if defined(__MINGW32__) +# define MINGW32 +# endif +# endif +# endif +#endif + +#ifdef __GNUC__ +//# define FALLTHRU [[fallthrough]] // c++17 +//# define FALLTHRU [[gnu:fallthrough]] // c++14 +# define FALLTHRU __attribute__((fallthrough)); // C and C++03 +#else +# define FALLTHRU +#endif + +#endif
\ No newline at end of file diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilsha.c b/Build/source/libs/pplib/pplib-src/src/util/utilsha.c new file mode 100644 index 00000000000..596bf76f72b --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilsha.c @@ -0,0 +1,1065 @@ +/* sha2 implementation excerpted from code by Aaron D. Gifford */ + +/* + * AUTHOR: Aaron D. Gifford - http://www.aarongifford.com/ + * + * Copyright (c) 2000-2001, Aaron D. Gifford + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTOR(S) ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTOR(S) BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: sha2.c,v 1.1 2001/11/08 00:01:51 adg Exp adg $ + */ + +#include <stdio.h> /* FILE */ +#include <string.h> /* memcpy()/memset() or bcopy()/bzero() */ +//#include <assert.h> /* assert() */ +#include "utilsha.h" + +/* + * UNROLLED TRANSFORM LOOP NOTE: + * You can define SHA2_UNROLL_TRANSFORM to use the unrolled transform + * loop version for the hash transform rounds (defined using macros + * later in this file). Either define on the command line, for example: + * + * cc -DSHA2_UNROLL_TRANSFORM -o sha2 sha2.c sha2prog.c + * + * or define below: + * + * #define SHA2_UNROLL_TRANSFORM + * + */ + +/*** SHA-256/384/512 Machine Architecture Definitions *****************/ +/* + * BYTE_ORDER NOTE: + * + * Please make sure that your system defines BYTE_ORDER. If your + * architecture is little-endian, make sure it also defines + * LITTLE_ENDIAN and that the two (BYTE_ORDER and LITTLE_ENDIAN) are + * equivilent. + * + * If your system does not define the above, then you can do so by + * hand like this: + * + * #define LITTLE_ENDIAN 1234 + * #define BIG_ENDIAN 4321 + * + * And for little-endian machines, add: + * + * #define BYTE_ORDER LITTLE_ENDIAN + * + * Or for big-endian machines: + * + * #define BYTE_ORDER BIG_ENDIAN + * + * The FreeBSD machine this was written on defines BYTE_ORDER + * appropriately by including <sys/types.h> (which in turn includes + * <machine/endian.h> where the appropriate definitions are actually + * made). + */ + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +//#if !defined(BYTE_ORDER) || (BYTE_ORDER != LITTLE_ENDIAN && BYTE_ORDER != BIG_ENDIAN) +//#error Define BYTE_ORDER to be equal to either LITTLE_ENDIAN or BIG_ENDIAN +//#endif + +/* + * Define the following sha2_* types to types of the correct length on + * the native archtecture. Most BSD systems and Linux define u_intXX_t + * types. Machines with very recent ANSI C headers, can use the + * uintXX_t definintions from inttypes.h by defining SHA2_USE_INTTYPES_H + * during compile or in the sha.h header file. + * + * Machines that support neither u_intXX_t nor inttypes.h's uintXX_t + * will need to define these three typedefs below (and the appropriate + * ones in sha.h too) by hand according to their system architecture. + * + * Thank you, Jun-ichiro itojun Hagino, for suggesting using u_intXX_t + * types and pointing out recent ANSI C support for uintXX_t in inttypes.h. + * + * PJ: replace by uintX_t + */ + +//typedef uint8_t sha2_byte; /* Exactly 1 byte */ +//typedef uint32_t sha2_word32; /* Exactly 4 bytes */ +//typedef uint64_t sha2_word64; /* Exactly 8 bytes */ + +/*** SHA-256/384/512 Various Length Definitions ***********************/ +/* NOTE: Most of these are in header */ +#define SHA256_SHORT_BLOCK_LENGTH (SHA256_BLOCK_LENGTH - 8) +#define SHA384_SHORT_BLOCK_LENGTH (SHA384_BLOCK_LENGTH - 16) +#define SHA512_SHORT_BLOCK_LENGTH (SHA512_BLOCK_LENGTH - 16) + + +/*** ENDIAN REVERSAL MACROS *******************************************/ +#if BYTE_ORDER == LITTLE_ENDIAN +#define REVERSE32(w, x) { \ + uint32_t tmp = (w); \ + tmp = (tmp >> 16) | (tmp << 16); \ + (x) = ((tmp & 0xff00ff00UL) >> 8) | ((tmp & 0x00ff00ffUL) << 8); \ +} +#define REVERSE64(w, x) { \ + uint64_t tmp = (w); \ + tmp = (tmp >> 32) | (tmp << 32); \ + tmp = ((tmp & 0xff00ff00ff00ff00ULL) >> 8) | \ + ((tmp & 0x00ff00ff00ff00ffULL) << 8); \ + (x) = ((tmp & 0xffff0000ffff0000ULL) >> 16) | \ + ((tmp & 0x0000ffff0000ffffULL) << 16); \ +} +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + +/* + * Macro for incrementally adding the unsigned 64-bit integer n to the + * unsigned 128-bit integer (represented using a two-element array of + * 64-bit words): + */ +#define ADDINC128(w,n) { \ + (w)[0] += (uint64_t)(n); \ + if ((w)[0] < (n)) { \ + (w)[1]++; \ + } \ +} + +#define MEMSET_BZERO(p,l) memset((p), 0, (l)) +#define MEMCPY_BCOPY(d,s,l) memcpy((d), (s), (l)) + +/*** THE SIX LOGICAL FUNCTIONS ****************************************/ +/* + * Bit shifting and rotation (used by the six SHA-XYZ logical functions: + * + * NOTE: The naming of R and S appears backwards here (R is a SHIFT and + * S is a ROTATION) because the SHA-256/384/512 description document + * (see http://csrc.nist.gov/cryptval/shs/sha256-384-512.pdf) uses this + * same "backwards" definition. + */ +/* Shift-right (used in SHA-256, SHA-384, and SHA-512): */ +#define R(b,x) ((x) >> (b)) +/* 32-bit Rotate-right (used in SHA-256): */ +#define S32(b,x) (((x) >> (b)) | ((x) << (32 - (b)))) +/* 64-bit Rotate-right (used in SHA-384 and SHA-512): */ +#define S64(b,x) (((x) >> (b)) | ((x) << (64 - (b)))) + +/* Two of six logical functions used in SHA-256, SHA-384, and SHA-512: */ +#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +/* Four of six logical functions used in SHA-256: */ +#define Sigma0_256(x) (S32(2, (x)) ^ S32(13, (x)) ^ S32(22, (x))) +#define Sigma1_256(x) (S32(6, (x)) ^ S32(11, (x)) ^ S32(25, (x))) +#define sigma0_256(x) (S32(7, (x)) ^ S32(18, (x)) ^ R(3 , (x))) +#define sigma1_256(x) (S32(17, (x)) ^ S32(19, (x)) ^ R(10, (x))) + +/* Four of six logical functions used in SHA-384 and SHA-512: */ +#define Sigma0_512(x) (S64(28, (x)) ^ S64(34, (x)) ^ S64(39, (x))) +#define Sigma1_512(x) (S64(14, (x)) ^ S64(18, (x)) ^ S64(41, (x))) +#define sigma0_512(x) (S64( 1, (x)) ^ S64( 8, (x)) ^ R( 7, (x))) +#define sigma1_512(x) (S64(19, (x)) ^ S64(61, (x)) ^ R( 6, (x))) + +static void sha512_last (sha512_state *state); +static void sha256_transform (sha256_state *state, const uint32_t idata[16]); +static void sha512_transform (sha512_state *state, const uint64_t idata[16]); + +/*** SHA-XYZ INITIAL HASH VALUES AND CONSTANTS ************************/ +/* Hash constant words K for SHA-256: */ +static const uint32_t K256[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, + 0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, + 0xd807aa98UL, 0x12835b01UL, 0x243185beUL, 0x550c7dc3UL, + 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, 0xc19bf174UL, + 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, + 0x983e5152UL, 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, + 0xc6e00bf3UL, 0xd5a79147UL, 0x06ca6351UL, 0x14292967UL, + 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, 0x53380d13UL, + 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, + 0xd192e819UL, 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, + 0x19a4c116UL, 0x1e376c08UL, 0x2748774cUL, 0x34b0bcb5UL, + 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL, + 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; + +/* Initial hash value H for SHA-256: */ +static const uint32_t sha256_initial_hash_value[8] = { + 0x6a09e667UL, + 0xbb67ae85UL, + 0x3c6ef372UL, + 0xa54ff53aUL, + 0x510e527fUL, + 0x9b05688cUL, + 0x1f83d9abUL, + 0x5be0cd19UL +}; + +/* Hash constant words K for SHA-384 and SHA-512: */ +static const uint64_t K512[80] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL +}; + +/* Initial hash value H for SHA-384 */ +static const uint64_t sha384_initial_hash_value[8] = { + 0xcbbb9d5dc1059ed8ULL, + 0x629a292a367cd507ULL, + 0x9159015a3070dd17ULL, + 0x152fecd8f70e5939ULL, + 0x67332667ffc00b31ULL, + 0x8eb44a8768581511ULL, + 0xdb0c2e0d64f98fa7ULL, + 0x47b5481dbefa4fa4ULL +}; + +/* Initial hash value H for SHA-512 */ +static const uint64_t sha512_initial_hash_value[8] = { + 0x6a09e667f3bcc908ULL, + 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, + 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, + 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, + 0x5be0cd19137e2179ULL +}; + +/*** SHA-256: *********************************************************/ +sha256_state * sha256_digest_init (sha256_state *state) +{ + MEMCPY_BCOPY(state->words, sha256_initial_hash_value, SHA256_DIGEST_LENGTH); + MEMSET_BZERO(state->buffer, SHA256_BLOCK_LENGTH); + state->bitcount = 0; + return state; +} + +#ifdef SHA2_UNROLL_TRANSFORM + +/* Unrolled SHA-256 round macros: */ + +#if BYTE_ORDER == LITTLE_ENDIAN + +#define ROUND256_0_TO_15(v, a, b, c, d, e, f, g, h) \ + REVERSE32(v, W256[j]); \ + T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + K256[j] + W256[j]; \ + (d) += T1; \ + (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)) + +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND256_0_TO_15(v, a, b, c, d, e, f, g, h) \ + T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + K256[j] + (W256[j] = v); \ + (d) += T1; \ + (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)) + +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND256(a, b, c, d, e, f, g, h) \ + s0 = W256[(j+1)&0x0f]; \ + s0 = sigma0_256(s0); \ + s1 = W256[(j+14)&0x0f]; \ + s1 = sigma1_256(s1); \ + T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + K256[j] + (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); \ + (d) += T1; \ + (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)) + +static void sha256_transform (sha256_state *state, const uint32_t idata[16]) { + uint32_t a, b, c, d, e, f, g, h, s0, s1; + uint32_t T1, *W256, v; + int j; + + W256 = state->buffer32; + + /* Initialize registers with the prev. intermediate value */ + a = state->words[0]; + b = state->words[1]; + c = state->words[2]; + d = state->words[3]; + e = state->words[4]; + f = state->words[5]; + g = state->words[6]; + h = state->words[7]; + + j = 0; + do { + /* Rounds 0 to 15 (unrolled): */ + v = idata[j]; ROUND256_0_TO_15(v, a, b, c, d, e, f, g, h); ++j; + v = idata[j]; ROUND256_0_TO_15(v, h, a, b, c, d, e, f, g); ++j; + v = idata[j]; ROUND256_0_TO_15(v, g, h, a, b, c, d, e, f); ++j; + v = idata[j]; ROUND256_0_TO_15(v, f, g, h, a, b, c, d, e); ++j; + v = idata[j]; ROUND256_0_TO_15(v, e, f, g, h, a, b, c, d); ++j; + v = idata[j]; ROUND256_0_TO_15(v, d, e, f, g, h, a, b, c); ++j; + v = idata[j]; ROUND256_0_TO_15(v, c, d, e, f, g, h, a, b); ++j; + v = idata[j]; ROUND256_0_TO_15(v, b, c, d, e, f, g, h, a); ++j; + } while (j < 16); + + /* Now for the remaining rounds to 64: */ + do { + ROUND256(a, b, c, d, e, f, g, h); ++j; + ROUND256(h, a, b, c, d, e, f, g); ++j; + ROUND256(g, h, a, b, c, d, e, f); ++j; + ROUND256(f, g, h, a, b, c, d, e); ++j; + ROUND256(e, f, g, h, a, b, c, d); ++j; + ROUND256(d, e, f, g, h, a, b, c); ++j; + ROUND256(c, d, e, f, g, h, a, b); ++j; + ROUND256(b, c, d, e, f, g, h, a); ++j; + } while (j < 64); + + /* Compute the current intermediate hash value */ + state->words[0] += a; + state->words[1] += b; + state->words[2] += c; + state->words[3] += d; + state->words[4] += e; + state->words[5] += f; + state->words[6] += g; + state->words[7] += h; +} + +#else /* SHA2_UNROLL_TRANSFORM */ + +static void sha256_transform (sha256_state *state, const uint32_t idata[16]) { + uint32_t a, b, c, d, e, f, g, h, s0, s1; + uint32_t T1, T2, *W256, v; + int j; + + W256 = state->buffer32; + + /* Initialize registers with the prev. intermediate value */ + a = state->words[0]; + b = state->words[1]; + c = state->words[2]; + d = state->words[3]; + e = state->words[4]; + f = state->words[5]; + g = state->words[6]; + h = state->words[7]; + + j = 0; + do { + v = idata[j]; +#if BYTE_ORDER == LITTLE_ENDIAN + /* Copy data while converting to host byte order */ + REVERSE32(v, W256[j]); + /* Apply the SHA-256 compression function to update a..h */ + T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + W256[j]; +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + /* Apply the SHA-256 compression function to update a..h with copy */ + T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + (W256[j] = v); +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + T2 = Sigma0_256(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 16); + + do { + /* Part of the message block expansion: */ + s0 = W256[(j+1)&0x0f]; + s0 = sigma0_256(s0); + s1 = W256[(j+14)&0x0f]; + s1 = sigma1_256(s1); + + /* Apply the SHA-256 compression function to update a..h */ + T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); + T2 = Sigma0_256(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 64); + + /* Compute the current intermediate hash value */ + state->words[0] += a; + state->words[1] += b; + state->words[2] += c; + state->words[3] += d; + state->words[4] += e; + state->words[5] += f; + state->words[6] += g; + state->words[7] += h; +} + +#endif /* SHA2_UNROLL_TRANSFORM */ + +/* PJ: alignment-safe version */ + +#define data_aligned4(data) (((data - (const uint8_t *)(0UL)) & 3) == 0) +#define data_aligned8(data) (((data - (const uint8_t *)(0ULL)) & 7) == 0) + +static void sha256_transform_aligned (sha256_state *state, const uint8_t *data) { + if (data_aligned4(data)) + { + sha256_transform(state, (const uint32_t *)((const void *)data)); // alignment ok + } + else + { + uint32_t idata[16]; + memcpy(&idata[0], data, 16 * sizeof(uint32_t)); + sha256_transform(state, idata); + } +} + +void sha256_digest_add (sha256_state *state, const void *vdata, size_t len) +{ + unsigned int freespace, usedspace; + const uint8_t *data; + + if (len == 0) /* Calling with no data is valid - we do nothing */ + return; + + data = (const uint8_t *)vdata; + + usedspace = (state->bitcount >> 3) % SHA256_BLOCK_LENGTH; + if (usedspace > 0) + { + /* Calculate how much free space is available in the buffer */ + freespace = SHA256_BLOCK_LENGTH - usedspace; + + if (len >= freespace) + { + /* Fill the buffer completely and process it */ + MEMCPY_BCOPY(&state->buffer[usedspace], data, freespace); + state->bitcount += freespace << 3; + len -= freespace; + data += freespace; + sha256_transform(state, state->buffer32); + } + else + { + /* The buffer is not yet full */ + MEMCPY_BCOPY(&state->buffer[usedspace], data, len); + state->bitcount += len << 3; + return; + } + } + while (len >= SHA256_BLOCK_LENGTH) + { + /* Process as many complete blocks as we can */ + sha256_transform_aligned(state, data); + + state->bitcount += SHA256_BLOCK_LENGTH << 3; + len -= SHA256_BLOCK_LENGTH; + data += SHA256_BLOCK_LENGTH; + } + if (len > 0) + { + /* There's left-overs, so save 'em */ + MEMCPY_BCOPY(state->buffer, data, len); + state->bitcount += len << 3; + } +} + +static void digest_hex (uint8_t digest[], const void *data, size_t size, int flags); + +void sha256_digest_get (sha256_state *state, uint8_t digest[], int flags) { + unsigned int usedspace; + + usedspace = (state->bitcount >> 3) % SHA256_BLOCK_LENGTH; +#if BYTE_ORDER == LITTLE_ENDIAN + /* Convert FROM host byte order */ + REVERSE64(state->bitcount,state->bitcount); +#endif + if (usedspace > 0) + { + /* Begin padding with a 1 bit: */ + state->buffer[usedspace++] = 0x80; + + if (usedspace <= SHA256_SHORT_BLOCK_LENGTH) { + /* Set-up for the last transform: */ + MEMSET_BZERO(&state->buffer[usedspace], SHA256_SHORT_BLOCK_LENGTH - usedspace); + } else { + if (usedspace < SHA256_BLOCK_LENGTH) { + MEMSET_BZERO(&state->buffer[usedspace], SHA256_BLOCK_LENGTH - usedspace); + } + /* Do second-to-last transform: */ + sha256_transform(state, state->buffer32); + + /* And set-up for the last transform: */ + MEMSET_BZERO(state->buffer, SHA256_SHORT_BLOCK_LENGTH); + } + } + else + { + /* Set-up for the last transform: */ + MEMSET_BZERO(state->buffer, SHA256_SHORT_BLOCK_LENGTH); + + /* Begin padding with a 1 bit: */ + *state->buffer = 0x80; + } + /* Set the bit count: */ + //*(uint64_t*)&state->buffer[SHA256_SHORT_BLOCK_LENGTH] = state->bitcount; // aliasing violation warning + state->buffer64[SHA256_SHORT_BLOCK_LENGTH / sizeof(uint64_t)] = state->bitcount; + + /* Final transform: */ + sha256_transform(state, state->buffer32); + +#if BYTE_ORDER == LITTLE_ENDIAN + { + /* Convert TO host byte order */ + int j; + for (j = 0; j < 8; j++) + { + REVERSE32(state->words[j], state->words[j]); + } + } +#endif + if (flags & SHA_HEX) + digest_hex(digest, state->words, SHA256_DIGEST_LENGTH, flags); + else + memcpy(digest, state->words, SHA256_DIGEST_LENGTH); +} + +/*** SHA-512: *********************************************************/ +sha512_state * sha512_digest_init (sha512_state *state) +{ + MEMCPY_BCOPY(state->words, sha512_initial_hash_value, SHA512_DIGEST_LENGTH); + MEMSET_BZERO(state->buffer, SHA512_BLOCK_LENGTH); + state->bitcount[0] = 0; + state->bitcount[1] = 0; + return state; +} + +#ifdef SHA2_UNROLL_TRANSFORM + +/* PJ: ++ operations moved out of macros! */ + +/* Unrolled SHA-512 round macros: */ +#if BYTE_ORDER == LITTLE_ENDIAN + +#define ROUND512_0_TO_15(v, a, b, c, d, e, f, g, h) \ + REVERSE64(v, W512[j]); \ + T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + K512[j] + W512[j]; \ + (d) += T1; \ + (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)) + +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND512_0_TO_15(v, a, b, c, d, e, f, g, h) \ + T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + K512[j] + (W512[j] = v); \ + (d) += T1; \ + (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)) + +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND512(a, b, c, d, e, f, g, h) \ + s0 = W512[(j+1)&0x0f]; \ + s0 = sigma0_512(s0); \ + s1 = W512[(j+14)&0x0f]; \ + s1 = sigma1_512(s1); \ + T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + K512[j] + (W512[j&0x0f] += s1 + W512[(j+9)&0x0f] + s0); \ + (d) += T1; \ + (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)) + +static void sha512_transform (sha512_state *state, const uint64_t idata[16]) +{ + uint64_t a, b, c, d, e, f, g, h, s0, s1; + uint64_t T1, *W512, v; + int j; + + W512 = state->buffer64; + + /* Initialize registers with the prev. intermediate value */ + a = state->words[0]; + b = state->words[1]; + c = state->words[2]; + d = state->words[3]; + e = state->words[4]; + f = state->words[5]; + g = state->words[6]; + h = state->words[7]; + + j = 0; + do { + v = idata[j]; ROUND512_0_TO_15(v, a, b, c, d, e, f, g, h); ++j; + v = idata[j]; ROUND512_0_TO_15(v, h, a, b, c, d, e, f, g); ++j; + v = idata[j]; ROUND512_0_TO_15(v, g, h, a, b, c, d, e, f); ++j; + v = idata[j]; ROUND512_0_TO_15(v, f, g, h, a, b, c, d, e); ++j; + v = idata[j]; ROUND512_0_TO_15(v, e, f, g, h, a, b, c, d); ++j; + v = idata[j]; ROUND512_0_TO_15(v, d, e, f, g, h, a, b, c); ++j; + v = idata[j]; ROUND512_0_TO_15(v, c, d, e, f, g, h, a, b); ++j; + v = idata[j]; ROUND512_0_TO_15(v, b, c, d, e, f, g, h, a); ++j; + } while (j < 16); + + /* Now for the remaining rounds up to 79: */ + do { + ROUND512(a, b, c, d, e, f, g, h); ++j; + ROUND512(h, a, b, c, d, e, f, g); ++j; + ROUND512(g, h, a, b, c, d, e, f); ++j; + ROUND512(f, g, h, a, b, c, d, e); ++j; + ROUND512(e, f, g, h, a, b, c, d); ++j; + ROUND512(d, e, f, g, h, a, b, c); ++j; + ROUND512(c, d, e, f, g, h, a, b); ++j; + ROUND512(b, c, d, e, f, g, h, a); ++j; + } while (j < 80); + + /* Compute the current intermediate hash value */ + state->words[0] += a; + state->words[1] += b; + state->words[2] += c; + state->words[3] += d; + state->words[4] += e; + state->words[5] += f; + state->words[6] += g; + state->words[7] += h; +} + +#else /* SHA2_UNROLL_TRANSFORM */ + +static void sha512_transform (sha512_state *state, const uint64_t idata[16]) +{ + uint64_t a, b, c, d, e, f, g, h, s0, s1; + uint64_t T1, T2, *W512, v; + int j; + + W512 = state->buffer64; + + /* Initialize registers with the prev. intermediate value */ + a = state->words[0]; + b = state->words[1]; + c = state->words[2]; + d = state->words[3]; + e = state->words[4]; + f = state->words[5]; + g = state->words[6]; + h = state->words[7]; + + j = 0; + do { + v = idata[j]; +#if BYTE_ORDER == LITTLE_ENDIAN + /* Convert TO host byte order */ + REVERSE64(v, W512[j]); + /* Apply the SHA-512 compression function to update a..h */ + T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + W512[j]; +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + /* Apply the SHA-512 compression function to update a..h with copy */ + T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + (W512[j] = v); +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + T2 = Sigma0_512(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 16); + + do { + /* Part of the message block expansion: */ + s0 = W512[(j+1)&0x0f]; + s0 = sigma0_512(s0); + s1 = W512[(j+14)&0x0f]; + s1 = sigma1_512(s1); + + /* Apply the SHA-512 compression function to update a..h */ + T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + (W512[j&0x0f] += s1 + W512[(j+9)&0x0f] + s0); + T2 = Sigma0_512(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 80); + + /* Compute the current intermediate hash value */ + state->words[0] += a; + state->words[1] += b; + state->words[2] += c; + state->words[3] += d; + state->words[4] += e; + state->words[5] += f; + state->words[6] += g; + state->words[7] += h; +} + +#endif /* SHA2_UNROLL_TRANSFORM */ + +static void sha512_transform_aligned (sha512_state *state, const uint8_t *data) +{ + if (data_aligned8(data)) + { + sha512_transform(state, (const uint64_t *)((const void *)data)); // alignment ok + } + else + { + uint64_t idata[16]; + memcpy(&idata[0], data, 16 * sizeof(uint64_t)); + sha512_transform(state, idata); + } +} + +void sha512_digest_add (sha512_state *state, const void *vdata, size_t len) +{ + unsigned int freespace, usedspace; + const uint8_t *data; + + if (len == 0) /* Calling with no data is valid - we do nothing */ + return; + + /* Sanity check: */ + data = (const uint8_t *)vdata; + + usedspace = (state->bitcount[0] >> 3) % SHA512_BLOCK_LENGTH; + if (usedspace > 0) + { + /* Calculate how much free space is available in the buffer */ + freespace = SHA512_BLOCK_LENGTH - usedspace; + + if (len >= freespace) + { + /* Fill the buffer completely and process it */ + MEMCPY_BCOPY(&state->buffer[usedspace], data, freespace); + ADDINC128(state->bitcount, freespace << 3); + len -= freespace; + data += freespace; + sha512_transform(state, state->buffer64); + } + else + { + /* The buffer is not yet full */ + MEMCPY_BCOPY(&state->buffer[usedspace], data, len); + ADDINC128(state->bitcount, len << 3); + return; + } + } + while (len >= SHA512_BLOCK_LENGTH) + { + /* Process as many complete blocks as we can */ + sha512_transform_aligned(state, data); + + ADDINC128(state->bitcount, SHA512_BLOCK_LENGTH << 3); + len -= SHA512_BLOCK_LENGTH; + data += SHA512_BLOCK_LENGTH; + } + if (len > 0) + { + /* There's left-overs, so save 'em */ + MEMCPY_BCOPY(state->buffer, data, len); + ADDINC128(state->bitcount, len << 3); + } +} + +static void sha512_last (sha512_state *state) +{ + unsigned int usedspace; + + usedspace = (state->bitcount[0] >> 3) % SHA512_BLOCK_LENGTH; +#if BYTE_ORDER == LITTLE_ENDIAN + /* Convert FROM host byte order */ + REVERSE64(state->bitcount[0],state->bitcount[0]); + REVERSE64(state->bitcount[1],state->bitcount[1]); +#endif + if (usedspace > 0) + { + /* Begin padding with a 1 bit: */ + state->buffer[usedspace++] = 0x80; + + if (usedspace <= SHA512_SHORT_BLOCK_LENGTH) { + /* Set-up for the last transform: */ + MEMSET_BZERO(&state->buffer[usedspace], SHA512_SHORT_BLOCK_LENGTH - usedspace); + } else { + if (usedspace < SHA512_BLOCK_LENGTH) { + MEMSET_BZERO(&state->buffer[usedspace], SHA512_BLOCK_LENGTH - usedspace); + } + /* Do second-to-last transform: */ + sha512_transform(state, state->buffer64); + + /* And set-up for the last transform: */ + //MEMSET_BZERO(state->buffer, SHA512_BLOCK_LENGTH - 2); // seems a typo, we overwrite last 16 bytes below + MEMSET_BZERO(state->buffer, SHA512_SHORT_BLOCK_LENGTH); + } + } + else + { + /* Prepare for final transform: */ + MEMSET_BZERO(state->buffer, SHA512_SHORT_BLOCK_LENGTH); + + /* Begin padding with a 1 bit: */ + *state->buffer = 0x80; + } + /* Store the length of input data (in bits): */ + //*(uint64_t*)&state->buffer[SHA512_SHORT_BLOCK_LENGTH] = state->bitcount[1]; // aliasing violation warning + //*(uint64_t*)&state->buffer[SHA512_SHORT_BLOCK_LENGTH+8] = state->bitcount[0]; + state->buffer64[SHA512_SHORT_BLOCK_LENGTH / sizeof(uint64_t)] = state->bitcount[1]; + state->buffer64[SHA512_SHORT_BLOCK_LENGTH / sizeof(uint64_t) + 1] = state->bitcount[0]; + + /* Final transform: */ + sha512_transform(state, state->buffer64); +} + +void sha512_digest_get (sha512_state *state, uint8_t digest[], int flags) +{ + /* If no digest buffer is passed, we don't bother doing this: */ + sha512_last(state); + + /* Save the hash data for output: */ +#if BYTE_ORDER == LITTLE_ENDIAN + { + /* Convert TO host byte order */ + int j; + for (j = 0; j < 8; j++) + { + REVERSE64(state->words[j], state->words[j]); + } + } +#endif + if (flags & SHA_HEX) + digest_hex(digest, state->words, SHA512_DIGEST_LENGTH, flags); + else + memcpy(digest, state->words, SHA512_DIGEST_LENGTH); +} + +/*** SHA-384: *********************************************************/ +sha384_state * sha384_digest_init (sha384_state *state) +{ + MEMCPY_BCOPY(state->words, sha384_initial_hash_value, SHA512_DIGEST_LENGTH); + MEMSET_BZERO(state->buffer, SHA384_BLOCK_LENGTH); + state->bitcount[0] = state->bitcount[1] = 0; + return state; +} + +void sha384_digest_add (sha384_state *state, const void *data, size_t len) +{ + sha512_digest_add((sha512_state *)state, data, len); +} + +void sha384_digest_get (sha384_state *state, uint8_t digest[], int flags) +{ + sha512_last((sha512_state *)state); + + /* Save the hash data for output: */ +#if BYTE_ORDER == LITTLE_ENDIAN + { + /* Convert TO host byte order */ + int j; + for (j = 0; j < 6; j++) + { + REVERSE64(state->words[j], state->words[j]); + } + } +#endif + if (flags & SHA_HEX) + digest_hex(digest, state->words, SHA384_DIGEST_LENGTH, flags); + else + memcpy(digest, state->words, SHA384_DIGEST_LENGTH); +} + +/* hex output */ + +static void digest_hex (uint8_t digest[], const void *data, size_t size, int flags) +{ + const char *alphabet; + const uint8_t *bytes; + size_t i; + + bytes = (const uint8_t *)data; + alphabet = (flags & SHA_LCHEX) ? "0123456789abcdef" : "0123456789ABCDEF"; + for (i = 0; i < size; ++i, ++bytes) + { + *digest++ = (uint8_t)alphabet[(*bytes) >> 4]; + *digest++ = (uint8_t)alphabet[(*bytes) & 15]; + } + *digest = 0; +} + +/* string checksum */ + +void sha256_digest (const void *data, size_t size, uint8_t digest[], int flags) +{ + sha256_state state; + sha256_digest_init(&state); + sha256_digest_add(&state, data, size); + sha256_digest_get(&state, digest, flags); +} + +void sha384_digest (const void *data, size_t size, uint8_t digest[], int flags) +{ + sha384_state state; + sha384_digest_init(&state); + sha384_digest_add(&state, data, size); + sha384_digest_get(&state, digest, flags); +} + +void sha512_digest (const void *data, size_t size, uint8_t digest[], int flags) +{ + sha512_state state; + sha512_digest_init(&state); + sha512_digest_add(&state, data, size); + sha512_digest_get(&state, digest, flags); +} + +/* file checksum */ + +#define DIGEST_BUFFER_SIZE 4096 + +int sha256_digest_add_file (sha256_state *state, const char *filename) +{ + FILE *fh; + uint8_t buffer[DIGEST_BUFFER_SIZE]; + size_t read; + + if ((fh = fopen(filename, "rb")) == NULL) + return 0; + do { + read = fread(buffer, 1, DIGEST_BUFFER_SIZE, fh); + sha256_digest_add(state, buffer, read); + } while (read == DIGEST_BUFFER_SIZE); + fclose(fh); + return 1; +} + +int sha256_digest_file (const char *filename, uint8_t digest[], int flags) +{ + sha256_state state; + + sha256_digest_init(&state); + if (sha256_digest_add_file(&state, filename)) + { + sha256_digest_get(&state, digest, flags); + return 1; + } + return 0; +} + +int sha384_digest_add_file (sha384_state *state, const char *filename) +{ + FILE *fh; + uint8_t buffer[DIGEST_BUFFER_SIZE]; + size_t read; + + if ((fh = fopen(filename, "rb")) == NULL) + return 0; + do { + read = fread(buffer, 1, DIGEST_BUFFER_SIZE, fh); + sha384_digest_add(state, buffer, read); + } while (read == DIGEST_BUFFER_SIZE); + fclose(fh); + return 1; +} + +int sha384_digest_file (const char *filename, uint8_t digest[], int flags) +{ + sha384_state state; + + sha384_digest_init(&state); + if (sha384_digest_add_file(&state, filename)) + { + sha384_digest_get(&state, digest, flags); + return 1; + } + return 0; +} + +int sha512_digest_add_file (sha512_state *state, const char *filename) +{ + FILE *fh; + uint8_t buffer[DIGEST_BUFFER_SIZE]; + size_t read; + + if ((fh = fopen(filename, "rb")) == NULL) + return 0; + do { + read = fread(buffer, 1, DIGEST_BUFFER_SIZE, fh); + sha512_digest_add(state, buffer, read); + } while (read == DIGEST_BUFFER_SIZE); + fclose(fh); + return 1; +} + +int sha512_digest_file (const char *filename, uint8_t digest[], int flags) +{ + sha512_state state; + + sha512_digest_init(&state); + if (sha512_digest_add_file(&state, filename)) + { + sha512_digest_get(&state, digest, flags); + return 1; + } + return 0; +} diff --git a/Build/source/libs/pplib/pplib-src/src/util/utilsha.h b/Build/source/libs/pplib/pplib-src/src/util/utilsha.h new file mode 100644 index 00000000000..6c9b1bdc9c0 --- /dev/null +++ b/Build/source/libs/pplib/pplib-src/src/util/utilsha.h @@ -0,0 +1,79 @@ +/* sha2 implementation excerpted from code by Aaron D. Gifford */ + +#ifndef UTIL_SHA_H +#define UTIL_SHA_H + +#include <stddef.h> +#include <stdint.h> +#include "utildecl.h" + +#define SHA256_BLOCK_LENGTH 64 +#define SHA256_DIGEST_LENGTH 32 +#define SHA256_STRING_LENGTH (SHA256_DIGEST_LENGTH * 2 + 1) +#define SHA384_BLOCK_LENGTH 128 +#define SHA384_DIGEST_LENGTH 48 +#define SHA384_STRING_LENGTH (SHA384_DIGEST_LENGTH * 2 + 1) +#define SHA512_BLOCK_LENGTH 128 +#define SHA512_DIGEST_LENGTH 64 +#define SHA512_STRING_LENGTH (SHA512_DIGEST_LENGTH * 2 + 1) + +//#define sha256_state sha256_state_t +//#define sha384_state sha384_state_t +//#define sha512_state sha512_state_t + +typedef struct { + uint32_t words[8]; + uint64_t bitcount; + union { + uint8_t buffer[SHA256_BLOCK_LENGTH]; + uint32_t buffer32[SHA256_BLOCK_LENGTH / sizeof(uint32_t)]; + uint64_t buffer64[SHA256_BLOCK_LENGTH / sizeof(uint64_t)]; + }; +} sha256_state; + +typedef struct { + uint64_t words[8]; + uint64_t bitcount[2]; + union { + uint8_t buffer[SHA512_BLOCK_LENGTH]; + uint64_t buffer64[SHA512_BLOCK_LENGTH / sizeof(uint64_t)]; + }; +} sha512_state; + +typedef sha512_state sha384_state; + +enum { + SHA_BYTES = 0, + SHA_UCHEX = (1<<0), + SHA_LCHEX = (1<<1) +}; + +#define SHA_DEFAULT SHA_BYTES +#define SHA_HEX (SHA_UCHEX|SHA_LCHEX) + +UTILAPI sha256_state * sha256_digest_init (sha256_state *state); +UTILAPI sha384_state * sha384_digest_init (sha384_state *state); +UTILAPI sha512_state * sha512_digest_init (sha512_state *state); + +UTILAPI void sha256_digest_add (sha256_state *state, const void *data, size_t size); +UTILAPI void sha384_digest_add (sha384_state *state, const void *data, size_t size); +UTILAPI void sha512_digest_add (sha512_state *state, const void *data, size_t size); + +UTILAPI void sha256_digest_get (sha256_state *state, uint8_t digest[], int flags); +UTILAPI void sha384_digest_get (sha384_state *state, uint8_t digest[], int flags); +UTILAPI void sha512_digest_get (sha512_state *state, uint8_t digest[], int flags); + +UTILAPI void sha256_digest (const void *data, size_t size, uint8_t digest[], int flags); +UTILAPI void sha384_digest (const void *data, size_t size, uint8_t digest[], int flags); +UTILAPI void sha512_digest (const void *data, size_t size, uint8_t digest[], int flags); + +UTILAPI int sha256_digest_add_file (sha256_state *state, const char *filename); +UTILAPI int sha256_digest_file (const char *filename, uint8_t digest[], int flags); + +UTILAPI int sha384_digest_add_file (sha384_state *state, const char *filename); +UTILAPI int sha384_digest_file (const char *filename, uint8_t digest[], int flags); + +UTILAPI int sha512_digest_add_file (sha512_state *state, const char *filename); +UTILAPI int sha512_digest_file (const char *filename, uint8_t digest[], int flags); + +#endif |