summaryrefslogtreecommitdiff
path: root/dviware/dvisvgm/libs/brotli
diff options
context:
space:
mode:
Diffstat (limited to 'dviware/dvisvgm/libs/brotli')
-rw-r--r--dviware/dvisvgm/libs/brotli/Makefile.am5
-rw-r--r--dviware/dvisvgm/libs/brotli/Makefile.in41
-rw-r--r--dviware/dvisvgm/libs/brotli/common/constants.c2
-rw-r--r--dviware/dvisvgm/libs/brotli/common/constants.h3
-rw-r--r--dviware/dvisvgm/libs/brotli/common/context.c2
-rw-r--r--dviware/dvisvgm/libs/brotli/common/dictionary.c8
-rw-r--r--dviware/dvisvgm/libs/brotli/common/platform.c3
-rw-r--r--dviware/dvisvgm/libs/brotli/common/platform.h173
-rw-r--r--dviware/dvisvgm/libs/brotli/common/shared_dictionary_internal.h75
-rw-r--r--dviware/dvisvgm/libs/brotli/common/transform.c2
-rw-r--r--dviware/dvisvgm/libs/brotli/common/version.h37
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/backward_references.c100
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/backward_references.h9
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/backward_references_hq.c200
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/backward_references_hq.h11
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/backward_references_inc.h36
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/bit_cost.c15
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/bit_cost.h9
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/block_splitter.c51
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/block_splitter.h9
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/block_splitter_inc.h131
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.c242
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.h17
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/cluster.c19
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/cluster.h13
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/cluster_inc.h45
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/command.c2
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/command.h9
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/compound_dictionary.c207
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/compound_dictionary.h74
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/compress_fragment.c176
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/compress_fragment.h39
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.c186
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.h24
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/dictionary_hash.c4
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/encode.c875
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/encoder_dict.c615
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/encoder_dict.h120
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/entropy_encode.c5
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/entropy_encode.h3
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/entropy_encode_static.h7
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/fast_log.c2
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/fast_log.h3
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/find_match_length.h37
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash.h330
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash_composite_inc.h37
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash_forgetful_chain_inc.h38
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash_longest_match64_inc.h73
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash_longest_match_inc.h16
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash_longest_match_quickly_inc.h8
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash_rolling_inc.h8
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/hash_to_binary_tree_inc.h11
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/histogram.c8
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/histogram.h13
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/literal_cost.c35
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/literal_cost.h6
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/memory.c34
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/memory.h21
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/metablock.c130
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/metablock.h17
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/metablock_inc.h14
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/params.h7
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/prefix.h5
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/quality.h43
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/ringbuffer.h7
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/state.h104
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/static_dict.c64
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/static_dict.h5
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/static_dict_lut.h2
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/utf8_util.c2
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/utf8_util.h3
-rw-r--r--dviware/dvisvgm/libs/brotli/enc/write_bits.h3
-rw-r--r--dviware/dvisvgm/libs/brotli/include/brotli/decode.h69
-rw-r--r--dviware/dvisvgm/libs/brotli/include/brotli/encode.h57
-rw-r--r--dviware/dvisvgm/libs/brotli/include/brotli/port.h39
-rw-r--r--dviware/dvisvgm/libs/brotli/include/brotli/shared_dictionary.h100
76 files changed, 3602 insertions, 1353 deletions
diff --git a/dviware/dvisvgm/libs/brotli/Makefile.am b/dviware/dvisvgm/libs/brotli/Makefile.am
index 64ad12e6aa..7d7bf1db32 100644
--- a/dviware/dvisvgm/libs/brotli/Makefile.am
+++ b/dviware/dvisvgm/libs/brotli/Makefile.am
@@ -9,6 +9,7 @@ libbrotli_a_SOURCES = \
common/dictionary.h \
common/platform.c \
common/platform.h \
+ common/shared_dictionary_internal.h \
common/transform.c \
common/transform.h \
common/version.h \
@@ -31,6 +32,8 @@ libbrotli_a_SOURCES = \
enc/cluster_inc.h \
enc/command.c \
enc/command.h \
+ enc/compound_dictionary.c \
+ enc/compound_dictionary.h \
enc/compress_fragment.c \
enc/compress_fragment.h \
enc/compress_fragment_two_pass.c \
@@ -68,6 +71,7 @@ libbrotli_a_SOURCES = \
enc/prefix.h \
enc/quality.h \
enc/ringbuffer.h \
+ enc/state.h \
enc/static_dict.c \
enc/static_dict.h \
enc/static_dict_lut.h \
@@ -77,6 +81,7 @@ libbrotli_a_SOURCES = \
include/brotli/decode.h \
include/brotli/encode.h \
include/brotli/port.h \
+ include/brotli/shared_dictionary.h \
include/brotli/types.h
AM_CFLAGS = -Wall -I$(srcdir)/include
diff --git a/dviware/dvisvgm/libs/brotli/Makefile.in b/dviware/dvisvgm/libs/brotli/Makefile.in
index 6518869128..075a208efb 100644
--- a/dviware/dvisvgm/libs/brotli/Makefile.in
+++ b/dviware/dvisvgm/libs/brotli/Makefile.in
@@ -120,15 +120,16 @@ libbrotli_a_LIBADD =
am__libbrotli_a_SOURCES_DIST = common/constants.c common/constants.h \
common/context.c common/context.h common/dictionary.c \
common/dictionary.h common/platform.c common/platform.h \
- common/transform.c common/transform.h common/version.h \
- enc/backward_references.c enc/backward_references.h \
- enc/backward_references_hq.c enc/backward_references_hq.h \
- enc/backward_references_inc.h enc/bit_cost.c enc/bit_cost.h \
- enc/bit_cost_inc.h enc/block_encoder_inc.h \
- enc/block_splitter.c enc/block_splitter.h \
- enc/block_splitter_inc.h enc/brotli_bit_stream.c \
- enc/brotli_bit_stream.h enc/cluster.c enc/cluster.h \
- enc/cluster_inc.h enc/command.c enc/command.h \
+ common/shared_dictionary_internal.h common/transform.c \
+ common/transform.h common/version.h enc/backward_references.c \
+ enc/backward_references.h enc/backward_references_hq.c \
+ enc/backward_references_hq.h enc/backward_references_inc.h \
+ enc/bit_cost.c enc/bit_cost.h enc/bit_cost_inc.h \
+ enc/block_encoder_inc.h enc/block_splitter.c \
+ enc/block_splitter.h enc/block_splitter_inc.h \
+ enc/brotli_bit_stream.c enc/brotli_bit_stream.h enc/cluster.c \
+ enc/cluster.h enc/cluster_inc.h enc/command.c enc/command.h \
+ enc/compound_dictionary.c enc/compound_dictionary.h \
enc/compress_fragment.c enc/compress_fragment.h \
enc/compress_fragment_two_pass.c \
enc/compress_fragment_two_pass.h enc/dictionary_hash.c \
@@ -143,11 +144,11 @@ am__libbrotli_a_SOURCES_DIST = common/constants.c common/constants.h \
enc/histogram_inc.h enc/literal_cost.c enc/literal_cost.h \
enc/memory.c enc/memory.h enc/metablock.c enc/metablock.h \
enc/metablock_inc.h enc/params.h enc/prefix.h enc/quality.h \
- enc/ringbuffer.h enc/static_dict.c enc/static_dict.h \
- enc/static_dict_lut.h enc/utf8_util.c enc/utf8_util.h \
- enc/write_bits.h include/brotli/decode.h \
+ enc/ringbuffer.h enc/state.h enc/static_dict.c \
+ enc/static_dict.h enc/static_dict_lut.h enc/utf8_util.c \
+ enc/utf8_util.h enc/write_bits.h include/brotli/decode.h \
include/brotli/encode.h include/brotli/port.h \
- include/brotli/types.h
+ include/brotli/shared_dictionary.h include/brotli/types.h
am__dirstamp = $(am__leading_dot)dirstamp
@HAVE_BROTLI_FALSE@am_libbrotli_a_OBJECTS = \
@HAVE_BROTLI_FALSE@ common/constants.$(OBJEXT) \
@@ -161,6 +162,7 @@ am__dirstamp = $(am__leading_dot)dirstamp
@HAVE_BROTLI_FALSE@ enc/block_splitter.$(OBJEXT) \
@HAVE_BROTLI_FALSE@ enc/brotli_bit_stream.$(OBJEXT) \
@HAVE_BROTLI_FALSE@ enc/cluster.$(OBJEXT) enc/command.$(OBJEXT) \
+@HAVE_BROTLI_FALSE@ enc/compound_dictionary.$(OBJEXT) \
@HAVE_BROTLI_FALSE@ enc/compress_fragment.$(OBJEXT) \
@HAVE_BROTLI_FALSE@ enc/compress_fragment_two_pass.$(OBJEXT) \
@HAVE_BROTLI_FALSE@ enc/dictionary_hash.$(OBJEXT) \
@@ -197,7 +199,8 @@ am__depfiles_remade = common/$(DEPDIR)/constants.Po \
enc/$(DEPDIR)/backward_references_hq.Po \
enc/$(DEPDIR)/bit_cost.Po enc/$(DEPDIR)/block_splitter.Po \
enc/$(DEPDIR)/brotli_bit_stream.Po enc/$(DEPDIR)/cluster.Po \
- enc/$(DEPDIR)/command.Po enc/$(DEPDIR)/compress_fragment.Po \
+ enc/$(DEPDIR)/command.Po enc/$(DEPDIR)/compound_dictionary.Po \
+ enc/$(DEPDIR)/compress_fragment.Po \
enc/$(DEPDIR)/compress_fragment_two_pass.Po \
enc/$(DEPDIR)/dictionary_hash.Po enc/$(DEPDIR)/encode.Po \
enc/$(DEPDIR)/encoder_dict.Po enc/$(DEPDIR)/entropy_encode.Po \
@@ -422,6 +425,7 @@ top_srcdir = @top_srcdir@
@HAVE_BROTLI_FALSE@ common/dictionary.h \
@HAVE_BROTLI_FALSE@ common/platform.c \
@HAVE_BROTLI_FALSE@ common/platform.h \
+@HAVE_BROTLI_FALSE@ common/shared_dictionary_internal.h \
@HAVE_BROTLI_FALSE@ common/transform.c \
@HAVE_BROTLI_FALSE@ common/transform.h \
@HAVE_BROTLI_FALSE@ common/version.h \
@@ -444,6 +448,8 @@ top_srcdir = @top_srcdir@
@HAVE_BROTLI_FALSE@ enc/cluster_inc.h \
@HAVE_BROTLI_FALSE@ enc/command.c \
@HAVE_BROTLI_FALSE@ enc/command.h \
+@HAVE_BROTLI_FALSE@ enc/compound_dictionary.c \
+@HAVE_BROTLI_FALSE@ enc/compound_dictionary.h \
@HAVE_BROTLI_FALSE@ enc/compress_fragment.c \
@HAVE_BROTLI_FALSE@ enc/compress_fragment.h \
@HAVE_BROTLI_FALSE@ enc/compress_fragment_two_pass.c \
@@ -481,6 +487,7 @@ top_srcdir = @top_srcdir@
@HAVE_BROTLI_FALSE@ enc/prefix.h \
@HAVE_BROTLI_FALSE@ enc/quality.h \
@HAVE_BROTLI_FALSE@ enc/ringbuffer.h \
+@HAVE_BROTLI_FALSE@ enc/state.h \
@HAVE_BROTLI_FALSE@ enc/static_dict.c \
@HAVE_BROTLI_FALSE@ enc/static_dict.h \
@HAVE_BROTLI_FALSE@ enc/static_dict_lut.h \
@@ -490,6 +497,7 @@ top_srcdir = @top_srcdir@
@HAVE_BROTLI_FALSE@ include/brotli/decode.h \
@HAVE_BROTLI_FALSE@ include/brotli/encode.h \
@HAVE_BROTLI_FALSE@ include/brotli/port.h \
+@HAVE_BROTLI_FALSE@ include/brotli/shared_dictionary.h \
@HAVE_BROTLI_FALSE@ include/brotli/types.h
@HAVE_BROTLI_FALSE@AM_CFLAGS = -Wall -I$(srcdir)/include
@@ -567,6 +575,8 @@ enc/cluster.$(OBJEXT): enc/$(am__dirstamp) \
enc/$(DEPDIR)/$(am__dirstamp)
enc/command.$(OBJEXT): enc/$(am__dirstamp) \
enc/$(DEPDIR)/$(am__dirstamp)
+enc/compound_dictionary.$(OBJEXT): enc/$(am__dirstamp) \
+ enc/$(DEPDIR)/$(am__dirstamp)
enc/compress_fragment.$(OBJEXT): enc/$(am__dirstamp) \
enc/$(DEPDIR)/$(am__dirstamp)
enc/compress_fragment_two_pass.$(OBJEXT): enc/$(am__dirstamp) \
@@ -619,6 +629,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@enc/$(DEPDIR)/brotli_bit_stream.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@enc/$(DEPDIR)/cluster.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@enc/$(DEPDIR)/command.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@enc/$(DEPDIR)/compound_dictionary.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@enc/$(DEPDIR)/compress_fragment.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@enc/$(DEPDIR)/compress_fragment_two_pass.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@enc/$(DEPDIR)/dictionary_hash.Po@am__quote@ # am--include-marker
@@ -810,6 +821,7 @@ distclean: distclean-am
-rm -f enc/$(DEPDIR)/brotli_bit_stream.Po
-rm -f enc/$(DEPDIR)/cluster.Po
-rm -f enc/$(DEPDIR)/command.Po
+ -rm -f enc/$(DEPDIR)/compound_dictionary.Po
-rm -f enc/$(DEPDIR)/compress_fragment.Po
-rm -f enc/$(DEPDIR)/compress_fragment_two_pass.Po
-rm -f enc/$(DEPDIR)/dictionary_hash.Po
@@ -880,6 +892,7 @@ maintainer-clean: maintainer-clean-am
-rm -f enc/$(DEPDIR)/brotli_bit_stream.Po
-rm -f enc/$(DEPDIR)/cluster.Po
-rm -f enc/$(DEPDIR)/command.Po
+ -rm -f enc/$(DEPDIR)/compound_dictionary.Po
-rm -f enc/$(DEPDIR)/compress_fragment.Po
-rm -f enc/$(DEPDIR)/compress_fragment_two_pass.Po
-rm -f enc/$(DEPDIR)/dictionary_hash.Po
diff --git a/dviware/dvisvgm/libs/brotli/common/constants.c b/dviware/dvisvgm/libs/brotli/common/constants.c
index 6bad9f613c..89866b1505 100644
--- a/dviware/dvisvgm/libs/brotli/common/constants.c
+++ b/dviware/dvisvgm/libs/brotli/common/constants.c
@@ -4,7 +4,7 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
-#include "./constants.h"
+#include "constants.h"
const BrotliPrefixCodeRange
_kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
diff --git a/dviware/dvisvgm/libs/brotli/common/constants.h b/dviware/dvisvgm/libs/brotli/common/constants.h
index e848195a0d..31e5bd376e 100644
--- a/dviware/dvisvgm/libs/brotli/common/constants.h
+++ b/dviware/dvisvgm/libs/brotli/common/constants.h
@@ -12,10 +12,11 @@
#ifndef BROTLI_COMMON_CONSTANTS_H_
#define BROTLI_COMMON_CONSTANTS_H_
-#include "./platform.h"
#include <brotli/port.h>
#include <brotli/types.h>
+#include "platform.h"
+
/* Specification: 7.3. Encoding of the context map */
#define BROTLI_CONTEXT_MAP_MAX_RLE 16
diff --git a/dviware/dvisvgm/libs/brotli/common/context.c b/dviware/dvisvgm/libs/brotli/common/context.c
index 2c2dceba9b..7f9c958699 100644
--- a/dviware/dvisvgm/libs/brotli/common/context.c
+++ b/dviware/dvisvgm/libs/brotli/common/context.c
@@ -1,4 +1,4 @@
-#include "./context.h"
+#include "context.h"
#include <brotli/types.h>
diff --git a/dviware/dvisvgm/libs/brotli/common/dictionary.c b/dviware/dvisvgm/libs/brotli/common/dictionary.c
index f9e30417c0..7c015ab0ba 100644
--- a/dviware/dvisvgm/libs/brotli/common/dictionary.c
+++ b/dviware/dvisvgm/libs/brotli/common/dictionary.c
@@ -4,8 +4,8 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
-#include "./dictionary.h"
-#include "./platform.h"
+#include "dictionary.h"
+#include "platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -13,6 +13,7 @@ extern "C" {
#if !defined(BROTLI_EXTERNAL_DICTIONARY_DATA)
static const uint8_t kBrotliDictionaryData[] =
+/* GENERATED CODE START */
{
116,105,109,101,100,111,119,110,108,105,102,101,108,101,102,116,98,97,99,107,99,
111,100,101,100,97,116,97,115,104,111,119,111,110,108,121,115,105,116,101,99,105
@@ -5860,6 +5861,7 @@ static const uint8_t kBrotliDictionaryData[] =
,164,181,224,164,190,224,164,136,224,164,184,224,164,149,224,165,141,224,164,176
,224,164,191,224,164,175,224,164,164,224,164,190
}
+/* GENERATED CODE END */
;
#endif /* !BROTLI_EXTERNAL_DICTIONARY_DATA */
@@ -5895,7 +5897,7 @@ static BrotliDictionary kBrotliDictionary = {
#endif
};
-const BrotliDictionary* BrotliGetDictionary() {
+const BrotliDictionary* BrotliGetDictionary(void) {
return &kBrotliDictionary;
}
diff --git a/dviware/dvisvgm/libs/brotli/common/platform.c b/dviware/dvisvgm/libs/brotli/common/platform.c
index aef39e93c4..25d84a9467 100644
--- a/dviware/dvisvgm/libs/brotli/common/platform.c
+++ b/dviware/dvisvgm/libs/brotli/common/platform.c
@@ -6,9 +6,10 @@
#include <stdlib.h>
-#include "./platform.h"
#include <brotli/types.h>
+#include "platform.h"
+
/* Default brotli_alloc_func */
void* BrotliDefaultAllocFunc(void* opaque, size_t size) {
BROTLI_UNUSED(opaque);
diff --git a/dviware/dvisvgm/libs/brotli/common/platform.h b/dviware/dvisvgm/libs/brotli/common/platform.h
index f5ca4435bc..7406f3fe69 100644
--- a/dviware/dvisvgm/libs/brotli/common/platform.h
+++ b/dviware/dvisvgm/libs/brotli/common/platform.h
@@ -12,12 +12,13 @@
* BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations
* BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations
* BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations
- * BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned
- read and overlapping memcpy; this reduces decompression speed by 5%
* BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs
+ * BROTLI_BUILD_NO_UNALIGNED_READ_FAST forces off the fast-unaligned-read
+ optimizations (mainly for testing purposes)
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
or memory error
* BROTLI_ENABLE_LOG enables asserts and dumps various state information
+ * BROTLI_ENABLE_DUMP overrides default "dump" behaviour
*/
#ifndef BROTLI_COMMON_PLATFORM_H_
@@ -40,7 +41,7 @@
#define BROTLI_X_BIG_ENDIAN BIG_ENDIAN
#endif
-#if BROTLI_MSVC_VERSION_CHECK(12, 0, 0)
+#if BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
#include <intrin.h>
#endif
@@ -156,24 +157,6 @@ OR:
#define BROTLI_NOINLINE
#endif
-/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
-#if !defined(BROTLI_INTERNAL)
-#if defined(_WIN32) || defined(__CYGWIN__)
-#define BROTLI_INTERNAL
-#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
- BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
- BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
- BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
- BROTLI_IBM_VERSION_CHECK(13, 1, 0) || \
- BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
- (BROTLI_TI_VERSION_CHECK(7, 3, 0) && \
- defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
-#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
-#else
-#define BROTLI_INTERNAL
-#endif
-#endif
-
/* <<< <<< <<< end of hedley macros. */
#if BROTLI_GNUC_HAS_ATTRIBUTE(unused, 2, 7, 0) || \
@@ -226,15 +209,24 @@ OR:
#define BROTLI_TARGET_RISCV64
#endif
+#if defined(__loongarch_lp64)
+#define BROTLI_TARGET_LOONGARCH64
+#endif
+
+#if defined(BROTLI_TARGET_X64) || defined(BROTLI_TARGET_ARMV8_64) || \
+ defined(BROTLI_TARGET_POWERPC64) || defined(BROTLI_TARGET_RISCV64) || \
+ defined(BROTLI_TARGET_LOONGARCH64)
+#define BROTLI_TARGET_64_BITS 1
+#else
+#define BROTLI_TARGET_64_BITS 0
+#endif
+
#if defined(BROTLI_BUILD_64_BIT)
#define BROTLI_64_BITS 1
#elif defined(BROTLI_BUILD_32_BIT)
#define BROTLI_64_BITS 0
-#elif defined(BROTLI_TARGET_X64) || defined(BROTLI_TARGET_ARMV8_64) || \
- defined(BROTLI_TARGET_POWERPC64) || defined(BROTLI_TARGET_RISCV64)
-#define BROTLI_64_BITS 1
#else
-#define BROTLI_64_BITS 0
+#define BROTLI_64_BITS BROTLI_TARGET_64_BITS
#endif
#if (BROTLI_64_BITS)
@@ -278,18 +270,19 @@ OR:
#undef BROTLI_X_BIG_ENDIAN
#endif
-#if defined(BROTLI_BUILD_PORTABLE)
-#define BROTLI_ALIGNED_READ (!!1)
-#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
+#if defined(BROTLI_BUILD_NO_UNALIGNED_READ_FAST)
+#define BROTLI_UNALIGNED_READ_FAST (!!0)
+#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY) || \
- defined(BROTLI_TARGET_RISCV64)
-/* Allow unaligned read only for white-listed CPUs. */
-#define BROTLI_ALIGNED_READ (!!0)
+ defined(BROTLI_TARGET_RISCV64) || defined(BROTLI_TARGET_LOONGARCH64)
+/* These targets are known to generate efficient code for unaligned reads
+ * (e.g. a single instruction, not multiple 1-byte loads, shifted and or'd
+ * together). */
+#define BROTLI_UNALIGNED_READ_FAST (!!1)
#else
-#define BROTLI_ALIGNED_READ (!!1)
+#define BROTLI_UNALIGNED_READ_FAST (!!0)
#endif
-#if BROTLI_ALIGNED_READ
/* Portable unaligned memory access: read / write values via memcpy. */
static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) {
uint16_t t;
@@ -309,75 +302,6 @@ static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
-#else /* BROTLI_ALIGNED_READ */
-/* Unaligned memory access is allowed: just cast pointer to requested type. */
-#if BROTLI_SANITIZED
-/* Consider we have an unaligned load/store of 4 bytes from address 0x...05.
- AddressSanitizer will treat it as a 3-byte access to the range 05:07 and
- will miss a bug if 08 is the first unaddressable byte.
- ThreadSanitizer will also treat this as a 3-byte access to 05:07 and will
- miss a race between this access and some other accesses to 08.
- MemorySanitizer will correctly propagate the shadow on unaligned stores
- and correctly report bugs on unaligned loads, but it may not properly
- update and report the origin of the uninitialized memory.
- For all three tools, replacing an unaligned access with a tool-specific
- callback solves the problem. */
-#if defined(__cplusplus)
-extern "C" {
-#endif /* __cplusplus */
- uint16_t __sanitizer_unaligned_load16(const void* p);
- uint32_t __sanitizer_unaligned_load32(const void* p);
- uint64_t __sanitizer_unaligned_load64(const void* p);
- void __sanitizer_unaligned_store64(void* p, uint64_t v);
-#if defined(__cplusplus)
-} /* extern "C" */
-#endif /* __cplusplus */
-#define BrotliUnalignedRead16 __sanitizer_unaligned_load16
-#define BrotliUnalignedRead32 __sanitizer_unaligned_load32
-#define BrotliUnalignedRead64 __sanitizer_unaligned_load64
-#define BrotliUnalignedWrite64 __sanitizer_unaligned_store64
-#else /* BROTLI_SANITIZED */
-static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) {
- return *(const uint16_t*)p;
-}
-static BROTLI_INLINE uint32_t BrotliUnalignedRead32(const void* p) {
- return *(const uint32_t*)p;
-}
-#if (BROTLI_64_BITS)
-static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
- return *(const uint64_t*)p;
-}
-static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
- *(uint64_t*)p = v;
-}
-#else /* BROTLI_64_BITS */
-/* Avoid emitting LDRD / STRD, which require properly aligned address. */
-/* If __attribute__(aligned) is available, use that. Otherwise, memcpy. */
-
-#if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)
-typedef BROTLI_ALIGNED(1) uint64_t brotli_unaligned_uint64_t;
-
-static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
- return (uint64_t) ((const brotli_unaligned_uint64_t*) p)[0];
-}
-static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
- brotli_unaligned_uint64_t* dwords = (brotli_unaligned_uint64_t*) p;
- dwords[0] = (brotli_unaligned_uint64_t) v;
-}
-#else /* BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) */
-static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
- uint64_t v;
- memcpy(&v, p, sizeof(uint64_t));
- return v;
-}
-
-static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
- memcpy(p, &v, sizeof(uint64_t));
-}
-#endif /* BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) */
-#endif /* BROTLI_64_BITS */
-#endif /* BROTLI_SANITIZED */
-#endif /* BROTLI_ALIGNED_READ */
#if BROTLI_LITTLE_ENDIAN
/* Straight endianness. Just read / write values. */
@@ -453,6 +377,16 @@ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
}
#endif /* BROTLI_LITTLE_ENDIAN */
+static BROTLI_INLINE void* BROTLI_UNALIGNED_LOAD_PTR(const void* p) {
+ void* v;
+ memcpy(&v, p, sizeof(void*));
+ return v;
+}
+
+static BROTLI_INLINE void BROTLI_UNALIGNED_STORE_PTR(void* p, const void* v) {
+ memcpy(p, &v, sizeof(void*));
+}
+
/* BROTLI_IS_CONSTANT macros returns true for compile-time constants. */
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_constant_p, 3, 0, 1) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
@@ -474,22 +408,34 @@ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
#endif
#if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG)
+#define BROTLI_ENABLE_DUMP_DEFAULT 1
#define BROTLI_DCHECK(x) assert(x)
+#else
+#define BROTLI_ENABLE_DUMP_DEFAULT 0
+#define BROTLI_DCHECK(x)
+#endif
+
+#if !defined(BROTLI_ENABLE_DUMP)
+#define BROTLI_ENABLE_DUMP BROTLI_ENABLE_DUMP_DEFAULT
+#endif
+
+#if BROTLI_ENABLE_DUMP
static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) {
fprintf(stderr, "%s:%d (%s)\n", f, l, fn);
fflush(stderr);
}
#define BROTLI_DUMP() BrotliDump(__FILE__, __LINE__, __FUNCTION__)
#else
-#define BROTLI_DCHECK(x)
#define BROTLI_DUMP() (void)(0)
#endif
-/* TODO: add appropriate icc/sunpro/arm/ibm/ti checks. */
+/* BrotliRBit assumes brotli_reg_t fits native CPU register type. */
+#if (BROTLI_64_BITS == BROTLI_TARGET_64_BITS)
+/* TODO(eustas): add appropriate icc/sunpro/arm/ibm/ti checks. */
#if (BROTLI_GNUC_VERSION_CHECK(3, 0, 0) || defined(__llvm__)) && \
!defined(BROTLI_BUILD_NO_RBIT)
#if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY)
-/* TODO: detect ARMv6T2 and enable this code for it. */
+/* TODO(eustas): detect ARMv6T2 and enable this code for it. */
static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) {
brotli_reg_t output;
__asm__("rbit %0, %1\n" : "=r"(output) : "r"(input));
@@ -498,15 +444,14 @@ static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) {
#define BROTLI_RBIT(x) BrotliRBit(x)
#endif /* armv7 / armv8 */
#endif /* gcc || clang */
+#endif /* brotli_reg_t is native */
#if !defined(BROTLI_RBIT)
static BROTLI_INLINE void BrotliRBit(void) { /* Should break build if used. */ }
#endif /* BROTLI_RBIT */
-#define BROTLI_REPEAT(N, X) { \
- if ((N & 1) != 0) {X;} \
- if ((N & 2) != 0) {X; X;} \
- if ((N & 4) != 0) {X; X; X; X;} \
-}
+#define BROTLI_REPEAT_4(X) {X; X; X; X;}
+#define BROTLI_REPEAT_5(X) {X; X; X; X; X;}
+#define BROTLI_REPEAT_6(X) {X; X; X; X; X; X;}
#define BROTLI_UNUSED(X) (void)(X)
@@ -529,7 +474,7 @@ BROTLI_MIN_MAX(size_t) BROTLI_MIN_MAX(uint32_t) BROTLI_MIN_MAX(uint8_t)
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_ctzll, 3, 4, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
#define BROTLI_TZCNT64 __builtin_ctzll
-#elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0)
+#elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
#if defined(BROTLI_TARGET_X64)
#define BROTLI_TZCNT64 _tzcnt_u64
#else /* BROTLI_TARGET_X64 */
@@ -546,7 +491,7 @@ static BROTLI_INLINE uint32_t BrotliBsf64Msvc(uint64_t x) {
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
#define BROTLI_BSR32(x) (31u ^ (uint32_t)__builtin_clz(x))
-#elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0)
+#elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
static BROTLI_INLINE uint32_t BrotliBsr32Msvc(uint32_t x) {
unsigned long msb;
_BitScanReverse(&msb, x);
@@ -571,6 +516,8 @@ BROTLI_UNUSED_FUNCTION void BrotliSuppressUnusedFunctions(void) {
BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD32LE);
BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD64LE);
BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE64LE);
+ BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD_PTR);
+ BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE_PTR);
BROTLI_UNUSED(&BrotliRBit);
BROTLI_UNUSED(&brotli_min_double);
BROTLI_UNUSED(&brotli_max_double);
@@ -586,7 +533,7 @@ BROTLI_UNUSED_FUNCTION void BrotliSuppressUnusedFunctions(void) {
BROTLI_UNUSED(&brotli_max_uint8_t);
BROTLI_UNUSED(&BrotliDefaultAllocFunc);
BROTLI_UNUSED(&BrotliDefaultFreeFunc);
-#if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG)
+#if BROTLI_ENABLE_DUMP
BROTLI_UNUSED(&BrotliDump);
#endif
}
diff --git a/dviware/dvisvgm/libs/brotli/common/shared_dictionary_internal.h b/dviware/dvisvgm/libs/brotli/common/shared_dictionary_internal.h
new file mode 100644
index 0000000000..963762e432
--- /dev/null
+++ b/dviware/dvisvgm/libs/brotli/common/shared_dictionary_internal.h
@@ -0,0 +1,75 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+ Distributed under MIT license.
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* (Transparent) Shared Dictionary definition. */
+
+#ifndef BROTLI_COMMON_SHARED_DICTIONARY_INTERNAL_H_
+#define BROTLI_COMMON_SHARED_DICTIONARY_INTERNAL_H_
+
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
+
+#include "dictionary.h"
+#include "transform.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+struct BrotliSharedDictionaryStruct {
+ /* LZ77 prefixes (compound dictionary). */
+ uint32_t num_prefix; /* max SHARED_BROTLI_MAX_COMPOUND_DICTS */
+ size_t prefix_size[SHARED_BROTLI_MAX_COMPOUND_DICTS];
+ const uint8_t* prefix[SHARED_BROTLI_MAX_COMPOUND_DICTS];
+
+ /* If set, the context map is used to select word and transform list from 64
+ contexts, if not set, the context map is not used and only words[0] and
+ transforms[0] are to be used. */
+ BROTLI_BOOL context_based;
+
+ uint8_t context_map[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
+
+ /* Amount of word_list+transform_list combinations. */
+ uint8_t num_dictionaries;
+
+ /* Must use num_dictionaries values. */
+ const BrotliDictionary* words[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
+
+ /* Must use num_dictionaries values. */
+ const BrotliTransforms* transforms[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
+
+ /* Amount of custom word lists. May be 0 if only Brotli's built-in is used */
+ uint8_t num_word_lists;
+
+ /* Contents of the custom words lists. Must be NULL if num_word_lists is 0. */
+ BrotliDictionary* words_instances;
+
+ /* Amount of custom transform lists. May be 0 if only Brotli's built-in is
+ used */
+ uint8_t num_transform_lists;
+
+ /* Contents of the custom transform lists. Must be NULL if num_transform_lists
+ is 0. */
+ BrotliTransforms* transforms_instances;
+
+ /* Concatenated prefix_suffix_maps of the custom transform lists. Must be NULL
+ if num_transform_lists is 0. */
+ uint16_t* prefix_suffix_maps;
+
+ /* Memory management */
+ brotli_alloc_func alloc_func;
+ brotli_free_func free_func;
+ void* memory_manager_opaque;
+};
+
+typedef struct BrotliSharedDictionaryStruct BrotliSharedDictionaryInternal;
+#define BrotliSharedDictionary BrotliSharedDictionaryInternal
+
+#if defined(__cplusplus) || defined(c_plusplus)
+} /* extern "C" */
+#endif
+
+#endif /* BROTLI_COMMON_SHARED_DICTIONARY_INTERNAL_H_ */
diff --git a/dviware/dvisvgm/libs/brotli/common/transform.c b/dviware/dvisvgm/libs/brotli/common/transform.c
index f8fa4335e1..49455fc496 100644
--- a/dviware/dvisvgm/libs/brotli/common/transform.c
+++ b/dviware/dvisvgm/libs/brotli/common/transform.c
@@ -4,7 +4,7 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
-#include "./transform.h"
+#include "transform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/common/version.h b/dviware/dvisvgm/libs/brotli/common/version.h
index 01b2998e25..8098040f64 100644
--- a/dviware/dvisvgm/libs/brotli/common/version.h
+++ b/dviware/dvisvgm/libs/brotli/common/version.h
@@ -9,18 +9,43 @@
#ifndef BROTLI_COMMON_VERSION_H_
#define BROTLI_COMMON_VERSION_H_
-/* This macro should only be used when library is compiled together with client.
- If library is dynamically linked, use BrotliDecoderVersion and
+/* Compose 3 components into a single number. In a hexadecimal representation
+ B and C components occupy exactly 3 digits. */
+#define BROTLI_MAKE_HEX_VERSION(A, B, C) ((A << 24) | (B << 12) | C)
+
+/* Those macros should only be used when library is compiled together with
+ the client. If library is dynamically linked, use BrotliDecoderVersion and
BrotliEncoderVersion methods. */
-/* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
-#define BROTLI_VERSION 0x1000009
+#define BROTLI_VERSION_MAJOR 1
+#define BROTLI_VERSION_MINOR 1
+#define BROTLI_VERSION_PATCH 0
+
+#define BROTLI_VERSION BROTLI_MAKE_HEX_VERSION( \
+ BROTLI_VERSION_MAJOR, BROTLI_VERSION_MINOR, BROTLI_VERSION_PATCH)
/* This macro is used by build system to produce Libtool-friendly soname. See
https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
+ Version evolution rules:
+ - interfaces added (or change is compatible) -> current+1:0:age+1
+ - interfaces removed (or changed is incompatible) -> current+1:0:0
+ - interfaces not changed -> current:revision+1:age
*/
-/* ABI version, calculated as (CURRENT << 24) | (REVISION << 12) | AGE */
-#define BROTLI_ABI_VERSION 0x1009000
+#define BROTLI_ABI_CURRENT 2
+#define BROTLI_ABI_REVISION 0
+#define BROTLI_ABI_AGE 1
+
+#if BROTLI_VERSION_MAJOR != (BROTLI_ABI_CURRENT - BROTLI_ABI_AGE)
+#error ABI/API version inconsistency
+#endif
+
+#if BROTLI_VERSION_MINOR != BROTLI_ABI_AGE
+#error ABI/API version inconsistency
+#endif
+
+#if BROTLI_VERSION_PATCH != BROTLI_ABI_REVISION
+#error ABI/API version inconsistency
+#endif
#endif /* BROTLI_COMMON_VERSION_H_ */
diff --git a/dviware/dvisvgm/libs/brotli/enc/backward_references.c b/dviware/dvisvgm/libs/brotli/enc/backward_references.c
index a07a617a09..f600e6452f 100644
--- a/dviware/dvisvgm/libs/brotli/enc/backward_references.c
+++ b/dviware/dvisvgm/libs/brotli/enc/backward_references.c
@@ -6,17 +6,19 @@
/* Function to find backward reference copies. */
-#include "./backward_references.h"
+#include "backward_references.h"
+
+#include <brotli/types.h>
#include "../common/constants.h"
-#include "../common/context.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./command.h"
-#include "./dictionary_hash.h"
-#include "./memory.h"
-#include "./quality.h"
+#include "command.h"
+#include "compound_dictionary.h"
+#include "dictionary_hash.h"
+#include "encoder_dict.h"
+#include "memory.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -52,67 +54,103 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
#define PREFIX() N
+#define ENABLE_COMPOUND_DICTIONARY 0
#define HASHER() H2
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H3
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H4
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H5
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H6
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H40
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H41
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H42
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H54
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H35
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H55
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H65
/* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
#undef HASHER
+#undef ENABLE_COMPOUND_DICTIONARY
+#undef PREFIX
+#define PREFIX() D
+#define ENABLE_COMPOUND_DICTIONARY 1
+
+#define HASHER() H5
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H6
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H40
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H41
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H42
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H55
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H65
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+
+#undef ENABLE_COMPOUND_DICTIONARY
#undef PREFIX
#undef EXPORT_FN
@@ -125,6 +163,29 @@ void BrotliCreateBackwardReferences(size_t num_bytes,
ContextLut literal_context_lut, const BrotliEncoderParams* params,
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals) {
+ if (params->dictionary.compound.num_chunks != 0) {
+ switch (params->hasher.type) {
+#define CASE_(N) \
+ case N: \
+ CreateBackwardReferencesDH ## N(num_bytes, \
+ position, ringbuffer, ringbuffer_mask, \
+ literal_context_lut, params, hasher, dist_cache, \
+ last_insert_len, commands, num_commands, num_literals); \
+ return;
+ CASE_(5)
+ CASE_(6)
+ CASE_(40)
+ CASE_(41)
+ CASE_(42)
+ CASE_(55)
+ CASE_(65)
+#undef CASE_
+ default:
+ BROTLI_DCHECK(false);
+ break;
+ }
+ }
+
switch (params->hasher.type) {
#define CASE_(N) \
case N: \
@@ -136,6 +197,7 @@ void BrotliCreateBackwardReferences(size_t num_bytes,
FOR_GENERIC_HASHERS(CASE_)
#undef CASE_
default:
+ BROTLI_DCHECK(false);
break;
}
}
diff --git a/dviware/dvisvgm/libs/brotli/enc/backward_references.h b/dviware/dvisvgm/libs/brotli/enc/backward_references.h
index 9589cc1541..20fb98a4d8 100644
--- a/dviware/dvisvgm/libs/brotli/enc/backward_references.h
+++ b/dviware/dvisvgm/libs/brotli/enc/backward_references.h
@@ -9,14 +9,15 @@
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
#define BROTLI_ENC_BACKWARD_REFERENCES_H_
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/context.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./command.h"
-#include "./hash.h"
-#include "./quality.h"
+#include "command.h"
+#include "hash.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.c b/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.c
index 5651caeb7a..6325032e1c 100644
--- a/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.c
+++ b/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.c
@@ -6,22 +6,24 @@
/* Function to find backward reference copies. */
-#include "./backward_references_hq.h"
+#include "backward_references_hq.h"
#include <string.h> /* memcpy, memset */
+#include <brotli/types.h>
+
#include "../common/constants.h"
-#include "../common/context.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./command.h"
-#include "./fast_log.h"
-#include "./find_match_length.h"
-#include "./literal_cost.h"
-#include "./memory.h"
-#include "./params.h"
-#include "./prefix.h"
-#include "./quality.h"
+#include "command.h"
+#include "compound_dictionary.h"
+#include "encoder_dict.h"
+#include "fast_log.h"
+#include "find_match_length.h"
+#include "literal_cost.h"
+#include "memory.h"
+#include "params.h"
+#include "prefix.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -73,6 +75,14 @@ static BROTLI_INLINE uint32_t ZopfliNodeCommandLength(const ZopfliNode* self) {
return ZopfliNodeCopyLength(self) + (self->dcode_insert_length & 0x7FFFFFF);
}
+/* Temporary data for ZopfliCostModelSetFromCommands. */
+typedef struct ZopfliCostModelArena {
+ uint32_t histogram_literal[BROTLI_NUM_LITERAL_SYMBOLS];
+ uint32_t histogram_cmd[BROTLI_NUM_COMMAND_SYMBOLS];
+ uint32_t histogram_dist[BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE];
+ float cost_literal[BROTLI_NUM_LITERAL_SYMBOLS];
+} ZopfliCostModelArena;
+
/* Histogram based cost model for zopflification. */
typedef struct ZopfliCostModel {
/* The insert and copy length symbols. */
@@ -83,6 +93,12 @@ typedef struct ZopfliCostModel {
float* literal_costs_;
float min_cost_cmd_;
size_t num_bytes_;
+
+ /* Temporary data. */
+ union {
+ size_t literal_histograms[3 * 256];
+ ZopfliCostModelArena arena;
+ };
} ZopfliCostModel;
static void InitZopfliCostModel(
@@ -139,18 +155,15 @@ static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
const Command* commands,
size_t num_commands,
size_t last_insert_len) {
- uint32_t histogram_literal[BROTLI_NUM_LITERAL_SYMBOLS];
- uint32_t histogram_cmd[BROTLI_NUM_COMMAND_SYMBOLS];
- uint32_t histogram_dist[BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE];
- float cost_literal[BROTLI_NUM_LITERAL_SYMBOLS];
+ ZopfliCostModelArena* arena = &self->arena;
size_t pos = position - last_insert_len;
float min_cost_cmd = kInfinity;
size_t i;
float* cost_cmd = self->cost_cmd_;
- memset(histogram_literal, 0, sizeof(histogram_literal));
- memset(histogram_cmd, 0, sizeof(histogram_cmd));
- memset(histogram_dist, 0, sizeof(histogram_dist));
+ memset(arena->histogram_literal, 0, sizeof(arena->histogram_literal));
+ memset(arena->histogram_cmd, 0, sizeof(arena->histogram_cmd));
+ memset(arena->histogram_dist, 0, sizeof(arena->histogram_dist));
for (i = 0; i < num_commands; i++) {
size_t inslength = commands[i].insert_len_;
@@ -159,21 +172,21 @@ static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
size_t cmdcode = commands[i].cmd_prefix_;
size_t j;
- histogram_cmd[cmdcode]++;
- if (cmdcode >= 128) histogram_dist[distcode]++;
+ arena->histogram_cmd[cmdcode]++;
+ if (cmdcode >= 128) arena->histogram_dist[distcode]++;
for (j = 0; j < inslength; j++) {
- histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
+ arena->histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
}
pos += inslength + copylength;
}
- SetCost(histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, BROTLI_TRUE,
- cost_literal);
- SetCost(histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, BROTLI_FALSE,
+ SetCost(arena->histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, BROTLI_TRUE,
+ arena->cost_literal);
+ SetCost(arena->histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, BROTLI_FALSE,
cost_cmd);
- SetCost(histogram_dist, self->distance_histogram_size, BROTLI_FALSE,
+ SetCost(arena->histogram_dist, self->distance_histogram_size, BROTLI_FALSE,
self->cost_dist_);
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i) {
@@ -188,7 +201,7 @@ static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
literal_costs[0] = 0.0;
for (i = 0; i < num_bytes; ++i) {
literal_carry +=
- cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
+ arena->cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
literal_costs[i + 1] = literal_costs[i] + literal_carry;
literal_carry -= literal_costs[i + 1] - literal_costs[i];
}
@@ -206,7 +219,8 @@ static void ZopfliCostModelSetFromLiteralCosts(ZopfliCostModel* self,
size_t num_bytes = self->num_bytes_;
size_t i;
BrotliEstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
- ringbuffer, &literal_costs[1]);
+ ringbuffer, self->literal_histograms,
+ &literal_costs[1]);
literal_costs[0] = 0.0;
for (i = 0; i < num_bytes; ++i) {
literal_carry += literal_costs[i + 1];
@@ -418,7 +432,8 @@ static size_t UpdateNodes(
size_t min_len;
size_t result = 0;
size_t k;
- size_t gap = 0;
+ const CompoundDictionary* addon = &params->dictionary.compound;
+ size_t gap = addon->total_size;
EvaluateNode(block_start + stream_offset, pos, max_backward_limit, gap,
starting_dist_cache, model, queue, nodes);
@@ -472,6 +487,24 @@ static size_t UpdateNodes(
len = FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
max_len);
+ } else if (backward > dictionary_start) {
+ size_t d = 0;
+ size_t offset;
+ size_t limit;
+ const uint8_t* source;
+ offset = dictionary_start + 1 + addon->total_size - 1;
+ while (offset >= backward + addon->chunk_offsets[d + 1]) d++;
+ source = addon->chunk_source[d];
+ offset = offset - addon->chunk_offsets[d] - backward;
+ limit = addon->chunk_offsets[d + 1] - addon->chunk_offsets[d] - offset;
+ limit = limit > max_len ? max_len : limit;
+ if (best_len >= limit ||
+ continuation != source[offset + best_len]) {
+ continue;
+ }
+ len = FindMatchLengthWithLimit(&source[offset],
+ &ringbuffer[cur_ix_masked],
+ limit);
} else {
/* "Gray" area. It is addressable by decoder, but this encoder
instance does not have that data -> should not touch it. */
@@ -577,7 +610,7 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
size_t pos = 0;
uint32_t offset = nodes[0].u.next;
size_t i;
- size_t gap = 0;
+ size_t gap = params->dictionary.compound.total_size;
for (i = 0; offset != BROTLI_UINT32_MAX; i++) {
const ZopfliNode* next = &nodes[pos + offset];
size_t copy_length = ZopfliNodeCopyLength(next);
@@ -653,6 +686,23 @@ static size_t ZopfliIterate(size_t num_bytes, size_t position,
return ComputeShortestPathFromNodes(num_bytes, nodes);
}
+static void MergeMatches(BackwardMatch* dst,
+ BackwardMatch* src1, size_t len1, BackwardMatch* src2, size_t len2) {
+ while (len1 > 0 && len2 > 0) {
+ size_t l1 = BackwardMatchLength(src1);
+ size_t l2 = BackwardMatchLength(src2);
+ if (l1 < l2 || ((l1 == l2) && (src1->distance < src2->distance))) {
+ *dst++ = *src1++;
+ len1--;
+ } else {
+ *dst++ = *src2++;
+ len2--;
+ }
+ }
+ while (len1-- > 0) *dst++ = *src1++;
+ while (len2-- > 0) *dst++ = *src2++;
+}
+
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
@@ -661,21 +711,26 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
const size_t stream_offset = params->stream_offset;
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
const size_t max_zopfli_len = MaxZopfliLen(params);
- ZopfliCostModel model;
StartPosQueue queue;
- BackwardMatch matches[2 * (MAX_NUM_MATCHES_H10 + 64)];
+ BackwardMatch* BROTLI_RESTRICT matches =
+ BROTLI_ALLOC(m, BackwardMatch, 2 * (MAX_NUM_MATCHES_H10 + 64));
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
position + num_bytes - StoreLookaheadH10() + 1 : position;
size_t i;
- size_t gap = 0;
- size_t lz_matches_offset = 0;
- BROTLI_UNUSED(literal_context_lut);
+ const CompoundDictionary* addon = &params->dictionary.compound;
+ size_t gap = addon->total_size;
+ size_t lz_matches_offset =
+ (addon->num_chunks != 0) ? (MAX_NUM_MATCHES_H10 + 128) : 0;
+ ZopfliCostModel* model = BROTLI_ALLOC(m, ZopfliCostModel, 1);
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(model) || BROTLI_IS_NULL(matches)) {
+ return 0;
+ }
nodes[0].length = 0;
nodes[0].u.cost = 0;
- InitZopfliCostModel(m, &model, &params->dist, num_bytes);
+ InitZopfliCostModel(m, model, &params->dist, num_bytes);
if (BROTLI_IS_OOM(m)) return 0;
ZopfliCostModelSetFromLiteralCosts(
- &model, position, ringbuffer, ringbuffer_mask);
+ model, position, ringbuffer, ringbuffer_mask);
InitStartPosQueue(&queue);
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; i++) {
const size_t pos = position + i;
@@ -684,17 +739,35 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
pos + stream_offset, max_backward_limit);
size_t skip;
size_t num_matches;
+ int dict_id = 0;
+ if (params->dictionary.contextual.context_based) {
+ uint8_t p1 = pos >= 1 ?
+ ringbuffer[(size_t)(pos - 1) & ringbuffer_mask] : 0;
+ uint8_t p2 = pos >= 2 ?
+ ringbuffer[(size_t)(pos - 2) & ringbuffer_mask] : 0;
+ dict_id = params->dictionary.contextual.context_map[
+ BROTLI_CONTEXT(p1, p2, literal_context_lut)];
+ }
num_matches = FindAllMatchesH10(&hasher->privat._H10,
- &params->dictionary,
+ params->dictionary.contextual.dict[dict_id],
ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance,
dictionary_start + gap, params, &matches[lz_matches_offset]);
+ if (addon->num_chunks != 0) {
+ size_t cd_matches = LookupAllCompoundDictionaryMatches(addon,
+ ringbuffer, ringbuffer_mask, pos, 3, num_bytes - i,
+ dictionary_start, params->dist.max_distance,
+ &matches[lz_matches_offset - 64], 64);
+ MergeMatches(matches, &matches[lz_matches_offset - 64], cd_matches,
+ &matches[lz_matches_offset], num_matches);
+ num_matches += cd_matches;
+ }
if (num_matches > 0 &&
BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
matches[0] = matches[num_matches - 1];
num_matches = 1;
}
skip = UpdateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask,
- params, max_backward_limit, dist_cache, num_matches, matches, &model,
+ params, max_backward_limit, dist_cache, num_matches, matches, model,
&queue, nodes);
if (skip < BROTLI_LONG_COPY_QUICK_STEP) skip = 0;
if (num_matches == 1 && BackwardMatchLength(&matches[0]) > max_zopfli_len) {
@@ -710,12 +783,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
i++;
if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
EvaluateNode(position + stream_offset, i, max_backward_limit, gap,
- dist_cache, &model, &queue, nodes);
+ dist_cache, model, &queue, nodes);
skip--;
}
}
}
- CleanupZopfliCostModel(m, &model);
+ CleanupZopfliCostModel(m, model);
+ BROTLI_FREE(m, model);
+ BROTLI_FREE(m, matches);
return ComputeShortestPathFromNodes(num_bytes, nodes);
}
@@ -753,14 +828,15 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
size_t orig_last_insert_len;
int orig_dist_cache[4];
size_t orig_num_commands;
- ZopfliCostModel model;
+ ZopfliCostModel* model = BROTLI_ALLOC(m, ZopfliCostModel, 1);
ZopfliNode* nodes;
BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
- size_t gap = 0;
- size_t shadow_matches = 0;
- BROTLI_UNUSED(literal_context_lut);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(num_matches) ||
- BROTLI_IS_NULL(matches)) {
+ const CompoundDictionary* addon = &params->dictionary.compound;
+ size_t gap = addon->total_size;
+ size_t shadow_matches =
+ (addon->num_chunks != 0) ? (MAX_NUM_MATCHES_H10 + 128) : 0;
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(model) ||
+ BROTLI_IS_NULL(num_matches) || BROTLI_IS_NULL(matches)) {
return;
}
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
@@ -772,15 +848,34 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
size_t num_found_matches;
size_t cur_match_end;
size_t j;
+ int dict_id = 0;
+ if (params->dictionary.contextual.context_based) {
+ uint8_t p1 = pos >= 1 ?
+ ringbuffer[(size_t)(pos - 1) & ringbuffer_mask] : 0;
+ uint8_t p2 = pos >= 2 ?
+ ringbuffer[(size_t)(pos - 2) & ringbuffer_mask] : 0;
+ dict_id = params->dictionary.contextual.context_map[
+ BROTLI_CONTEXT(p1, p2, literal_context_lut)];
+ }
/* Ensure that we have enough free slots. */
BROTLI_ENSURE_CAPACITY(m, BackwardMatch, matches, matches_size,
cur_match_pos + MAX_NUM_MATCHES_H10 + shadow_matches);
if (BROTLI_IS_OOM(m)) return;
num_found_matches = FindAllMatchesH10(&hasher->privat._H10,
- &params->dictionary,
+ params->dictionary.contextual.dict[dict_id],
ringbuffer, ringbuffer_mask, pos, max_length,
max_distance, dictionary_start + gap, params,
&matches[cur_match_pos + shadow_matches]);
+ if (addon->num_chunks != 0) {
+ size_t cd_matches = LookupAllCompoundDictionaryMatches(addon,
+ ringbuffer, ringbuffer_mask, pos, 3, max_length,
+ dictionary_start, params->dist.max_distance,
+ &matches[cur_match_pos + shadow_matches - 64], 64);
+ MergeMatches(&matches[cur_match_pos],
+ &matches[cur_match_pos + shadow_matches - 64], cd_matches,
+ &matches[cur_match_pos + shadow_matches], num_found_matches);
+ num_found_matches += cd_matches;
+ }
cur_match_end = cur_match_pos + num_found_matches;
for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
BROTLI_DCHECK(BackwardMatchLength(&matches[j]) <=
@@ -810,15 +905,15 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
orig_num_commands = *num_commands;
nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) return;
- InitZopfliCostModel(m, &model, &params->dist, num_bytes);
+ InitZopfliCostModel(m, model, &params->dist, num_bytes);
if (BROTLI_IS_OOM(m)) return;
for (i = 0; i < 2; i++) {
BrotliInitZopfliNodes(nodes, num_bytes + 1);
if (i == 0) {
ZopfliCostModelSetFromLiteralCosts(
- &model, position, ringbuffer, ringbuffer_mask);
+ model, position, ringbuffer, ringbuffer_mask);
} else {
- ZopfliCostModelSetFromCommands(&model, position, ringbuffer,
+ ZopfliCostModelSetFromCommands(model, position, ringbuffer,
ringbuffer_mask, commands, *num_commands - orig_num_commands,
orig_last_insert_len);
}
@@ -827,12 +922,13 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
*last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
*num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
- ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches,
+ ringbuffer_mask, params, gap, dist_cache, model, num_matches, matches,
nodes);
BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache,
last_insert_len, params, commands, num_literals);
}
- CleanupZopfliCostModel(m, &model);
+ CleanupZopfliCostModel(m, model);
+ BROTLI_FREE(m, model);
BROTLI_FREE(m, nodes);
BROTLI_FREE(m, matches);
BROTLI_FREE(m, num_matches);
diff --git a/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.h b/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.h
index 36b75f250d..8acf975ab9 100644
--- a/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.h
+++ b/dviware/dvisvgm/libs/brotli/enc/backward_references_hq.h
@@ -9,15 +9,16 @@
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
#define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/context.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./command.h"
-#include "./hash.h"
-#include "./memory.h"
-#include "./quality.h"
+#include "command.h"
+#include "hash.h"
+#include "memory.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/backward_references_inc.h b/dviware/dvisvgm/libs/brotli/enc/backward_references_inc.h
index 766bf91ffd..752c12e9fd 100644
--- a/dviware/dvisvgm/libs/brotli/enc/backward_references_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/backward_references_inc.h
@@ -28,13 +28,11 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
const size_t random_heuristics_window_size =
LiteralSpreeLengthForSparseSearch(params);
size_t apply_random_heuristics = position + random_heuristics_window_size;
- const size_t gap = 0;
+ const size_t gap = params->dictionary.compound.total_size;
/* Minimum score to accept a backward reference. */
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
- BROTLI_UNUSED(literal_context_lut);
-
FN(PrepareDistanceCache)(privat, dist_cache);
while (position + FN(HashTypeLength)() < pos_end) {
@@ -43,13 +41,29 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
size_t dictionary_start = BROTLI_MIN(size_t,
position + position_offset, max_backward_limit);
HasherSearchResult sr;
+ int dict_id = 0;
+ uint8_t p1 = 0;
+ uint8_t p2 = 0;
+ if (params->dictionary.contextual.context_based) {
+ p1 = position >= 1 ?
+ ringbuffer[(size_t)(position - 1) & ringbuffer_mask] : 0;
+ p2 = position >= 2 ?
+ ringbuffer[(size_t)(position - 2) & ringbuffer_mask] : 0;
+ dict_id = params->dictionary.contextual.context_map[
+ BROTLI_CONTEXT(p1, p2, literal_context_lut)];
+ }
sr.len = 0;
sr.len_code_delta = 0;
sr.distance = 0;
sr.score = kMinScore;
- FN(FindLongestMatch)(privat, &params->dictionary,
+ FN(FindLongestMatch)(privat, params->dictionary.contextual.dict[dict_id],
ringbuffer, ringbuffer_mask, dist_cache, position, max_length,
max_distance, dictionary_start + gap, params->dist.max_distance, &sr);
+ if (ENABLE_COMPOUND_DICTIONARY) {
+ LookupCompoundDictionaryMatch(&params->dictionary.compound, ringbuffer,
+ ringbuffer_mask, dist_cache, position, max_length,
+ dictionary_start, params->dist.max_distance, &sr);
+ }
if (sr.score > kMinScore) {
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
@@ -65,11 +79,23 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
dictionary_start = BROTLI_MIN(size_t,
position + 1 + position_offset, max_backward_limit);
+ if (params->dictionary.contextual.context_based) {
+ p2 = p1;
+ p1 = ringbuffer[position & ringbuffer_mask];
+ dict_id = params->dictionary.contextual.context_map[
+ BROTLI_CONTEXT(p1, p2, literal_context_lut)];
+ }
FN(FindLongestMatch)(privat,
- &params->dictionary,
+ params->dictionary.contextual.dict[dict_id],
ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
max_distance, dictionary_start + gap, params->dist.max_distance,
&sr2);
+ if (ENABLE_COMPOUND_DICTIONARY) {
+ LookupCompoundDictionaryMatch(
+ &params->dictionary.compound, ringbuffer,
+ ringbuffer_mask, dist_cache, position + 1, max_length,
+ dictionary_start, params->dist.max_distance, &sr2);
+ }
if (sr2.score >= sr.score + cost_diff_lazy) {
/* Ok, let's just write one byte for now and start a match from the
next byte. */
diff --git a/dviware/dvisvgm/libs/brotli/enc/bit_cost.c b/dviware/dvisvgm/libs/brotli/enc/bit_cost.c
index 1f3f7ad5c9..6b7c904ced 100644
--- a/dviware/dvisvgm/libs/brotli/enc/bit_cost.c
+++ b/dviware/dvisvgm/libs/brotli/enc/bit_cost.c
@@ -6,28 +6,29 @@
/* Functions to estimate the bit cost of Huffman trees. */
-#include "./bit_cost.h"
+#include "bit_cost.h"
+
+#include <brotli/types.h>
#include "../common/constants.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./fast_log.h"
-#include "./histogram.h"
+#include "fast_log.h"
+#include "histogram.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#define FN(X) X ## Literal
-#include "./bit_cost_inc.h" /* NOLINT(build/include) */
+#include "bit_cost_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Command
-#include "./bit_cost_inc.h" /* NOLINT(build/include) */
+#include "bit_cost_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Distance
-#include "./bit_cost_inc.h" /* NOLINT(build/include) */
+#include "bit_cost_inc.h" /* NOLINT(build/include) */
#undef FN
#if defined(__cplusplus) || defined(c_plusplus)
diff --git a/dviware/dvisvgm/libs/brotli/enc/bit_cost.h b/dviware/dvisvgm/libs/brotli/enc/bit_cost.h
index 6586469e62..f6f2773994 100644
--- a/dviware/dvisvgm/libs/brotli/enc/bit_cost.h
+++ b/dviware/dvisvgm/libs/brotli/enc/bit_cost.h
@@ -9,10 +9,11 @@
#ifndef BROTLI_ENC_BIT_COST_H_
#define BROTLI_ENC_BIT_COST_H_
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./fast_log.h"
-#include "./histogram.h"
+
+#include "../common/platform.h"
+#include "fast_log.h"
+#include "histogram.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -45,7 +46,7 @@ static BROTLI_INLINE double BitsEntropy(
const uint32_t* population, size_t size) {
size_t sum;
double retval = ShannonEntropy(population, size, &sum);
- if (retval < sum) {
+ if (retval < (double)sum) {
/* At least one bit per literal is needed. */
retval = (double)sum;
}
diff --git a/dviware/dvisvgm/libs/brotli/enc/block_splitter.c b/dviware/dvisvgm/libs/brotli/enc/block_splitter.c
index deb7c2e151..eba1b691e5 100644
--- a/dviware/dvisvgm/libs/brotli/enc/block_splitter.c
+++ b/dviware/dvisvgm/libs/brotli/enc/block_splitter.c
@@ -6,18 +6,18 @@
/* Block split point selection utilities. */
-#include "./block_splitter.h"
+#include "block_splitter.h"
#include <string.h> /* memcpy, memset */
#include "../common/platform.h"
-#include "./bit_cost.h"
-#include "./cluster.h"
-#include "./command.h"
-#include "./fast_log.h"
-#include "./histogram.h"
-#include "./memory.h"
-#include "./quality.h"
+#include "bit_cost.h"
+#include "cluster.h"
+#include "command.h"
+#include "fast_log.h"
+#include "histogram.h"
+#include "memory.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -30,6 +30,7 @@ static const double kCommandBlockSwitchCost = 13.5;
static const double kDistanceBlockSwitchCost = 14.6;
static const size_t kLiteralStrideLength = 70;
static const size_t kCommandStrideLength = 40;
+static const size_t kDistanceStrideLength = 40;
static const size_t kSymbolsPerLiteralHistogram = 544;
static const size_t kSymbolsPerCommandHistogram = 530;
static const size_t kSymbolsPerDistanceHistogram = 544;
@@ -89,19 +90,19 @@ static BROTLI_INLINE double BitCost(size_t count) {
#define FN(X) X ## Literal
#define DataType uint8_t
/* NOLINTNEXTLINE(build/include) */
-#include "./block_splitter_inc.h"
+#include "block_splitter_inc.h"
#undef DataType
#undef FN
#define FN(X) X ## Command
#define DataType uint16_t
/* NOLINTNEXTLINE(build/include) */
-#include "./block_splitter_inc.h"
+#include "block_splitter_inc.h"
#undef FN
#define FN(X) X ## Distance
/* NOLINTNEXTLINE(build/include) */
-#include "./block_splitter_inc.h"
+#include "block_splitter_inc.h"
#undef DataType
#undef FN
@@ -119,6 +120,8 @@ void BrotliDestroyBlockSplit(MemoryManager* m, BlockSplit* self) {
BROTLI_FREE(m, self->lengths);
}
+/* Extracts literals, command distance and prefix codes, then applies
+ * SplitByteVector to create partitioning. */
void BrotliSplitBlock(MemoryManager* m,
const Command* cmds,
const size_t num_commands,
@@ -136,7 +139,9 @@ void BrotliSplitBlock(MemoryManager* m,
/* Create a continuous array of literals. */
CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
/* Create the block split on the array of literals.
- Literal histograms have alphabet size 256. */
+ * Literal histograms can have alphabet size up to 256.
+ * Though, to accomodate context modeling, less than half of maximum size
+ * is allowed. */
SplitByteVectorLiteral(
m, literals, literals_count,
kSymbolsPerLiteralHistogram, kMaxLiteralHistograms,
@@ -144,6 +149,10 @@ void BrotliSplitBlock(MemoryManager* m,
literal_split);
if (BROTLI_IS_OOM(m)) return;
BROTLI_FREE(m, literals);
+ /* NB: this might be a good place for injecting extra splitting without
+ * increasing encoder complexity; however, output parition would be less
+ * optimal than one produced with forced splitting inside
+ * SplitByteVector (FindBlocks / ClusterBlocks). */
}
{
@@ -161,7 +170,7 @@ void BrotliSplitBlock(MemoryManager* m,
kCommandStrideLength, kCommandBlockSwitchCost, params,
insert_and_copy_split);
if (BROTLI_IS_OOM(m)) return;
- /* TODO: reuse for distances? */
+ /* TODO(eustas): reuse for distances? */
BROTLI_FREE(m, insert_and_copy_codes);
}
@@ -181,13 +190,27 @@ void BrotliSplitBlock(MemoryManager* m,
SplitByteVectorDistance(
m, distance_prefixes, j,
kSymbolsPerDistanceHistogram, kMaxCommandHistograms,
- kCommandStrideLength, kDistanceBlockSwitchCost, params,
+ kDistanceStrideLength, kDistanceBlockSwitchCost, params,
dist_split);
if (BROTLI_IS_OOM(m)) return;
BROTLI_FREE(m, distance_prefixes);
}
}
+#if defined(BROTLI_TEST)
+size_t CountLiteralsForTest(const Command*, const size_t);
+size_t CountLiteralsForTest(const Command* cmds, const size_t num_commands) {
+ return CountLiterals(cmds, num_commands);
+}
+
+void CopyLiteralsToByteArrayForTest(const Command*,
+ const size_t, const uint8_t*, const size_t, const size_t, uint8_t*);
+void CopyLiteralsToByteArrayForTest(const Command* cmds,
+ const size_t num_commands, const uint8_t* data, const size_t offset,
+ const size_t mask, uint8_t* literals) {
+ CopyLiteralsToByteArray(cmds, num_commands, data, offset, mask, literals);
+}
+#endif
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
diff --git a/dviware/dvisvgm/libs/brotli/enc/block_splitter.h b/dviware/dvisvgm/libs/brotli/enc/block_splitter.h
index a5e006c4b3..6046b90a5d 100644
--- a/dviware/dvisvgm/libs/brotli/enc/block_splitter.h
+++ b/dviware/dvisvgm/libs/brotli/enc/block_splitter.h
@@ -9,11 +9,12 @@
#ifndef BROTLI_ENC_BLOCK_SPLITTER_H_
#define BROTLI_ENC_BLOCK_SPLITTER_H_
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./command.h"
-#include "./memory.h"
-#include "./quality.h"
+
+#include "../common/platform.h"
+#include "command.h"
+#include "memory.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/block_splitter_inc.h b/dviware/dvisvgm/libs/brotli/enc/block_splitter_inc.h
index e612d6a370..aa40bfd329 100644
--- a/dviware/dvisvgm/libs/brotli/enc/block_splitter_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/block_splitter_inc.h
@@ -46,17 +46,17 @@ static void FN(RandomSample)(uint32_t* seed,
static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
size_t stride,
size_t num_histograms,
- HistogramType* histograms) {
+ HistogramType* histograms,
+ HistogramType* tmp) {
size_t iters =
kIterMulForRefining * length / stride + kMinItersForRefining;
uint32_t seed = 7;
size_t iter;
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms;
for (iter = 0; iter < iters; ++iter) {
- HistogramType sample;
- FN(HistogramClear)(&sample);
- FN(RandomSample)(&seed, data, length, stride, &sample);
- FN(HistogramAddHistogram)(&histograms[iter % num_histograms], &sample);
+ FN(HistogramClear)(tmp);
+ FN(RandomSample)(&seed, data, length, stride, tmp);
+ FN(HistogramAddHistogram)(&histograms[iter % num_histograms], tmp);
}
}
@@ -71,46 +71,56 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
double* cost,
uint8_t* switch_signal,
uint8_t* block_id) {
- const size_t data_size = FN(HistogramDataSize)();
- const size_t bitmaplen = (num_histograms + 7) >> 3;
+ const size_t alphabet_size = FN(HistogramDataSize)();
+ const size_t bitmap_len = (num_histograms + 7) >> 3;
size_t num_blocks = 1;
+ size_t byte_ix;
size_t i;
size_t j;
BROTLI_DCHECK(num_histograms <= 256);
+
+ /* Trivial case: single historgram -> single block type. */
if (num_histograms <= 1) {
for (i = 0; i < length; ++i) {
block_id[i] = 0;
}
return 1;
}
- memset(insert_cost, 0, sizeof(insert_cost[0]) * data_size * num_histograms);
+
+ /* Fill bitcost for each symbol of all histograms.
+ * Non-existing symbol cost: 2 + log2(total_count).
+ * Regular symbol cost: -log2(symbol_count / total_count). */
+ memset(insert_cost, 0,
+ sizeof(insert_cost[0]) * alphabet_size * num_histograms);
for (i = 0; i < num_histograms; ++i) {
insert_cost[i] = FastLog2((uint32_t)histograms[i].total_count_);
}
- for (i = data_size; i != 0;) {
+ for (i = alphabet_size; i != 0;) {
+ /* Reverse order to use the 0-th row as a temporary storage. */
--i;
for (j = 0; j < num_histograms; ++j) {
insert_cost[i * num_histograms + j] =
insert_cost[j] - BitCost(histograms[j].data_[i]);
}
}
- memset(cost, 0, sizeof(cost[0]) * num_histograms);
- memset(switch_signal, 0, sizeof(switch_signal[0]) * length * bitmaplen);
+
/* After each iteration of this loop, cost[k] will contain the difference
between the minimum cost of arriving at the current byte position using
entropy code k, and the minimum cost of arriving at the current byte
position. This difference is capped at the block switch cost, and if it
reaches block switch cost, it means that when we trace back from the last
position, we need to switch here. */
- for (i = 0; i < length; ++i) {
- const size_t byte_ix = i;
- size_t ix = byte_ix * bitmaplen;
- size_t insert_cost_ix = data[byte_ix] * num_histograms;
+ memset(cost, 0, sizeof(cost[0]) * num_histograms);
+ memset(switch_signal, 0, sizeof(switch_signal[0]) * length * bitmap_len);
+ for (byte_ix = 0; byte_ix < length; ++byte_ix) {
+ size_t ix = byte_ix * bitmap_len;
+ size_t symbol = data[byte_ix];
+ size_t insert_cost_ix = symbol * num_histograms;
double min_cost = 1e99;
double block_switch_cost = block_switch_bitcost;
size_t k;
for (k = 0; k < num_histograms; ++k) {
- /* We are coding the symbol in data[byte_ix] with entropy code k. */
+ /* We are coding the symbol with entropy code k. */
cost[k] += insert_cost[insert_cost_ix + k];
if (cost[k] < min_cost) {
min_cost = cost[k];
@@ -126,20 +136,21 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
if (cost[k] >= block_switch_cost) {
const uint8_t mask = (uint8_t)(1u << (k & 7));
cost[k] = block_switch_cost;
- BROTLI_DCHECK((k >> 3) < bitmaplen);
+ BROTLI_DCHECK((k >> 3) < bitmap_len);
switch_signal[ix + (k >> 3)] |= mask;
}
}
}
+
+ byte_ix = length - 1;
{ /* Trace back from the last position and switch at the marked places. */
- size_t byte_ix = length - 1;
- size_t ix = byte_ix * bitmaplen;
+ size_t ix = byte_ix * bitmap_len;
uint8_t cur_id = block_id[byte_ix];
while (byte_ix > 0) {
const uint8_t mask = (uint8_t)(1u << (cur_id & 7));
- BROTLI_DCHECK(((size_t)cur_id >> 3) < bitmaplen);
+ BROTLI_DCHECK(((size_t)cur_id >> 3) < bitmap_len);
--byte_ix;
- ix -= bitmaplen;
+ ix -= bitmap_len;
if (switch_signal[ix + (cur_id >> 3)] & mask) {
if (cur_id != block_id[byte_ix]) {
cur_id = block_id[byte_ix];
@@ -185,13 +196,16 @@ static void FN(BuildBlockHistograms)(const DataType* data, const size_t length,
}
}
+/* Given the initial partitioning build partitioning with limited number
+ * of histograms (and block types). */
static void FN(ClusterBlocks)(MemoryManager* m,
const DataType* data, const size_t length,
const size_t num_blocks,
uint8_t* block_ids,
BlockSplit* split) {
uint32_t* histogram_symbols = BROTLI_ALLOC(m, uint32_t, num_blocks);
- uint32_t* block_lengths = BROTLI_ALLOC(m, uint32_t, num_blocks);
+ uint32_t* u32 =
+ BROTLI_ALLOC(m, uint32_t, num_blocks + 4 * HISTOGRAMS_PER_BATCH);
const size_t expected_num_clusters = CLUSTERS_PER_BATCH *
(num_blocks + HISTOGRAMS_PER_BATCH - 1) / HISTOGRAMS_PER_BATCH;
size_t all_histograms_size = 0;
@@ -214,20 +228,25 @@ static void FN(ClusterBlocks)(MemoryManager* m,
static const uint32_t kInvalidIndex = BROTLI_UINT32_MAX;
uint32_t* new_index;
size_t i;
- uint32_t sizes[HISTOGRAMS_PER_BATCH] = { 0 };
- uint32_t new_clusters[HISTOGRAMS_PER_BATCH] = { 0 };
- uint32_t symbols[HISTOGRAMS_PER_BATCH] = { 0 };
- uint32_t remap[HISTOGRAMS_PER_BATCH] = { 0 };
+ uint32_t* BROTLI_RESTRICT const sizes = u32 + 0 * HISTOGRAMS_PER_BATCH;
+ uint32_t* BROTLI_RESTRICT const new_clusters = u32 + 1 * HISTOGRAMS_PER_BATCH;
+ uint32_t* BROTLI_RESTRICT const symbols = u32 + 2 * HISTOGRAMS_PER_BATCH;
+ uint32_t* BROTLI_RESTRICT const remap = u32 + 3 * HISTOGRAMS_PER_BATCH;
+ uint32_t* BROTLI_RESTRICT const block_lengths =
+ u32 + 4 * HISTOGRAMS_PER_BATCH;
+ /* TODO(eustas): move to arena? */
+ HistogramType* tmp = BROTLI_ALLOC(m, HistogramType, 2);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histogram_symbols) ||
- BROTLI_IS_NULL(block_lengths) || BROTLI_IS_NULL(all_histograms) ||
+ BROTLI_IS_NULL(u32) || BROTLI_IS_NULL(all_histograms) ||
BROTLI_IS_NULL(cluster_size) || BROTLI_IS_NULL(histograms) ||
- BROTLI_IS_NULL(pairs)) {
+ BROTLI_IS_NULL(pairs) || BROTLI_IS_NULL(tmp)) {
return;
}
- memset(block_lengths, 0, num_blocks * sizeof(uint32_t));
+ memset(u32, 0, (num_blocks + 4 * HISTOGRAMS_PER_BATCH) * sizeof(uint32_t));
+ /* Calculate block lengths (convert repeating values -> series length). */
{
size_t block_idx = 0;
for (i = 0; i < length; ++i) {
@@ -240,6 +259,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
BROTLI_DCHECK(block_idx == num_blocks);
}
+ /* Pre-cluster blocks (cluster batches). */
for (i = 0; i < num_blocks; i += HISTOGRAMS_PER_BATCH) {
const size_t num_to_combine =
BROTLI_MIN(size_t, num_blocks - i, HISTOGRAMS_PER_BATCH);
@@ -247,8 +267,9 @@ static void FN(ClusterBlocks)(MemoryManager* m,
size_t j;
for (j = 0; j < num_to_combine; ++j) {
size_t k;
+ size_t block_length = block_lengths[i + j];
FN(HistogramClear)(&histograms[j]);
- for (k = 0; k < block_lengths[i + j]; ++k) {
+ for (k = 0; k < block_length; ++k) {
FN(HistogramAdd)(&histograms[j], data[pos++]);
}
histograms[j].bit_cost_ = FN(BrotliPopulationCost)(&histograms[j]);
@@ -257,7 +278,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
sizes[j] = 1;
}
num_new_clusters = FN(BrotliHistogramCombine)(
- histograms, sizes, symbols, new_clusters, pairs, num_to_combine,
+ histograms, tmp, sizes, symbols, new_clusters, pairs, num_to_combine,
num_to_combine, HISTOGRAMS_PER_BATCH, max_num_pairs);
BROTLI_ENSURE_CAPACITY(m, HistogramType, all_histograms,
all_histograms_capacity, all_histograms_size + num_new_clusters);
@@ -278,6 +299,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
}
BROTLI_FREE(m, histograms);
+ /* Final clustering. */
max_num_pairs =
BROTLI_MIN(size_t, 64 * num_clusters, (num_clusters / 2) * num_clusters);
if (pairs_capacity < max_num_pairs + 1) {
@@ -285,19 +307,19 @@ static void FN(ClusterBlocks)(MemoryManager* m,
pairs = BROTLI_ALLOC(m, HistogramPair, max_num_pairs + 1);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(pairs)) return;
}
-
clusters = BROTLI_ALLOC(m, uint32_t, num_clusters);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(clusters)) return;
for (i = 0; i < num_clusters; ++i) {
clusters[i] = (uint32_t)i;
}
num_final_clusters = FN(BrotliHistogramCombine)(
- all_histograms, cluster_size, histogram_symbols, clusters, pairs,
+ all_histograms, tmp, cluster_size, histogram_symbols, clusters, pairs,
num_clusters, num_blocks, BROTLI_MAX_NUMBER_OF_BLOCK_TYPES,
max_num_pairs);
BROTLI_FREE(m, pairs);
BROTLI_FREE(m, cluster_size);
+ /* Assign blocks to final histograms. */
new_index = BROTLI_ALLOC(m, uint32_t, num_clusters);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_index)) return;
for (i = 0; i < num_clusters; ++i) new_index[i] = kInvalidIndex;
@@ -305,20 +327,21 @@ static void FN(ClusterBlocks)(MemoryManager* m,
{
uint32_t next_index = 0;
for (i = 0; i < num_blocks; ++i) {
- HistogramType histo;
size_t j;
uint32_t best_out;
double best_bits;
- FN(HistogramClear)(&histo);
+ FN(HistogramClear)(tmp);
for (j = 0; j < block_lengths[i]; ++j) {
- FN(HistogramAdd)(&histo, data[pos++]);
+ FN(HistogramAdd)(tmp, data[pos++]);
}
+ /* Among equally good histograms prefer last used. */
+ /* TODO(eustas): should we give a block-switch discount here? */
best_out = (i == 0) ? histogram_symbols[0] : histogram_symbols[i - 1];
- best_bits =
- FN(BrotliHistogramBitCostDistance)(&histo, &all_histograms[best_out]);
+ best_bits = FN(BrotliHistogramBitCostDistance)(
+ tmp, &all_histograms[best_out], tmp + 1);
for (j = 0; j < num_final_clusters; ++j) {
const double cur_bits = FN(BrotliHistogramBitCostDistance)(
- &histo, &all_histograms[clusters[j]]);
+ tmp, &all_histograms[clusters[j]], tmp + 1);
if (cur_bits < best_bits) {
best_bits = cur_bits;
best_out = clusters[j];
@@ -330,6 +353,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
}
}
}
+ BROTLI_FREE(m, tmp);
BROTLI_FREE(m, clusters);
BROTLI_FREE(m, all_histograms);
BROTLI_ENSURE_CAPACITY(
@@ -337,6 +361,9 @@ static void FN(ClusterBlocks)(MemoryManager* m,
BROTLI_ENSURE_CAPACITY(
m, uint32_t, split->lengths, split->lengths_alloc_size, num_blocks);
if (BROTLI_IS_OOM(m)) return;
+
+ /* Rewrite final assignment to block-split. There might be less blocks
+ * than |num_blocks| due to clustering. */
{
uint32_t cur_length = 0;
size_t block_idx = 0;
@@ -357,28 +384,41 @@ static void FN(ClusterBlocks)(MemoryManager* m,
split->num_types = (size_t)max_type + 1;
}
BROTLI_FREE(m, new_index);
- BROTLI_FREE(m, block_lengths);
+ BROTLI_FREE(m, u32);
BROTLI_FREE(m, histogram_symbols);
}
+/* Create BlockSplit (partitioning) given the limits, estimates and "effort"
+ * parameters.
+ *
+ * NB: max_histograms is often less than number of histograms allowed by format;
+ * this is done intentionally, to save some "space" for context-aware
+ * clustering (here entropy is estimated for context-free symbols). */
static void FN(SplitByteVector)(MemoryManager* m,
const DataType* data, const size_t length,
- const size_t literals_per_histogram,
+ const size_t symbols_per_histogram,
const size_t max_histograms,
const size_t sampling_stride_length,
const double block_switch_cost,
const BrotliEncoderParams* params,
BlockSplit* split) {
const size_t data_size = FN(HistogramDataSize)();
- size_t num_histograms = length / literals_per_histogram + 1;
HistogramType* histograms;
+ HistogramType* tmp;
+ /* Calculate number of histograms; initial estimate is one histogram per
+ * specified amount of symbols; however, this value is capped. */
+ size_t num_histograms = length / symbols_per_histogram + 1;
if (num_histograms > max_histograms) {
num_histograms = max_histograms;
}
+
+ /* Corner case: no input. */
if (length == 0) {
split->num_types = 1;
return;
- } else if (length < kMinLengthForBlockSplitting) {
+ }
+
+ if (length < kMinLengthForBlockSplitting) {
BROTLI_ENSURE_CAPACITY(m, uint8_t,
split->types, split->types_alloc_size, split->num_blocks + 1);
BROTLI_ENSURE_CAPACITY(m, uint32_t,
@@ -390,7 +430,8 @@ static void FN(SplitByteVector)(MemoryManager* m,
split->num_blocks++;
return;
}
- histograms = BROTLI_ALLOC(m, HistogramType, num_histograms);
+ histograms = BROTLI_ALLOC(m, HistogramType, num_histograms + 1);
+ tmp = histograms + num_histograms;
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histograms)) return;
/* Find good entropy codes. */
FN(InitialEntropyCodes)(data, length,
@@ -398,7 +439,7 @@ static void FN(SplitByteVector)(MemoryManager* m,
num_histograms, histograms);
FN(RefineEntropyCodes)(data, length,
sampling_stride_length,
- num_histograms, histograms);
+ num_histograms, histograms, tmp);
{
/* Find a good path through literals with the good entropy codes. */
uint8_t* block_ids = BROTLI_ALLOC(m, uint8_t, length);
diff --git a/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.c b/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.c
index 9348a97e1b..5fa0c69aa8 100644
--- a/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.c
+++ b/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.c
@@ -8,20 +8,21 @@
compression algorithms here, just the right ordering of bits to match the
specs. */
-#include "./brotli_bit_stream.h"
+#include "brotli_bit_stream.h"
#include <string.h> /* memcpy, memset */
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/context.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./entropy_encode.h"
-#include "./entropy_encode_static.h"
-#include "./fast_log.h"
-#include "./histogram.h"
-#include "./memory.h"
-#include "./write_bits.h"
+#include "entropy_encode.h"
+#include "entropy_encode_static.h"
+#include "fast_log.h"
+#include "histogram.h"
+#include "memory.h"
+#include "write_bits.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -286,6 +287,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
/* Write the Huffman tree into the brotli-representation.
The command alphabet is the largest, so this allocation will fit all
alphabets. */
+ /* TODO(eustas): fix me */
uint8_t huffman_tree[BROTLI_NUM_COMMAND_SYMBOLS];
uint8_t huffman_tree_extra_bits[BROTLI_NUM_COMMAND_SYMBOLS];
size_t huffman_tree_size = 0;
@@ -400,7 +402,7 @@ static BROTLI_INLINE BROTLI_BOOL SortHuffmanTree(
return TO_BROTLI_BOOL(v0->total_count_ < v1->total_count_);
}
-void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
+void BrotliBuildAndStoreHuffmanTreeFast(HuffmanTree* tree,
const uint32_t* histogram,
const size_t histogram_total,
const size_t max_bits,
@@ -432,10 +434,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
memset(depth, 0, length * sizeof(depth[0]));
{
- const size_t max_tree_size = 2 * length + 1;
- HuffmanTree* tree = BROTLI_ALLOC(m, HuffmanTree, max_tree_size);
uint32_t count_limit;
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
for (count_limit = 1; ; count_limit *= 2) {
HuffmanTree* node = tree;
size_t l;
@@ -500,7 +499,6 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
}
}
}
- BROTLI_FREE(m, tree);
}
BrotliConvertBitDepthsToSymbols(depth, length, bits);
if (count <= 4) {
@@ -677,7 +675,14 @@ static void RunLengthCodeZeros(const size_t in_size,
#define SYMBOL_BITS 9
+typedef struct EncodeContextMapArena {
+ uint32_t histogram[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
+ uint8_t depths[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
+ uint16_t bits[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
+} EncodeContextMapArena;
+
static void EncodeContextMap(MemoryManager* m,
+ EncodeContextMapArena* arena,
const uint32_t* context_map,
size_t context_map_size,
size_t num_clusters,
@@ -687,10 +692,10 @@ static void EncodeContextMap(MemoryManager* m,
uint32_t* rle_symbols;
uint32_t max_run_length_prefix = 6;
size_t num_rle_symbols = 0;
- uint32_t histogram[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
+ uint32_t* BROTLI_RESTRICT const histogram = arena->histogram;
static const uint32_t kSymbolMask = (1u << SYMBOL_BITS) - 1u;
- uint8_t depths[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
- uint16_t bits[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
+ uint8_t* BROTLI_RESTRICT const depths = arena->depths;
+ uint16_t* BROTLI_RESTRICT const bits = arena->bits;
StoreVarLenUint8(num_clusters - 1, storage_ix, storage);
@@ -703,7 +708,7 @@ static void EncodeContextMap(MemoryManager* m,
MoveToFrontTransform(context_map, context_map_size, rle_symbols);
RunLengthCodeZeros(context_map_size, rle_symbols,
&num_rle_symbols, &max_run_length_prefix);
- memset(histogram, 0, sizeof(histogram));
+ memset(histogram, 0, sizeof(arena->histogram));
for (i = 0; i < num_rle_symbols; ++i) {
++histogram[rle_symbols[i] & kSymbolMask];
}
@@ -774,7 +779,7 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
++length_histo[BlockLengthPrefixCode(lengths[i])];
}
StoreVarLenUint8(num_types - 1, storage_ix, storage);
- if (num_types > 1) { /* TODO: else? could StoreBlockSwitch occur? */
+ if (num_types > 1) { /* TODO(eustas): else? could StoreBlockSwitch occur? */
BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, num_types + 2, tree,
&code->type_depths[0], &code->type_bits[0],
storage_ix, storage);
@@ -787,7 +792,8 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
}
/* Stores a context map where the histogram type is always the block type. */
-static void StoreTrivialContextMap(size_t num_types,
+static void StoreTrivialContextMap(EncodeContextMapArena* arena,
+ size_t num_types,
size_t context_bits,
HuffmanTree* tree,
size_t* storage_ix,
@@ -797,9 +803,9 @@ static void StoreTrivialContextMap(size_t num_types,
size_t repeat_code = context_bits - 1u;
size_t repeat_bits = (1u << repeat_code) - 1u;
size_t alphabet_size = num_types + repeat_code;
- uint32_t histogram[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
- uint8_t depths[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
- uint16_t bits[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
+ uint32_t* BROTLI_RESTRICT const histogram = arena->histogram;
+ uint8_t* BROTLI_RESTRICT const depths = arena->depths;
+ uint16_t* BROTLI_RESTRICT const bits = arena->bits;
size_t i;
memset(histogram, 0, alphabet_size * sizeof(histogram[0]));
/* Write RLEMAX. */
@@ -914,17 +920,17 @@ static void StoreSymbolWithContext(BlockEncoder* self, size_t symbol,
#define FN(X) X ## Literal
/* NOLINTNEXTLINE(build/include) */
-#include "./block_encoder_inc.h"
+#include "block_encoder_inc.h"
#undef FN
#define FN(X) X ## Command
/* NOLINTNEXTLINE(build/include) */
-#include "./block_encoder_inc.h"
+#include "block_encoder_inc.h"
#undef FN
#define FN(X) X ## Distance
/* NOLINTNEXTLINE(build/include) */
-#include "./block_encoder_inc.h"
+#include "block_encoder_inc.h"
#undef FN
static void JumpToByteBoundary(size_t* storage_ix, uint8_t* storage) {
@@ -932,6 +938,13 @@ static void JumpToByteBoundary(size_t* storage_ix, uint8_t* storage) {
storage[*storage_ix >> 3] = 0;
}
+typedef struct StoreMetablockArena {
+ BlockEncoder literal_enc;
+ BlockEncoder command_enc;
+ BlockEncoder distance_enc;
+ EncodeContextMapArena context_map_arena;
+} StoreMetablockArena;
+
void BrotliStoreMetaBlock(MemoryManager* m,
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last,
@@ -945,9 +958,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
uint32_t num_effective_distance_symbols = params->dist.alphabet_size_limit;
HuffmanTree* tree;
ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
- BlockEncoder literal_enc;
- BlockEncoder command_enc;
- BlockEncoder distance_enc;
+ StoreMetablockArena* arena = NULL;
+ BlockEncoder* literal_enc = NULL;
+ BlockEncoder* command_enc = NULL;
+ BlockEncoder* distance_enc = NULL;
const BrotliDistanceParams* dist = &params->dist;
BROTLI_DCHECK(
num_effective_distance_symbols <= BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS);
@@ -955,21 +969,24 @@ void BrotliStoreMetaBlock(MemoryManager* m,
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
- InitBlockEncoder(&literal_enc, BROTLI_NUM_LITERAL_SYMBOLS,
+ arena = BROTLI_ALLOC(m, StoreMetablockArena, 1);
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree) || BROTLI_IS_NULL(arena)) return;
+ literal_enc = &arena->literal_enc;
+ command_enc = &arena->command_enc;
+ distance_enc = &arena->distance_enc;
+ InitBlockEncoder(literal_enc, BROTLI_NUM_LITERAL_SYMBOLS,
mb->literal_split.num_types, mb->literal_split.types,
mb->literal_split.lengths, mb->literal_split.num_blocks);
- InitBlockEncoder(&command_enc, BROTLI_NUM_COMMAND_SYMBOLS,
+ InitBlockEncoder(command_enc, BROTLI_NUM_COMMAND_SYMBOLS,
mb->command_split.num_types, mb->command_split.types,
mb->command_split.lengths, mb->command_split.num_blocks);
- InitBlockEncoder(&distance_enc, num_effective_distance_symbols,
+ InitBlockEncoder(distance_enc, num_effective_distance_symbols,
mb->distance_split.num_types, mb->distance_split.types,
mb->distance_split.lengths, mb->distance_split.num_blocks);
- BuildAndStoreBlockSwitchEntropyCodes(&literal_enc, tree, storage_ix, storage);
- BuildAndStoreBlockSwitchEntropyCodes(&command_enc, tree, storage_ix, storage);
- BuildAndStoreBlockSwitchEntropyCodes(
- &distance_enc, tree, storage_ix, storage);
+ BuildAndStoreBlockSwitchEntropyCodes(literal_enc, tree, storage_ix, storage);
+ BuildAndStoreBlockSwitchEntropyCodes(command_enc, tree, storage_ix, storage);
+ BuildAndStoreBlockSwitchEntropyCodes(distance_enc, tree, storage_ix, storage);
BrotliWriteBits(2, dist->distance_postfix_bits, storage_ix, storage);
BrotliWriteBits(
@@ -980,34 +997,36 @@ void BrotliStoreMetaBlock(MemoryManager* m,
}
if (mb->literal_context_map_size == 0) {
- StoreTrivialContextMap(mb->literal_histograms_size,
+ StoreTrivialContextMap(
+ &arena->context_map_arena, mb->literal_histograms_size,
BROTLI_LITERAL_CONTEXT_BITS, tree, storage_ix, storage);
} else {
- EncodeContextMap(m,
+ EncodeContextMap(m, &arena->context_map_arena,
mb->literal_context_map, mb->literal_context_map_size,
mb->literal_histograms_size, tree, storage_ix, storage);
if (BROTLI_IS_OOM(m)) return;
}
if (mb->distance_context_map_size == 0) {
- StoreTrivialContextMap(mb->distance_histograms_size,
+ StoreTrivialContextMap(
+ &arena->context_map_arena, mb->distance_histograms_size,
BROTLI_DISTANCE_CONTEXT_BITS, tree, storage_ix, storage);
} else {
- EncodeContextMap(m,
+ EncodeContextMap(m, &arena->context_map_arena,
mb->distance_context_map, mb->distance_context_map_size,
mb->distance_histograms_size, tree, storage_ix, storage);
if (BROTLI_IS_OOM(m)) return;
}
- BuildAndStoreEntropyCodesLiteral(m, &literal_enc, mb->literal_histograms,
+ BuildAndStoreEntropyCodesLiteral(m, literal_enc, mb->literal_histograms,
mb->literal_histograms_size, BROTLI_NUM_LITERAL_SYMBOLS, tree,
storage_ix, storage);
if (BROTLI_IS_OOM(m)) return;
- BuildAndStoreEntropyCodesCommand(m, &command_enc, mb->command_histograms,
+ BuildAndStoreEntropyCodesCommand(m, command_enc, mb->command_histograms,
mb->command_histograms_size, BROTLI_NUM_COMMAND_SYMBOLS, tree,
storage_ix, storage);
if (BROTLI_IS_OOM(m)) return;
- BuildAndStoreEntropyCodesDistance(m, &distance_enc, mb->distance_histograms,
+ BuildAndStoreEntropyCodesDistance(m, distance_enc, mb->distance_histograms,
mb->distance_histograms_size, num_distance_symbols, tree,
storage_ix, storage);
if (BROTLI_IS_OOM(m)) return;
@@ -1016,12 +1035,12 @@ void BrotliStoreMetaBlock(MemoryManager* m,
for (i = 0; i < n_commands; ++i) {
const Command cmd = commands[i];
size_t cmd_code = cmd.cmd_prefix_;
- StoreSymbol(&command_enc, cmd_code, storage_ix, storage);
+ StoreSymbol(command_enc, cmd_code, storage_ix, storage);
StoreCommandExtra(&cmd, storage_ix, storage);
if (mb->literal_context_map_size == 0) {
size_t j;
for (j = cmd.insert_len_; j != 0; --j) {
- StoreSymbol(&literal_enc, input[pos & mask], storage_ix, storage);
+ StoreSymbol(literal_enc, input[pos & mask], storage_ix, storage);
++pos;
}
} else {
@@ -1030,7 +1049,7 @@ void BrotliStoreMetaBlock(MemoryManager* m,
size_t context =
BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
uint8_t literal = input[pos & mask];
- StoreSymbolWithContext(&literal_enc, literal, context,
+ StoreSymbolWithContext(literal_enc, literal, context,
mb->literal_context_map, storage_ix, storage,
BROTLI_LITERAL_CONTEXT_BITS);
prev_byte2 = prev_byte;
@@ -1047,10 +1066,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
uint32_t distnumextra = cmd.dist_prefix_ >> 10;
uint64_t distextra = cmd.dist_extra_;
if (mb->distance_context_map_size == 0) {
- StoreSymbol(&distance_enc, dist_code, storage_ix, storage);
+ StoreSymbol(distance_enc, dist_code, storage_ix, storage);
} else {
size_t context = CommandDistanceContext(&cmd);
- StoreSymbolWithContext(&distance_enc, dist_code, context,
+ StoreSymbolWithContext(distance_enc, dist_code, context,
mb->distance_context_map, storage_ix, storage,
BROTLI_DISTANCE_CONTEXT_BITS);
}
@@ -1058,9 +1077,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
}
}
}
- CleanupBlockEncoder(m, &distance_enc);
- CleanupBlockEncoder(m, &command_enc);
- CleanupBlockEncoder(m, &literal_enc);
+ CleanupBlockEncoder(m, distance_enc);
+ CleanupBlockEncoder(m, command_enc);
+ CleanupBlockEncoder(m, literal_enc);
+ BROTLI_FREE(m, arena);
if (is_last) {
JumpToByteBoundary(storage_ix, storage);
}
@@ -1131,54 +1151,60 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
}
}
-void BrotliStoreMetaBlockTrivial(MemoryManager* m,
- const uint8_t* input, size_t start_pos, size_t length, size_t mask,
- BROTLI_BOOL is_last, const BrotliEncoderParams* params,
- const Command* commands, size_t n_commands,
- size_t* storage_ix, uint8_t* storage) {
+/* TODO(eustas): pull alloc/dealloc to caller? */
+typedef struct MetablockArena {
HistogramLiteral lit_histo;
HistogramCommand cmd_histo;
HistogramDistance dist_histo;
+ /* TODO(eustas): merge bits and depth? */
uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
- HuffmanTree* tree;
+ HuffmanTree tree[MAX_HUFFMAN_TREE_SIZE];
+} MetablockArena;
+
+void BrotliStoreMetaBlockTrivial(MemoryManager* m,
+ const uint8_t* input, size_t start_pos, size_t length, size_t mask,
+ BROTLI_BOOL is_last, const BrotliEncoderParams* params,
+ const Command* commands, size_t n_commands,
+ size_t* storage_ix, uint8_t* storage) {
+ MetablockArena* arena = BROTLI_ALLOC(m, MetablockArena, 1);
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
- HistogramClearLiteral(&lit_histo);
- HistogramClearCommand(&cmd_histo);
- HistogramClearDistance(&dist_histo);
+ HistogramClearLiteral(&arena->lit_histo);
+ HistogramClearCommand(&arena->cmd_histo);
+ HistogramClearDistance(&arena->dist_histo);
BuildHistograms(input, start_pos, mask, commands, n_commands,
- &lit_histo, &cmd_histo, &dist_histo);
+ &arena->lit_histo, &arena->cmd_histo, &arena->dist_histo);
BrotliWriteBits(13, 0, storage_ix, storage);
- tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
- BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS,
- BROTLI_NUM_LITERAL_SYMBOLS, tree,
- lit_depth, lit_bits,
+ BuildAndStoreHuffmanTree(arena->lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS,
+ BROTLI_NUM_LITERAL_SYMBOLS, arena->tree,
+ arena->lit_depth, arena->lit_bits,
storage_ix, storage);
- BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS,
- BROTLI_NUM_COMMAND_SYMBOLS, tree,
- cmd_depth, cmd_bits,
+ BuildAndStoreHuffmanTree(arena->cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS,
+ BROTLI_NUM_COMMAND_SYMBOLS, arena->tree,
+ arena->cmd_depth, arena->cmd_bits,
storage_ix, storage);
- BuildAndStoreHuffmanTree(dist_histo.data_, MAX_SIMPLE_DISTANCE_ALPHABET_SIZE,
- num_distance_symbols, tree,
- dist_depth, dist_bits,
+ BuildAndStoreHuffmanTree(arena->dist_histo.data_,
+ MAX_SIMPLE_DISTANCE_ALPHABET_SIZE,
+ num_distance_symbols, arena->tree,
+ arena->dist_depth, arena->dist_bits,
storage_ix, storage);
- BROTLI_FREE(m, tree);
StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
- n_commands, lit_depth, lit_bits,
- cmd_depth, cmd_bits,
- dist_depth, dist_bits,
+ n_commands, arena->lit_depth, arena->lit_bits,
+ arena->cmd_depth, arena->cmd_bits,
+ arena->dist_depth, arena->dist_bits,
storage_ix, storage);
+ BROTLI_FREE(m, arena);
if (is_last) {
JumpToByteBoundary(storage_ix, storage);
}
@@ -1189,9 +1215,11 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
const Command* commands, size_t n_commands,
size_t* storage_ix, uint8_t* storage) {
+ MetablockArena* arena = BROTLI_ALLOC(m, MetablockArena, 1);
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
uint32_t distance_alphabet_bits =
Log2FloorNonZero(num_distance_symbols - 1) + 1;
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
@@ -1202,8 +1230,6 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
size_t pos = start_pos;
size_t num_literals = 0;
size_t i;
- uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
- uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
for (i = 0; i < n_commands; ++i) {
const Command cmd = commands[i];
size_t j;
@@ -1214,61 +1240,50 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
num_literals += cmd.insert_len_;
pos += CommandCopyLen(&cmd);
}
- BrotliBuildAndStoreHuffmanTreeFast(m, histogram, num_literals,
+ BrotliBuildAndStoreHuffmanTreeFast(arena->tree, histogram, num_literals,
/* max_bits = */ 8,
- lit_depth, lit_bits,
+ arena->lit_depth, arena->lit_bits,
storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
StoreStaticCommandHuffmanTree(storage_ix, storage);
StoreStaticDistanceHuffmanTree(storage_ix, storage);
StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
- n_commands, lit_depth, lit_bits,
+ n_commands, arena->lit_depth, arena->lit_bits,
kStaticCommandCodeDepth,
kStaticCommandCodeBits,
kStaticDistanceCodeDepth,
kStaticDistanceCodeBits,
storage_ix, storage);
} else {
- HistogramLiteral lit_histo;
- HistogramCommand cmd_histo;
- HistogramDistance dist_histo;
- uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
- uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
- uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
- uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
- uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
- uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
- HistogramClearLiteral(&lit_histo);
- HistogramClearCommand(&cmd_histo);
- HistogramClearDistance(&dist_histo);
+ HistogramClearLiteral(&arena->lit_histo);
+ HistogramClearCommand(&arena->cmd_histo);
+ HistogramClearDistance(&arena->dist_histo);
BuildHistograms(input, start_pos, mask, commands, n_commands,
- &lit_histo, &cmd_histo, &dist_histo);
- BrotliBuildAndStoreHuffmanTreeFast(m, lit_histo.data_,
- lit_histo.total_count_,
+ &arena->lit_histo, &arena->cmd_histo, &arena->dist_histo);
+ BrotliBuildAndStoreHuffmanTreeFast(arena->tree, arena->lit_histo.data_,
+ arena->lit_histo.total_count_,
/* max_bits = */ 8,
- lit_depth, lit_bits,
+ arena->lit_depth, arena->lit_bits,
storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
- BrotliBuildAndStoreHuffmanTreeFast(m, cmd_histo.data_,
- cmd_histo.total_count_,
+ BrotliBuildAndStoreHuffmanTreeFast(arena->tree, arena->cmd_histo.data_,
+ arena->cmd_histo.total_count_,
/* max_bits = */ 10,
- cmd_depth, cmd_bits,
+ arena->cmd_depth, arena->cmd_bits,
storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
- BrotliBuildAndStoreHuffmanTreeFast(m, dist_histo.data_,
- dist_histo.total_count_,
+ BrotliBuildAndStoreHuffmanTreeFast(arena->tree, arena->dist_histo.data_,
+ arena->dist_histo.total_count_,
/* max_bits = */
distance_alphabet_bits,
- dist_depth, dist_bits,
+ arena->dist_depth, arena->dist_bits,
storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
- n_commands, lit_depth, lit_bits,
- cmd_depth, cmd_bits,
- dist_depth, dist_bits,
+ n_commands, arena->lit_depth, arena->lit_bits,
+ arena->cmd_depth, arena->cmd_bits,
+ arena->dist_depth, arena->dist_bits,
storage_ix, storage);
}
+ BROTLI_FREE(m, arena);
+
if (is_last) {
JumpToByteBoundary(storage_ix, storage);
}
@@ -1309,6 +1324,13 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
}
}
+#if defined(BROTLI_TEST)
+void GetBlockLengthPrefixCodeForTest(uint32_t len, size_t* code,
+ uint32_t* n_extra, uint32_t* extra) {
+ GetBlockLengthPrefixCode(len, code, n_extra, extra);
+}
+#endif
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.h b/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.h
index 2ed703bf79..a289509af3 100644
--- a/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.h
+++ b/dviware/dvisvgm/libs/brotli/enc/brotli_bit_stream.h
@@ -16,13 +16,14 @@
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
+#include <brotli/types.h>
+
#include "../common/context.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./command.h"
-#include "./entropy_encode.h"
-#include "./memory.h"
-#include "./metablock.h"
+#include "command.h"
+#include "entropy_encode.h"
+#include "memory.h"
+#include "metablock.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -35,7 +36,7 @@ BROTLI_INTERNAL void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage);
BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
- MemoryManager* m, const uint32_t* histogram, const size_t histogram_total,
+ HuffmanTree* tree, const uint32_t* histogram, const size_t histogram_total,
const size_t max_bits, uint8_t* depth, uint16_t* bits, size_t* storage_ix,
uint8_t* storage);
@@ -77,6 +78,10 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
size_t position, size_t mask, size_t len,
size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage);
+#if defined(BROTLI_TEST)
+void GetBlockLengthPrefixCodeForTest(uint32_t, size_t*, uint32_t*, uint32_t*);
+#endif
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/cluster.c b/dviware/dvisvgm/libs/brotli/enc/cluster.c
index a20dfd385f..b0faf8114c 100644
--- a/dviware/dvisvgm/libs/brotli/enc/cluster.c
+++ b/dviware/dvisvgm/libs/brotli/enc/cluster.c
@@ -6,14 +6,15 @@
/* Functions for clustering similar histograms together. */
-#include "./cluster.h"
+#include "cluster.h"
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./bit_cost.h" /* BrotliPopulationCost */
-#include "./fast_log.h"
-#include "./histogram.h"
-#include "./memory.h"
+
+#include "../common/platform.h"
+#include "bit_cost.h" /* BrotliPopulationCost */
+#include "fast_log.h"
+#include "histogram.h"
+#include "memory.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -38,15 +39,15 @@ static BROTLI_INLINE double ClusterCostDiff(size_t size_a, size_t size_b) {
#define CODE(X) X
#define FN(X) X ## Literal
-#include "./cluster_inc.h" /* NOLINT(build/include) */
+#include "cluster_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Command
-#include "./cluster_inc.h" /* NOLINT(build/include) */
+#include "cluster_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Distance
-#include "./cluster_inc.h" /* NOLINT(build/include) */
+#include "cluster_inc.h" /* NOLINT(build/include) */
#undef FN
#undef CODE
diff --git a/dviware/dvisvgm/libs/brotli/enc/cluster.h b/dviware/dvisvgm/libs/brotli/enc/cluster.h
index bb26124d24..013629c6d9 100644
--- a/dviware/dvisvgm/libs/brotli/enc/cluster.h
+++ b/dviware/dvisvgm/libs/brotli/enc/cluster.h
@@ -9,10 +9,11 @@
#ifndef BROTLI_ENC_CLUSTER_H_
#define BROTLI_ENC_CLUSTER_H_
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./histogram.h"
-#include "./memory.h"
+
+#include "../common/platform.h"
+#include "histogram.h"
+#include "memory.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -28,15 +29,15 @@ typedef struct HistogramPair {
#define CODE(X) /* Declaration */;
#define FN(X) X ## Literal
-#include "./cluster_inc.h" /* NOLINT(build/include) */
+#include "cluster_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Command
-#include "./cluster_inc.h" /* NOLINT(build/include) */
+#include "cluster_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Distance
-#include "./cluster_inc.h" /* NOLINT(build/include) */
+#include "cluster_inc.h" /* NOLINT(build/include) */
#undef FN
#undef CODE
diff --git a/dviware/dvisvgm/libs/brotli/enc/cluster_inc.h b/dviware/dvisvgm/libs/brotli/enc/cluster_inc.h
index 3d4f40e601..d6215ef06e 100644
--- a/dviware/dvisvgm/libs/brotli/enc/cluster_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/cluster_inc.h
@@ -12,8 +12,8 @@
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
- const HistogramType* out, const uint32_t* cluster_size, uint32_t idx1,
- uint32_t idx2, size_t max_num_pairs, HistogramPair* pairs,
+ const HistogramType* out, HistogramType* tmp, const uint32_t* cluster_size,
+ uint32_t idx1, uint32_t idx2, size_t max_num_pairs, HistogramPair* pairs,
size_t* num_pairs) CODE({
BROTLI_BOOL is_good_pair = BROTLI_FALSE;
HistogramPair p;
@@ -42,10 +42,10 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
} else {
double threshold = *num_pairs == 0 ? 1e99 :
BROTLI_MAX(double, 0.0, pairs[0].cost_diff);
- HistogramType combo = out[idx1];
double cost_combo;
- FN(HistogramAddHistogram)(&combo, &out[idx2]);
- cost_combo = FN(BrotliPopulationCost)(&combo);
+ *tmp = out[idx1];
+ FN(HistogramAddHistogram)(tmp, &out[idx2]);
+ cost_combo = FN(BrotliPopulationCost)(tmp);
if (cost_combo < threshold - p.cost_diff) {
p.cost_combo = cost_combo;
is_good_pair = BROTLI_TRUE;
@@ -68,6 +68,7 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
})
BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
+ HistogramType* tmp,
uint32_t* cluster_size,
uint32_t* symbols,
uint32_t* clusters,
@@ -87,7 +88,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
for (idx1 = 0; idx1 < num_clusters; ++idx1) {
size_t idx2;
for (idx2 = idx1 + 1; idx2 < num_clusters; ++idx2) {
- FN(BrotliCompareAndPushToQueue)(out, cluster_size, clusters[idx1],
+ FN(BrotliCompareAndPushToQueue)(out, tmp, cluster_size, clusters[idx1],
clusters[idx2], max_num_pairs, &pairs[0], &num_pairs);
}
}
@@ -146,8 +147,8 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
/* Push new pairs formed with the combined histogram to the heap. */
for (i = 0; i < num_clusters; ++i) {
- FN(BrotliCompareAndPushToQueue)(out, cluster_size, best_idx1, clusters[i],
- max_num_pairs, &pairs[0], &num_pairs);
+ FN(BrotliCompareAndPushToQueue)(out, tmp, cluster_size, best_idx1,
+ clusters[i], max_num_pairs, &pairs[0], &num_pairs);
}
}
return num_clusters;
@@ -155,13 +156,14 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
BROTLI_INTERNAL double FN(BrotliHistogramBitCostDistance)(
- const HistogramType* histogram, const HistogramType* candidate) CODE({
+ const HistogramType* histogram, const HistogramType* candidate,
+ HistogramType* tmp) CODE({
if (histogram->total_count_ == 0) {
return 0.0;
} else {
- HistogramType tmp = *histogram;
- FN(HistogramAddHistogram)(&tmp, candidate);
- return FN(BrotliPopulationCost)(&tmp) - candidate->bit_cost_;
+ *tmp = *histogram;
+ FN(HistogramAddHistogram)(tmp, candidate);
+ return FN(BrotliPopulationCost)(tmp) - candidate->bit_cost_;
}
})
@@ -171,16 +173,16 @@ BROTLI_INTERNAL double FN(BrotliHistogramBitCostDistance)(
Note: we assume that out[]->bit_cost_ is already up-to-date. */
BROTLI_INTERNAL void FN(BrotliHistogramRemap)(const HistogramType* in,
size_t in_size, const uint32_t* clusters, size_t num_clusters,
- HistogramType* out, uint32_t* symbols) CODE({
+ HistogramType* out, HistogramType* tmp, uint32_t* symbols) CODE({
size_t i;
for (i = 0; i < in_size; ++i) {
uint32_t best_out = i == 0 ? symbols[0] : symbols[i - 1];
double best_bits =
- FN(BrotliHistogramBitCostDistance)(&in[i], &out[best_out]);
+ FN(BrotliHistogramBitCostDistance)(&in[i], &out[best_out], tmp);
size_t j;
for (j = 0; j < num_clusters; ++j) {
const double cur_bits =
- FN(BrotliHistogramBitCostDistance)(&in[i], &out[clusters[j]]);
+ FN(BrotliHistogramBitCostDistance)(&in[i], &out[clusters[j]], tmp);
if (cur_bits < best_bits) {
best_bits = cur_bits;
best_out = clusters[j];
@@ -226,7 +228,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
++next_index;
}
}
- /* TODO: by using idea of "cycle-sort" we can avoid allocation of
+ /* TODO(eustas): by using idea of "cycle-sort" we can avoid allocation of
tmp and reduce the number of copying by the factor of 2. */
tmp = BROTLI_ALLOC(m, HistogramType, next_index);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return 0;
@@ -257,10 +259,12 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
size_t pairs_capacity = max_input_histograms * max_input_histograms / 2;
/* For the first pass of clustering, we allow all pairs. */
HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity + 1);
+ /* TODO(eustas): move to "persistent" arena? */
+ HistogramType* tmp = BROTLI_ALLOC(m, HistogramType, 1);
size_t i;
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(cluster_size) ||
- BROTLI_IS_NULL(clusters) || BROTLI_IS_NULL(pairs)) {
+ BROTLI_IS_NULL(clusters) || BROTLI_IS_NULL(pairs)|| BROTLI_IS_NULL(tmp)) {
return;
}
@@ -283,7 +287,7 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
clusters[num_clusters + j] = (uint32_t)(i + j);
}
num_new_clusters =
- FN(BrotliHistogramCombine)(out, cluster_size,
+ FN(BrotliHistogramCombine)(out, tmp, cluster_size,
&histogram_symbols[i],
&clusters[num_clusters], pairs,
num_to_combine, num_to_combine,
@@ -301,7 +305,7 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
if (BROTLI_IS_OOM(m)) return;
/* Collapse similar histograms. */
- num_clusters = FN(BrotliHistogramCombine)(out, cluster_size,
+ num_clusters = FN(BrotliHistogramCombine)(out, tmp, cluster_size,
histogram_symbols, clusters,
pairs, num_clusters, in_size,
max_histograms, max_num_pairs);
@@ -310,7 +314,8 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
BROTLI_FREE(m, cluster_size);
/* Find the optimal map from original histograms to the final ones. */
FN(BrotliHistogramRemap)(in, in_size, clusters, num_clusters,
- out, histogram_symbols);
+ out, tmp, histogram_symbols);
+ BROTLI_FREE(m, tmp);
BROTLI_FREE(m, clusters);
/* Convert the context map to a canonical form. */
*out_size = FN(BrotliHistogramReindex)(m, out, histogram_symbols, in_size);
diff --git a/dviware/dvisvgm/libs/brotli/enc/command.c b/dviware/dvisvgm/libs/brotli/enc/command.c
index 5e6c249196..bf80561bca 100644
--- a/dviware/dvisvgm/libs/brotli/enc/command.c
+++ b/dviware/dvisvgm/libs/brotli/enc/command.c
@@ -4,7 +4,7 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
-#include "./command.h"
+#include "command.h"
#include <brotli/types.h>
diff --git a/dviware/dvisvgm/libs/brotli/enc/command.h b/dviware/dvisvgm/libs/brotli/enc/command.h
index d84e373d00..ba4de7eab3 100644
--- a/dviware/dvisvgm/libs/brotli/enc/command.h
+++ b/dviware/dvisvgm/libs/brotli/enc/command.h
@@ -9,12 +9,13 @@
#ifndef BROTLI_ENC_COMMAND_H_
#define BROTLI_ENC_COMMAND_H_
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./fast_log.h"
-#include "./params.h"
-#include "./prefix.h"
+#include "fast_log.h"
+#include "params.h"
+#include "prefix.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/compound_dictionary.c b/dviware/dvisvgm/libs/brotli/enc/compound_dictionary.c
new file mode 100644
index 0000000000..a3b5e6933d
--- /dev/null
+++ b/dviware/dvisvgm/libs/brotli/enc/compound_dictionary.c
@@ -0,0 +1,207 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+ Distributed under MIT license.
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+#include "compound_dictionary.h"
+
+#include <brotli/types.h>
+
+#include "../common/platform.h"
+#include "memory.h"
+#include "quality.h"
+
+static PreparedDictionary* CreatePreparedDictionaryWithParams(MemoryManager* m,
+ const uint8_t* source, size_t source_size, uint32_t bucket_bits,
+ uint32_t slot_bits, uint32_t hash_bits, uint16_t bucket_limit) {
+ /* Step 1: create "bloated" hasher. */
+ uint32_t num_slots = 1u << slot_bits;
+ uint32_t num_buckets = 1u << bucket_bits;
+ uint32_t hash_shift = 64u - bucket_bits;
+ uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
+ uint32_t slot_mask = num_slots - 1;
+ size_t alloc_size = (sizeof(uint32_t) << slot_bits) +
+ (sizeof(uint32_t) << slot_bits) +
+ (sizeof(uint16_t) << bucket_bits) +
+ (sizeof(uint32_t) << bucket_bits) +
+ (sizeof(uint32_t) * source_size);
+ uint8_t* flat = NULL;
+ PreparedDictionary* result = NULL;
+ uint16_t* num = NULL;
+ uint32_t* bucket_heads = NULL;
+ uint32_t* next_bucket = NULL;
+ uint32_t* slot_offsets = NULL;
+ uint16_t* heads = NULL;
+ uint32_t* items = NULL;
+ uint8_t** source_ref = NULL;
+ uint32_t i;
+ uint32_t* slot_size = NULL;
+ uint32_t* slot_limit = NULL;
+ uint32_t total_items = 0;
+ if (slot_bits > 16) return NULL;
+ if (slot_bits > bucket_bits) return NULL;
+ if (bucket_bits - slot_bits >= 16) return NULL;
+
+ flat = BROTLI_ALLOC(m, uint8_t, alloc_size);
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(flat)) return NULL;
+
+ slot_size = (uint32_t*)flat;
+ slot_limit = (uint32_t*)(&slot_size[num_slots]);
+ num = (uint16_t*)(&slot_limit[num_slots]);
+ bucket_heads = (uint32_t*)(&num[num_buckets]);
+ next_bucket = (uint32_t*)(&bucket_heads[num_buckets]);
+ memset(num, 0, num_buckets * sizeof(num[0]));
+
+ /* TODO(eustas): apply custom "store" order. */
+ for (i = 0; i + 7 < source_size; ++i) {
+ const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(&source[i]) & hash_mask) *
+ kPreparedDictionaryHashMul64Long;
+ const uint32_t key = (uint32_t)(h >> hash_shift);
+ uint16_t count = num[key];
+ next_bucket[i] = (count == 0) ? ((uint32_t)(-1)) : bucket_heads[key];
+ bucket_heads[key] = i;
+ count++;
+ if (count > bucket_limit) count = bucket_limit;
+ num[key] = count;
+ }
+
+ /* Step 2: find slot limits. */
+ for (i = 0; i < num_slots; ++i) {
+ BROTLI_BOOL overflow = BROTLI_FALSE;
+ slot_limit[i] = bucket_limit;
+ while (BROTLI_TRUE) {
+ uint32_t limit = slot_limit[i];
+ size_t j;
+ uint32_t count = 0;
+ overflow = BROTLI_FALSE;
+ for (j = i; j < num_buckets; j += num_slots) {
+ uint32_t size = num[j];
+ /* Last chain may span behind 64K limit; overflow happens only if
+ we are about to use 0xFFFF+ as item offset. */
+ if (count >= 0xFFFF) {
+ overflow = BROTLI_TRUE;
+ break;
+ }
+ if (size > limit) size = limit;
+ count += size;
+ }
+ if (!overflow) {
+ slot_size[i] = count;
+ total_items += count;
+ break;
+ }
+ slot_limit[i]--;
+ }
+ }
+
+ /* Step 3: transfer data to "slim" hasher. */
+ alloc_size = sizeof(PreparedDictionary) + (sizeof(uint32_t) << slot_bits) +
+ (sizeof(uint16_t) << bucket_bits) + (sizeof(uint32_t) * total_items) +
+ sizeof(uint8_t*);
+
+ result = (PreparedDictionary*)BROTLI_ALLOC(m, uint8_t, alloc_size);
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(result)) {
+ BROTLI_FREE(m, flat);
+ return NULL;
+ }
+ slot_offsets = (uint32_t*)(&result[1]);
+ heads = (uint16_t*)(&slot_offsets[num_slots]);
+ items = (uint32_t*)(&heads[num_buckets]);
+ source_ref = (uint8_t**)(&items[total_items]);
+
+ result->magic = kLeanPreparedDictionaryMagic;
+ result->num_items = total_items;
+ result->source_size = (uint32_t)source_size;
+ result->hash_bits = hash_bits;
+ result->bucket_bits = bucket_bits;
+ result->slot_bits = slot_bits;
+ BROTLI_UNALIGNED_STORE_PTR(source_ref, source);
+
+ total_items = 0;
+ for (i = 0; i < num_slots; ++i) {
+ slot_offsets[i] = total_items;
+ total_items += slot_size[i];
+ slot_size[i] = 0;
+ }
+ for (i = 0; i < num_buckets; ++i) {
+ uint32_t slot = i & slot_mask;
+ uint32_t count = num[i];
+ uint32_t pos;
+ size_t j;
+ size_t cursor = slot_size[slot];
+ if (count > slot_limit[slot]) count = slot_limit[slot];
+ if (count == 0) {
+ heads[i] = 0xFFFF;
+ continue;
+ }
+ heads[i] = (uint16_t)cursor;
+ cursor += slot_offsets[slot];
+ slot_size[slot] += count;
+ pos = bucket_heads[i];
+ for (j = 0; j < count; j++) {
+ items[cursor++] = pos;
+ pos = next_bucket[pos];
+ }
+ items[cursor - 1] |= 0x80000000;
+ }
+
+ BROTLI_FREE(m, flat);
+ return result;
+}
+
+PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
+ const uint8_t* source, size_t source_size) {
+ uint32_t bucket_bits = 17;
+ uint32_t slot_bits = 7;
+ uint32_t hash_bits = 40;
+ uint16_t bucket_limit = 32;
+ size_t volume = 16u << bucket_bits;
+ /* Tune parameters to fit dictionary size. */
+ while (volume < source_size && bucket_bits < 22) {
+ bucket_bits++;
+ slot_bits++;
+ volume <<= 1;
+ }
+ return CreatePreparedDictionaryWithParams(m,
+ source, source_size, bucket_bits, slot_bits, hash_bits, bucket_limit);
+}
+
+void DestroyPreparedDictionary(MemoryManager* m,
+ PreparedDictionary* dictionary) {
+ if (!dictionary) return;
+ BROTLI_FREE(m, dictionary);
+}
+
+BROTLI_BOOL AttachPreparedDictionary(
+ CompoundDictionary* compound, const PreparedDictionary* dictionary) {
+ size_t length = 0;
+ size_t index = 0;
+
+ if (compound->num_chunks == SHARED_BROTLI_MAX_COMPOUND_DICTS) {
+ return BROTLI_FALSE;
+ }
+
+ if (!dictionary) return BROTLI_FALSE;
+
+ length = dictionary->source_size;
+ index = compound->num_chunks;
+ compound->total_size += length;
+ compound->chunks[index] = dictionary;
+ compound->chunk_offsets[index + 1] = compound->total_size;
+ {
+ uint32_t* slot_offsets = (uint32_t*)(&dictionary[1]);
+ uint16_t* heads = (uint16_t*)(&slot_offsets[1u << dictionary->slot_bits]);
+ uint32_t* items = (uint32_t*)(&heads[1u << dictionary->bucket_bits]);
+ const void* tail = (void*)&items[dictionary->num_items];
+ if (dictionary->magic == kPreparedDictionaryMagic) {
+ compound->chunk_source[index] = (const uint8_t*)tail;
+ } else {
+ /* dictionary->magic == kLeanPreparedDictionaryMagic */
+ compound->chunk_source[index] =
+ (const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
+ }
+ }
+ compound->num_chunks++;
+ return BROTLI_TRUE;
+}
diff --git a/dviware/dvisvgm/libs/brotli/enc/compound_dictionary.h b/dviware/dvisvgm/libs/brotli/enc/compound_dictionary.h
new file mode 100644
index 0000000000..9c531d5b19
--- /dev/null
+++ b/dviware/dvisvgm/libs/brotli/enc/compound_dictionary.h
@@ -0,0 +1,74 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+ Distributed under MIT license.
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+#ifndef BROTLI_ENC_PREPARED_DICTIONARY_H_
+#define BROTLI_ENC_PREPARED_DICTIONARY_H_
+
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
+
+#include "../common/platform.h"
+#include "../common/constants.h"
+#include "memory.h"
+
+/* "Fat" prepared dictionary, could be cooked outside of C implementation,
+ * e.g. on Java side. LZ77 data is copied inside PreparedDictionary struct. */
+static const uint32_t kPreparedDictionaryMagic = 0xDEBCEDE0;
+
+static const uint32_t kSharedDictionaryMagic = 0xDEBCEDE1;
+
+static const uint32_t kManagedDictionaryMagic = 0xDEBCEDE2;
+
+/* "Lean" prepared dictionary. LZ77 data is referenced. It is the responsibility
+ * of caller of "prepare dictionary" to keep the LZ77 data while prepared
+ * dictionary is in use. */
+static const uint32_t kLeanPreparedDictionaryMagic = 0xDEBCEDE3;
+
+static const uint64_t kPreparedDictionaryHashMul64Long =
+ BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
+
+typedef struct PreparedDictionary {
+ uint32_t magic;
+ uint32_t num_items;
+ uint32_t source_size;
+ uint32_t hash_bits;
+ uint32_t bucket_bits;
+ uint32_t slot_bits;
+
+ /* --- Dynamic size members --- */
+
+ /* uint32_t slot_offsets[1 << slot_bits]; */
+ /* uint16_t heads[1 << bucket_bits]; */
+ /* uint32_t items[variable]; */
+
+ /* [maybe] uint8_t* source_ref, depending on magic. */
+ /* [maybe] uint8_t source[source_size], depending on magic. */
+} PreparedDictionary;
+
+BROTLI_INTERNAL PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
+ const uint8_t* source, size_t source_size);
+
+BROTLI_INTERNAL void DestroyPreparedDictionary(MemoryManager* m,
+ PreparedDictionary* dictionary);
+
+typedef struct CompoundDictionary {
+ /* LZ77 prefix, compound dictionary */
+ size_t num_chunks;
+ size_t total_size;
+ /* Client instances. */
+ const PreparedDictionary* chunks[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+ const uint8_t* chunk_source[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+ size_t chunk_offsets[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+
+ size_t num_prepared_instances_;
+ /* Owned instances. */
+ PreparedDictionary* prepared_instances_[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+} CompoundDictionary;
+
+BROTLI_INTERNAL BROTLI_BOOL AttachPreparedDictionary(
+ CompoundDictionary* compound, const PreparedDictionary* dictionary);
+
+#endif /* BROTLI_ENC_PREPARED_DICTIONARY */
diff --git a/dviware/dvisvgm/libs/brotli/enc/compress_fragment.c b/dviware/dvisvgm/libs/brotli/enc/compress_fragment.c
index 9e50b2098a..13890eabf6 100644
--- a/dviware/dvisvgm/libs/brotli/enc/compress_fragment.c
+++ b/dviware/dvisvgm/libs/brotli/enc/compress_fragment.c
@@ -12,19 +12,18 @@
Adapted from the CompressFragment() function in
https://github.com/google/snappy/blob/master/snappy.cc */
-#include "./compress_fragment.h"
+#include "compress_fragment.h"
#include <string.h> /* memcmp, memcpy, memset */
-#include "../common/constants.h"
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./brotli_bit_stream.h"
-#include "./entropy_encode.h"
-#include "./fast_log.h"
-#include "./find_match_length.h"
-#include "./memory.h"
-#include "./write_bits.h"
+
+#include "../common/platform.h"
+#include "brotli_bit_stream.h"
+#include "entropy_encode.h"
+#include "fast_log.h"
+#include "find_match_length.h"
+#include "write_bits.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -69,16 +68,18 @@ static BROTLI_INLINE BROTLI_BOOL IsMatch(const uint8_t* p1, const uint8_t* p2) {
and thus have to assign a non-zero depth for each literal.
Returns estimated compression ratio millibytes/char for encoding given input
with generated code. */
-static size_t BuildAndStoreLiteralPrefixCode(MemoryManager* m,
+static size_t BuildAndStoreLiteralPrefixCode(BrotliOnePassArena* s,
const uint8_t* input,
const size_t input_size,
uint8_t depths[256],
uint16_t bits[256],
size_t* storage_ix,
uint8_t* storage) {
- uint32_t histogram[256] = { 0 };
+ uint32_t* BROTLI_RESTRICT const histogram = s->histogram;
size_t histogram_total;
size_t i;
+ memset(histogram, 0, sizeof(s->histogram));
+
if (input_size < (1 << 15)) {
for (i = 0; i < input_size; ++i) {
++histogram[input[i]];
@@ -108,10 +109,9 @@ static size_t BuildAndStoreLiteralPrefixCode(MemoryManager* m,
histogram_total += adjust;
}
}
- BrotliBuildAndStoreHuffmanTreeFast(m, histogram, histogram_total,
+ BrotliBuildAndStoreHuffmanTreeFast(s->tree, histogram, histogram_total,
/* max_bits = */ 8,
depths, bits, storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return 0;
{
size_t literal_ratio = 0;
for (i = 0; i < 256; ++i) {
@@ -124,53 +124,56 @@ static size_t BuildAndStoreLiteralPrefixCode(MemoryManager* m,
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
"bits" based on "histogram" and stores it into the bit stream. */
-static void BuildAndStoreCommandPrefixCode(const uint32_t histogram[128],
- uint8_t depth[128], uint16_t bits[128], size_t* storage_ix,
- uint8_t* storage) {
- /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
- HuffmanTree tree[129];
- uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS] = { 0 };
- uint16_t cmd_bits[64];
-
- BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
- BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
+static void BuildAndStoreCommandPrefixCode(BrotliOnePassArena* s,
+ size_t* storage_ix, uint8_t* storage) {
+ const uint32_t* const histogram = s->cmd_histo;
+ uint8_t* const depth = s->cmd_depth;
+ uint16_t* const bits = s->cmd_bits;
+ uint8_t* BROTLI_RESTRICT const tmp_depth = s->tmp_depth;
+ uint16_t* BROTLI_RESTRICT const tmp_bits = s->tmp_bits;
+ /* TODO(eustas): do only once on initialization. */
+ memset(tmp_depth, 0, BROTLI_NUM_COMMAND_SYMBOLS);
+
+ BrotliCreateHuffmanTree(histogram, 64, 15, s->tree, depth);
+ BrotliCreateHuffmanTree(&histogram[64], 64, 14, s->tree, &depth[64]);
/* We have to jump through a few hoops here in order to compute
the command bits because the symbols are in a different order than in
the full alphabet. This looks complicated, but having the symbols
in this order in the command bits saves a few branches in the Emit*
functions. */
- memcpy(cmd_depth, depth, 24);
- memcpy(cmd_depth + 24, depth + 40, 8);
- memcpy(cmd_depth + 32, depth + 24, 8);
- memcpy(cmd_depth + 40, depth + 48, 8);
- memcpy(cmd_depth + 48, depth + 32, 8);
- memcpy(cmd_depth + 56, depth + 56, 8);
- BrotliConvertBitDepthsToSymbols(cmd_depth, 64, cmd_bits);
- memcpy(bits, cmd_bits, 48);
- memcpy(bits + 24, cmd_bits + 32, 16);
- memcpy(bits + 32, cmd_bits + 48, 16);
- memcpy(bits + 40, cmd_bits + 24, 16);
- memcpy(bits + 48, cmd_bits + 40, 16);
- memcpy(bits + 56, cmd_bits + 56, 16);
+ memcpy(tmp_depth, depth, 24);
+ memcpy(tmp_depth + 24, depth + 40, 8);
+ memcpy(tmp_depth + 32, depth + 24, 8);
+ memcpy(tmp_depth + 40, depth + 48, 8);
+ memcpy(tmp_depth + 48, depth + 32, 8);
+ memcpy(tmp_depth + 56, depth + 56, 8);
+ BrotliConvertBitDepthsToSymbols(tmp_depth, 64, tmp_bits);
+ memcpy(bits, tmp_bits, 48);
+ memcpy(bits + 24, tmp_bits + 32, 16);
+ memcpy(bits + 32, tmp_bits + 48, 16);
+ memcpy(bits + 40, tmp_bits + 24, 16);
+ memcpy(bits + 48, tmp_bits + 40, 16);
+ memcpy(bits + 56, tmp_bits + 56, 16);
BrotliConvertBitDepthsToSymbols(&depth[64], 64, &bits[64]);
{
/* Create the bit length array for the full command alphabet. */
size_t i;
- memset(cmd_depth, 0, 64); /* only 64 first values were used */
- memcpy(cmd_depth, depth, 8);
- memcpy(cmd_depth + 64, depth + 8, 8);
- memcpy(cmd_depth + 128, depth + 16, 8);
- memcpy(cmd_depth + 192, depth + 24, 8);
- memcpy(cmd_depth + 384, depth + 32, 8);
+ memset(tmp_depth, 0, 64); /* only 64 first values were used */
+ memcpy(tmp_depth, depth, 8);
+ memcpy(tmp_depth + 64, depth + 8, 8);
+ memcpy(tmp_depth + 128, depth + 16, 8);
+ memcpy(tmp_depth + 192, depth + 24, 8);
+ memcpy(tmp_depth + 384, depth + 32, 8);
for (i = 0; i < 8; ++i) {
- cmd_depth[128 + 8 * i] = depth[40 + i];
- cmd_depth[256 + 8 * i] = depth[48 + i];
- cmd_depth[448 + 8 * i] = depth[56 + i];
+ tmp_depth[128 + 8 * i] = depth[40 + i];
+ tmp_depth[256 + 8 * i] = depth[48 + i];
+ tmp_depth[448 + 8 * i] = depth[56 + i];
}
+ /* TODO(eustas): could/should full-length machinery be avoided? */
BrotliStoreHuffmanTree(
- cmd_depth, BROTLI_NUM_COMMAND_SYMBOLS, tree, storage_ix, storage);
+ tmp_depth, BROTLI_NUM_COMMAND_SYMBOLS, s->tree, storage_ix, storage);
}
- BrotliStoreHuffmanTree(&depth[64], 64, tree, storage_ix, storage);
+ BrotliStoreHuffmanTree(&depth[64], 64, s->tree, storage_ix, storage);
}
/* REQUIRES: insertlen < 6210 */
@@ -369,11 +372,12 @@ static void RewindBitPosition(const size_t new_storage_ix,
*storage_ix = new_storage_ix;
}
-static BROTLI_BOOL ShouldMergeBlock(
+static BROTLI_BOOL ShouldMergeBlock(BrotliOnePassArena* s,
const uint8_t* data, size_t len, const uint8_t* depths) {
- size_t histo[256] = { 0 };
+ uint32_t* BROTLI_RESTRICT const histo = s->histogram;
static const size_t kSampleRate = 43;
size_t i;
+ memset(histo, 0, sizeof(s->histogram));
for (i = 0; i < len; i += kSampleRate) {
++histo[data[i]];
}
@@ -423,11 +427,14 @@ static uint32_t kCmdHistoSeed[128] = {
};
static BROTLI_INLINE void BrotliCompressFragmentFastImpl(
- MemoryManager* m, const uint8_t* input, size_t input_size,
- BROTLI_BOOL is_last, int* table, size_t table_bits, uint8_t cmd_depth[128],
- uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code,
+ BrotliOnePassArena* s, const uint8_t* input, size_t input_size,
+ BROTLI_BOOL is_last, int* table, size_t table_bits,
size_t* storage_ix, uint8_t* storage) {
- uint32_t cmd_histo[128];
+ uint8_t* BROTLI_RESTRICT const cmd_depth = s->cmd_depth;
+ uint16_t* BROTLI_RESTRICT const cmd_bits = s->cmd_bits;
+ uint32_t* BROTLI_RESTRICT const cmd_histo = s->cmd_histo;
+ uint8_t* BROTLI_RESTRICT const lit_depth = s->lit_depth;
+ uint16_t* BROTLI_RESTRICT const lit_bits = s->lit_bits;
const uint8_t* ip_end;
/* "next_emit" is a pointer to the first byte that is not covered by a
@@ -451,9 +458,6 @@ static BROTLI_INLINE void BrotliCompressFragmentFastImpl(
we can update it later if we decide to extend this meta-block. */
size_t mlen_storage_ix = *storage_ix + 3;
- uint8_t lit_depth[256];
- uint16_t lit_bits[256];
-
size_t literal_ratio;
const uint8_t* ip;
@@ -466,25 +470,24 @@ static BROTLI_INLINE void BrotliCompressFragmentFastImpl(
BrotliWriteBits(13, 0, storage_ix, storage);
literal_ratio = BuildAndStoreLiteralPrefixCode(
- m, input, block_size, lit_depth, lit_bits, storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
+ s, input, block_size, s->lit_depth, s->lit_bits, storage_ix, storage);
{
/* Store the pre-compressed command and distance prefix codes. */
size_t i;
- for (i = 0; i + 7 < *cmd_code_numbits; i += 8) {
- BrotliWriteBits(8, cmd_code[i >> 3], storage_ix, storage);
+ for (i = 0; i + 7 < s->cmd_code_numbits; i += 8) {
+ BrotliWriteBits(8, s->cmd_code[i >> 3], storage_ix, storage);
}
}
- BrotliWriteBits(*cmd_code_numbits & 7, cmd_code[*cmd_code_numbits >> 3],
- storage_ix, storage);
+ BrotliWriteBits(s->cmd_code_numbits & 7,
+ s->cmd_code[s->cmd_code_numbits >> 3], storage_ix, storage);
emit_commands:
/* Initialize the command and distance histograms. We will gather
statistics of command and distance codes during the processing
of this block and use it to update the command and distance
prefix codes for the next block. */
- memcpy(cmd_histo, kCmdHistoSeed, sizeof(kCmdHistoSeed));
+ memcpy(s->cmd_histo, kCmdHistoSeed, sizeof(kCmdHistoSeed));
/* "ip" is the input pointer. */
ip = input;
@@ -565,6 +568,8 @@ trawl:
int distance = (int)(base - candidate); /* > 0 */
size_t insert = (size_t)(base - next_emit);
ip += matched;
+ BROTLI_LOG(("[CompressFragment] pos = %d insert = %lu copy = %d\n",
+ (int)(next_emit - base_ip), (unsigned long)insert, 2));
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
if (BROTLI_PREDICT_TRUE(insert < 6210)) {
EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
@@ -593,6 +598,12 @@ trawl:
}
EmitCopyLenLastDistance(matched, cmd_depth, cmd_bits, cmd_histo,
storage_ix, storage);
+ BROTLI_LOG(("[CompressFragment] pos = %d distance = %d\n"
+ "[CompressFragment] pos = %d insert = %d copy = %d\n"
+ "[CompressFragment] pos = %d distance = %d\n",
+ (int)(base - base_ip), (int)distance,
+ (int)(base - base_ip) + 2, 0, (int)matched - 2,
+ (int)(base - base_ip) + 2, (int)distance));
next_emit = ip;
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -630,6 +641,10 @@ trawl:
storage_ix, storage);
EmitDistance((size_t)last_distance, cmd_depth, cmd_bits,
cmd_histo, storage_ix, storage);
+ BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n"
+ "[CompressFragment] pos = %d distance = %d\n",
+ (int)(base - base_ip), 0, (int)matched,
+ (int)(base - base_ip), (int)last_distance));
next_emit = ip;
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -667,7 +682,7 @@ trawl:
last insert-only command. */
if (input_size > 0 &&
total_block_size + block_size <= (1 << 20) &&
- ShouldMergeBlock(input, block_size, lit_depth)) {
+ ShouldMergeBlock(s, input, block_size, lit_depth)) {
BROTLI_DCHECK(total_block_size > (1 << 16));
/* Update the size of the current meta-block and continue emitting commands.
We can do this because the current size and the new size both have 5
@@ -680,6 +695,8 @@ trawl:
/* Emit the remaining bytes as literals. */
if (next_emit < ip_end) {
const size_t insert = (size_t)(ip_end - next_emit);
+ BROTLI_LOG(("[CompressFragment] pos = %d insert = %lu copy = %d\n",
+ (int)(next_emit - base_ip), (unsigned long)insert, 2));
if (BROTLI_PREDICT_TRUE(insert < 6210)) {
EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
storage_ix, storage);
@@ -711,20 +728,17 @@ next_block:
/* No block splits, no contexts. */
BrotliWriteBits(13, 0, storage_ix, storage);
literal_ratio = BuildAndStoreLiteralPrefixCode(
- m, input, block_size, lit_depth, lit_bits, storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
- BuildAndStoreCommandPrefixCode(cmd_histo, cmd_depth, cmd_bits,
- storage_ix, storage);
+ s, input, block_size, lit_depth, lit_bits, storage_ix, storage);
+ BuildAndStoreCommandPrefixCode(s, storage_ix, storage);
goto emit_commands;
}
if (!is_last) {
/* If this is not the last block, update the command and distance prefix
codes for the next block and store the compressed forms. */
- cmd_code[0] = 0;
- *cmd_code_numbits = 0;
- BuildAndStoreCommandPrefixCode(cmd_histo, cmd_depth, cmd_bits,
- cmd_code_numbits, cmd_code);
+ s->cmd_code[0] = 0;
+ s->cmd_code_numbits = 0;
+ BuildAndStoreCommandPrefixCode(s, &s->cmd_code_numbits, s->cmd_code);
}
}
@@ -732,20 +746,17 @@ next_block:
#define BAKE_METHOD_PARAM_(B) \
static BROTLI_NOINLINE void BrotliCompressFragmentFastImpl ## B( \
- MemoryManager* m, const uint8_t* input, size_t input_size, \
- BROTLI_BOOL is_last, int* table, uint8_t cmd_depth[128], \
- uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code, \
- size_t* storage_ix, uint8_t* storage) { \
- BrotliCompressFragmentFastImpl(m, input, input_size, is_last, table, B, \
- cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage); \
+ BrotliOnePassArena* s, const uint8_t* input, size_t input_size, \
+ BROTLI_BOOL is_last, int* table, size_t* storage_ix, uint8_t* storage) { \
+ BrotliCompressFragmentFastImpl(s, input, input_size, is_last, table, B, \
+ storage_ix, storage); \
}
FOR_TABLE_BITS_(BAKE_METHOD_PARAM_)
#undef BAKE_METHOD_PARAM_
void BrotliCompressFragmentFast(
- MemoryManager* m, const uint8_t* input, size_t input_size,
- BROTLI_BOOL is_last, int* table, size_t table_size, uint8_t cmd_depth[128],
- uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code,
+ BrotliOnePassArena* s, const uint8_t* input, size_t input_size,
+ BROTLI_BOOL is_last, int* table, size_t table_size,
size_t* storage_ix, uint8_t* storage) {
const size_t initial_storage_ix = *storage_ix;
const size_t table_bits = Log2FloorNonZero(table_size);
@@ -762,8 +773,7 @@ void BrotliCompressFragmentFast(
#define CASE_(B) \
case B: \
BrotliCompressFragmentFastImpl ## B( \
- m, input, input_size, is_last, table, cmd_depth, cmd_bits, \
- cmd_code_numbits, cmd_code, storage_ix, storage); \
+ s, input, input_size, is_last, table, storage_ix, storage);\
break;
FOR_TABLE_BITS_(CASE_)
#undef CASE_
diff --git a/dviware/dvisvgm/libs/brotli/enc/compress_fragment.h b/dviware/dvisvgm/libs/brotli/enc/compress_fragment.h
index 80007f5dca..9c0780f8c9 100644
--- a/dviware/dvisvgm/libs/brotli/enc/compress_fragment.h
+++ b/dviware/dvisvgm/libs/brotli/enc/compress_fragment.h
@@ -12,14 +12,43 @@
#ifndef BROTLI_ENC_COMPRESS_FRAGMENT_H_
#define BROTLI_ENC_COMPRESS_FRAGMENT_H_
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./memory.h"
+
+#include "../common/constants.h"
+#include "../common/platform.h"
+#include "entropy_encode.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
+typedef struct BrotliOnePassArena {
+ uint8_t lit_depth[256];
+ uint16_t lit_bits[256];
+
+ /* Command and distance prefix codes (each 64 symbols, stored back-to-back)
+ used for the next block. The command prefix code is over a smaller alphabet
+ with the following 64 symbols:
+ 0 - 15: insert length code 0, copy length code 0 - 15, same distance
+ 16 - 39: insert length code 0, copy length code 0 - 23
+ 40 - 63: insert length code 0 - 23, copy length code 0
+ Note that symbols 16 and 40 represent the same code in the full alphabet,
+ but we do not use either of them. */
+ uint8_t cmd_depth[128];
+ uint16_t cmd_bits[128];
+ uint32_t cmd_histo[128];
+
+ /* The compressed form of the command and distance prefix codes for the next
+ block. */
+ uint8_t cmd_code[512];
+ size_t cmd_code_numbits;
+
+ HuffmanTree tree[2 * BROTLI_NUM_LITERAL_SYMBOLS + 1];
+ uint32_t histogram[256];
+ uint8_t tmp_depth[BROTLI_NUM_COMMAND_SYMBOLS];
+ uint16_t tmp_bits[64];
+} BrotliOnePassArena;
+
/* Compresses "input" string to the "*storage" buffer as one or more complete
meta-blocks, and updates the "*storage_ix" bit position.
@@ -42,15 +71,11 @@ extern "C" {
REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
OUTPUT: maximal copy distance <= |input_size|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
-BROTLI_INTERNAL void BrotliCompressFragmentFast(MemoryManager* m,
+BROTLI_INTERNAL void BrotliCompressFragmentFast(BrotliOnePassArena* s,
const uint8_t* input,
size_t input_size,
BROTLI_BOOL is_last,
int* table, size_t table_size,
- uint8_t cmd_depth[128],
- uint16_t cmd_bits[128],
- size_t* cmd_code_numbits,
- uint8_t* cmd_code,
size_t* storage_ix,
uint8_t* storage);
diff --git a/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.c b/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.c
index ca68b38139..a762679c1f 100644
--- a/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.c
+++ b/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.c
@@ -10,20 +10,20 @@
second pass we emit them into the bit stream using prefix codes built based
on the actual command and literal byte histograms. */
-#include "./compress_fragment_two_pass.h"
+#include "compress_fragment_two_pass.h"
#include <string.h> /* memcmp, memcpy, memset */
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./bit_cost.h"
-#include "./brotli_bit_stream.h"
-#include "./entropy_encode.h"
-#include "./fast_log.h"
-#include "./find_match_length.h"
-#include "./memory.h"
-#include "./write_bits.h"
+#include "bit_cost.h"
+#include "brotli_bit_stream.h"
+#include "entropy_encode.h"
+#include "fast_log.h"
+#include "find_match_length.h"
+#include "write_bits.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -66,53 +66,53 @@ static BROTLI_INLINE BROTLI_BOOL IsMatch(const uint8_t* p1, const uint8_t* p2,
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
"bits" based on "histogram" and stores it into the bit stream. */
-static void BuildAndStoreCommandPrefixCode(
- const uint32_t histogram[128],
- uint8_t depth[128], uint16_t bits[128],
- size_t* storage_ix, uint8_t* storage) {
+static void BuildAndStoreCommandPrefixCode(BrotliTwoPassArena* s,
+ size_t* storage_ix,
+ uint8_t* storage) {
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
- HuffmanTree tree[129];
- uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS] = { 0 };
- uint16_t cmd_bits[64];
- BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
- BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
+ /* TODO(eustas): initialize once. */
+ memset(s->tmp_depth, 0, sizeof(s->tmp_depth));
+ BrotliCreateHuffmanTree(s->cmd_histo, 64, 15, s->tmp_tree, s->cmd_depth);
+ BrotliCreateHuffmanTree(&s->cmd_histo[64], 64, 14, s->tmp_tree,
+ &s->cmd_depth[64]);
/* We have to jump through a few hoops here in order to compute
the command bits because the symbols are in a different order than in
the full alphabet. This looks complicated, but having the symbols
in this order in the command bits saves a few branches in the Emit*
functions. */
- memcpy(cmd_depth, depth + 24, 24);
- memcpy(cmd_depth + 24, depth, 8);
- memcpy(cmd_depth + 32, depth + 48, 8);
- memcpy(cmd_depth + 40, depth + 8, 8);
- memcpy(cmd_depth + 48, depth + 56, 8);
- memcpy(cmd_depth + 56, depth + 16, 8);
- BrotliConvertBitDepthsToSymbols(cmd_depth, 64, cmd_bits);
- memcpy(bits, cmd_bits + 24, 16);
- memcpy(bits + 8, cmd_bits + 40, 16);
- memcpy(bits + 16, cmd_bits + 56, 16);
- memcpy(bits + 24, cmd_bits, 48);
- memcpy(bits + 48, cmd_bits + 32, 16);
- memcpy(bits + 56, cmd_bits + 48, 16);
- BrotliConvertBitDepthsToSymbols(&depth[64], 64, &bits[64]);
+ memcpy(s->tmp_depth, s->cmd_depth + 24, 24);
+ memcpy(s->tmp_depth + 24, s->cmd_depth, 8);
+ memcpy(s->tmp_depth + 32, s->cmd_depth + 48, 8);
+ memcpy(s->tmp_depth + 40, s->cmd_depth + 8, 8);
+ memcpy(s->tmp_depth + 48, s->cmd_depth + 56, 8);
+ memcpy(s->tmp_depth + 56, s->cmd_depth + 16, 8);
+ BrotliConvertBitDepthsToSymbols(s->tmp_depth, 64, s->tmp_bits);
+ memcpy(s->cmd_bits, s->tmp_bits + 24, 16);
+ memcpy(s->cmd_bits + 8, s->tmp_bits + 40, 16);
+ memcpy(s->cmd_bits + 16, s->tmp_bits + 56, 16);
+ memcpy(s->cmd_bits + 24, s->tmp_bits, 48);
+ memcpy(s->cmd_bits + 48, s->tmp_bits + 32, 16);
+ memcpy(s->cmd_bits + 56, s->tmp_bits + 48, 16);
+ BrotliConvertBitDepthsToSymbols(&s->cmd_depth[64], 64, &s->cmd_bits[64]);
{
/* Create the bit length array for the full command alphabet. */
size_t i;
- memset(cmd_depth, 0, 64); /* only 64 first values were used */
- memcpy(cmd_depth, depth + 24, 8);
- memcpy(cmd_depth + 64, depth + 32, 8);
- memcpy(cmd_depth + 128, depth + 40, 8);
- memcpy(cmd_depth + 192, depth + 48, 8);
- memcpy(cmd_depth + 384, depth + 56, 8);
+ memset(s->tmp_depth, 0, 64); /* only 64 first values were used */
+ memcpy(s->tmp_depth, s->cmd_depth + 24, 8);
+ memcpy(s->tmp_depth + 64, s->cmd_depth + 32, 8);
+ memcpy(s->tmp_depth + 128, s->cmd_depth + 40, 8);
+ memcpy(s->tmp_depth + 192, s->cmd_depth + 48, 8);
+ memcpy(s->tmp_depth + 384, s->cmd_depth + 56, 8);
for (i = 0; i < 8; ++i) {
- cmd_depth[128 + 8 * i] = depth[i];
- cmd_depth[256 + 8 * i] = depth[8 + i];
- cmd_depth[448 + 8 * i] = depth[16 + i];
+ s->tmp_depth[128 + 8 * i] = s->cmd_depth[i];
+ s->tmp_depth[256 + 8 * i] = s->cmd_depth[8 + i];
+ s->tmp_depth[448 + 8 * i] = s->cmd_depth[16 + i];
}
- BrotliStoreHuffmanTree(
- cmd_depth, BROTLI_NUM_COMMAND_SYMBOLS, tree, storage_ix, storage);
+ BrotliStoreHuffmanTree(s->tmp_depth, BROTLI_NUM_COMMAND_SYMBOLS,
+ s->tmp_tree, storage_ix, storage);
}
- BrotliStoreHuffmanTree(&depth[64], 64, tree, storage_ix, storage);
+ BrotliStoreHuffmanTree(&s->cmd_depth[64], 64, s->tmp_tree, storage_ix,
+ storage);
}
static BROTLI_INLINE void EmitInsertLen(
@@ -330,6 +330,8 @@ trawl:
ip += matched;
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
EmitInsertLen((uint32_t)insert, commands);
+ BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
+ (int)(next_emit - base_ip), insert, 2));
memcpy(*literals, next_emit, (size_t)insert);
*literals += insert;
if (distance == last_distance) {
@@ -340,6 +342,12 @@ trawl:
last_distance = distance;
}
EmitCopyLenLastDistance(matched, commands);
+ BROTLI_LOG(("[CompressFragment] pos = %d distance = %d\n"
+ "[CompressFragment] pos = %d insert = %d copy = %d\n"
+ "[CompressFragment] pos = %d distance = %d\n",
+ (int)(base - base_ip), (int)distance,
+ (int)(base - base_ip) + 2, 0, (int)matched - 2,
+ (int)(base - base_ip) + 2, (int)distance));
next_emit = ip;
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -395,6 +403,10 @@ trawl:
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
EmitCopyLen(matched, commands);
EmitDistance((uint32_t)last_distance, commands);
+ BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n"
+ "[CompressFragment] pos = %d distance = %d\n",
+ (int)(base - base_ip), 0, (int)matched,
+ (int)(base - base_ip), (int)last_distance));
next_emit = ip;
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -447,70 +459,71 @@ emit_remainder:
if (next_emit < ip_end) {
const uint32_t insert = (uint32_t)(ip_end - next_emit);
EmitInsertLen(insert, commands);
+ BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
+ (int)(next_emit - base_ip), insert, 2));
memcpy(*literals, next_emit, insert);
*literals += insert;
}
}
-static void StoreCommands(MemoryManager* m,
+static void StoreCommands(BrotliTwoPassArena* s,
const uint8_t* literals, const size_t num_literals,
const uint32_t* commands, const size_t num_commands,
size_t* storage_ix, uint8_t* storage) {
static const uint32_t kNumExtraBits[128] = {
- 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24,
- 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4,
- 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
- 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16,
- 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24,
+ 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5,
+ 6, 7, 8, 9, 10, 12, 14, 24, 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16,
+ 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24,
};
static const uint32_t kInsertOffset[24] = {
- 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578,
- 1090, 2114, 6210, 22594,
+ 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26,
+ 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594,
};
- uint8_t lit_depths[256];
- uint16_t lit_bits[256];
- uint32_t lit_histo[256] = { 0 };
- uint8_t cmd_depths[128] = { 0 };
- uint16_t cmd_bits[128] = { 0 };
- uint32_t cmd_histo[128] = { 0 };
size_t i;
+ memset(s->lit_histo, 0, sizeof(s->lit_histo));
+ /* TODO(eustas): is that necessary? */
+ memset(s->cmd_depth, 0, sizeof(s->cmd_depth));
+ /* TODO(eustas): is that necessary? */
+ memset(s->cmd_bits, 0, sizeof(s->cmd_bits));
+ memset(s->cmd_histo, 0, sizeof(s->cmd_histo));
for (i = 0; i < num_literals; ++i) {
- ++lit_histo[literals[i]];
+ ++s->lit_histo[literals[i]];
}
- BrotliBuildAndStoreHuffmanTreeFast(m, lit_histo, num_literals,
- /* max_bits = */ 8,
- lit_depths, lit_bits,
- storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
+ BrotliBuildAndStoreHuffmanTreeFast(s->tmp_tree, s->lit_histo, num_literals,
+ /* max_bits = */ 8, s->lit_depth,
+ s->lit_bits, storage_ix, storage);
for (i = 0; i < num_commands; ++i) {
const uint32_t code = commands[i] & 0xFF;
BROTLI_DCHECK(code < 128);
- ++cmd_histo[code];
+ ++s->cmd_histo[code];
}
- cmd_histo[1] += 1;
- cmd_histo[2] += 1;
- cmd_histo[64] += 1;
- cmd_histo[84] += 1;
- BuildAndStoreCommandPrefixCode(cmd_histo, cmd_depths, cmd_bits,
- storage_ix, storage);
+ s->cmd_histo[1] += 1;
+ s->cmd_histo[2] += 1;
+ s->cmd_histo[64] += 1;
+ s->cmd_histo[84] += 1;
+ BuildAndStoreCommandPrefixCode(s, storage_ix, storage);
for (i = 0; i < num_commands; ++i) {
const uint32_t cmd = commands[i];
const uint32_t code = cmd & 0xFF;
const uint32_t extra = cmd >> 8;
BROTLI_DCHECK(code < 128);
- BrotliWriteBits(cmd_depths[code], cmd_bits[code], storage_ix, storage);
+ BrotliWriteBits(s->cmd_depth[code], s->cmd_bits[code], storage_ix, storage);
BrotliWriteBits(kNumExtraBits[code], extra, storage_ix, storage);
if (code < 24) {
const uint32_t insert = kInsertOffset[code] + extra;
uint32_t j;
for (j = 0; j < insert; ++j) {
const uint8_t lit = *literals;
- BrotliWriteBits(lit_depths[lit], lit_bits[lit], storage_ix, storage);
+ BrotliWriteBits(s->lit_depth[lit], s->lit_bits[lit], storage_ix,
+ storage);
++literals;
}
}
@@ -521,19 +534,19 @@ static void StoreCommands(MemoryManager* m,
#define MIN_RATIO 0.98
#define SAMPLE_RATE 43
-static BROTLI_BOOL ShouldCompress(
+static BROTLI_BOOL ShouldCompress(BrotliTwoPassArena* s,
const uint8_t* input, size_t input_size, size_t num_literals) {
double corpus_size = (double)input_size;
if ((double)num_literals < MIN_RATIO * corpus_size) {
return BROTLI_TRUE;
} else {
- uint32_t literal_histo[256] = { 0 };
const double max_total_bit_cost = corpus_size * 8 * MIN_RATIO / SAMPLE_RATE;
size_t i;
+ memset(s->lit_histo, 0, sizeof(s->lit_histo));
for (i = 0; i < input_size; i += SAMPLE_RATE) {
- ++literal_histo[input[i]];
+ ++s->lit_histo[input[i]];
}
- return TO_BROTLI_BOOL(BitsEntropy(literal_histo, 256) < max_total_bit_cost);
+ return TO_BROTLI_BOOL(BitsEntropy(s->lit_histo, 256) < max_total_bit_cost);
}
}
@@ -555,7 +568,7 @@ static void EmitUncompressedMetaBlock(const uint8_t* input, size_t input_size,
}
static BROTLI_INLINE void BrotliCompressFragmentTwoPassImpl(
- MemoryManager* m, const uint8_t* input, size_t input_size,
+ BrotliTwoPassArena* s, const uint8_t* input, size_t input_size,
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf,
int* table, size_t table_bits, size_t min_match,
size_t* storage_ix, uint8_t* storage) {
@@ -573,14 +586,13 @@ static BROTLI_INLINE void BrotliCompressFragmentTwoPassImpl(
CreateCommands(input, block_size, input_size, base_ip, table,
table_bits, min_match, &literals, &commands);
num_literals = (size_t)(literals - literal_buf);
- if (ShouldCompress(input, block_size, num_literals)) {
+ if (ShouldCompress(s, input, block_size, num_literals)) {
const size_t num_commands = (size_t)(commands - command_buf);
BrotliStoreMetaBlockHeader(block_size, 0, storage_ix, storage);
/* No block splits, no contexts. */
BrotliWriteBits(13, 0, storage_ix, storage);
- StoreCommands(m, literal_buf, num_literals, command_buf, num_commands,
+ StoreCommands(s, literal_buf, num_literals, command_buf, num_commands,
storage_ix, storage);
- if (BROTLI_IS_OOM(m)) return;
} else {
/* Since we did not find many backward references and the entropy of
the data is close to 8 bits, we can simply emit an uncompressed block.
@@ -597,18 +609,18 @@ static BROTLI_INLINE void BrotliCompressFragmentTwoPassImpl(
#define BAKE_METHOD_PARAM_(B) \
static BROTLI_NOINLINE void BrotliCompressFragmentTwoPassImpl ## B( \
- MemoryManager* m, const uint8_t* input, size_t input_size, \
+ BrotliTwoPassArena* s, const uint8_t* input, size_t input_size, \
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf, \
int* table, size_t* storage_ix, uint8_t* storage) { \
size_t min_match = (B <= 15) ? 4 : 6; \
- BrotliCompressFragmentTwoPassImpl(m, input, input_size, is_last, command_buf,\
+ BrotliCompressFragmentTwoPassImpl(s, input, input_size, is_last, command_buf,\
literal_buf, table, B, min_match, storage_ix, storage); \
}
FOR_TABLE_BITS_(BAKE_METHOD_PARAM_)
#undef BAKE_METHOD_PARAM_
void BrotliCompressFragmentTwoPass(
- MemoryManager* m, const uint8_t* input, size_t input_size,
+ BrotliTwoPassArena* s, const uint8_t* input, size_t input_size,
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf,
int* table, size_t table_size, size_t* storage_ix, uint8_t* storage) {
const size_t initial_storage_ix = *storage_ix;
@@ -617,7 +629,7 @@ void BrotliCompressFragmentTwoPass(
#define CASE_(B) \
case B: \
BrotliCompressFragmentTwoPassImpl ## B( \
- m, input, input_size, is_last, command_buf, \
+ s, input, input_size, is_last, command_buf, \
literal_buf, table, storage_ix, storage); \
break;
FOR_TABLE_BITS_(CASE_)
diff --git a/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.h b/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.h
index 928677df42..6d28d9bb78 100644
--- a/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.h
+++ b/dviware/dvisvgm/libs/brotli/enc/compress_fragment_two_pass.h
@@ -13,16 +13,34 @@
#ifndef BROTLI_ENC_COMPRESS_FRAGMENT_TWO_PASS_H_
#define BROTLI_ENC_COMPRESS_FRAGMENT_TWO_PASS_H_
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./memory.h"
+
+#include "../common/constants.h"
+#include "../common/platform.h"
+#include "entropy_encode.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
+/* TODO(eustas): turn to macro. */
static const size_t kCompressFragmentTwoPassBlockSize = 1 << 17;
+typedef struct BrotliTwoPassArena {
+ uint32_t lit_histo[256];
+ uint8_t lit_depth[256];
+ uint16_t lit_bits[256];
+
+ uint32_t cmd_histo[128];
+ uint8_t cmd_depth[128];
+ uint16_t cmd_bits[128];
+
+ /* BuildAndStoreCommandPrefixCode */
+ HuffmanTree tmp_tree[2 * BROTLI_NUM_LITERAL_SYMBOLS + 1];
+ uint8_t tmp_depth[BROTLI_NUM_COMMAND_SYMBOLS];
+ uint16_t tmp_bits[64];
+} BrotliTwoPassArena;
+
/* Compresses "input" string to the "*storage" buffer as one or more complete
meta-blocks, and updates the "*storage_ix" bit position.
@@ -36,7 +54,7 @@ static const size_t kCompressFragmentTwoPassBlockSize = 1 << 17;
REQUIRES: "table_size" is a power of two
OUTPUT: maximal copy distance <= |input_size|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
-BROTLI_INTERNAL void BrotliCompressFragmentTwoPass(MemoryManager* m,
+BROTLI_INTERNAL void BrotliCompressFragmentTwoPass(BrotliTwoPassArena* s,
const uint8_t* input,
size_t input_size,
BROTLI_BOOL is_last,
diff --git a/dviware/dvisvgm/libs/brotli/enc/dictionary_hash.c b/dviware/dvisvgm/libs/brotli/enc/dictionary_hash.c
index 16d853fe5a..1a60eb3cda 100644
--- a/dviware/dvisvgm/libs/brotli/enc/dictionary_hash.c
+++ b/dviware/dvisvgm/libs/brotli/enc/dictionary_hash.c
@@ -7,12 +7,13 @@
/* Hash table on the 4-byte prefixes of static dictionary words. */
#include "../common/platform.h"
-#include "./dictionary_hash.h"
+#include "dictionary_hash.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
+/* GENERATED CODE START */
BROTLI_INTERNAL const uint16_t kStaticDictionaryHashWords[32768] = {
1002,0,0,0,0,0,0,0,0,683,0,0,0,0,0,0,0,1265,0,0,0,0,0,1431,0,0,0,0,0,0,40,0,0,0,
0,155,8,741,0,624,0,0,0,0,0,0,0,0,0,0,0,0,66,503,0,0,0,451,0,0,0,0,0,0,0,835,70,
@@ -1840,6 +1841,7 @@ BROTLI_INTERNAL const uint8_t kStaticDictionaryHashLengths[32768] = {
0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,
10,7,0,0,0,0,0,0,0,0,9,0,0,0,0,4,0,0,0,0,0,0,0,0,0,5,11,0,0,0,0,0,0,0,8,6,0,0,9,
7,0,0,12,4,0,0,0,0,0,0,12,6,0,6,0,7,0,0,8,5,0,0,0,0};
+/* GENERATED CODE END */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
diff --git a/dviware/dvisvgm/libs/brotli/enc/encode.c b/dviware/dvisvgm/libs/brotli/enc/encode.c
index e77009e39a..95ff2eaf50 100644
--- a/dviware/dvisvgm/libs/brotli/enc/encode.c
+++ b/dviware/dvisvgm/libs/brotli/enc/encode.c
@@ -7,6 +7,8 @@
/* Implementation of Brotli compressor. */
#include <brotli/encode.h>
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
#include <stdlib.h> /* free, malloc */
#include <string.h> /* memcpy, memset */
@@ -15,24 +17,26 @@
#include "../common/context.h"
#include "../common/platform.h"
#include "../common/version.h"
-#include "./backward_references.h"
-#include "./backward_references_hq.h"
-#include "./bit_cost.h"
-#include "./brotli_bit_stream.h"
-#include "./compress_fragment.h"
-#include "./compress_fragment_two_pass.h"
-#include "./encoder_dict.h"
-#include "./entropy_encode.h"
-#include "./fast_log.h"
-#include "./hash.h"
-#include "./histogram.h"
-#include "./memory.h"
-#include "./metablock.h"
-#include "./prefix.h"
-#include "./quality.h"
-#include "./ringbuffer.h"
-#include "./utf8_util.h"
-#include "./write_bits.h"
+#include "backward_references.h"
+#include "backward_references_hq.h"
+#include "bit_cost.h"
+#include "brotli_bit_stream.h"
+#include "compress_fragment.h"
+#include "compress_fragment_two_pass.h"
+#include "dictionary_hash.h"
+#include "encoder_dict.h"
+#include "entropy_encode.h"
+#include "fast_log.h"
+#include "hash.h"
+#include "histogram.h"
+#include "memory.h"
+#include "metablock.h"
+#include "prefix.h"
+#include "state.h"
+#include "quality.h"
+#include "ringbuffer.h"
+#include "utf8_util.h"
+#include "write_bits.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -40,94 +44,6 @@ extern "C" {
#define COPY_ARRAY(dst, src) memcpy(dst, src, sizeof(src));
-typedef enum BrotliEncoderStreamState {
- /* Default state. */
- BROTLI_STREAM_PROCESSING = 0,
- /* Intermediate state; after next block is emitted, byte-padding should be
- performed before getting back to default state. */
- BROTLI_STREAM_FLUSH_REQUESTED = 1,
- /* Last metablock was produced; no more input is acceptable. */
- BROTLI_STREAM_FINISHED = 2,
- /* Flushing compressed block and writing meta-data block header. */
- BROTLI_STREAM_METADATA_HEAD = 3,
- /* Writing metadata block body. */
- BROTLI_STREAM_METADATA_BODY = 4
-} BrotliEncoderStreamState;
-
-typedef enum BrotliEncoderFlintState {
- BROTLI_FLINT_NEEDS_2_BYTES = 2,
- BROTLI_FLINT_NEEDS_1_BYTE = 1,
- BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
- BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
- BROTLI_FLINT_DONE = -2
-} BrotliEncoderFlintState;
-
-typedef struct BrotliEncoderStateStruct {
- BrotliEncoderParams params;
-
- MemoryManager memory_manager_;
-
- uint64_t input_pos_;
- RingBuffer ringbuffer_;
- size_t cmd_alloc_size_;
- Command* commands_;
- size_t num_commands_;
- size_t num_literals_;
- size_t last_insert_len_;
- uint64_t last_flush_pos_;
- uint64_t last_processed_pos_;
- int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
- int saved_dist_cache_[4];
- uint16_t last_bytes_;
- uint8_t last_bytes_bits_;
- /* "Flint" is a tiny uncompressed block emitted before the continuation
- block to unwire literal context from previous data. Despite being int8_t,
- field is actually BrotliEncoderFlintState enum. */
- int8_t flint_;
- uint8_t prev_byte_;
- uint8_t prev_byte2_;
- size_t storage_size_;
- uint8_t* storage_;
-
- Hasher hasher_;
-
- /* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
- int small_table_[1 << 10]; /* 4KiB */
- int* large_table_; /* Allocated only when needed */
- size_t large_table_size_;
- /* Command and distance prefix codes (each 64 symbols, stored back-to-back)
- used for the next block in FAST_ONE_PASS_COMPRESSION_QUALITY. The command
- prefix code is over a smaller alphabet with the following 64 symbols:
- 0 - 15: insert length code 0, copy length code 0 - 15, same distance
- 16 - 39: insert length code 0, copy length code 0 - 23
- 40 - 63: insert length code 0 - 23, copy length code 0
- Note that symbols 16 and 40 represent the same code in the full alphabet,
- but we do not use either of them in FAST_ONE_PASS_COMPRESSION_QUALITY. */
- uint8_t cmd_depths_[128];
- uint16_t cmd_bits_[128];
- /* The compressed form of the command and distance prefix codes for the next
- block in FAST_ONE_PASS_COMPRESSION_QUALITY. */
- uint8_t cmd_code_[512];
- size_t cmd_code_numbits_;
- /* Command and literal buffers for FAST_TWO_PASS_COMPRESSION_QUALITY. */
- uint32_t* command_buf_;
- uint8_t* literal_buf_;
-
- uint8_t* next_out_;
- size_t available_out_;
- size_t total_out_;
- /* Temporary buffer for padding flush bits or metadata block header / body. */
- union {
- uint64_t u64[2];
- uint8_t u8[16];
- } tiny_buf_;
- uint32_t remaining_metadata_bytes_;
- BrotliEncoderStreamState stream_state_;
-
- BROTLI_BOOL is_last_block_emitted_;
- BROTLI_BOOL is_initialized_;
-} BrotliEncoderStateStruct;
-
static size_t InputBlockSize(BrotliEncoderState* s) {
return (size_t)1 << s->params.lgblock;
}
@@ -147,7 +63,7 @@ BROTLI_BOOL BrotliEncoderSetParameter(
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) {
/* Changing parameters on the fly is not implemented yet. */
if (state->is_initialized_) return BROTLI_FALSE;
- /* TODO: Validate/clamp parameters here. */
+ /* TODO(eustas): Validate/clamp parameters here. */
switch (p) {
case BROTLI_PARAM_MODE:
state->params.mode = (BrotliEncoderMode)value;
@@ -283,11 +199,9 @@ static void EncodeWindowBits(int lgwin, BROTLI_BOOL large_window,
}
}
+/* TODO(eustas): move to compress_fragment.c? */
/* Initializes the command and distance prefix codes for the first block. */
-static void InitCommandPrefixCodes(uint8_t cmd_depths[128],
- uint16_t cmd_bits[128],
- uint8_t cmd_code[512],
- size_t* cmd_code_numbits) {
+static void InitCommandPrefixCodes(BrotliOnePassArena* s) {
static const uint8_t kDefaultCommandDepths[128] = {
0, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8,
0, 0, 0, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7,
@@ -320,13 +234,13 @@ static void InitCommandPrefixCodes(uint8_t cmd_depths[128],
0x46, 0xe1, 0xb0, 0xd0, 0x4e, 0xb2, 0xf7, 0x04, 0x00,
};
static const size_t kDefaultCommandCodeNumBits = 448;
- COPY_ARRAY(cmd_depths, kDefaultCommandDepths);
- COPY_ARRAY(cmd_bits, kDefaultCommandBits);
+ COPY_ARRAY(s->cmd_depth, kDefaultCommandDepths);
+ COPY_ARRAY(s->cmd_bits, kDefaultCommandBits);
/* Initialize the pre-compressed form of the command and distance prefix
codes. */
- COPY_ARRAY(cmd_code, kDefaultCommandCode);
- *cmd_code_numbits = kDefaultCommandCodeNumBits;
+ COPY_ARRAY(s->cmd_code, kDefaultCommandCode);
+ s->cmd_code_numbits = kDefaultCommandCodeNumBits;
}
/* Decide about the context map based on the ability of the prediction
@@ -401,7 +315,8 @@ static void ChooseContextMap(int quality,
first 5 bits of literals. */
static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
- size_t* num_literal_contexts, const uint32_t** literal_context_map) {
+ size_t* num_literal_contexts, const uint32_t** literal_context_map,
+ uint32_t* arena) {
static const uint32_t kStaticContextMapComplexUTF8[64] = {
11, 11, 12, 12, /* 0 special */
0, 0, 0, 0, /* 4 lf */
@@ -426,16 +341,17 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
return BROTLI_FALSE;
} else {
const size_t end_pos = start_pos + length;
- /* To make entropy calculations faster and to fit on the stack, we collect
- histograms over the 5 most significant bits of literals. One histogram
+ /* To make entropy calculations faster, we collect histograms
+ over the 5 most significant bits of literals. One histogram
without context and 13 additional histograms for each context value. */
- uint32_t combined_histo[32] = { 0 };
- uint32_t context_histo[13][32] = { { 0 } };
+ uint32_t* BROTLI_RESTRICT const combined_histo = arena;
+ uint32_t* BROTLI_RESTRICT const context_histo = arena + 32;
uint32_t total = 0;
double entropy[3];
size_t dummy;
size_t i;
ContextLut utf8_lut = BROTLI_CONTEXT_LUT(CONTEXT_UTF8);
+ memset(arena, 0, sizeof(arena[0]) * 32 * 14);
for (; start_pos + 64 <= end_pos; start_pos += 4096) {
const size_t stride_end_pos = start_pos + 64;
uint8_t prev2 = input[start_pos & mask];
@@ -449,7 +365,7 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
BROTLI_CONTEXT(prev1, prev2, utf8_lut)];
++total;
++combined_histo[literal >> 3];
- ++context_histo[context][literal >> 3];
+ ++context_histo[(context << 5) + (literal >> 3)];
prev2 = prev1;
prev1 = literal;
}
@@ -457,7 +373,7 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
entropy[1] = ShannonEntropy(combined_histo, 32, &dummy);
entropy[2] = 0;
for (i = 0; i < 13; ++i) {
- entropy[2] += ShannonEntropy(&context_histo[i][0], 32, &dummy);
+ entropy[2] += ShannonEntropy(context_histo + (i << 5), 32, &dummy);
}
entropy[0] = 1.0 / (double)total;
entropy[1] *= entropy[0];
@@ -481,19 +397,21 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
static void DecideOverLiteralContextModeling(const uint8_t* input,
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
- size_t* num_literal_contexts, const uint32_t** literal_context_map) {
+ size_t* num_literal_contexts, const uint32_t** literal_context_map,
+ uint32_t* arena) {
if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
return;
} else if (ShouldUseComplexStaticContextMap(
input, start_pos, length, mask, quality, size_hint,
- num_literal_contexts, literal_context_map)) {
+ num_literal_contexts, literal_context_map, arena)) {
/* Context map was already set, nothing else to do. */
} else {
/* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
UTF8 data faster we only examine 64 byte long strides at every 4kB
intervals. */
const size_t end_pos = start_pos + length;
- uint32_t bigram_prefix_histo[9] = { 0 };
+ uint32_t* BROTLI_RESTRICT const bigram_prefix_histo = arena;
+ memset(bigram_prefix_histo, 0, sizeof(arena[0]) * 9);
for (; start_pos + 64 <= end_pos; start_pos += 4096) {
static const int lut[4] = { 0, 0, 1, 2 };
const size_t stride_end_pos = start_pos + 64;
@@ -513,7 +431,7 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
static BROTLI_BOOL ShouldCompress(
const uint8_t* data, const size_t mask, const uint64_t last_flush_pos,
const size_t bytes, const size_t num_literals, const size_t num_commands) {
- /* TODO: find more precise minimal block overhead. */
+ /* TODO(eustas): find more precise minimal block overhead. */
if (bytes <= 2) return BROTLI_FALSE;
if (num_commands < (bytes >> 8) + 2) {
if ((double)num_literals > 0.99 * (double)bytes) {
@@ -613,10 +531,14 @@ static void WriteMetaBlockInternal(MemoryManager* m,
size_t num_literal_contexts = 1;
const uint32_t* literal_context_map = NULL;
if (!params->disable_literal_context_modeling) {
+ /* TODO(eustas): pull to higher level and reuse. */
+ uint32_t* arena = BROTLI_ALLOC(m, uint32_t, 14 * 32);
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
DecideOverLiteralContextModeling(
data, wrapped_last_flush_pos, bytes, mask, params->quality,
params->size_hint, &num_literal_contexts,
- &literal_context_map);
+ &literal_context_map, arena);
+ BROTLI_FREE(m, arena);
}
BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,
prev_byte, prev_byte2, literal_context_lut, num_literal_contexts,
@@ -681,12 +603,13 @@ static void ChooseDistanceParams(BrotliEncoderParams* params) {
}
}
- BrotliInitDistanceParams(
- params, distance_postfix_bits, num_direct_distance_codes);
+ BrotliInitDistanceParams(&params->dist, distance_postfix_bits,
+ num_direct_distance_codes, params->large_window);
}
static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
- if (BROTLI_IS_OOM(&s->memory_manager_)) return BROTLI_FALSE;
+ MemoryManager* m = &s->memory_manager_;
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
if (s->is_initialized_) return BROTLI_TRUE;
s->last_bytes_bits_ = 0;
@@ -728,8 +651,12 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
}
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
- InitCommandPrefixCodes(s->cmd_depths_, s->cmd_bits_,
- s->cmd_code_, &s->cmd_code_numbits_);
+ s->one_pass_arena_ = BROTLI_ALLOC(m, BrotliOnePassArena, 1);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ InitCommandPrefixCodes(s->one_pass_arena_);
+ } else if (s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
+ s->two_pass_arena_ = BROTLI_ALLOC(m, BrotliTwoPassArena, 1);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
}
s->is_initialized_ = BROTLI_TRUE;
@@ -745,7 +672,7 @@ static void BrotliEncoderInitParams(BrotliEncoderParams* params) {
params->stream_offset = 0;
params->size_hint = 0;
params->disable_literal_context_modeling = BROTLI_FALSE;
- BrotliInitEncoderDictionary(&params->dictionary);
+ BrotliInitSharedEncoderDictionary(&params->dictionary);
params->dist.distance_postfix_bits = 0;
params->dist.num_direct_distance_codes = 0;
params->dist.alphabet_size_max =
@@ -754,6 +681,11 @@ static void BrotliEncoderInitParams(BrotliEncoderParams* params) {
params->dist.max_distance = BROTLI_MAX_DISTANCE;
}
+static void BrotliEncoderCleanupParams(MemoryManager* m,
+ BrotliEncoderParams* params) {
+ BrotliCleanupSharedEncoderDictionary(m, &params->dictionary);
+}
+
static void BrotliEncoderInitState(BrotliEncoderState* s) {
BrotliEncoderInitParams(&s->params);
s->input_pos_ = 0;
@@ -769,9 +701,11 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
HasherInit(&s->hasher_);
s->large_table_ = NULL;
s->large_table_size_ = 0;
- s->cmd_code_numbits_ = 0;
+ s->one_pass_arena_ = NULL;
+ s->two_pass_arena_ = NULL;
s->command_buf_ = NULL;
s->literal_buf_ = NULL;
+ s->total_in_ = 0;
s->next_out_ = NULL;
s->available_out_ = 0;
s->total_out_ = 0;
@@ -796,13 +730,9 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
BrotliEncoderState* BrotliEncoderCreateInstance(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
- BrotliEncoderState* state = 0;
- if (!alloc_func && !free_func) {
- state = (BrotliEncoderState*)malloc(sizeof(BrotliEncoderState));
- } else if (alloc_func && free_func) {
- state = (BrotliEncoderState*)alloc_func(opaque, sizeof(BrotliEncoderState));
- }
- if (state == 0) {
+ BrotliEncoderState* state = (BrotliEncoderState*)BrotliBootstrapAlloc(
+ sizeof(BrotliEncoderState), alloc_func, free_func, opaque);
+ if (state == NULL) {
/* BROTLI_DUMP(); */
return 0;
}
@@ -812,19 +742,36 @@ BrotliEncoderState* BrotliEncoderCreateInstance(
return state;
}
+#ifdef BROTLI_REPORTING
+/* When BROTLI_REPORTING is defined extra reporting module have to be linked. */
+void BrotliEncoderOnFinish(const BrotliEncoderState* s);
+#define BROTLI_ENCODER_ON_FINISH(s) BrotliEncoderOnFinish(s);
+#else
+#if !defined(BROTLI_ENCODER_ON_FINISH)
+#define BROTLI_ENCODER_ON_FINISH(s) (void)(s);
+#endif
+#endif
+
static void BrotliEncoderCleanupState(BrotliEncoderState* s) {
MemoryManager* m = &s->memory_manager_;
+
+ BROTLI_ENCODER_ON_FINISH(s);
+
if (BROTLI_IS_OOM(m)) {
BrotliWipeOutMemoryManager(m);
return;
}
+
BROTLI_FREE(m, s->storage_);
BROTLI_FREE(m, s->commands_);
RingBufferFree(m, &s->ringbuffer_);
DestroyHasher(m, &s->hasher_);
BROTLI_FREE(m, s->large_table_);
+ BROTLI_FREE(m, s->one_pass_arena_);
+ BROTLI_FREE(m, s->two_pass_arena_);
BROTLI_FREE(m, s->command_buf_);
BROTLI_FREE(m, s->literal_buf_);
+ BrotliEncoderCleanupParams(m, &s->params);
}
/* Deinitializes and frees BrotliEncoderState instance. */
@@ -832,11 +779,8 @@ void BrotliEncoderDestroyInstance(BrotliEncoderState* state) {
if (!state) {
return;
} else {
- MemoryManager* m = &state->memory_manager_;
- brotli_free_func free_func = m->free_func;
- void* opaque = m->opaque;
BrotliEncoderCleanupState(state);
- free_func(opaque, state);
+ BrotliBootstrapFree(state, &state->memory_manager_);
}
}
@@ -925,6 +869,8 @@ static void ExtendLastCommand(BrotliEncoderState* s, uint32_t* bytes,
uint64_t cmd_dist = (uint64_t)s->dist_cache_[0];
uint32_t distance_code = CommandRestoreDistanceCode(last_command,
&s->params.dist);
+ const CompoundDictionary* dict = &s->params.dictionary.compound;
+ size_t compound_dictionary_size = dict->total_size;
if (distance_code < BROTLI_NUM_DISTANCE_SHORT_CODES ||
distance_code - (BROTLI_NUM_DISTANCE_SHORT_CODES - 1) == cmd_dist) {
if (cmd_dist <= max_distance) {
@@ -935,6 +881,38 @@ static void ExtendLastCommand(BrotliEncoderState* s, uint32_t* bytes,
(*wrapped_last_processed_pos)++;
}
} else {
+ if ((cmd_dist - max_distance - 1) < compound_dictionary_size &&
+ last_copy_len < cmd_dist - max_distance) {
+ size_t address =
+ compound_dictionary_size - (size_t)(cmd_dist - max_distance) +
+ (size_t)last_copy_len;
+ size_t br_index = 0;
+ size_t br_offset;
+ const uint8_t* chunk;
+ size_t chunk_length;
+ while (address >= dict->chunk_offsets[br_index + 1]) br_index++;
+ br_offset = address - dict->chunk_offsets[br_index];
+ chunk = dict->chunk_source[br_index];
+ chunk_length =
+ dict->chunk_offsets[br_index + 1] - dict->chunk_offsets[br_index];
+ while (*bytes != 0 && data[*wrapped_last_processed_pos & mask] ==
+ chunk[br_offset]) {
+ last_command->copy_len_++;
+ (*bytes)--;
+ (*wrapped_last_processed_pos)++;
+ if (++br_offset == chunk_length) {
+ br_index++;
+ br_offset = 0;
+ if (br_index != dict->num_chunks) {
+ chunk = dict->chunk_source[br_index];
+ chunk_length = dict->chunk_offsets[br_index + 1] -
+ dict->chunk_offsets[br_index];
+ } else {
+ break;
+ }
+ }
+ }
+ }
}
/* The copy length is at most the metablock size, and thus expressible. */
GetLengthCode(last_command->insert_len_,
@@ -952,8 +930,8 @@ static void ExtendLastCommand(BrotliEncoderState* s, uint32_t* bytes,
If |*out_size| is positive, |*output| points to the start of the output
data. If |is_last| or |force_flush| is BROTLI_TRUE, an output meta-block is
always created. However, until |is_last| is BROTLI_TRUE encoder may retain up
- to 7 bits of the last byte of output. To force encoder to dump the remaining
- bits use WriteMetadata() to append an empty meta-data block.
+ to 7 bits of the last byte of output. Byte-alignment could be enforced by
+ emitting an empty meta-data block.
Returns BROTLI_FALSE if the size of the input data is larger than
input_block_size().
*/
@@ -968,10 +946,39 @@ static BROTLI_BOOL EncodeData(
MemoryManager* m = &s->memory_manager_;
ContextType literal_context_mode;
ContextLut literal_context_lut;
+ BROTLI_BOOL fast_compress =
+ s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
+ s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY;
data = s->ringbuffer_.buffer_;
mask = s->ringbuffer_.mask_;
+ if (delta == 0) { /* No new input; still might want to flush or finish. */
+ if (!data) { /* No input has been processed so far. */
+ if (is_last) { /* Emit complete finalized stream. */
+ BROTLI_DCHECK(s->last_bytes_bits_ <= 14);
+ s->last_bytes_ |= (uint16_t)(3u << s->last_bytes_bits_);
+ s->last_bytes_bits_ = (uint8_t)(s->last_bytes_bits_ + 2u);
+ s->tiny_buf_.u8[0] = (uint8_t)s->last_bytes_;
+ s->tiny_buf_.u8[1] = (uint8_t)(s->last_bytes_ >> 8);
+ *output = s->tiny_buf_.u8;
+ *out_size = (s->last_bytes_bits_ + 7u) >> 3u;
+ return BROTLI_TRUE;
+ } else { /* No data, not last -> no-op. */
+ *out_size = 0;
+ return BROTLI_TRUE;
+ }
+ } else {
+ /* Fast compress performs flush every block -> flush is no-op. */
+ if (!is_last && (!force_flush || fast_compress)) { /* Another no-op. */
+ *out_size = 0;
+ return BROTLI_TRUE;
+ }
+ }
+ }
+ BROTLI_DCHECK(data);
+
+ if (s->params.quality > s->params.dictionary.max_quality) return BROTLI_FALSE;
/* Adding more blocks after "last" block is forbidden. */
if (s->is_last_block_emitted_) return BROTLI_FALSE;
if (is_last) s->is_last_block_emitted_ = BROTLI_TRUE;
@@ -991,19 +998,12 @@ static BROTLI_BOOL EncodeData(
}
}
- if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
- s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
+ if (fast_compress) {
uint8_t* storage;
size_t storage_ix = s->last_bytes_bits_;
size_t table_size;
int* table;
- if (delta == 0 && !is_last) {
- /* We have no new input data and we don't have to finish the stream, so
- nothing to do. */
- *out_size = 0;
- return BROTLI_TRUE;
- }
storage = GetBrotliStorage(s, 2 * bytes + 503);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
storage[0] = (uint8_t)s->last_bytes_;
@@ -1012,16 +1012,14 @@ static BROTLI_BOOL EncodeData(
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
BrotliCompressFragmentFast(
- m, &data[wrapped_last_processed_pos & mask],
+ s->one_pass_arena_, &data[wrapped_last_processed_pos & mask],
bytes, is_last,
table, table_size,
- s->cmd_depths_, s->cmd_bits_,
- &s->cmd_code_numbits_, s->cmd_code_,
&storage_ix, storage);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
} else {
BrotliCompressFragmentTwoPass(
- m, &data[wrapped_last_processed_pos & mask],
+ s->two_pass_arena_, &data[wrapped_last_processed_pos & mask],
bytes, is_last,
s->command_buf_, s->literal_buf_,
table, table_size,
@@ -1099,7 +1097,7 @@ static BROTLI_BOOL EncodeData(
const size_t max_commands = max_length / 8;
const size_t processed_bytes = (size_t)(s->input_pos_ - s->last_flush_pos_);
/* If maximal possible additional block doesn't fit metablock, flush now. */
- /* TODO: Postpone decision until next block arrives? */
+ /* TODO(eustas): Postpone decision until next block arrives? */
const BROTLI_BOOL next_input_fits_metablock = TO_BROTLI_BOOL(
processed_bytes + InputBlockSize(s) <= max_length);
/* If block splitting is not used, then flush as soon as there is some
@@ -1192,7 +1190,7 @@ static size_t WriteMetadataHeader(
if (block_size == 0) {
BrotliWriteBits(2, 0, &storage_ix, header);
} else {
- uint32_t nbits = (block_size == 1) ? 0 :
+ uint32_t nbits = (block_size == 1) ? 1 :
(Log2FloorNonZero((uint32_t)block_size - 1) + 1);
uint32_t nbytes = (nbits + 7) / 8;
BrotliWriteBits(2, nbytes, &storage_ix, header);
@@ -1201,228 +1199,6 @@ static size_t WriteMetadataHeader(
return (storage_ix + 7u) >> 3;
}
-static BROTLI_BOOL BrotliCompressBufferQuality10(
- int lgwin, size_t input_size, const uint8_t* input_buffer,
- size_t* encoded_size, uint8_t* encoded_buffer) {
- MemoryManager memory_manager;
- MemoryManager* m = &memory_manager;
-
- const size_t mask = BROTLI_SIZE_MAX >> 1;
- int dist_cache[4] = { 4, 11, 15, 16 };
- int saved_dist_cache[4] = { 4, 11, 15, 16 };
- BROTLI_BOOL ok = BROTLI_TRUE;
- const size_t max_out_size = *encoded_size;
- size_t total_out_size = 0;
- uint16_t last_bytes;
- uint8_t last_bytes_bits;
-
- const size_t hasher_eff_size = BROTLI_MIN(size_t,
- input_size, BROTLI_MAX_BACKWARD_LIMIT(lgwin) + BROTLI_WINDOW_GAP);
-
- BrotliEncoderParams params;
-
- const int lgmetablock = BROTLI_MIN(int, 24, lgwin + 1);
- size_t max_block_size;
- const size_t max_metablock_size = (size_t)1 << lgmetablock;
- const size_t max_literals_per_metablock = max_metablock_size / 8;
- const size_t max_commands_per_metablock = max_metablock_size / 8;
- size_t metablock_start = 0;
- uint8_t prev_byte = 0;
- uint8_t prev_byte2 = 0;
-
- Hasher hasher;
- HasherInit(&hasher);
-
- BrotliEncoderInitParams(&params);
- params.quality = 10;
- params.lgwin = lgwin;
- if (lgwin > BROTLI_MAX_WINDOW_BITS) {
- params.large_window = BROTLI_TRUE;
- }
- SanitizeParams(&params);
- params.lgblock = ComputeLgBlock(&params);
- ChooseDistanceParams(&params);
- max_block_size = (size_t)1 << params.lgblock;
-
- BrotliInitMemoryManager(m, 0, 0, 0);
-
- BROTLI_DCHECK(input_size <= mask + 1);
- EncodeWindowBits(lgwin, params.large_window, &last_bytes, &last_bytes_bits);
- InitOrStitchToPreviousBlock(m, &hasher, input_buffer, mask, &params,
- 0, hasher_eff_size, BROTLI_TRUE);
- if (BROTLI_IS_OOM(m)) goto oom;
-
- while (ok && metablock_start < input_size) {
- const size_t metablock_end =
- BROTLI_MIN(size_t, input_size, metablock_start + max_metablock_size);
- const size_t expected_num_commands =
- (metablock_end - metablock_start) / 12 + 16;
- Command* commands = 0;
- size_t num_commands = 0;
- size_t last_insert_len = 0;
- size_t num_literals = 0;
- size_t metablock_size = 0;
- size_t cmd_alloc_size = 0;
- BROTLI_BOOL is_last;
- uint8_t* storage;
- size_t storage_ix;
-
- ContextType literal_context_mode = ChooseContextMode(&params,
- input_buffer, metablock_start, mask, metablock_end - metablock_start);
- ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
-
- size_t block_start;
- for (block_start = metablock_start; block_start < metablock_end; ) {
- size_t block_size =
- BROTLI_MIN(size_t, metablock_end - block_start, max_block_size);
- ZopfliNode* nodes = BROTLI_ALLOC(m, ZopfliNode, block_size + 1);
- size_t path_size;
- size_t new_cmd_alloc_size;
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) goto oom;
- BrotliInitZopfliNodes(nodes, block_size + 1);
- StitchToPreviousBlockH10(&hasher.privat._H10, block_size, block_start,
- input_buffer, mask);
- path_size = BrotliZopfliComputeShortestPath(m, block_size, block_start,
- input_buffer, mask, literal_context_lut, &params, dist_cache, &hasher,
- nodes);
- if (BROTLI_IS_OOM(m)) goto oom;
- /* We allocate a command buffer in the first iteration of this loop that
- will be likely big enough for the whole metablock, so that for most
- inputs we will not have to reallocate in later iterations. We do the
- allocation here and not before the loop, because if the input is small,
- this will be allocated after the Zopfli cost model is freed, so this
- will not increase peak memory usage.
- TODO: If the first allocation is too small, increase command
- buffer size exponentially. */
- new_cmd_alloc_size = BROTLI_MAX(size_t, expected_num_commands,
- num_commands + path_size + 1);
- if (cmd_alloc_size != new_cmd_alloc_size) {
- Command* new_commands = BROTLI_ALLOC(m, Command, new_cmd_alloc_size);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_commands)) goto oom;
- cmd_alloc_size = new_cmd_alloc_size;
- if (commands) {
- memcpy(new_commands, commands, sizeof(Command) * num_commands);
- BROTLI_FREE(m, commands);
- }
- commands = new_commands;
- }
- BrotliZopfliCreateCommands(block_size, block_start, &nodes[0], dist_cache,
- &last_insert_len, &params, &commands[num_commands], &num_literals);
- num_commands += path_size;
- block_start += block_size;
- metablock_size += block_size;
- BROTLI_FREE(m, nodes);
- if (num_literals > max_literals_per_metablock ||
- num_commands > max_commands_per_metablock) {
- break;
- }
- }
-
- if (last_insert_len > 0) {
- InitInsertCommand(&commands[num_commands++], last_insert_len);
- num_literals += last_insert_len;
- }
-
- is_last = TO_BROTLI_BOOL(metablock_start + metablock_size == input_size);
- storage = NULL;
- storage_ix = last_bytes_bits;
-
- if (metablock_size == 0) {
- /* Write the ISLAST and ISEMPTY bits. */
- storage = BROTLI_ALLOC(m, uint8_t, 16);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
- storage[0] = (uint8_t)last_bytes;
- storage[1] = (uint8_t)(last_bytes >> 8);
- BrotliWriteBits(2, 3, &storage_ix, storage);
- storage_ix = (storage_ix + 7u) & ~7u;
- } else if (!ShouldCompress(input_buffer, mask, metablock_start,
- metablock_size, num_literals, num_commands)) {
- /* Restore the distance cache, as its last update by
- CreateBackwardReferences is now unused. */
- memcpy(dist_cache, saved_dist_cache, 4 * sizeof(dist_cache[0]));
- storage = BROTLI_ALLOC(m, uint8_t, metablock_size + 16);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
- storage[0] = (uint8_t)last_bytes;
- storage[1] = (uint8_t)(last_bytes >> 8);
- BrotliStoreUncompressedMetaBlock(is_last, input_buffer,
- metablock_start, mask, metablock_size,
- &storage_ix, storage);
- } else {
- MetaBlockSplit mb;
- BrotliEncoderParams block_params = params;
- InitMetaBlockSplit(&mb);
- BrotliBuildMetaBlock(m, input_buffer, metablock_start, mask,
- &block_params,
- prev_byte, prev_byte2,
- commands, num_commands,
- literal_context_mode,
- &mb);
- if (BROTLI_IS_OOM(m)) goto oom;
- {
- /* The number of distance symbols effectively used for distance
- histograms. It might be less than distance alphabet size
- for "Large Window Brotli" (32-bit). */
- BrotliOptimizeHistograms(block_params.dist.alphabet_size_limit, &mb);
- }
- storage = BROTLI_ALLOC(m, uint8_t, 2 * metablock_size + 503);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
- storage[0] = (uint8_t)last_bytes;
- storage[1] = (uint8_t)(last_bytes >> 8);
- BrotliStoreMetaBlock(m, input_buffer, metablock_start, metablock_size,
- mask, prev_byte, prev_byte2,
- is_last,
- &block_params,
- literal_context_mode,
- commands, num_commands,
- &mb,
- &storage_ix, storage);
- if (BROTLI_IS_OOM(m)) goto oom;
- if (metablock_size + 4 < (storage_ix >> 3)) {
- /* Restore the distance cache and last byte. */
- memcpy(dist_cache, saved_dist_cache, 4 * sizeof(dist_cache[0]));
- storage[0] = (uint8_t)last_bytes;
- storage[1] = (uint8_t)(last_bytes >> 8);
- storage_ix = last_bytes_bits;
- BrotliStoreUncompressedMetaBlock(is_last, input_buffer,
- metablock_start, mask,
- metablock_size, &storage_ix, storage);
- }
- DestroyMetaBlockSplit(m, &mb);
- }
- last_bytes = (uint16_t)(storage[storage_ix >> 3]);
- last_bytes_bits = storage_ix & 7u;
- metablock_start += metablock_size;
- if (metablock_start < input_size) {
- prev_byte = input_buffer[metablock_start - 1];
- prev_byte2 = input_buffer[metablock_start - 2];
- }
- /* Save the state of the distance cache in case we need to restore it for
- emitting an uncompressed block. */
- memcpy(saved_dist_cache, dist_cache, 4 * sizeof(dist_cache[0]));
-
- {
- const size_t out_size = storage_ix >> 3;
- total_out_size += out_size;
- if (total_out_size <= max_out_size) {
- memcpy(encoded_buffer, storage, out_size);
- encoded_buffer += out_size;
- } else {
- ok = BROTLI_FALSE;
- }
- }
- BROTLI_FREE(m, storage);
- BROTLI_FREE(m, commands);
- }
-
- *encoded_size = total_out_size;
- DestroyHasher(m, &hasher);
- return ok;
-
-oom:
- BrotliWipeOutMemoryManager(m);
- return BROTLI_FALSE;
-}
-
size_t BrotliEncoderMaxCompressedSize(size_t input_size) {
/* [window bits / empty metadata] + N * [uncompressed] + [last empty] */
size_t num_large_blocks = input_size >> 14;
@@ -1470,7 +1246,8 @@ static size_t MakeUncompressedStream(
BROTLI_BOOL BrotliEncoderCompress(
int quality, int lgwin, BrotliEncoderMode mode, size_t input_size,
- const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)], size_t* encoded_size,
+ const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)],
+ size_t* encoded_size,
uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(*encoded_size)]) {
BrotliEncoderState* s;
size_t out_size = *encoded_size;
@@ -1487,17 +1264,6 @@ BROTLI_BOOL BrotliEncoderCompress(
*encoded_buffer = 6;
return BROTLI_TRUE;
}
- if (quality == 10) {
- /* TODO: Implement this direct path for all quality levels. */
- const int lg_win = BROTLI_MIN(int, BROTLI_LARGE_MAX_WINDOW_BITS,
- BROTLI_MAX(int, 16, lgwin));
- int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer,
- encoded_size, encoded_buffer);
- if (!ok || (max_out_size && *encoded_size > max_out_size)) {
- goto fallback;
- }
- return BROTLI_TRUE;
- }
s = BrotliEncoderCreateInstance(0, 0, 0);
if (!s) {
@@ -1509,6 +1275,7 @@ BROTLI_BOOL BrotliEncoderCompress(
uint8_t* next_out = encoded_buffer;
size_t total_out = 0;
BROTLI_BOOL result = BROTLI_FALSE;
+ /* TODO(eustas): check that parameters are sane. */
BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)quality);
BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);
BrotliEncoderSetParameter(s, BROTLI_PARAM_MODE, (uint32_t)mode);
@@ -1560,6 +1327,18 @@ static void InjectBytePaddingBlock(BrotliEncoderState* s) {
s->available_out_ += (seal_bits + 7) >> 3;
}
+/* Fills the |total_out|, if it is not NULL. */
+static void SetTotalOut(BrotliEncoderState* s, size_t* total_out) {
+ if (total_out) {
+ /* Saturating conversion uint64_t -> size_t */
+ size_t result = (size_t)-1;
+ if (s->total_out_ < result) {
+ result = (size_t)s->total_out_;
+ }
+ *total_out = result;
+ }
+}
+
/* Injects padding bits or pushes compressed data to output.
Returns false if nothing is done. */
static BROTLI_BOOL InjectFlushOrPushOutput(BrotliEncoderState* s,
@@ -1579,7 +1358,7 @@ static BROTLI_BOOL InjectFlushOrPushOutput(BrotliEncoderState* s,
s->next_out_ += copy_output_size;
s->available_out_ -= copy_output_size;
s->total_out_ += copy_output_size;
- if (total_out) *total_out = s->total_out_;
+ SetTotalOut(s, total_out);
return BROTLI_TRUE;
}
@@ -1676,19 +1455,19 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
- BrotliCompressFragmentFast(m, *next_in, block_size, is_last, table,
- table_size, s->cmd_depths_, s->cmd_bits_, &s->cmd_code_numbits_,
- s->cmd_code_, &storage_ix, storage);
+ BrotliCompressFragmentFast(s->one_pass_arena_, *next_in, block_size,
+ is_last, table, table_size, &storage_ix, storage);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
} else {
- BrotliCompressFragmentTwoPass(m, *next_in, block_size, is_last,
- command_buf, literal_buf, table, table_size,
+ BrotliCompressFragmentTwoPass(s->two_pass_arena_, *next_in, block_size,
+ is_last, command_buf, literal_buf, table, table_size,
&storage_ix, storage);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
}
if (block_size != 0) {
*next_in += block_size;
*available_in -= block_size;
+ s->total_in_ += block_size;
}
if (inplace) {
size_t out_bytes = storage_ix >> 3;
@@ -1697,7 +1476,7 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
*next_out += out_bytes;
*available_out -= out_bytes;
s->total_out_ += out_bytes;
- if (total_out) *total_out = s->total_out_;
+ SetTotalOut(s, total_out);
} else {
size_t out_bytes = storage_ix >> 3;
s->next_out_ = storage;
@@ -1766,6 +1545,7 @@ static BROTLI_BOOL ProcessMetadata(
memcpy(*next_out, *next_in, copy);
*next_in += copy;
*available_in -= copy;
+ s->total_in_ += copy; /* not actually data input, though */
s->remaining_metadata_bytes_ -= copy;
*next_out += copy;
*available_out -= copy;
@@ -1776,6 +1556,7 @@ static BROTLI_BOOL ProcessMetadata(
memcpy(s->next_out_, *next_in, copy);
*next_in += copy;
*available_in -= copy;
+ s->total_in_ += copy; /* not actually data input, though */
s->remaining_metadata_bytes_ -= copy;
s->available_out_ = copy;
}
@@ -1803,7 +1584,7 @@ static void UpdateSizeHint(BrotliEncoderState* s, size_t available_in) {
BROTLI_BOOL BrotliEncoderCompressStream(
BrotliEncoderState* s, BrotliEncoderOperation op, size_t* available_in,
- const uint8_t** next_in, size_t* available_out,uint8_t** next_out,
+ const uint8_t** next_in, size_t* available_out, uint8_t** next_out,
size_t* total_out) {
if (!EnsureInitialized(s)) return BROTLI_FALSE;
@@ -1845,6 +1626,7 @@ BROTLI_BOOL BrotliEncoderCompressStream(
CopyInputToRingBuffer(s, copy_input_size, *next_in);
*next_in += copy_input_size;
*available_in -= copy_input_size;
+ s->total_in_ += copy_input_size;
if (s->flint_ > 0) s->flint_ = (int8_t)(s->flint_ - (int)copy_input_size);
continue;
}
@@ -1922,6 +1704,293 @@ uint32_t BrotliEncoderVersion(void) {
return BROTLI_VERSION;
}
+BrotliEncoderPreparedDictionary* BrotliEncoderPrepareDictionary(
+ BrotliSharedDictionaryType type, size_t size,
+ const uint8_t data[BROTLI_ARRAY_PARAM(size)], int quality,
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
+ ManagedDictionary* managed_dictionary = NULL;
+ BROTLI_BOOL type_is_known = BROTLI_FALSE;
+ type_is_known |= (type == BROTLI_SHARED_DICTIONARY_RAW);
+#if defined(BROTLI_EXPERIMENTAL)
+ type_is_known |= (type == BROTLI_SHARED_DICTIONARY_SERIALIZED);
+#endif /* BROTLI_EXPERIMENTAL */
+ if (!type_is_known) {
+ return NULL;
+ }
+ managed_dictionary =
+ BrotliCreateManagedDictionary(alloc_func, free_func, opaque);
+ if (managed_dictionary == NULL) {
+ return NULL;
+ }
+ if (type == BROTLI_SHARED_DICTIONARY_RAW) {
+ managed_dictionary->dictionary = (uint32_t*)CreatePreparedDictionary(
+ &managed_dictionary->memory_manager_, data, size);
+ }
+#if defined(BROTLI_EXPERIMENTAL)
+ if (type == BROTLI_SHARED_DICTIONARY_SERIALIZED) {
+ SharedEncoderDictionary* dict = (SharedEncoderDictionary*)BrotliAllocate(
+ &managed_dictionary->memory_manager_, sizeof(SharedEncoderDictionary));
+ managed_dictionary->dictionary = (uint32_t*)dict;
+ if (dict != NULL) {
+ BROTLI_BOOL ok = BrotliInitCustomSharedEncoderDictionary(
+ &managed_dictionary->memory_manager_, data, size, quality, dict);
+ if (!ok) {
+ BrotliFree(&managed_dictionary->memory_manager_, dict);
+ managed_dictionary->dictionary = NULL;
+ }
+ }
+ }
+#else /* BROTLI_EXPERIMENTAL */
+ (void)quality;
+#endif /* BROTLI_EXPERIMENTAL */
+ if (managed_dictionary->dictionary == NULL) {
+ BrotliDestroyManagedDictionary(managed_dictionary);
+ return NULL;
+ }
+ return (BrotliEncoderPreparedDictionary*)managed_dictionary;
+}
+
+void BrotliEncoderDestroyPreparedDictionary(
+ BrotliEncoderPreparedDictionary* dictionary) {
+ ManagedDictionary* dict = (ManagedDictionary*)dictionary;
+ if (!dictionary) return;
+ /* First field of dictionary structs. */
+ /* Only managed dictionaries are eligible for destruction by this method. */
+ if (dict->magic != kManagedDictionaryMagic) {
+ return;
+ }
+ if (dict->dictionary == NULL) {
+ /* This should never ever happen. */
+ } else if (*dict->dictionary == kLeanPreparedDictionaryMagic) {
+ DestroyPreparedDictionary(
+ &dict->memory_manager_, (PreparedDictionary*)dict->dictionary);
+ } else if (*dict->dictionary == kSharedDictionaryMagic) {
+ BrotliCleanupSharedEncoderDictionary(&dict->memory_manager_,
+ (SharedEncoderDictionary*)dict->dictionary);
+ BrotliFree(&dict->memory_manager_, dict->dictionary);
+ } else {
+ /* There is also kPreparedDictionaryMagic, but such instances should be
+ * constructed and destroyed by different means. */
+ }
+ dict->dictionary = NULL;
+ BrotliDestroyManagedDictionary(dict);
+}
+
+BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(BrotliEncoderState* state,
+ const BrotliEncoderPreparedDictionary* dictionary) {
+ /* First field of dictionary structs */
+ const BrotliEncoderPreparedDictionary* dict = dictionary;
+ uint32_t magic = *((const uint32_t*)dict);
+ SharedEncoderDictionary* current = NULL;
+ if (magic == kManagedDictionaryMagic) {
+ /* Unwrap managed dictionary. */
+ ManagedDictionary* managed_dictionary = (ManagedDictionary*)dict;
+ magic = *managed_dictionary->dictionary;
+ dict = (BrotliEncoderPreparedDictionary*)managed_dictionary->dictionary;
+ }
+ current = &state->params.dictionary;
+ if (magic == kPreparedDictionaryMagic ||
+ magic == kLeanPreparedDictionaryMagic) {
+ const PreparedDictionary* prepared = (const PreparedDictionary*)dict;
+ if (!AttachPreparedDictionary(&current->compound, prepared)) {
+ return BROTLI_FALSE;
+ }
+ } else if (magic == kSharedDictionaryMagic) {
+ const SharedEncoderDictionary* attached =
+ (const SharedEncoderDictionary*)dict;
+ BROTLI_BOOL was_default = !current->contextual.context_based &&
+ current->contextual.num_dictionaries == 1 &&
+ current->contextual.dict[0]->hash_table_words ==
+ kStaticDictionaryHashWords &&
+ current->contextual.dict[0]->hash_table_lengths ==
+ kStaticDictionaryHashLengths;
+ BROTLI_BOOL new_default = !attached->contextual.context_based &&
+ attached->contextual.num_dictionaries == 1 &&
+ attached->contextual.dict[0]->hash_table_words ==
+ kStaticDictionaryHashWords &&
+ attached->contextual.dict[0]->hash_table_lengths ==
+ kStaticDictionaryHashLengths;
+ size_t i;
+ if (state->is_initialized_) return BROTLI_FALSE;
+ current->max_quality =
+ BROTLI_MIN(int, current->max_quality, attached->max_quality);
+ for (i = 0; i < attached->compound.num_chunks; i++) {
+ if (!AttachPreparedDictionary(&current->compound,
+ attached->compound.chunks[i])) {
+ return BROTLI_FALSE;
+ }
+ }
+ if (!new_default) {
+ if (!was_default) return BROTLI_FALSE;
+ /* Copy by value, but then set num_instances_ to 0 because their memory
+ is managed by attached, not by current */
+ current->contextual = attached->contextual;
+ current->contextual.num_instances_ = 0;
+ }
+ } else {
+ return BROTLI_FALSE;
+ }
+ return BROTLI_TRUE;
+}
+
+size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin,
+ size_t input_size) {
+ BrotliEncoderParams params;
+ size_t memory_manager_slots = BROTLI_ENCODER_MEMORY_MANAGER_SLOTS;
+ size_t memory_manager_size = memory_manager_slots * sizeof(void*);
+ BrotliEncoderInitParams(&params);
+ params.quality = quality;
+ params.lgwin = lgwin;
+ params.size_hint = input_size;
+ params.large_window = lgwin > BROTLI_MAX_WINDOW_BITS;
+ SanitizeParams(&params);
+ params.lgblock = ComputeLgBlock(&params);
+ ChooseHasher(&params, &params.hasher);
+ if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
+ params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
+ size_t state_size = sizeof(BrotliEncoderState);
+ size_t block_size = BROTLI_MIN(size_t, input_size, (1ul << params.lgwin));
+ size_t hash_table_size =
+ HashTableSize(MaxHashTableSize(params.quality), block_size);
+ size_t hash_size =
+ (hash_table_size < (1u << 10)) ? 0 : sizeof(int) * hash_table_size;
+ size_t cmdbuf_size = params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY ?
+ 5 * BROTLI_MIN(size_t, block_size, 1ul << 17) : 0;
+ if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
+ state_size += sizeof(BrotliOnePassArena);
+ } else {
+ state_size += sizeof(BrotliTwoPassArena);
+ }
+ return hash_size + cmdbuf_size + state_size;
+ } else {
+ size_t short_ringbuffer_size = (size_t)1 << params.lgblock;
+ int ringbuffer_bits = ComputeRbBits(&params);
+ size_t ringbuffer_size = input_size < short_ringbuffer_size ?
+ input_size : (1u << ringbuffer_bits) + short_ringbuffer_size;
+ size_t hash_size[4] = {0};
+ size_t metablock_size =
+ BROTLI_MIN(size_t, input_size, MaxMetablockSize(&params));
+ size_t inputblock_size =
+ BROTLI_MIN(size_t, input_size, (size_t)1 << params.lgblock);
+ size_t cmdbuf_size = metablock_size * 2 + inputblock_size * 6;
+ size_t outbuf_size = metablock_size * 2 + 503;
+ size_t histogram_size = 0;
+ HasherSize(&params, BROTLI_TRUE, input_size, hash_size);
+ if (params.quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
+ cmdbuf_size = BROTLI_MIN(size_t, cmdbuf_size,
+ MAX_NUM_DELAYED_SYMBOLS * sizeof(Command) + inputblock_size * 12);
+ }
+ if (params.quality >= MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING) {
+ /* Only a very rough estimation, based on enwik8. */
+ histogram_size = 200 << 20;
+ } else if (params.quality >= MIN_QUALITY_FOR_BLOCK_SPLIT) {
+ size_t literal_histograms =
+ BROTLI_MIN(size_t, metablock_size / 6144, 256);
+ size_t command_histograms =
+ BROTLI_MIN(size_t, metablock_size / 6144, 256);
+ size_t distance_histograms =
+ BROTLI_MIN(size_t, metablock_size / 6144, 256);
+ histogram_size = literal_histograms * sizeof(HistogramLiteral) +
+ command_histograms * sizeof(HistogramCommand) +
+ distance_histograms * sizeof(HistogramDistance);
+ }
+ return (memory_manager_size + ringbuffer_size +
+ hash_size[0] + hash_size[1] + hash_size[2] + hash_size[3] +
+ cmdbuf_size +
+ outbuf_size +
+ histogram_size);
+ }
+}
+size_t BrotliEncoderGetPreparedDictionarySize(
+ const BrotliEncoderPreparedDictionary* prepared_dictionary) {
+ /* First field of dictionary structs */
+ const BrotliEncoderPreparedDictionary* prepared = prepared_dictionary;
+ uint32_t magic = *((const uint32_t*)prepared);
+ size_t overhead = 0;
+ if (magic == kManagedDictionaryMagic) {
+ const ManagedDictionary* managed = (const ManagedDictionary*)prepared;
+ overhead = sizeof(ManagedDictionary);
+ magic = *managed->dictionary;
+ prepared = (const BrotliEncoderPreparedDictionary*)managed->dictionary;
+ }
+
+ if (magic == kPreparedDictionaryMagic) {
+ const PreparedDictionary* dictionary =
+ (const PreparedDictionary*)prepared;
+ /* Keep in sync with step 3 of CreatePreparedDictionary */
+ return sizeof(PreparedDictionary) + dictionary->source_size +
+ (sizeof(uint32_t) << dictionary->slot_bits) +
+ (sizeof(uint16_t) << dictionary->bucket_bits) +
+ (sizeof(uint32_t) * dictionary->num_items) + overhead;
+ } else if (magic == kLeanPreparedDictionaryMagic) {
+ const PreparedDictionary* dictionary =
+ (const PreparedDictionary*)prepared;
+ /* Keep in sync with step 3 of CreatePreparedDictionary */
+ return sizeof(PreparedDictionary) + sizeof(uint8_t*) +
+ (sizeof(uint32_t) << dictionary->slot_bits) +
+ (sizeof(uint16_t) << dictionary->bucket_bits) +
+ (sizeof(uint32_t) * dictionary->num_items) + overhead;
+ } else if (magic == kSharedDictionaryMagic) {
+ const SharedEncoderDictionary* dictionary =
+ (const SharedEncoderDictionary*)prepared;
+ const CompoundDictionary* compound = &dictionary->compound;
+ const ContextualEncoderDictionary* contextual = &dictionary->contextual;
+ size_t result = sizeof(*dictionary);
+ size_t i;
+ size_t num_instances;
+ const BrotliEncoderDictionary* instances;
+ for (i = 0; i < compound->num_prepared_instances_; i++) {
+ size_t size = BrotliEncoderGetPreparedDictionarySize(
+ (const BrotliEncoderPreparedDictionary*)
+ compound->prepared_instances_[i]);
+ if (!size) return 0; /* error */
+ result += size;
+ }
+ if (contextual->context_based) {
+ num_instances = contextual->num_instances_;
+ instances = contextual->instances_;
+ result += sizeof(*instances) * num_instances;
+ } else {
+ num_instances = 1;
+ instances = &contextual->instance_;
+ }
+ for (i = 0; i < num_instances; i++) {
+ const BrotliEncoderDictionary* dict = &instances[i];
+ result += dict->trie.pool_capacity * sizeof(BrotliTrieNode);
+ if (dict->hash_table_data_words_) {
+ result += sizeof(kStaticDictionaryHashWords);
+ }
+ if (dict->hash_table_data_lengths_) {
+ result += sizeof(kStaticDictionaryHashLengths);
+ }
+ if (dict->buckets_data_) {
+ result += sizeof(*dict->buckets_data_) * dict->buckets_alloc_size_;
+ }
+ if (dict->dict_words_data_) {
+ result += sizeof(*dict->dict_words) * dict->dict_words_alloc_size_;
+ }
+ if (dict->words_instance_) {
+ result += sizeof(*dict->words_instance_);
+ /* data_size not added here: it is never allocated by the
+ SharedEncoderDictionary, instead it always points to the file
+ already loaded in memory. So if the caller wants to include
+ this memory as well, add the size of the loaded dictionary
+ file to this. */
+ }
+ }
+ return result + overhead;
+ }
+ return 0; /* error */
+}
+
+#if defined(BROTLI_TEST)
+size_t MakeUncompressedStreamForTest(const uint8_t*, size_t, uint8_t*);
+size_t MakeUncompressedStreamForTest(
+ const uint8_t* input, size_t input_size, uint8_t* output) {
+ return MakeUncompressedStream(input, input_size, output);
+}
+#endif
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/encoder_dict.c b/dviware/dvisvgm/libs/brotli/enc/encoder_dict.c
index c9e963b89d..6602c55a24 100644
--- a/dviware/dvisvgm/libs/brotli/enc/encoder_dict.c
+++ b/dviware/dvisvgm/libs/brotli/enc/encoder_dict.c
@@ -4,18 +4,45 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
-#include "./encoder_dict.h"
+#include "encoder_dict.h"
+
+#include <stdlib.h> /* malloc, free */
#include "../common/dictionary.h"
+#include "../common/platform.h"
+#include "../common/shared_dictionary_internal.h"
#include "../common/transform.h"
-#include "./dictionary_hash.h"
-#include "./hash.h"
+#include "compound_dictionary.h"
+#include "dictionary_hash.h"
+#include "memory.h"
+#include "quality.h"
+#include "hash.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
-void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
+#define NUM_HASH_BITS 15u
+#define NUM_HASH_BUCKETS (1u << NUM_HASH_BITS)
+
+static void BrotliTrieInit(BrotliTrie* trie) {
+ trie->pool_capacity = 0;
+ trie->pool_size = 0;
+ trie->pool = 0;
+
+ /* Set up the root node */
+ trie->root.single = 0;
+ trie->root.len_ = 0;
+ trie->root.idx_ = 0;
+ trie->root.sub = 0;
+}
+
+static void BrotliTrieFree(MemoryManager* m, BrotliTrie* trie) {
+ BrotliFree(m, trie->pool);
+}
+
+/* Initializes to RFC 7932 static dictionary / transforms. */
+static void InitEncoderDictionary(BrotliEncoderDictionary* dict) {
dict->words = BrotliGetDictionary();
dict->num_transforms = (uint32_t)BrotliGetTransforms()->num_transforms;
@@ -26,8 +53,588 @@ void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
dict->cutoffTransformsCount = kCutoffTransformsCount;
dict->cutoffTransforms = kCutoffTransforms;
+
+ dict->parent = 0;
+
+ dict->hash_table_data_words_ = 0;
+ dict->hash_table_data_lengths_ = 0;
+ dict->buckets_alloc_size_ = 0;
+ dict->buckets_data_ = 0;
+ dict->dict_words_alloc_size_ = 0;
+ dict->dict_words_data_ = 0;
+ dict->words_instance_ = 0;
+ dict->has_words_heavy = BROTLI_FALSE;
+ BrotliTrieInit(&dict->trie);
+}
+
+static void BrotliDestroyEncoderDictionary(MemoryManager* m,
+ BrotliEncoderDictionary* dict) {
+ BrotliFree(m, dict->hash_table_data_words_);
+ BrotliFree(m, dict->hash_table_data_lengths_);
+ BrotliFree(m, dict->buckets_data_);
+ BrotliFree(m, dict->dict_words_data_);
+ BrotliFree(m, dict->words_instance_);
+ BrotliTrieFree(m, &dict->trie);
+}
+
+#if defined(BROTLI_EXPERIMENTAL)
+/* Word length must be at least 4 bytes */
+static uint32_t Hash(const uint8_t* data, int bits) {
+ uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
+ /* The higher bits contain more mixture from the multiplication,
+ so we take our results from there. */
+ return h >> (32 - bits);
+}
+
+/* Theoretical max possible word size after transform */
+#define kTransformedBufferSize \
+ (256 + 256 + SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH)
+
+/* To be safe buffer must have at least kTransformedBufferSize */
+static void TransformedDictionaryWord(uint32_t word_idx, int len, int transform,
+ const BrotliTransforms* transforms,
+ const BrotliEncoderDictionary* dict,
+ uint8_t* buffer, size_t* size) {
+ const uint8_t* dict_word = &dict->words->data[
+ dict->words->offsets_by_length[len] + (uint32_t)len * word_idx];
+ *size = (size_t)BrotliTransformDictionaryWord(buffer, dict_word, len,
+ transforms, transform);
+}
+
+static DictWord MakeDictWord(uint8_t len, uint8_t transform, uint16_t idx) {
+ DictWord result;
+ result.len = len;
+ result.transform = transform;
+ result.idx = idx;
+ return result;
+}
+
+static uint32_t BrotliTrieAlloc(MemoryManager* m, size_t num, BrotliTrie* trie,
+ BrotliTrieNode** keep) {
+ uint32_t result;
+ uint32_t keep_index = 0;
+ if (keep && *keep != &trie->root) {
+ /* Optional node to keep, since address may change after re-allocating */
+ keep_index = (uint32_t)(*keep - trie->pool);
+ }
+ if (trie->pool_size == 0) {
+ /* Have a dummy node in the front. We do not want the result to be 0, it
+ must be at least 1, 0 represents "null pointer" */
+ trie->pool_size = 1;
+ }
+ BROTLI_ENSURE_CAPACITY(m, BrotliTrieNode, trie->pool, trie->pool_capacity,
+ trie->pool_size + num);
+ if (BROTLI_IS_OOM(m)) return 0;
+ /* Init the new nodes to empty */
+ memset(trie->pool + trie->pool_size, 0, sizeof(*trie->pool) * num);
+ result = (uint32_t)trie->pool_size;
+ trie->pool_size += num;
+ if (keep && *keep != &trie->root) {
+ *keep = trie->pool + keep_index;
+ }
+ return result;
+}
+
+/**
+ * len and idx: payload for last node
+ * word, size: the string
+ * index: position in the string
+ */
+static BROTLI_BOOL BrotliTrieNodeAdd(MemoryManager* m, uint8_t len,
+ uint32_t idx, const uint8_t* word, size_t size, int index,
+ BrotliTrieNode* node, BrotliTrie* trie) {
+ BrotliTrieNode* child = 0;
+ uint8_t c;
+ if ((size_t)index == size) {
+ if (!node->len_ || idx < node->idx_) {
+ node->len_ = len;
+ node->idx_ = idx;
+ }
+ return BROTLI_TRUE;
+ }
+ c = word[index];
+ if (node->single && c != node->c) {
+ BrotliTrieNode old = trie->pool[node->sub];
+ uint32_t new_nodes = BrotliTrieAlloc(m, 32, trie, &node);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ node->single = 0;
+ node->sub = new_nodes;
+ trie->pool[node->sub + (node->c >> 4)].sub = new_nodes + 16;
+ trie->pool[trie->pool[node->sub + (node->c >> 4)].sub + (node->c & 15)] =
+ old;
+ }
+ if (!node->sub) {
+ uint32_t new_node = BrotliTrieAlloc(m, 1, trie, &node);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ node->single = 1;
+ node->c = c;
+ node->sub = new_node;
+ }
+ if (node->single) {
+ child = &trie->pool[node->sub];
+ } else {
+ if (!trie->pool[node->sub + (c >> 4)].sub) {
+ uint32_t new_nodes = BrotliTrieAlloc(m, 16, trie, &node);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ trie->pool[node->sub + (c >> 4)].sub = new_nodes;
+ }
+ child = &trie->pool[trie->pool[node->sub + (c >> 4)].sub + (c & 15)];
+ }
+ return BrotliTrieNodeAdd(m, len, idx, word, size, index + 1, child, trie);
+}
+
+static BROTLI_BOOL BrotliTrieAdd(MemoryManager* m, uint8_t len, uint32_t idx,
+ const uint8_t* word, size_t size, BrotliTrie* trie) {
+ return BrotliTrieNodeAdd(m, len, idx, word, size, 0, &trie->root, trie);
+}
+
+const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie,
+ const BrotliTrieNode* node, uint8_t c) {
+ BrotliTrieNode* temp_node;
+ if (node->single) {
+ if (node->c == c) return &trie->pool[node->sub];
+ return 0;
+ }
+ if (!node->sub) return 0;
+ temp_node = &trie->pool[node->sub + (c >> 4)];
+ if (!temp_node->sub) return 0;
+ return &trie->pool[temp_node->sub + (c & 15)];
+}
+
+static const BrotliTrieNode* BrotliTrieFind(const BrotliTrie* trie,
+ const uint8_t* word, size_t size) {
+ const BrotliTrieNode* node = &trie->root;
+ size_t i;
+ for (i = 0; i < size; i++) {
+ node = BrotliTrieSub(trie, node, word[i]);
+ if (!node) return 0;
+ }
+ return node;
+}
+
+static BROTLI_BOOL BuildDictionaryLut(MemoryManager* m,
+ const BrotliTransforms* transforms,
+ BrotliEncoderDictionary* dict) {
+ uint32_t i;
+ DictWord* dict_words;
+ uint16_t* buckets;
+ DictWord** words_by_hash;
+ size_t* words_by_hash_size;
+ size_t* words_by_hash_capacity;
+ BrotliTrie dedup;
+ uint8_t word[kTransformedBufferSize];
+ size_t word_size;
+ size_t total = 0;
+ uint8_t l;
+ uint16_t idx;
+
+ BrotliTrieInit(&dedup);
+
+ words_by_hash = (DictWord**)BrotliAllocate(m,
+ sizeof(*words_by_hash) * NUM_HASH_BUCKETS);
+ words_by_hash_size = (size_t*)BrotliAllocate(m,
+ sizeof(*words_by_hash_size) * NUM_HASH_BUCKETS);
+ words_by_hash_capacity = (size_t*)BrotliAllocate(m,
+ sizeof(*words_by_hash_capacity) * NUM_HASH_BUCKETS);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ memset(words_by_hash, 0, sizeof(*words_by_hash) * NUM_HASH_BUCKETS);
+ memset(words_by_hash_size, 0, sizeof(*words_by_hash_size) * NUM_HASH_BUCKETS);
+ memset(words_by_hash_capacity, 0,
+ sizeof(*words_by_hash_capacity) * NUM_HASH_BUCKETS);
+
+ if (transforms->num_transforms > 0) {
+ for (l = SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH;
+ l <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; ++l) {
+ uint16_t n = dict->words->size_bits_by_length[l] ?
+ (uint16_t)(1 << dict->words->size_bits_by_length[l]) : 0u;
+ for (idx = 0; idx < n; ++idx) {
+ uint32_t key;
+ /* First transform (usually identity) */
+ TransformedDictionaryWord(idx, l, 0, transforms, dict, word,
+ &word_size);
+ /* Cannot hash words smaller than 4 bytes */
+ if (word_size < 4) {
+ /* Break instead of continue, all next words of this length will have
+ same length after transform */
+ break;
+ }
+ if (!BrotliTrieAdd(m, 0, idx, word, word_size, &dedup)) {
+ return BROTLI_FALSE;
+ }
+ key = Hash(word, NUM_HASH_BITS);
+ BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
+ words_by_hash_capacity[key], words_by_hash_size[key],
+ MakeDictWord(l, 0, idx));
+ ++total;
+ }
+ }
+ }
+
+ /* These LUT transforms only supported if no custom transforms. This is
+ ok, we will use the heavy trie instead. */
+ if (transforms == BrotliGetTransforms()) {
+ for (l = SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH;
+ l <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; ++l) {
+ uint16_t n = dict->words->size_bits_by_length[l] ?
+ (uint16_t)(1 << dict->words->size_bits_by_length[l]) : 0u;
+ for (idx = 0; idx < n; ++idx) {
+ int k;
+ BROTLI_BOOL is_ascii = BROTLI_TRUE;
+ size_t offset = dict->words->offsets_by_length[l] + (size_t)l * idx;
+ const uint8_t* data = &dict->words->data[offset];
+ for (k = 0; k < l; ++k) {
+ if (data[k] >= 128) is_ascii = BROTLI_FALSE;
+ }
+ if (data[0] < 128) {
+ int transform = 9; /* {empty, uppercase first, empty} */
+ uint32_t ix = idx + (uint32_t)transform * n;
+ const BrotliTrieNode* it;
+ TransformedDictionaryWord(idx, l, transform, transforms,
+ dict, word, &word_size);
+ it = BrotliTrieFind(&dedup, word, word_size);
+ if (!it || it->idx_ > ix) {
+ uint32_t key = Hash(word, NUM_HASH_BITS);
+ if (!BrotliTrieAdd(m, 0, ix, word, word_size, &dedup)) {
+ return BROTLI_FALSE;
+ }
+ BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
+ words_by_hash_capacity[key], words_by_hash_size[key],
+ MakeDictWord(l, BROTLI_TRANSFORM_UPPERCASE_FIRST, idx));
+ ++total;
+ }
+ }
+ if (is_ascii) {
+ int transform = 44; /* {empty, uppercase all, empty} */
+ uint32_t ix = idx + (uint32_t)transform * n;
+ const BrotliTrieNode* it;
+ TransformedDictionaryWord(idx, l, transform, transforms,
+ dict, word, &word_size);
+ it = BrotliTrieFind(&dedup, word, word_size);
+ if (!it || it->idx_ > ix) {
+ uint32_t key = Hash(word, NUM_HASH_BITS);
+ if (!BrotliTrieAdd(m, 0, ix, word, word_size, &dedup)) {
+ return BROTLI_FALSE;
+ }
+ BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
+ words_by_hash_capacity[key], words_by_hash_size[key],
+ MakeDictWord(l, BROTLI_TRANSFORM_UPPERCASE_ALL, idx));
+ ++total;
+ }
+ }
+ }
+ }
+ }
+
+ dict_words = (DictWord*)BrotliAllocate(m,
+ sizeof(*dict->dict_words) * (total + 1));
+ buckets = (uint16_t*)BrotliAllocate(m,
+ sizeof(*dict->buckets) * NUM_HASH_BUCKETS);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ dict->dict_words_alloc_size_ = total + 1;
+ dict->dict_words = dict->dict_words_data_ = dict_words;
+ dict->buckets_alloc_size_ = NUM_HASH_BUCKETS;
+ dict->buckets = dict->buckets_data_ = buckets;
+
+ /* Unused; makes offsets start from 1. */
+ dict_words[0] = MakeDictWord(0, 0, 0);
+ total = 1;
+ for (i = 0; i < NUM_HASH_BUCKETS; ++i) {
+ size_t num_words = words_by_hash_size[i];
+ if (num_words > 0) {
+ buckets[i] = (uint16_t)(total);
+ memcpy(&dict_words[total], &words_by_hash[i][0],
+ sizeof(dict_words[0]) * num_words);
+ total += num_words;
+ dict_words[total - 1].len |= 0x80;
+ } else {
+ buckets[i] = 0;
+ }
+ }
+
+ for (i = 0; i < NUM_HASH_BUCKETS; ++i) {
+ BrotliFree(m, words_by_hash[i]);
+ }
+ BrotliFree(m, words_by_hash);
+ BrotliFree(m, words_by_hash_size);
+ BrotliFree(m, words_by_hash_capacity);
+ BrotliTrieFree(m, &dedup);
+
+ return BROTLI_TRUE;
+}
+
+static void BuildDictionaryHashTable(uint16_t* hash_table_words,
+ uint8_t* hash_table_lengths, const BrotliDictionary* dict) {
+ int j, len;
+ /* The order of the loops is such that in case of collision, words with
+ shorter length are preferred, and in case of same length, words with
+ smaller index. There is only a single word per bucket. */
+ /* TODO(lode): consider adding optional user-supplied frequency_map to use
+ for preferred words instead, this can make the encoder better for
+ quality 9 and below without affecting the decoder */
+ memset(hash_table_words, 0, sizeof(kStaticDictionaryHashWords));
+ memset(hash_table_lengths, 0, sizeof(kStaticDictionaryHashLengths));
+ for (len = SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH;
+ len >= SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH; --len) {
+ const size_t num_words = dict->size_bits_by_length[len] ?
+ (1u << dict->size_bits_by_length[len]) : 0;
+ for (j = (int)num_words - 1; j >= 0; --j) {
+ size_t offset = dict->offsets_by_length[len] +
+ (size_t)len * (size_t)j;
+ const uint8_t* word = &dict->data[offset];
+ const uint32_t key = Hash(word, 14);
+ int idx = (int)(key << 1) + (len < 8 ? 1 : 0);
+ BROTLI_DCHECK(idx < (int)NUM_HASH_BUCKETS);
+ hash_table_words[idx] = (uint16_t)j;
+ hash_table_lengths[idx] = (uint8_t)len;
+ }
+ }
}
+static BROTLI_BOOL GenerateWordsHeavy(MemoryManager* m,
+ const BrotliTransforms* transforms,
+ BrotliEncoderDictionary* dict) {
+ int i, j, l;
+ for (j = (int)transforms->num_transforms - 1; j >= 0 ; --j) {
+ for (l = 0; l < 32; l++) {
+ int num = (int)((1u << dict->words->size_bits_by_length[l]) & ~1u);
+ for (i = 0; i < num; i++) {
+ uint8_t transformed[kTransformedBufferSize];
+ size_t size;
+ TransformedDictionaryWord(
+ (uint32_t)i, l, j, transforms, dict, transformed, &size);
+ if (size < 4) continue;
+ if (!BrotliTrieAdd(m, (uint8_t)l, (uint32_t)(i + num * j),
+ transformed, size, &dict->trie)) {
+ return BROTLI_FALSE;
+ }
+ }
+ }
+ }
+ return BROTLI_TRUE;
+}
+
+/* Computes cutoffTransformsCount (in count) and cutoffTransforms (in data) for
+ the custom transforms, where possible within the limits of the
+ cutoffTransforms encoding. The fast encoder uses this to do fast lookup for
+ transforms that remove the N last characters (OmitLast). */
+static void ComputeCutoffTransforms(
+ const BrotliTransforms* transforms,
+ uint32_t* count, uint64_t* data) {
+ int i;
+ /* The encoding in a 64-bit integer of transform N in the data is: (N << 2) +
+ ((cutoffTransforms >> (N * 6)) & 0x3F), so for example the identity
+ transform code must be 0-63, for N=1 the transform code must be 4-67, ...,
+ for N=9 it must be 36-99.
+ TODO(lode): consider a simple flexible uint8_t[10] instead of the uint64_t
+ for the cutoff transforms, so that shared dictionaries can have the
+ OmitLast transforms anywhere without loss. */
+ *count = 0;
+ *data = 0;
+ for (i = 0; i < BROTLI_TRANSFORMS_MAX_CUT_OFF + 1; i++) {
+ int idx = transforms->cutOffTransforms[i];
+ if (idx == -1) break; /* Not found */
+ if (idx < (i << 2)) break; /* Too small for the encoding */
+ if (idx >= (i << 2) + 64) break; /* Too large for the encoding */
+ (*count)++;
+ *data |= (uint64_t)(((uint64_t)idx -
+ ((uint64_t)i << 2u)) << ((uint64_t)i * 6u));
+ }
+}
+
+static BROTLI_BOOL ComputeDictionary(MemoryManager* m, int quality,
+ const BrotliTransforms* transforms,
+ BrotliEncoderDictionary* current) {
+ int default_words = current->words == BrotliGetDictionary();
+ int default_transforms = transforms == BrotliGetTransforms();
+
+ if (default_words && default_transforms) {
+ /* hashes are already set to Brotli defaults */
+ return BROTLI_TRUE;
+ }
+
+ current->hash_table_data_words_ = (uint16_t*)BrotliAllocate(
+ m, sizeof(kStaticDictionaryHashWords));
+ current->hash_table_data_lengths_ = (uint8_t*)BrotliAllocate(
+ m, sizeof(kStaticDictionaryHashLengths));
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ current->hash_table_words = current->hash_table_data_words_;
+ current->hash_table_lengths = current->hash_table_data_lengths_;
+
+ BuildDictionaryHashTable(current->hash_table_data_words_,
+ current->hash_table_data_lengths_, current->words);
+
+ ComputeCutoffTransforms(transforms,
+ &current->cutoffTransformsCount, &current->cutoffTransforms);
+
+ /* Only compute the data for slow encoder if the requested quality is high
+ enough to need it */
+ if (quality >= ZOPFLIFICATION_QUALITY) {
+ if (!BuildDictionaryLut(m, transforms, current)) return BROTLI_FALSE;
+
+ /* For the built-in Brotli transforms, there is a hard-coded function to
+ handle all transforms, but for custom transforms, we use the following
+ large hammer instead */
+ current->has_words_heavy = !default_transforms;
+ if (current->has_words_heavy) {
+ if (!GenerateWordsHeavy(m, transforms, current)) return BROTLI_FALSE;
+ }
+ }
+
+ return BROTLI_TRUE;
+}
+#endif /* BROTLI_EXPERIMENTAL */
+
+void BrotliInitSharedEncoderDictionary(SharedEncoderDictionary* dict) {
+ dict->magic = kSharedDictionaryMagic;
+
+ dict->compound.num_chunks = 0;
+ dict->compound.total_size = 0;
+ dict->compound.chunk_offsets[0] = 0;
+ dict->compound.num_prepared_instances_ = 0;
+
+ dict->contextual.context_based = 0;
+ dict->contextual.num_dictionaries = 1;
+ dict->contextual.instances_ = 0;
+ dict->contextual.num_instances_ = 1; /* The instance_ field */
+ dict->contextual.dict[0] = &dict->contextual.instance_;
+ InitEncoderDictionary(&dict->contextual.instance_);
+ dict->contextual.instance_.parent = &dict->contextual;
+
+ dict->max_quality = BROTLI_MAX_QUALITY;
+}
+
+#if defined(BROTLI_EXPERIMENTAL)
+/* TODO(eustas): make sure that tooling will warn user if not all the cutoff
+ transforms are available (for low-quality encoder). */
+static BROTLI_BOOL InitCustomSharedEncoderDictionary(
+ MemoryManager* m, const BrotliSharedDictionary* decoded_dict,
+ int quality, SharedEncoderDictionary* dict) {
+ ContextualEncoderDictionary* contextual;
+ CompoundDictionary* compound;
+ BrotliEncoderDictionary* instances;
+ int i;
+ BrotliInitSharedEncoderDictionary(dict);
+
+ contextual = &dict->contextual;
+ compound = &dict->compound;
+
+ for (i = 0; i < (int)decoded_dict->num_prefix; i++) {
+ PreparedDictionary* prepared = CreatePreparedDictionary(m,
+ decoded_dict->prefix[i], decoded_dict->prefix_size[i]);
+ AttachPreparedDictionary(compound, prepared);
+ /* remember for cleanup */
+ compound->prepared_instances_[
+ compound->num_prepared_instances_++] = prepared;
+ }
+
+ dict->max_quality = quality;
+ contextual->context_based = decoded_dict->context_based;
+ if (decoded_dict->context_based) {
+ memcpy(contextual->context_map, decoded_dict->context_map,
+ SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS);
+ }
+
+ contextual->num_dictionaries = decoded_dict->num_dictionaries;
+ contextual->num_instances_ = decoded_dict->num_dictionaries;
+ if (contextual->num_instances_ == 1) {
+ instances = &contextual->instance_;
+ } else {
+ contextual->instances_ = (BrotliEncoderDictionary*)
+ BrotliAllocate(m, sizeof(*contextual->instances_) *
+ contextual->num_instances_);
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ instances = contextual->instances_;
+ }
+ for (i = 0; i < (int)contextual->num_instances_; i++) {
+ BrotliEncoderDictionary* current = &instances[i];
+ InitEncoderDictionary(current);
+ current->parent = &dict->contextual;
+ if (decoded_dict->words[i] == BrotliGetDictionary()) {
+ current->words = BrotliGetDictionary();
+ } else {
+ current->words_instance_ = (BrotliDictionary*)BrotliAllocate(
+ m, sizeof(BrotliDictionary));
+ if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+ *current->words_instance_ = *decoded_dict->words[i];
+ current->words = current->words_instance_;
+ }
+ current->num_transforms =
+ (uint32_t)decoded_dict->transforms[i]->num_transforms;
+ if (!ComputeDictionary(
+ m, quality, decoded_dict->transforms[i], current)) {
+ return BROTLI_FALSE;
+ }
+
+ contextual->dict[i] = current;
+ }
+
+ return BROTLI_TRUE; /* success */
+}
+
+BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary(
+ MemoryManager* m, const uint8_t* encoded_dict, size_t size,
+ int quality, SharedEncoderDictionary* dict) {
+ BROTLI_BOOL success = BROTLI_FALSE;
+ BrotliSharedDictionary* decoded_dict = BrotliSharedDictionaryCreateInstance(
+ m->alloc_func, m->free_func, m->opaque);
+ if (!decoded_dict) { /* OOM */
+ return BROTLI_FALSE;
+ }
+ success = BrotliSharedDictionaryAttach(
+ decoded_dict, BROTLI_SHARED_DICTIONARY_SERIALIZED, size, encoded_dict);
+ if (success) {
+ success = InitCustomSharedEncoderDictionary(m,
+ decoded_dict, quality, dict);
+ }
+ BrotliSharedDictionaryDestroyInstance(decoded_dict);
+ return success;
+}
+#endif /* BROTLI_EXPERIMENTAL */
+
+void BrotliCleanupSharedEncoderDictionary(MemoryManager* m,
+ SharedEncoderDictionary* dict) {
+ size_t i;
+ for (i = 0; i < dict->compound.num_prepared_instances_; i++) {
+ DestroyPreparedDictionary(m,
+ (PreparedDictionary*)dict->compound.prepared_instances_[i]);
+ }
+ if (dict->contextual.num_instances_ == 1) {
+ BrotliDestroyEncoderDictionary(m, &dict->contextual.instance_);
+ } else if (dict->contextual.num_instances_ > 1) {
+ for (i = 0; i < dict->contextual.num_instances_; i++) {
+ BrotliDestroyEncoderDictionary(m, &dict->contextual.instances_[i]);
+ }
+ BrotliFree(m, dict->contextual.instances_);
+ }
+}
+
+ManagedDictionary* BrotliCreateManagedDictionary(
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
+ ManagedDictionary* result = (ManagedDictionary*)BrotliBootstrapAlloc(
+ sizeof(ManagedDictionary), alloc_func, free_func, opaque);
+ if (result == NULL) return NULL;
+
+ result->magic = kManagedDictionaryMagic;
+ BrotliInitMemoryManager(
+ &result->memory_manager_, alloc_func, free_func, opaque);
+ result->dictionary = NULL;
+
+ return result;
+}
+
+void BrotliDestroyManagedDictionary(ManagedDictionary* dictionary) {
+ if (!dictionary) return;
+ BrotliBootstrapFree(dictionary, &dictionary->memory_manager_);
+}
+
+/* Escalate internal functions visibility; for testing purposes only. */
+#if defined(BROTLI_TEST)
+void InitEncoderDictionaryForTest(BrotliEncoderDictionary*);
+void InitEncoderDictionaryForTest(BrotliEncoderDictionary* d) {
+ InitEncoderDictionary(d);
+}
+#endif
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/encoder_dict.h b/dviware/dvisvgm/libs/brotli/enc/encoder_dict.h
index a1c329fbf4..27dcbcd2f3 100644
--- a/dviware/dvisvgm/libs/brotli/enc/encoder_dict.h
+++ b/dviware/dvisvgm/libs/brotli/enc/encoder_dict.h
@@ -7,15 +7,56 @@
#ifndef BROTLI_ENC_ENCODER_DICT_H_
#define BROTLI_ENC_ENCODER_DICT_H_
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
+
#include "../common/dictionary.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./static_dict_lut.h"
+#include "compound_dictionary.h"
+#include "memory.h"
+#include "static_dict_lut.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
+/*
+Dictionary hierarchy for Encoder:
+-SharedEncoderDictionary
+--CompoundDictionary
+---PreparedDictionary [up to 15x]
+ = prefix dictionary with precomputed hashes
+--ContextualEncoderDictionary
+---BrotliEncoderDictionary [up to 64x]
+ = for each context, precomputed static dictionary with words + transforms
+
+Dictionary hiearchy from common: similar, but without precomputed hashes
+-BrotliSharedDictionary
+--BrotliDictionary [up to 64x]
+--BrotliTransforms [up to 64x]
+--const uint8_t* prefix [up to 15x]: compound dictionaries
+*/
+
+typedef struct BrotliTrieNode {
+ uint8_t single; /* if 1, sub is a single node for c instead of 256 */
+ uint8_t c;
+ uint8_t len_; /* untransformed length */
+ uint32_t idx_; /* word index + num words * transform index */
+ uint32_t sub; /* index of sub node(s) in the pool */
+} BrotliTrieNode;
+
+typedef struct BrotliTrie {
+ BrotliTrieNode* pool;
+ size_t pool_capacity;
+ size_t pool_size;
+ BrotliTrieNode root;
+} BrotliTrie;
+
+#if defined(BROTLI_EXPERIMENTAL)
+BROTLI_INTERNAL const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie,
+ const BrotliTrieNode* node, uint8_t c);
+#endif /* BROTLI_EXPERIMENTAL */
+
/* Dictionary data (words and transforms) for 1 possible context */
typedef struct BrotliEncoderDictionary {
const BrotliDictionary* words;
@@ -32,9 +73,82 @@ typedef struct BrotliEncoderDictionary {
/* from static_dict_lut.h, for slow encoder */
const uint16_t* buckets;
const DictWord* dict_words;
+ /* Heavy version, for use by slow encoder when there are custom transforms.
+ Contains every possible transformed dictionary word in a trie. It encodes
+ about as fast as the non-heavy encoder but consumes a lot of memory and
+ takes time to build. */
+ BrotliTrie trie;
+ BROTLI_BOOL has_words_heavy;
+
+ /* Reference to other dictionaries. */
+ const struct ContextualEncoderDictionary* parent;
+
+ /* Allocated memory, used only when not using the Brotli defaults */
+ uint16_t* hash_table_data_words_;
+ uint8_t* hash_table_data_lengths_;
+ size_t buckets_alloc_size_;
+ uint16_t* buckets_data_;
+ size_t dict_words_alloc_size_;
+ DictWord* dict_words_data_;
+ BrotliDictionary* words_instance_;
} BrotliEncoderDictionary;
-BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict);
+/* Dictionary data for all 64 contexts */
+typedef struct ContextualEncoderDictionary {
+ BROTLI_BOOL context_based;
+ uint8_t num_dictionaries;
+ uint8_t context_map[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
+ const BrotliEncoderDictionary* dict[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
+
+ /* If num_instances_ is 1, instance_ is used, else dynamic allocation with
+ instances_ is used. */
+ size_t num_instances_;
+ BrotliEncoderDictionary instance_;
+ BrotliEncoderDictionary* instances_;
+} ContextualEncoderDictionary;
+
+typedef struct SharedEncoderDictionary {
+ /* Magic value to distinguish this struct from PreparedDictionary for
+ certain external usages. */
+ uint32_t magic;
+
+ /* LZ77 prefix, compound dictionary */
+ CompoundDictionary compound;
+
+ /* Custom static dictionary (optionally context-based) */
+ ContextualEncoderDictionary contextual;
+
+ /* The maximum quality the dictionary was computed for */
+ int max_quality;
+} SharedEncoderDictionary;
+
+typedef struct ManagedDictionary {
+ uint32_t magic;
+ MemoryManager memory_manager_;
+ uint32_t* dictionary;
+} ManagedDictionary;
+
+/* Initializes to the brotli built-in dictionary */
+BROTLI_INTERNAL void BrotliInitSharedEncoderDictionary(
+ SharedEncoderDictionary* dict);
+
+#if defined(BROTLI_EXPERIMENTAL)
+/* Initializes to shared dictionary that will be parsed from
+ encoded_dict. Requires that you keep the encoded_dict buffer
+ around, parts of data will point to it. */
+BROTLI_INTERNAL BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary(
+ MemoryManager* m, const uint8_t* encoded_dict, size_t size,
+ int quality, SharedEncoderDictionary* dict);
+#endif /* BROTLI_EXPERIMENTAL */
+
+BROTLI_INTERNAL void BrotliCleanupSharedEncoderDictionary(
+ MemoryManager* m, SharedEncoderDictionary* dict);
+
+BROTLI_INTERNAL ManagedDictionary* BrotliCreateManagedDictionary(
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
+
+BROTLI_INTERNAL void BrotliDestroyManagedDictionary(
+ ManagedDictionary* dictionary);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
diff --git a/dviware/dvisvgm/libs/brotli/enc/entropy_encode.c b/dviware/dvisvgm/libs/brotli/enc/entropy_encode.c
index b50ccb5d1f..9aed43b6eb 100644
--- a/dviware/dvisvgm/libs/brotli/enc/entropy_encode.c
+++ b/dviware/dvisvgm/libs/brotli/enc/entropy_encode.c
@@ -6,13 +6,14 @@
/* Entropy encoding (Huffman) utilities. */
-#include "./entropy_encode.h"
+#include "entropy_encode.h"
#include <string.h> /* memset */
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/platform.h"
-#include <brotli/types.h>
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/entropy_encode.h b/dviware/dvisvgm/libs/brotli/enc/entropy_encode.h
index 9618e1d359..e1c779cc6f 100644
--- a/dviware/dvisvgm/libs/brotli/enc/entropy_encode.h
+++ b/dviware/dvisvgm/libs/brotli/enc/entropy_encode.h
@@ -9,9 +9,10 @@
#ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
#define BROTLI_ENC_ENTROPY_ENCODE_H_
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/entropy_encode_static.h b/dviware/dvisvgm/libs/brotli/enc/entropy_encode_static.h
index 62b99a954c..ecff1fe9ee 100644
--- a/dviware/dvisvgm/libs/brotli/enc/entropy_encode_static.h
+++ b/dviware/dvisvgm/libs/brotli/enc/entropy_encode_static.h
@@ -9,10 +9,11 @@
#ifndef BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
#define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./write_bits.h"
+#include "write_bits.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -76,6 +77,7 @@ static const uint8_t kStaticDistanceCodeDepth[64] = {
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
};
+/* GENERATED CODE START */
static const uint32_t kCodeLengthBits[18] = {
0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 15, 31, 0, 11, 7,
};
@@ -531,6 +533,7 @@ static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
size_t* storage_ix, uint8_t* storage) {
BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
}
+/* GENERATED CODE END */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
diff --git a/dviware/dvisvgm/libs/brotli/enc/fast_log.c b/dviware/dvisvgm/libs/brotli/enc/fast_log.c
index 2319baeb74..2fa0efcf86 100644
--- a/dviware/dvisvgm/libs/brotli/enc/fast_log.c
+++ b/dviware/dvisvgm/libs/brotli/enc/fast_log.c
@@ -4,7 +4,7 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
-#include "./fast_log.h"
+#include "fast_log.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/fast_log.h b/dviware/dvisvgm/libs/brotli/enc/fast_log.h
index 2094f13e55..f82f4cffc8 100644
--- a/dviware/dvisvgm/libs/brotli/enc/fast_log.h
+++ b/dviware/dvisvgm/libs/brotli/enc/fast_log.h
@@ -11,9 +11,10 @@
#include <math.h>
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/find_match_length.h b/dviware/dvisvgm/libs/brotli/enc/find_match_length.h
index f8853a70fb..f3de0bdb6b 100644
--- a/dviware/dvisvgm/libs/brotli/enc/find_match_length.h
+++ b/dviware/dvisvgm/libs/brotli/enc/find_match_length.h
@@ -9,9 +9,10 @@
#ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
#define BROTLI_ENC_FIND_MATCH_LENGTH_H_
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
@@ -21,31 +22,23 @@ extern "C" {
static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
const uint8_t* s2,
size_t limit) {
- size_t matched = 0;
- size_t limit2 = (limit >> 3) + 1; /* + 1 is for pre-decrement in while */
- while (BROTLI_PREDICT_TRUE(--limit2)) {
- if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) ==
- BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) {
- s2 += 8;
- matched += 8;
- } else {
- uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
- BROTLI_UNALIGNED_LOAD64LE(s1 + matched);
+ const uint8_t *s1_orig = s1;
+ for (; limit >= 8; limit -= 8) {
+ uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
+ BROTLI_UNALIGNED_LOAD64LE(s1);
+ s2 += 8;
+ if (x != 0) {
size_t matching_bits = (size_t)BROTLI_TZCNT64(x);
- matched += matching_bits >> 3;
- return matched;
+ return (size_t)(s1 - s1_orig) + (matching_bits >> 3);
}
+ s1 += 8;
}
- limit = (limit & 7) + 1; /* + 1 is for pre-decrement in while */
- while (--limit) {
- if (BROTLI_PREDICT_TRUE(s1[matched] == *s2)) {
- ++s2;
- ++matched;
- } else {
- return matched;
- }
+ while (limit && *s1 == *s2) {
+ limit--;
+ ++s2;
+ ++s1;
}
- return matched;
+ return (size_t)(s1 - s1_orig);
}
#else
static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash.h b/dviware/dvisvgm/libs/brotli/enc/hash.h
index 6362f69b9f..5677d82920 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash.h
@@ -10,33 +10,49 @@
#ifndef BROTLI_ENC_HASH_H_
#define BROTLI_ENC_HASH_H_
+#include <stdlib.h> /* exit */
#include <string.h> /* memcmp, memset */
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./encoder_dict.h"
-#include "./fast_log.h"
-#include "./find_match_length.h"
-#include "./memory.h"
-#include "./quality.h"
-#include "./static_dict.h"
+#include "compound_dictionary.h"
+#include "encoder_dict.h"
+#include "fast_log.h"
+#include "find_match_length.h"
+#include "memory.h"
+#include "quality.h"
+#include "static_dict.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct {
- /* Dynamically allocated area; first member for quickest access. */
- void* extra;
+ /**
+ * Dynamically allocated areas; regular hasher uses one or two allocations;
+ * "composite" hasher uses up to 4 allocations.
+ */
+ void* extra[4];
+
+ /**
+ * False before the fisrt invocation of HasherSetup (where "extra" memory)
+ * is allocated.
+ */
+ BROTLI_BOOL is_setup_;
size_t dict_num_lookups;
size_t dict_num_matches;
BrotliHasherParams params;
- /* False if hasher needs to be "prepared" before use. */
+ /**
+ * False if hasher needs to be "prepared" before use (before the first
+ * invocation of HasherSetup or after HasherReset). "preparation" is hasher
+ * data initialization (using input ringbuffer).
+ */
BROTLI_BOOL is_prepared_;
} HasherCommon;
@@ -62,8 +78,7 @@ typedef struct HasherSearchResult {
for this use.
* The number has been tuned heuristically against compression benchmarks. */
static const uint32_t kHashMul32 = 0x1E35A7BD;
-static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1E35A7BD, 0x1E35A7BD);
-static const uint64_t kHashMul64Long =
+static const uint64_t kHashMul64 =
BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
@@ -232,7 +247,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define BUCKET_BITS 17
#define MAX_TREE_SEARCH_DEPTH 64
#define MAX_TREE_COMP_LENGTH 128
-#include "./hash_to_binary_tree_inc.h" /* NOLINT(build/include) */
+#include "hash_to_binary_tree_inc.h" /* NOLINT(build/include) */
#undef MAX_TREE_SEARCH_DEPTH
#undef MAX_TREE_COMP_LENGTH
#undef BUCKET_BITS
@@ -249,7 +264,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define BUCKET_SWEEP_BITS 0
#define HASH_LEN 5
#define USE_DICTIONARY 1
-#include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
+#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef BUCKET_SWEEP_BITS
#undef USE_DICTIONARY
#undef HASHER
@@ -257,7 +272,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define HASHER() H3
#define BUCKET_SWEEP_BITS 1
#define USE_DICTIONARY 0
-#include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
+#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef USE_DICTIONARY
#undef BUCKET_SWEEP_BITS
#undef BUCKET_BITS
@@ -267,7 +282,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define BUCKET_BITS 17
#define BUCKET_SWEEP_BITS 2
#define USE_DICTIONARY 1
-#include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
+#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef USE_DICTIONARY
#undef HASH_LEN
#undef BUCKET_SWEEP_BITS
@@ -275,11 +290,11 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#undef HASHER
#define HASHER() H5
-#include "./hash_longest_match_inc.h" /* NOLINT(build/include) */
+#include "hash_longest_match_inc.h" /* NOLINT(build/include) */
#undef HASHER
#define HASHER() H6
-#include "./hash_longest_match64_inc.h" /* NOLINT(build/include) */
+#include "hash_longest_match64_inc.h" /* NOLINT(build/include) */
#undef HASHER
#define BUCKET_BITS 15
@@ -288,13 +303,13 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define NUM_BANKS 1
#define BANK_BITS 16
#define HASHER() H40
-#include "./hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
+#include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
#undef HASHER
#undef NUM_LAST_DISTANCES_TO_CHECK
#define NUM_LAST_DISTANCES_TO_CHECK 10
#define HASHER() H41
-#include "./hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
+#include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
#undef HASHER
#undef NUM_LAST_DISTANCES_TO_CHECK
#undef NUM_BANKS
@@ -304,7 +319,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define NUM_BANKS 512
#define BANK_BITS 9
#define HASHER() H42
-#include "./hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
+#include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
#undef HASHER
#undef NUM_LAST_DISTANCES_TO_CHECK
#undef NUM_BANKS
@@ -317,7 +332,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define BUCKET_SWEEP_BITS 2
#define HASH_LEN 7
#define USE_DICTIONARY 0
-#include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
+#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef USE_DICTIONARY
#undef HASH_LEN
#undef BUCKET_SWEEP_BITS
@@ -331,14 +346,14 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define JUMP 4
#define NUMBUCKETS 16777216
#define MASK ((NUMBUCKETS * 64) - 1)
-#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
+#include "hash_rolling_inc.h" /* NOLINT(build/include) */
#undef JUMP
#undef HASHER
#define HASHER() HROLLING
#define JUMP 1
-#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
+#include "hash_rolling_inc.h" /* NOLINT(build/include) */
#undef MASK
#undef NUMBUCKETS
#undef JUMP
@@ -348,7 +363,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define HASHER() H35
#define HASHER_A H3
#define HASHER_B HROLLING_FAST
-#include "./hash_composite_inc.h" /* NOLINT(build/include) */
+#include "hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
@@ -356,7 +371,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define HASHER() H55
#define HASHER_A H54
#define HASHER_B HROLLING_FAST
-#include "./hash_composite_inc.h" /* NOLINT(build/include) */
+#include "hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
@@ -364,7 +379,7 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#define HASHER() H65
#define HASHER_A H6
#define HASHER_B HROLLING
-#include "./hash_composite_inc.h" /* NOLINT(build/include) */
+#include "hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
@@ -391,43 +406,55 @@ typedef struct {
/* MUST be invoked before any other method. */
static BROTLI_INLINE void HasherInit(Hasher* hasher) {
- hasher->common.extra = NULL;
+ hasher->common.is_setup_ = BROTLI_FALSE;
+ hasher->common.extra[0] = NULL;
+ hasher->common.extra[1] = NULL;
+ hasher->common.extra[2] = NULL;
+ hasher->common.extra[3] = NULL;
}
static BROTLI_INLINE void DestroyHasher(MemoryManager* m, Hasher* hasher) {
- if (hasher->common.extra == NULL) return;
- BROTLI_FREE(m, hasher->common.extra);
+ if (hasher->common.extra[0] != NULL) BROTLI_FREE(m, hasher->common.extra[0]);
+ if (hasher->common.extra[1] != NULL) BROTLI_FREE(m, hasher->common.extra[1]);
+ if (hasher->common.extra[2] != NULL) BROTLI_FREE(m, hasher->common.extra[2]);
+ if (hasher->common.extra[3] != NULL) BROTLI_FREE(m, hasher->common.extra[3]);
}
static BROTLI_INLINE void HasherReset(Hasher* hasher) {
hasher->common.is_prepared_ = BROTLI_FALSE;
}
-static BROTLI_INLINE size_t HasherSize(const BrotliEncoderParams* params,
- BROTLI_BOOL one_shot, const size_t input_size) {
+static BROTLI_INLINE void HasherSize(const BrotliEncoderParams* params,
+ BROTLI_BOOL one_shot, const size_t input_size, size_t* alloc_size) {
switch (params->hasher.type) {
-#define SIZE_(N) \
- case N: \
- return HashMemAllocInBytesH ## N(params, one_shot, input_size);
+#define SIZE_(N) \
+ case N: \
+ HashMemAllocInBytesH ## N(params, one_shot, input_size, alloc_size); \
+ break;
FOR_ALL_HASHERS(SIZE_)
#undef SIZE_
default:
break;
}
- return 0; /* Default case. */
}
static BROTLI_INLINE void HasherSetup(MemoryManager* m, Hasher* hasher,
BrotliEncoderParams* params, const uint8_t* data, size_t position,
size_t input_size, BROTLI_BOOL is_last) {
BROTLI_BOOL one_shot = (position == 0 && is_last);
- if (hasher->common.extra == NULL) {
- size_t alloc_size;
+ if (!hasher->common.is_setup_) {
+ size_t alloc_size[4] = {0};
+ size_t i;
ChooseHasher(params, &params->hasher);
- alloc_size = HasherSize(params, one_shot, input_size);
- hasher->common.extra = BROTLI_ALLOC(m, uint8_t, alloc_size);
- if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(hasher->common.extra)) return;
hasher->common.params = params->hasher;
+ hasher->common.dict_num_lookups = 0;
+ hasher->common.dict_num_matches = 0;
+ HasherSize(params, one_shot, input_size, alloc_size);
+ for (i = 0; i < 4; ++i) {
+ if (alloc_size[i] == 0) continue;
+ hasher->common.extra[i] = BROTLI_ALLOC(m, uint8_t, alloc_size[i]);
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(hasher->common.extra[i])) return;
+ }
switch (hasher->common.params.type) {
#define INITIALIZE_(N) \
case N: \
@@ -440,6 +467,7 @@ static BROTLI_INLINE void HasherSetup(MemoryManager* m, Hasher* hasher,
break;
}
HasherReset(hasher);
+ hasher->common.is_setup_ = BROTLI_TRUE;
}
if (!hasher->common.is_prepared_) {
@@ -454,10 +482,6 @@ static BROTLI_INLINE void HasherSetup(MemoryManager* m, Hasher* hasher,
#undef PREPARE_
default: break;
}
- if (position == 0) {
- hasher->common.dict_num_lookups = 0;
- hasher->common.dict_num_matches = 0;
- }
hasher->common.is_prepared_ = BROTLI_TRUE;
}
}
@@ -481,6 +505,222 @@ static BROTLI_INLINE void InitOrStitchToPreviousBlock(
}
}
+/* NB: when seamless dictionary-ring-buffer copies are implemented, don't forget
+ to add proper guards for non-zero-BROTLI_PARAM_STREAM_OFFSET. */
+static BROTLI_INLINE void FindCompoundDictionaryMatch(
+ const PreparedDictionary* self, const uint8_t* BROTLI_RESTRICT data,
+ const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
+ const size_t cur_ix, const size_t max_length, const size_t distance_offset,
+ const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
+ const uint32_t source_size = self->source_size;
+ const size_t boundary = distance_offset - source_size;
+ const uint32_t hash_bits = self->hash_bits;
+ const uint32_t bucket_bits = self->bucket_bits;
+ const uint32_t slot_bits = self->slot_bits;
+
+ const uint32_t hash_shift = 64u - bucket_bits;
+ const uint32_t slot_mask = (~((uint32_t)0U)) >> (32 - slot_bits);
+ const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
+
+ const uint32_t* slot_offsets = (uint32_t*)(&self[1]);
+ const uint16_t* heads = (uint16_t*)(&slot_offsets[1u << slot_bits]);
+ const uint32_t* items = (uint32_t*)(&heads[1u << bucket_bits]);
+ const uint8_t* source = NULL;
+
+ const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
+ score_t best_score = out->score;
+ size_t best_len = out->len;
+ size_t i;
+ const uint64_t h =
+ (BROTLI_UNALIGNED_LOAD64LE(&data[cur_ix_masked]) & hash_mask) *
+ kPreparedDictionaryHashMul64Long;
+ const uint32_t key = (uint32_t)(h >> hash_shift);
+ const uint32_t slot = key & slot_mask;
+ const uint32_t head = heads[key];
+ const uint32_t* BROTLI_RESTRICT chain = &items[slot_offsets[slot] + head];
+ uint32_t item = (head == 0xFFFF) ? 1 : 0;
+
+ const void* tail = (void*)&items[self->num_items];
+ if (self->magic == kPreparedDictionaryMagic) {
+ source = (const uint8_t*)tail;
+ } else {
+ /* kLeanPreparedDictionaryMagic */
+ source = (const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
+ }
+
+ for (i = 0; i < 4; ++i) {
+ const size_t distance = (size_t)distance_cache[i];
+ size_t offset;
+ size_t limit;
+ size_t len;
+ if (distance <= boundary || distance > distance_offset) continue;
+ offset = distance_offset - distance;
+ limit = source_size - offset;
+ limit = limit > max_length ? max_length : limit;
+ len = FindMatchLengthWithLimit(&source[offset], &data[cur_ix_masked],
+ limit);
+ if (len >= 2) {
+ score_t score = BackwardReferenceScoreUsingLastDistance(len);
+ if (best_score < score) {
+ if (i != 0) score -= BackwardReferencePenaltyUsingLastDistance(i);
+ if (best_score < score) {
+ best_score = score;
+ if (len > best_len) best_len = len;
+ out->len = len;
+ out->len_code_delta = 0;
+ out->distance = distance;
+ out->score = best_score;
+ }
+ }
+ }
+ }
+ while (item == 0) {
+ size_t offset;
+ size_t distance;
+ size_t limit;
+ item = *chain;
+ chain++;
+ offset = item & 0x7FFFFFFF;
+ item &= 0x80000000;
+ distance = distance_offset - offset;
+ limit = source_size - offset;
+ limit = (limit > max_length) ? max_length : limit;
+ if (distance > max_distance) continue;
+ if (cur_ix_masked + best_len > ring_buffer_mask ||
+ best_len >= limit ||
+ data[cur_ix_masked + best_len] != source[offset + best_len]) {
+ continue;
+ }
+ {
+ const size_t len = FindMatchLengthWithLimit(&source[offset],
+ &data[cur_ix_masked],
+ limit);
+ if (len >= 4) {
+ score_t score = BackwardReferenceScore(len, distance);
+ if (best_score < score) {
+ best_score = score;
+ best_len = len;
+ out->len = best_len;
+ out->len_code_delta = 0;
+ out->distance = distance;
+ out->score = best_score;
+ }
+ }
+ }
+ }
+}
+
+/* NB: when seamless dictionary-ring-buffer copies are implemented, don't forget
+ to add proper guards for non-zero-BROTLI_PARAM_STREAM_OFFSET. */
+static BROTLI_INLINE size_t FindAllCompoundDictionaryMatches(
+ const PreparedDictionary* self, const uint8_t* BROTLI_RESTRICT data,
+ const size_t ring_buffer_mask, const size_t cur_ix, const size_t min_length,
+ const size_t max_length, const size_t distance_offset,
+ const size_t max_distance, BackwardMatch* matches, size_t match_limit) {
+ const uint32_t source_size = self->source_size;
+ const uint32_t hash_bits = self->hash_bits;
+ const uint32_t bucket_bits = self->bucket_bits;
+ const uint32_t slot_bits = self->slot_bits;
+
+ const uint32_t hash_shift = 64u - bucket_bits;
+ const uint32_t slot_mask = (~((uint32_t)0U)) >> (32 - slot_bits);
+ const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
+
+ const uint32_t* slot_offsets = (uint32_t*)(&self[1]);
+ const uint16_t* heads = (uint16_t*)(&slot_offsets[1u << slot_bits]);
+ const uint32_t* items = (uint32_t*)(&heads[1u << bucket_bits]);
+ const uint8_t* source = NULL;
+
+ const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
+ size_t best_len = min_length;
+ const uint64_t h =
+ (BROTLI_UNALIGNED_LOAD64LE(&data[cur_ix_masked]) & hash_mask) *
+ kPreparedDictionaryHashMul64Long;
+ const uint32_t key = (uint32_t)(h >> hash_shift);
+ const uint32_t slot = key & slot_mask;
+ const uint32_t head = heads[key];
+ const uint32_t* BROTLI_RESTRICT chain = &items[slot_offsets[slot] + head];
+ uint32_t item = (head == 0xFFFF) ? 1 : 0;
+ size_t found = 0;
+
+ const void* tail = (void*)&items[self->num_items];
+ if (self->magic == kPreparedDictionaryMagic) {
+ source = (const uint8_t*)tail;
+ } else {
+ /* kLeanPreparedDictionaryMagic */
+ source = (const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
+ }
+
+ while (item == 0) {
+ size_t offset;
+ size_t distance;
+ size_t limit;
+ size_t len;
+ item = *chain;
+ chain++;
+ offset = item & 0x7FFFFFFF;
+ item &= 0x80000000;
+ distance = distance_offset - offset;
+ limit = source_size - offset;
+ limit = (limit > max_length) ? max_length : limit;
+ if (distance > max_distance) continue;
+ if (cur_ix_masked + best_len > ring_buffer_mask ||
+ best_len >= limit ||
+ data[cur_ix_masked + best_len] != source[offset + best_len]) {
+ continue;
+ }
+ len = FindMatchLengthWithLimit(
+ &source[offset], &data[cur_ix_masked], limit);
+ if (len > best_len) {
+ best_len = len;
+ InitBackwardMatch(matches++, distance, len);
+ found++;
+ if (found == match_limit) break;
+ }
+ }
+ return found;
+}
+
+static BROTLI_INLINE void LookupCompoundDictionaryMatch(
+ const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data,
+ const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
+ const size_t cur_ix, const size_t max_length,
+ const size_t max_ring_buffer_distance, const size_t max_distance,
+ HasherSearchResult* sr) {
+ size_t base_offset = max_ring_buffer_distance + 1 + addon->total_size - 1;
+ size_t d;
+ for (d = 0; d < addon->num_chunks; ++d) {
+ /* Only one prepared dictionary type is currently supported. */
+ FindCompoundDictionaryMatch(
+ (const PreparedDictionary*)addon->chunks[d], data, ring_buffer_mask,
+ distance_cache, cur_ix, max_length,
+ base_offset - addon->chunk_offsets[d], max_distance, sr);
+ }
+}
+
+static BROTLI_INLINE size_t LookupAllCompoundDictionaryMatches(
+ const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data,
+ const size_t ring_buffer_mask, const size_t cur_ix, size_t min_length,
+ const size_t max_length, const size_t max_ring_buffer_distance,
+ const size_t max_distance, BackwardMatch* matches,
+ size_t match_limit) {
+ size_t base_offset = max_ring_buffer_distance + 1 + addon->total_size - 1;
+ size_t d;
+ size_t total_found = 0;
+ for (d = 0; d < addon->num_chunks; ++d) {
+ /* Only one prepared dictionary type is currently supported. */
+ total_found += FindAllCompoundDictionaryMatches(
+ (const PreparedDictionary*)addon->chunks[d], data, ring_buffer_mask,
+ cur_ix, min_length, max_length, base_offset - addon->chunk_offsets[d],
+ max_distance, matches + total_found, match_limit - total_found);
+ if (total_found == match_limit) break;
+ if (total_found > 0) {
+ min_length = BackwardMatchLength(&matches[total_found - 1]);
+ }
+ }
+ return total_found;
+}
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash_composite_inc.h b/dviware/dvisvgm/libs/brotli/enc/hash_composite_inc.h
index cba156c0e2..3923bc72c8 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash_composite_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash_composite_inc.h
@@ -30,10 +30,10 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
typedef struct HashComposite {
HASHER_A ha;
HASHER_B hb;
+ HasherCommon ha_common;
HasherCommon hb_common;
/* Shortcuts. */
- void* extra;
HasherCommon* common;
BROTLI_BOOL fresh;
@@ -43,12 +43,12 @@ typedef struct HashComposite {
static void FN(Initialize)(HasherCommon* common,
HashComposite* BROTLI_RESTRICT self, const BrotliEncoderParams* params) {
self->common = common;
- self->extra = common->extra;
+ self->ha_common = *self->common;
self->hb_common = *self->common;
self->fresh = BROTLI_TRUE;
self->params = params;
- /* TODO: Initialize of the hashers is defered to Prepare (and params
+ /* TODO(lode): Initialize of the hashers is deferred to Prepare (and params
remembered here) because we don't get the one_shot and input_size params
here that are needed to know the memory size of them. Instead provide
those params to all hashers FN(Initialize) */
@@ -59,21 +59,36 @@ static void FN(Prepare)(
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
if (self->fresh) {
self->fresh = BROTLI_FALSE;
- self->hb_common.extra = (uint8_t*)self->extra +
- FN_A(HashMemAllocInBytes)(self->params, one_shot, input_size);
-
- FN_A(Initialize)(self->common, &self->ha, self->params);
+ self->ha_common.extra[0] = self->common->extra[0];
+ self->ha_common.extra[1] = self->common->extra[1];
+ self->ha_common.extra[2] = NULL;
+ self->ha_common.extra[3] = NULL;
+ self->hb_common.extra[0] = self->common->extra[2];
+ self->hb_common.extra[1] = self->common->extra[3];
+ self->hb_common.extra[2] = NULL;
+ self->hb_common.extra[3] = NULL;
+
+ FN_A(Initialize)(&self->ha_common, &self->ha, self->params);
FN_B(Initialize)(&self->hb_common, &self->hb, self->params);
}
FN_A(Prepare)(&self->ha, one_shot, input_size, data);
FN_B(Prepare)(&self->hb, one_shot, input_size, data);
}
-static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
+static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
- size_t input_size) {
- return FN_A(HashMemAllocInBytes)(params, one_shot, input_size) +
- FN_B(HashMemAllocInBytes)(params, one_shot, input_size);
+ size_t input_size, size_t* alloc_size) {
+ size_t alloc_size_a[4] = {0};
+ size_t alloc_size_b[4] = {0};
+ FN_A(HashMemAllocInBytes)(params, one_shot, input_size, alloc_size_a);
+ FN_B(HashMemAllocInBytes)(params, one_shot, input_size, alloc_size_b);
+ /* Should never happen. */
+ if (alloc_size_a[2] != 0 || alloc_size_a[3] != 0) exit(EXIT_FAILURE);
+ if (alloc_size_b[2] != 0 || alloc_size_b[3] != 0) exit(EXIT_FAILURE);
+ alloc_size[0] = alloc_size_a[0];
+ alloc_size[1] = alloc_size_a[1];
+ alloc_size[2] = alloc_size_b[0];
+ alloc_size[3] = alloc_size_b[1];
}
static BROTLI_INLINE void FN(Store)(HashComposite* BROTLI_RESTRICT self,
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash_forgetful_chain_inc.h b/dviware/dvisvgm/libs/brotli/enc/hash_forgetful_chain_inc.h
index bfae6ba6a2..48e1cdcdf2 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash_forgetful_chain_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash_forgetful_chain_inc.h
@@ -49,7 +49,7 @@ typedef struct HashForgetfulChain {
size_t max_hops;
/* Shortcuts. */
- void* extra;
+ void* extra[2];
HasherCommon* common;
/* --- Dynamic size members --- */
@@ -77,14 +77,15 @@ static uint8_t* FN(TinyHash)(void* extra) {
}
static FN(Bank)* FN(Banks)(void* extra) {
- return (FN(Bank)*)(&FN(TinyHash)(extra)[65536]);
+ return (FN(Bank)*)(extra);
}
static void FN(Initialize)(
HasherCommon* common, HashForgetfulChain* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->common = common;
- self->extra = common->extra;
+ self->extra[0] = common->extra[0];
+ self->extra[1] = common->extra[1];
self->max_hops = (params->quality > 6 ? 7u : 8u) << (params->quality - 4);
}
@@ -92,9 +93,9 @@ static void FN(Initialize)(
static void FN(Prepare)(
HashForgetfulChain* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
- uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra);
- uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra);
- uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra);
+ uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra[0]);
+ uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra[0]);
+ uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra[0]);
/* Partial preparation is 100 times slower (per socket). */
size_t partial_prepare_threshold = BUCKET_SIZE >> 6;
if (one_shot && input_size <= partial_prepare_threshold) {
@@ -116,24 +117,25 @@ static void FN(Prepare)(
memset(self->free_slot_idx, 0, sizeof(self->free_slot_idx));
}
-static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
+static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
- size_t input_size) {
+ size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
- return sizeof(uint32_t) * BUCKET_SIZE + sizeof(uint16_t) * BUCKET_SIZE +
- sizeof(uint8_t) * 65536 + sizeof(FN(Bank)) * NUM_BANKS;
+ alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE +
+ sizeof(uint16_t) * BUCKET_SIZE + sizeof(uint8_t) * 65536;
+ alloc_size[1] = sizeof(FN(Bank)) * NUM_BANKS;
}
/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend
node to corresponding chain; also update tiny_hash for current position. */
static BROTLI_INLINE void FN(Store)(HashForgetfulChain* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
- uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra);
- uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra);
- uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra);
- FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra);
+ uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra[0]);
+ uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra[0]);
+ uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra[0]);
+ FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra[1]);
const size_t key = FN(HashBytes)(&data[ix & mask]);
const size_t bank = key & (NUM_BANKS - 1);
const size_t idx = self->free_slot_idx[bank]++ & (BANK_SIZE - 1);
@@ -196,10 +198,10 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
const size_t cur_ix, const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
- uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra);
- uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra);
- uint8_t* BROTLI_RESTRICT tiny_hashes = FN(TinyHash)(self->extra);
- FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra);
+ uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra[0]);
+ uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra[0]);
+ uint8_t* BROTLI_RESTRICT tiny_hashes = FN(TinyHash)(self->extra[0]);
+ FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra[1]);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash_longest_match64_inc.h b/dviware/dvisvgm/libs/brotli/enc/hash_longest_match64_inc.h
index 956fb304b3..e48fc61554 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash_longest_match64_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash_longest_match64_inc.h
@@ -20,13 +20,12 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
/* HashBytes is the function that chooses the bucket to place the address in. */
-static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data,
- const uint64_t mask,
- const int shift) {
- const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
+static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data,
+ uint64_t hash_mul) {
+ const uint64_t h = BROTLI_UNALIGNED_LOAD64LE(data) * hash_mul;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
- return (uint32_t)(h >> shift);
+ return (size_t)(h >> (64 - 15));
}
typedef struct HashLongestMatch {
@@ -35,10 +34,8 @@ typedef struct HashLongestMatch {
/* Only block_size_ newest backward references are kept,
and the older are forgotten. */
size_t block_size_;
- /* Left-shift for computing hash bucket index from hash value. */
- int hash_shift_;
- /* Mask for selecting the next 4-8 bytes of input */
- uint64_t hash_mask_;
+ /* Hash multiplier tuned to match length. */
+ uint64_t hash_mul_;
/* Mask for accessing entries in a block (in a ring-buffer manner). */
uint32_t block_mask_;
@@ -63,16 +60,16 @@ static void FN(Initialize)(
self->common_ = common;
BROTLI_UNUSED(params);
- self->hash_shift_ = 64 - common->params.bucket_bits;
- self->hash_mask_ = (~((uint64_t)0U)) >> (64 - 8 * common->params.hash_len);
+ self->hash_mul_ = kHashMul64 << (64 - 5 * 8);
+ BROTLI_DCHECK(common->params.bucket_bits == 15);
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
self->block_bits_ = common->params.block_bits;
self->block_size_ = (size_t)1 << common->params.block_bits;
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
- self->num_ = (uint16_t*)common->extra;
- self->buckets_ = (uint32_t*)&self->num_[self->bucket_size_];
+ self->num_ = (uint16_t*)common->extra[0];
+ self->buckets_ = (uint32_t*)common->extra[1];
}
static void FN(Prepare)(
@@ -84,8 +81,7 @@ static void FN(Prepare)(
if (one_shot && input_size <= partial_prepare_threshold) {
size_t i;
for (i = 0; i < input_size; ++i) {
- const uint32_t key = FN(HashBytes)(&data[i], self->hash_mask_,
- self->hash_shift_);
+ const size_t key = FN(HashBytes)(&data[i], self->hash_mul_);
num[key] = 0;
}
} else {
@@ -93,15 +89,15 @@ static void FN(Prepare)(
}
}
-static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
+static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
- size_t input_size) {
+ size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
- return sizeof(uint16_t) * bucket_size +
- sizeof(uint32_t) * bucket_size * block_size;
+ alloc_size[0] = sizeof(uint16_t) * bucket_size;
+ alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].
@@ -111,8 +107,7 @@ static BROTLI_INLINE void FN(Store)(
const size_t mask, const size_t ix) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
- const uint32_t key = FN(HashBytes)(&data[ix & mask], self->hash_mask_,
- self->hash_shift_);
+ const size_t key = FN(HashBytes)(&data[ix & mask], self->hash_mul_);
const size_t minor_ix = num[key] & self->block_mask_;
const size_t offset = minor_ix + (key << self->block_bits_);
++num[key];
@@ -217,14 +212,17 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
}
}
{
- const uint32_t key = FN(HashBytes)(
- &data[cur_ix_masked], self->hash_mask_, self->hash_shift_);
+ const size_t key = FN(HashBytes)(&data[cur_ix_masked], self->hash_mul_);
uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
const size_t down =
(num[key] > self->block_size_) ?
(num[key] - self->block_size_) : 0u;
- for (i = num[key]; i > down;) {
+ const uint32_t first4 = BrotliUnalignedRead32(data + cur_ix_masked);
+ const size_t max_length_m4 = max_length - 4;
+ i = num[key];
+ for (; i > down;) {
size_t prev_ix = bucket[--i & self->block_mask_];
+ uint32_t current4;
const size_t backward = cur_ix - prev_ix;
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
break;
@@ -235,22 +233,19 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
continue;
}
+ current4 = BrotliUnalignedRead32(data + prev_ix);
+ if (first4 != current4) continue;
{
- const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
- &data[cur_ix_masked],
- max_length);
- if (len >= 4) {
- /* Comparing for >= 3 does not change the semantics, but just saves
- for a few unnecessary binary logarithms in backward reference
- score, since we are not interested in such short matches. */
- score_t score = BackwardReferenceScore(len, backward);
- if (best_score < score) {
- best_score = score;
- best_len = len;
- out->len = best_len;
- out->distance = backward;
- out->score = best_score;
- }
+ const size_t len = FindMatchLengthWithLimit(&data[prev_ix + 4],
+ &data[cur_ix_masked + 4],
+ max_length_m4) + 4;
+ const score_t score = BackwardReferenceScore(len, backward);
+ if (best_score < score) {
+ best_score = score;
+ best_len = len;
+ out->len = best_len;
+ out->distance = backward;
+ out->score = best_score;
}
}
}
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_inc.h b/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_inc.h
index 27f4463d7f..788e9ef993 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_inc.h
@@ -54,10 +54,6 @@ typedef struct HashLongestMatch {
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
} HashLongestMatch;
-static BROTLI_INLINE uint16_t* FN(Num)(void* extra) {
- return (uint16_t*)extra;
-}
-
static void FN(Initialize)(
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
@@ -68,8 +64,8 @@ static void FN(Initialize)(
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
self->block_size_ = (size_t)1 << common->params.block_bits;
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
- self->num_ = (uint16_t*)common->extra;
- self->buckets_ = (uint32_t*)(&self->num_[self->bucket_size_]);
+ self->num_ = (uint16_t*)common->extra[0];
+ self->buckets_ = (uint32_t*)common->extra[1];
self->block_bits_ = common->params.block_bits;
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
@@ -92,15 +88,15 @@ static void FN(Prepare)(
}
}
-static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
+static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
- size_t input_size) {
+ size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
- return sizeof(uint16_t) * bucket_size +
- sizeof(uint32_t) * bucket_size * block_size;
+ alloc_size[0] = sizeof(uint16_t) * bucket_size;
+ alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_quickly_inc.h b/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_quickly_inc.h
index e5ba840ab9..54397ef891 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_quickly_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash_longest_match_quickly_inc.h
@@ -49,7 +49,7 @@ static void FN(Initialize)(
self->common = common;
BROTLI_UNUSED(params);
- self->buckets_ = (uint32_t*)common->extra;
+ self->buckets_ = (uint32_t*)common->extra[0];
}
static void FN(Prepare)(
@@ -80,13 +80,13 @@ static void FN(Prepare)(
}
}
-static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
+static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
- size_t input_size) {
+ size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
- return sizeof(uint32_t) * BUCKET_SIZE;
+ alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
}
/* Look at 5 bytes at &data[ix & mask].
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash_rolling_inc.h b/dviware/dvisvgm/libs/brotli/enc/hash_rolling_inc.h
index 586ae73859..4c7a6b199a 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash_rolling_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash_rolling_inc.h
@@ -67,7 +67,7 @@ static void FN(Initialize)(
self->factor_remove *= self->factor;
}
- self->table = (uint32_t*)common->extra;
+ self->table = (uint32_t*)common->extra[0];
for (i = 0; i < NUMBUCKETS; i++) {
self->table[i] = FN(kInvalidPos);
}
@@ -88,13 +88,13 @@ static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
BROTLI_UNUSED(one_shot);
}
-static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
+static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
- size_t input_size) {
- return NUMBUCKETS * sizeof(uint32_t);
+ size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
+ alloc_size[0] = NUMBUCKETS * sizeof(uint32_t);
}
static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self,
diff --git a/dviware/dvisvgm/libs/brotli/enc/hash_to_binary_tree_inc.h b/dviware/dvisvgm/libs/brotli/enc/hash_to_binary_tree_inc.h
index 9880e0aef6..a639d2d4b0 100644
--- a/dviware/dvisvgm/libs/brotli/enc/hash_to_binary_tree_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/hash_to_binary_tree_inc.h
@@ -57,8 +57,8 @@ typedef struct HashToBinaryTree {
static void FN(Initialize)(
HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
- self->buckets_ = (uint32_t*)common->extra;
- self->forest_ = &self->buckets_[BUCKET_SIZE];
+ self->buckets_ = (uint32_t*)common->extra[0];
+ self->forest_ = (uint32_t*)common->extra[1];
self->window_mask_ = (1u << params->lgwin) - 1u;
self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
@@ -78,14 +78,15 @@ static void FN(Prepare)
}
}
-static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
+static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
- size_t input_size) {
+ size_t input_size, size_t* alloc_size) {
size_t num_nodes = (size_t)1 << params->lgwin;
if (one_shot && input_size < num_nodes) {
num_nodes = input_size;
}
- return sizeof(uint32_t) * BUCKET_SIZE + 2 * sizeof(uint32_t) * num_nodes;
+ alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
+ alloc_size[1] = 2 * sizeof(uint32_t) * num_nodes;
}
static BROTLI_INLINE size_t FN(LeftChildIndex)(
diff --git a/dviware/dvisvgm/libs/brotli/enc/histogram.c b/dviware/dvisvgm/libs/brotli/enc/histogram.c
index 6da2ff6bb4..4dbb87f907 100644
--- a/dviware/dvisvgm/libs/brotli/enc/histogram.c
+++ b/dviware/dvisvgm/libs/brotli/enc/histogram.c
@@ -6,11 +6,11 @@
/* Build per-context histograms of literals, commands and distance codes. */
-#include "./histogram.h"
+#include "histogram.h"
#include "../common/context.h"
-#include "./block_splitter.h"
-#include "./command.h"
+#include "block_splitter.h"
+#include "command.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -63,7 +63,7 @@ void BrotliBuildHistogramsWithContext(
BlockSplitIteratorNext(&insert_and_copy_it);
HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
cmd->cmd_prefix_);
- /* TODO: unwrap iterator blocks. */
+ /* TODO(eustas): unwrap iterator blocks. */
for (j = cmd->insert_len_; j != 0; --j) {
size_t context;
BlockSplitIteratorNext(&literal_it);
diff --git a/dviware/dvisvgm/libs/brotli/enc/histogram.h b/dviware/dvisvgm/libs/brotli/enc/histogram.h
index 42af3c3f9d..d1abd973c1 100644
--- a/dviware/dvisvgm/libs/brotli/enc/histogram.h
+++ b/dviware/dvisvgm/libs/brotli/enc/histogram.h
@@ -11,12 +11,13 @@
#include <string.h> /* memset */
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/context.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./block_splitter.h"
-#include "./command.h"
+#include "block_splitter.h"
+#include "command.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -28,7 +29,7 @@ extern "C" {
#define FN(X) X ## Literal
#define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
#define DataType uint8_t
-#include "./histogram_inc.h" /* NOLINT(build/include) */
+#include "histogram_inc.h" /* NOLINT(build/include) */
#undef DataType
#undef DATA_SIZE
#undef FN
@@ -36,13 +37,13 @@ extern "C" {
#define FN(X) X ## Command
#define DataType uint16_t
#define DATA_SIZE BROTLI_NUM_COMMAND_SYMBOLS
-#include "./histogram_inc.h" /* NOLINT(build/include) */
+#include "histogram_inc.h" /* NOLINT(build/include) */
#undef DATA_SIZE
#undef FN
#define FN(X) X ## Distance
#define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
-#include "./histogram_inc.h" /* NOLINT(build/include) */
+#include "histogram_inc.h" /* NOLINT(build/include) */
#undef DataType
#undef DATA_SIZE
#undef FN
diff --git a/dviware/dvisvgm/libs/brotli/enc/literal_cost.c b/dviware/dvisvgm/libs/brotli/enc/literal_cost.c
index c231100e34..2ac847f3c9 100644
--- a/dviware/dvisvgm/libs/brotli/enc/literal_cost.c
+++ b/dviware/dvisvgm/libs/brotli/enc/literal_cost.c
@@ -7,12 +7,15 @@
/* Literal cost model to allow backward reference replacement to be efficient.
*/
-#include "./literal_cost.h"
+#include "literal_cost.h"
+
+#include <string.h> /* memset */
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./fast_log.h"
-#include "./utf8_util.h"
+
+#include "../common/platform.h"
+#include "fast_log.h"
+#include "utf8_util.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -54,22 +57,23 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
}
static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
- const uint8_t* data, float* cost) {
+ const uint8_t* data,
+ size_t* histogram, float* cost) {
/* max_utf8 is 0 (normal ASCII single byte modeling),
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
- size_t histogram[3][256] = { { 0 } };
size_t window_half = 495;
size_t in_window = BROTLI_MIN(size_t, window_half, len);
size_t in_window_utf8[3] = { 0 };
-
size_t i;
+ memset(histogram, 0, 3 * 256 * sizeof(histogram[0]));
+
{ /* Bootstrap histograms. */
size_t last_c = 0;
size_t utf8_pos = 0;
for (i = 0; i < in_window; ++i) {
size_t c = data[(pos + i) & mask];
- ++histogram[utf8_pos][c];
+ ++histogram[256 * utf8_pos + c];
++in_window_utf8[utf8_pos];
utf8_pos = UTF8Position(last_c, c, max_utf8);
last_c = c;
@@ -85,7 +89,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
size_t last_c =
i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask];
size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
- --histogram[utf8_pos2][data[(pos + i - window_half) & mask]];
+ --histogram[256 * utf8_pos2 + data[(pos + i - window_half) & mask]];
--in_window_utf8[utf8_pos2];
}
if (i + window_half < len) {
@@ -93,7 +97,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
size_t c = data[(pos + i + window_half - 1) & mask];
size_t last_c = data[(pos + i + window_half - 2) & mask];
size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
- ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]];
+ ++histogram[256 * utf8_pos2 + data[(pos + i + window_half) & mask]];
++in_window_utf8[utf8_pos2];
}
{
@@ -101,7 +105,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
size_t utf8_pos = UTF8Position(last_c, c, max_utf8);
size_t masked_pos = (pos + i) & mask;
- size_t histo = histogram[utf8_pos][data[masked_pos]];
+ size_t histo = histogram[256 * utf8_pos + data[masked_pos]];
double lit_cost;
if (histo == 0) {
histo = 1;
@@ -125,17 +129,18 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
}
void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
- const uint8_t* data, float* cost) {
+ const uint8_t* data,
+ size_t* histogram, float* cost) {
if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
- EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
+ EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, histogram, cost);
return;
} else {
- size_t histogram[256] = { 0 };
size_t window_half = 2000;
size_t in_window = BROTLI_MIN(size_t, window_half, len);
+ size_t i;
+ memset(histogram, 0, 256 * sizeof(histogram[0]));
/* Bootstrap histogram. */
- size_t i;
for (i = 0; i < in_window; ++i) {
++histogram[data[(pos + i) & mask]];
}
diff --git a/dviware/dvisvgm/libs/brotli/enc/literal_cost.h b/dviware/dvisvgm/libs/brotli/enc/literal_cost.h
index 8f53f39d3f..284a8e5af7 100644
--- a/dviware/dvisvgm/libs/brotli/enc/literal_cost.h
+++ b/dviware/dvisvgm/libs/brotli/enc/literal_cost.h
@@ -10,9 +10,10 @@
#ifndef BROTLI_ENC_LITERAL_COST_H_
#define BROTLI_ENC_LITERAL_COST_H_
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
@@ -21,7 +22,8 @@ extern "C" {
ring-buffer (data, mask) will take entropy coded and writes these estimates
to the cost[0..len) array. */
BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
- size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost);
+ size_t pos, size_t len, size_t mask, const uint8_t* data, size_t* histogram,
+ float* cost);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
diff --git a/dviware/dvisvgm/libs/brotli/enc/memory.c b/dviware/dvisvgm/libs/brotli/enc/memory.c
index f6ed7e3cb7..bb5e364198 100644
--- a/dviware/dvisvgm/libs/brotli/enc/memory.c
+++ b/dviware/dvisvgm/libs/brotli/enc/memory.c
@@ -7,21 +7,22 @@
/* Algorithms for distributing the literals and commands of a metablock between
block types and contexts. */
-#include "./memory.h"
+#include "memory.h"
#include <stdlib.h> /* exit, free, malloc */
#include <string.h> /* memcpy */
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
-#define MAX_PERM_ALLOCATED 128
-#define MAX_NEW_ALLOCATED 64
-#define MAX_NEW_FREED 64
+#define MAX_NEW_ALLOCATED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 2)
+#define MAX_NEW_FREED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 2)
+#define MAX_PERM_ALLOCATED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 1)
#define PERM_ALLOCATED_OFFSET 0
#define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
@@ -67,6 +68,7 @@ void BrotliWipeOutMemoryManager(MemoryManager* m) {
static void SortPointers(void** items, const size_t n) {
/* Shell sort. */
+ /* TODO(eustas): fine-tune for "many slots" case */
static const size_t gaps[] = {23, 10, 4, 1};
int g = 0;
for (; g < 4; ++g) {
@@ -165,6 +167,28 @@ void BrotliWipeOutMemoryManager(MemoryManager* m) {
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
+void* BrotliBootstrapAlloc(size_t size,
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
+ if (!alloc_func && !free_func) {
+ return malloc(size);
+ } else if (alloc_func && free_func) {
+ return alloc_func(opaque, size);
+ }
+ return NULL;
+}
+
+void BrotliBootstrapFree(void* address, MemoryManager* m) {
+ if (!address) {
+ /* Should not happen! */
+ return;
+ } else {
+ /* Copy values, as those would be freed. */
+ brotli_free_func free_func = m->free_func;
+ void* opaque = m->opaque;
+ free_func(opaque, address);
+ }
+}
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/memory.h b/dviware/dvisvgm/libs/brotli/enc/memory.h
index 832e7b2b6e..a4417df4cb 100644
--- a/dviware/dvisvgm/libs/brotli/enc/memory.h
+++ b/dviware/dvisvgm/libs/brotli/enc/memory.h
@@ -11,9 +11,10 @@
#include <string.h> /* memcpy */
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
@@ -23,6 +24,16 @@ extern "C" {
#define BROTLI_ENCODER_EXIT_ON_OOM
#endif
+#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#if defined(BROTLI_EXPERIMENTAL)
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS (48*1024)
+#else /* BROTLI_EXPERIMENTAL */
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 256
+#endif /* BROTLI_EXPERIMENTAL */
+#else /* BROTLI_ENCODER_EXIT_ON_OOM */
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 0
+#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
+
typedef struct MemoryManager {
brotli_alloc_func alloc_func;
brotli_free_func free_func;
@@ -32,7 +43,7 @@ typedef struct MemoryManager {
size_t perm_allocated;
size_t new_allocated;
size_t new_freed;
- void* pointers[256];
+ void* pointers[BROTLI_ENCODER_MEMORY_MANAGER_SLOTS];
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
} MemoryManager;
@@ -107,6 +118,12 @@ V: value to append
A[(S) - 1] = (V); \
}
+/* "Bootstrap" allocations are not tracked by memory manager; should be used
+ only to allocate MemoryManager itself (or structure containing it). */
+BROTLI_INTERNAL void* BrotliBootstrapAlloc(size_t size,
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
+BROTLI_INTERNAL void BrotliBootstrapFree(void* address, MemoryManager* m);
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/metablock.c b/dviware/dvisvgm/libs/brotli/enc/metablock.c
index 5aa4d4f17c..0c5c078d05 100644
--- a/dviware/dvisvgm/libs/brotli/enc/metablock.c
+++ b/dviware/dvisvgm/libs/brotli/enc/metablock.c
@@ -7,27 +7,27 @@
/* Algorithms for distributing the literals and commands of a metablock between
block types and contexts. */
-#include "./metablock.h"
+#include "metablock.h"
+
+#include <brotli/types.h>
#include "../common/constants.h"
#include "../common/context.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./bit_cost.h"
-#include "./block_splitter.h"
-#include "./cluster.h"
-#include "./entropy_encode.h"
-#include "./histogram.h"
-#include "./memory.h"
-#include "./quality.h"
+#include "bit_cost.h"
+#include "block_splitter.h"
+#include "cluster.h"
+#include "entropy_encode.h"
+#include "histogram.h"
+#include "memory.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
-void BrotliInitDistanceParams(BrotliEncoderParams* params,
- uint32_t npostfix, uint32_t ndirect) {
- BrotliDistanceParams* dist_params = &params->dist;
+void BrotliInitDistanceParams(BrotliDistanceParams* dist_params,
+ uint32_t npostfix, uint32_t ndirect, BROTLI_BOOL large_window) {
uint32_t alphabet_size_max;
uint32_t alphabet_size_limit;
uint32_t max_distance;
@@ -41,7 +41,7 @@ void BrotliInitDistanceParams(BrotliEncoderParams* params,
max_distance = ndirect + (1U << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) -
(1U << (npostfix + 2));
- if (params->large_window) {
+ if (large_window) {
BrotliDistanceCodeLimit limit = BrotliCalculateDistanceCodeLimit(
BROTLI_MAX_ALLOWED_DISTANCE, npostfix, ndirect);
alphabet_size_max = BROTLI_DISTANCE_ALPHABET_SIZE(
@@ -83,14 +83,14 @@ static BROTLI_BOOL ComputeDistanceCost(const Command* cmds,
size_t num_commands,
const BrotliDistanceParams* orig_params,
const BrotliDistanceParams* new_params,
- double* cost) {
+ double* cost,
+ HistogramDistance* tmp) {
size_t i;
BROTLI_BOOL equal_params = BROTLI_FALSE;
uint16_t dist_prefix;
uint32_t dist_extra;
double extra_bits = 0.0;
- HistogramDistance histo;
- HistogramClearDistance(&histo);
+ HistogramClearDistance(tmp);
if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
orig_params->num_direct_distance_codes ==
@@ -114,12 +114,12 @@ static BROTLI_BOOL ComputeDistanceCost(const Command* cmds,
&dist_prefix,
&dist_extra);
}
- HistogramAddDistance(&histo, dist_prefix & 0x3FF);
+ HistogramAddDistance(tmp, dist_prefix & 0x3FF);
extra_bits += dist_prefix >> 10;
}
}
- *cost = BrotliPopulationCostDistance(&histo) + extra_bits;
+ *cost = BrotliPopulationCostDistance(tmp) + extra_bits;
return BROTLI_TRUE;
}
@@ -147,43 +147,46 @@ void BrotliBuildMetaBlock(MemoryManager* m,
uint32_t ndirect_msb = 0;
BROTLI_BOOL check_orig = BROTLI_TRUE;
double best_dist_cost = 1e99;
- BrotliEncoderParams orig_params = *params;
- BrotliEncoderParams new_params = *params;
+ BrotliDistanceParams orig_params = params->dist;
+ BrotliDistanceParams new_params = params->dist;
+ HistogramDistance* tmp = BROTLI_ALLOC(m, HistogramDistance, 1);
+
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return;
for (npostfix = 0; npostfix <= BROTLI_MAX_NPOSTFIX; npostfix++) {
for (; ndirect_msb < 16; ndirect_msb++) {
uint32_t ndirect = ndirect_msb << npostfix;
BROTLI_BOOL skip;
double dist_cost;
- BrotliInitDistanceParams(&new_params, npostfix, ndirect);
- if (npostfix == orig_params.dist.distance_postfix_bits &&
- ndirect == orig_params.dist.num_direct_distance_codes) {
+ BrotliInitDistanceParams(&new_params, npostfix, ndirect,
+ params->large_window);
+ if (npostfix == orig_params.distance_postfix_bits &&
+ ndirect == orig_params.num_direct_distance_codes) {
check_orig = BROTLI_FALSE;
}
skip = !ComputeDistanceCost(
- cmds, num_commands,
- &orig_params.dist, &new_params.dist, &dist_cost);
+ cmds, num_commands, &orig_params, &new_params, &dist_cost, tmp);
if (skip || (dist_cost > best_dist_cost)) {
break;
}
best_dist_cost = dist_cost;
- params->dist = new_params.dist;
+ params->dist = new_params;
}
if (ndirect_msb > 0) ndirect_msb--;
ndirect_msb /= 2;
}
if (check_orig) {
double dist_cost;
- ComputeDistanceCost(cmds, num_commands,
- &orig_params.dist, &orig_params.dist, &dist_cost);
+ ComputeDistanceCost(cmds, num_commands, &orig_params, &orig_params,
+ &dist_cost, tmp);
if (dist_cost < best_dist_cost) {
/* NB: currently unused; uncomment when more param tuning is added. */
/* best_dist_cost = dist_cost; */
- params->dist = orig_params.dist;
+ params->dist = orig_params;
}
}
- RecomputeDistancePrefixes(cmds, num_commands,
- &orig_params.dist, &params->dist);
+ BROTLI_FREE(m, tmp);
+ RecomputeDistancePrefixes(cmds, num_commands, &orig_params, &params->dist);
BrotliSplitBlock(m, cmds, num_commands,
ringbuffer, pos, mask, params,
@@ -284,15 +287,15 @@ void BrotliBuildMetaBlock(MemoryManager* m,
}
#define FN(X) X ## Literal
-#include "./metablock_inc.h" /* NOLINT(build/include) */
+#include "metablock_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Command
-#include "./metablock_inc.h" /* NOLINT(build/include) */
+#include "metablock_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Distance
-#include "./metablock_inc.h" /* NOLINT(build/include) */
+#include "metablock_inc.h" /* NOLINT(build/include) */
#undef FN
#define BROTLI_MAX_STATIC_CONTEXTS 13
@@ -535,17 +538,21 @@ static void MapStaticContexts(MemoryManager* m,
}
}
-static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
- MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
- uint8_t prev_byte, uint8_t prev_byte2, ContextLut literal_context_lut,
- const size_t num_contexts, const uint32_t* static_context_map,
- const Command* commands, size_t n_commands, MetaBlockSplit* mb) {
+typedef struct GreedyMetablockArena {
union {
BlockSplitterLiteral plain;
ContextBlockSplitter ctx;
} lit_blocks;
BlockSplitterCommand cmd_blocks;
BlockSplitterDistance dist_blocks;
+} GreedyMetablockArena;
+
+static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
+ MemoryManager* m, GreedyMetablockArena* arena, const uint8_t* ringbuffer,
+ size_t pos, size_t mask, uint8_t prev_byte, uint8_t prev_byte2,
+ ContextLut literal_context_lut, const size_t num_contexts,
+ const uint32_t* static_context_map, const Command* commands,
+ size_t n_commands, MetaBlockSplit* mb) {
size_t num_literals = 0;
size_t i;
for (i = 0; i < n_commands; ++i) {
@@ -553,20 +560,20 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
}
if (num_contexts == 1) {
- InitBlockSplitterLiteral(m, &lit_blocks.plain, 256, 512, 400.0,
+ InitBlockSplitterLiteral(m, &arena->lit_blocks.plain, 256, 512, 400.0,
num_literals, &mb->literal_split, &mb->literal_histograms,
&mb->literal_histograms_size);
} else {
- InitContextBlockSplitter(m, &lit_blocks.ctx, 256, num_contexts, 512, 400.0,
- num_literals, &mb->literal_split, &mb->literal_histograms,
+ InitContextBlockSplitter(m, &arena->lit_blocks.ctx, 256, num_contexts, 512,
+ 400.0, num_literals, &mb->literal_split, &mb->literal_histograms,
&mb->literal_histograms_size);
}
if (BROTLI_IS_OOM(m)) return;
- InitBlockSplitterCommand(m, &cmd_blocks, BROTLI_NUM_COMMAND_SYMBOLS, 1024,
- 500.0, n_commands, &mb->command_split, &mb->command_histograms,
+ InitBlockSplitterCommand(m, &arena->cmd_blocks, BROTLI_NUM_COMMAND_SYMBOLS,
+ 1024, 500.0, n_commands, &mb->command_split, &mb->command_histograms,
&mb->command_histograms_size);
if (BROTLI_IS_OOM(m)) return;
- InitBlockSplitterDistance(m, &dist_blocks, 64, 512, 100.0, n_commands,
+ InitBlockSplitterDistance(m, &arena->dist_blocks, 64, 512, 100.0, n_commands,
&mb->distance_split, &mb->distance_histograms,
&mb->distance_histograms_size);
if (BROTLI_IS_OOM(m)) return;
@@ -574,15 +581,15 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
for (i = 0; i < n_commands; ++i) {
const Command cmd = commands[i];
size_t j;
- BlockSplitterAddSymbolCommand(&cmd_blocks, cmd.cmd_prefix_);
+ BlockSplitterAddSymbolCommand(&arena->cmd_blocks, cmd.cmd_prefix_);
for (j = cmd.insert_len_; j != 0; --j) {
uint8_t literal = ringbuffer[pos & mask];
if (num_contexts == 1) {
- BlockSplitterAddSymbolLiteral(&lit_blocks.plain, literal);
+ BlockSplitterAddSymbolLiteral(&arena->lit_blocks.plain, literal);
} else {
size_t context =
BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
- ContextBlockSplitterAddSymbol(&lit_blocks.ctx, m, literal,
+ ContextBlockSplitterAddSymbol(&arena->lit_blocks.ctx, m, literal,
static_context_map[context]);
if (BROTLI_IS_OOM(m)) return;
}
@@ -595,21 +602,24 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
prev_byte2 = ringbuffer[(pos - 2) & mask];
prev_byte = ringbuffer[(pos - 1) & mask];
if (cmd.cmd_prefix_ >= 128) {
- BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_ & 0x3FF);
+ BlockSplitterAddSymbolDistance(
+ &arena->dist_blocks, cmd.dist_prefix_ & 0x3FF);
}
}
}
if (num_contexts == 1) {
BlockSplitterFinishBlockLiteral(
- &lit_blocks.plain, /* is_final = */ BROTLI_TRUE);
+ &arena->lit_blocks.plain, /* is_final = */ BROTLI_TRUE);
} else {
ContextBlockSplitterFinishBlock(
- &lit_blocks.ctx, m, /* is_final = */ BROTLI_TRUE);
+ &arena->lit_blocks.ctx, m, /* is_final = */ BROTLI_TRUE);
if (BROTLI_IS_OOM(m)) return;
}
- BlockSplitterFinishBlockCommand(&cmd_blocks, /* is_final = */ BROTLI_TRUE);
- BlockSplitterFinishBlockDistance(&dist_blocks, /* is_final = */ BROTLI_TRUE);
+ BlockSplitterFinishBlockCommand(
+ &arena->cmd_blocks, /* is_final = */ BROTLI_TRUE);
+ BlockSplitterFinishBlockDistance(
+ &arena->dist_blocks, /* is_final = */ BROTLI_TRUE);
if (num_contexts > 1) {
MapStaticContexts(m, num_contexts, static_context_map, mb);
@@ -628,14 +638,18 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
const Command* commands,
size_t n_commands,
MetaBlockSplit* mb) {
+ GreedyMetablockArena* arena = BROTLI_ALLOC(m, GreedyMetablockArena, 1);
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
if (num_contexts == 1) {
- BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
- prev_byte2, literal_context_lut, 1, NULL, commands, n_commands, mb);
+ BrotliBuildMetaBlockGreedyInternal(m, arena, ringbuffer, pos, mask,
+ prev_byte, prev_byte2, literal_context_lut, 1, NULL, commands,
+ n_commands, mb);
} else {
- BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
- prev_byte2, literal_context_lut, num_contexts, static_context_map,
- commands, n_commands, mb);
+ BrotliBuildMetaBlockGreedyInternal(m, arena, ringbuffer, pos, mask,
+ prev_byte, prev_byte2, literal_context_lut, num_contexts,
+ static_context_map, commands, n_commands, mb);
}
+ BROTLI_FREE(m, arena);
}
void BrotliOptimizeHistograms(uint32_t num_distance_codes,
diff --git a/dviware/dvisvgm/libs/brotli/enc/metablock.h b/dviware/dvisvgm/libs/brotli/enc/metablock.h
index 334a79a443..db38f8fd21 100644
--- a/dviware/dvisvgm/libs/brotli/enc/metablock.h
+++ b/dviware/dvisvgm/libs/brotli/enc/metablock.h
@@ -10,14 +10,15 @@
#ifndef BROTLI_ENC_METABLOCK_H_
#define BROTLI_ENC_METABLOCK_H_
+#include <brotli/types.h>
+
#include "../common/context.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./block_splitter.h"
-#include "./command.h"
-#include "./histogram.h"
-#include "./memory.h"
-#include "./quality.h"
+#include "block_splitter.h"
+#include "command.h"
+#include "histogram.h"
+#include "memory.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -95,8 +96,8 @@ BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy(
BROTLI_INTERNAL void BrotliOptimizeHistograms(uint32_t num_distance_codes,
MetaBlockSplit* mb);
-BROTLI_INTERNAL void BrotliInitDistanceParams(BrotliEncoderParams* params,
- uint32_t npostfix, uint32_t ndirect);
+BROTLI_INTERNAL void BrotliInitDistanceParams(BrotliDistanceParams* params,
+ uint32_t npostfix, uint32_t ndirect, BROTLI_BOOL large_window);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
diff --git a/dviware/dvisvgm/libs/brotli/enc/metablock_inc.h b/dviware/dvisvgm/libs/brotli/enc/metablock_inc.h
index ed507ef5ef..f9393869ab 100644
--- a/dviware/dvisvgm/libs/brotli/enc/metablock_inc.h
+++ b/dviware/dvisvgm/libs/brotli/enc/metablock_inc.h
@@ -27,6 +27,9 @@ typedef struct FN(BlockSplitter) {
HistogramType* histograms_; /* not owned */
size_t* histograms_size_; /* not owned */
+ /* Temporary storage for BlockSplitterFinishBlock. */
+ HistogramType combined_histo[2];
+
/* The number of symbols that we want to collect before deciding on whether
or not to merge the block with a previous one or emit a new block. */
size_t target_block_size_;
@@ -104,17 +107,16 @@ static void FN(BlockSplitterFinishBlock)(
} else if (self->block_size_ > 0) {
double entropy = BitsEntropy(histograms[self->curr_histogram_ix_].data_,
self->alphabet_size_);
- HistogramType combined_histo[2];
double combined_entropy[2];
double diff[2];
size_t j;
for (j = 0; j < 2; ++j) {
size_t last_histogram_ix = self->last_histogram_ix_[j];
- combined_histo[j] = histograms[self->curr_histogram_ix_];
- FN(HistogramAddHistogram)(&combined_histo[j],
+ self->combined_histo[j] = histograms[self->curr_histogram_ix_];
+ FN(HistogramAddHistogram)(&self->combined_histo[j],
&histograms[last_histogram_ix]);
combined_entropy[j] = BitsEntropy(
- &combined_histo[j].data_[0], self->alphabet_size_);
+ &self->combined_histo[j].data_[0], self->alphabet_size_);
diff[j] = combined_entropy[j] - entropy - last_entropy[j];
}
@@ -141,7 +143,7 @@ static void FN(BlockSplitterFinishBlock)(
split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;
split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];
BROTLI_SWAP(size_t, self->last_histogram_ix_, 0, 1);
- histograms[self->last_histogram_ix_[0]] = combined_histo[1];
+ histograms[self->last_histogram_ix_[0]] = self->combined_histo[1];
last_entropy[1] = last_entropy[0];
last_entropy[0] = combined_entropy[1];
++self->num_blocks_;
@@ -152,7 +154,7 @@ static void FN(BlockSplitterFinishBlock)(
} else {
/* Combine this block with last block. */
split->lengths[self->num_blocks_ - 1] += (uint32_t)self->block_size_;
- histograms[self->last_histogram_ix_[0]] = combined_histo[0];
+ histograms[self->last_histogram_ix_[0]] = self->combined_histo[0];
last_entropy[0] = combined_entropy[0];
if (split->num_types == 1) {
last_entropy[1] = last_entropy[0];
diff --git a/dviware/dvisvgm/libs/brotli/enc/params.h b/dviware/dvisvgm/libs/brotli/enc/params.h
index 54a7f00736..78b2ab6082 100644
--- a/dviware/dvisvgm/libs/brotli/enc/params.h
+++ b/dviware/dvisvgm/libs/brotli/enc/params.h
@@ -10,13 +10,13 @@
#define BROTLI_ENC_PARAMS_H_
#include <brotli/encode.h>
-#include "./encoder_dict.h"
+
+#include "encoder_dict.h"
typedef struct BrotliHasherParams {
int type;
int bucket_bits;
int block_bits;
- int hash_len;
int num_last_distances_to_check;
} BrotliHasherParams;
@@ -40,7 +40,8 @@ typedef struct BrotliEncoderParams {
BROTLI_BOOL large_window;
BrotliHasherParams hasher;
BrotliDistanceParams dist;
- BrotliEncoderDictionary dictionary;
+ /* TODO(eustas): rename to BrotliShared... */
+ SharedEncoderDictionary dictionary;
} BrotliEncoderParams;
#endif /* BROTLI_ENC_PARAMS_H_ */
diff --git a/dviware/dvisvgm/libs/brotli/enc/prefix.h b/dviware/dvisvgm/libs/brotli/enc/prefix.h
index fd359a478d..0f006f1614 100644
--- a/dviware/dvisvgm/libs/brotli/enc/prefix.h
+++ b/dviware/dvisvgm/libs/brotli/enc/prefix.h
@@ -10,10 +10,11 @@
#ifndef BROTLI_ENC_PREFIX_H_
#define BROTLI_ENC_PREFIX_H_
+#include <brotli/types.h>
+
#include "../common/constants.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./fast_log.h"
+#include "fast_log.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/quality.h b/dviware/dvisvgm/libs/brotli/enc/quality.h
index 5f4d034503..ffdfd72fb4 100644
--- a/dviware/dvisvgm/libs/brotli/enc/quality.h
+++ b/dviware/dvisvgm/libs/brotli/enc/quality.h
@@ -10,9 +10,10 @@
#ifndef BROTLI_ENC_QUALITY_H_
#define BROTLI_ENC_QUALITY_H_
-#include "../common/platform.h"
#include <brotli/encode.h>
-#include "./params.h"
+
+#include "../common/platform.h"
+#include "params.h"
#define FAST_ONE_PASS_COMPRESSION_QUALITY 0
#define FAST_TWO_PASS_COMPRESSION_QUALITY 1
@@ -118,6 +119,41 @@ static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch(
return params->quality < 9 ? 64 : 512;
}
+/* Quality to hasher mapping:
+
+ - q02: h02 (longest_match_quickly), b16, l5
+
+ - q03: h03 (longest_match_quickly), b17, l5
+
+ - q04: h04 (longest_match_quickly), b17, l5
+ - q04: h54 (longest_match_quickly), b20, l7 | for large files
+
+ - q05: h05 (longest_match ), b14, l4
+ - q05: h06 (longest_match64 ), b15, l5 | for large files
+ - q05: h40 (forgetful_chain ), b15, l4 | for small window
+
+ - q06: h05 (longest_match ), b14, l4
+ - q06: h06 (longest_match64 ), b15, l5 | for large files
+ - q06: h40 (forgetful_chain ), b15, l4 | for small window
+
+ - q07: h05 (longest_match ), b15, l4
+ - q07: h06 (longest_match64 ), b15, l5 | for large files
+ - q07: h41 (forgetful_chain ), b15, l4 | for small window
+
+ - q08: h05 (longest_match ), b15, l4
+ - q08: h06 (longest_match64 ), b15, l5 | for large files
+ - q08: h41 (forgetful_chain ), b15, l4 | for small window
+
+ - q09: h05 (longest_match ), b15, l4
+ - q09: h06 (longest_match64 ), b15, l5 | for large files
+ - q09: h42 (forgetful_chain ), b15, l4 | for small window
+
+ - q10: t10 (to_binary_tree ), b17, l128
+
+ - q11: t10 (to_binary_tree ), b17, l128
+
+ Where "q" is quality, "h" is hasher type, "b" is bucket bits,
+ "l" is source len. */
static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
BrotliHasherParams* hparams) {
if (params->quality > 9) {
@@ -132,10 +168,11 @@ static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
hparams->type = 6;
hparams->block_bits = params->quality - 1;
hparams->bucket_bits = 15;
- hparams->hash_len = 5;
hparams->num_last_distances_to_check =
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
} else {
+ /* TODO(eustas): often previous setting (H6) is faster and denser; consider
+ adding an option to use it. */
hparams->type = 5;
hparams->block_bits = params->quality - 1;
hparams->bucket_bits = params->quality < 7 ? 14 : 15;
diff --git a/dviware/dvisvgm/libs/brotli/enc/ringbuffer.h b/dviware/dvisvgm/libs/brotli/enc/ringbuffer.h
index 8dce148039..27245b7f39 100644
--- a/dviware/dvisvgm/libs/brotli/enc/ringbuffer.h
+++ b/dviware/dvisvgm/libs/brotli/enc/ringbuffer.h
@@ -11,10 +11,11 @@
#include <string.h> /* memcpy */
-#include "../common/platform.h"
#include <brotli/types.h>
-#include "./memory.h"
-#include "./quality.h"
+
+#include "../common/platform.h"
+#include "memory.h"
+#include "quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/state.h b/dviware/dvisvgm/libs/brotli/enc/state.h
new file mode 100644
index 0000000000..cb82987701
--- /dev/null
+++ b/dviware/dvisvgm/libs/brotli/enc/state.h
@@ -0,0 +1,104 @@
+/* Copyright 2022 Google Inc. All Rights Reserved.
+
+ Distributed under MIT license.
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Encoder state. */
+
+#ifndef BROTLI_ENC_STATE_H_
+#define BROTLI_ENC_STATE_H_
+
+#include <brotli/types.h>
+
+#include "command.h"
+#include "compress_fragment.h"
+#include "compress_fragment_two_pass.h"
+#include "hash.h"
+#include "memory.h"
+#include "params.h"
+#include "ringbuffer.h"
+
+typedef enum BrotliEncoderStreamState {
+ /* Default state. */
+ BROTLI_STREAM_PROCESSING = 0,
+ /* Intermediate state; after next block is emitted, byte-padding should be
+ performed before getting back to default state. */
+ BROTLI_STREAM_FLUSH_REQUESTED = 1,
+ /* Last metablock was produced; no more input is acceptable. */
+ BROTLI_STREAM_FINISHED = 2,
+ /* Flushing compressed block and writing meta-data block header. */
+ BROTLI_STREAM_METADATA_HEAD = 3,
+ /* Writing metadata block body. */
+ BROTLI_STREAM_METADATA_BODY = 4
+} BrotliEncoderStreamState;
+
+typedef enum BrotliEncoderFlintState {
+ BROTLI_FLINT_NEEDS_2_BYTES = 2,
+ BROTLI_FLINT_NEEDS_1_BYTE = 1,
+ BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
+ BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
+ BROTLI_FLINT_DONE = -2
+} BrotliEncoderFlintState;
+
+typedef struct BrotliEncoderStateStruct {
+ BrotliEncoderParams params;
+
+ MemoryManager memory_manager_;
+
+ uint64_t input_pos_;
+ RingBuffer ringbuffer_;
+ size_t cmd_alloc_size_;
+ Command* commands_;
+ size_t num_commands_;
+ size_t num_literals_;
+ size_t last_insert_len_;
+ uint64_t last_flush_pos_;
+ uint64_t last_processed_pos_;
+ int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
+ int saved_dist_cache_[4];
+ uint16_t last_bytes_;
+ uint8_t last_bytes_bits_;
+ /* "Flint" is a tiny uncompressed block emitted before the continuation
+ block to unwire literal context from previous data. Despite being int8_t,
+ field is actually BrotliEncoderFlintState enum. */
+ int8_t flint_;
+ uint8_t prev_byte_;
+ uint8_t prev_byte2_;
+ size_t storage_size_;
+ uint8_t* storage_;
+
+ Hasher hasher_;
+
+ /* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
+ int small_table_[1 << 10]; /* 4KiB */
+ int* large_table_; /* Allocated only when needed */
+ size_t large_table_size_;
+
+ BrotliOnePassArena* one_pass_arena_;
+ BrotliTwoPassArena* two_pass_arena_;
+
+ /* Command and literal buffers for FAST_TWO_PASS_COMPRESSION_QUALITY. */
+ uint32_t* command_buf_;
+ uint8_t* literal_buf_;
+
+ uint64_t total_in_;
+ uint8_t* next_out_;
+ size_t available_out_;
+ uint64_t total_out_;
+ /* Temporary buffer for padding flush bits or metadata block header / body. */
+ union {
+ uint64_t u64[2];
+ uint8_t u8[16];
+ } tiny_buf_;
+ uint32_t remaining_metadata_bytes_;
+ BrotliEncoderStreamState stream_state_;
+
+ BROTLI_BOOL is_last_block_emitted_;
+ BROTLI_BOOL is_initialized_;
+} BrotliEncoderStateStruct;
+
+typedef struct BrotliEncoderStateStruct BrotliEncoderStateInternal;
+#define BrotliEncoderState BrotliEncoderStateInternal
+
+#endif // BROTLI_ENC_STATE_H_
diff --git a/dviware/dvisvgm/libs/brotli/enc/static_dict.c b/dviware/dvisvgm/libs/brotli/enc/static_dict.c
index 7299ab7203..291d283354 100644
--- a/dviware/dvisvgm/libs/brotli/enc/static_dict.c
+++ b/dviware/dvisvgm/libs/brotli/enc/static_dict.c
@@ -4,13 +4,13 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
-#include "./static_dict.h"
+#include "static_dict.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "../common/transform.h"
-#include "./encoder_dict.h"
-#include "./find_match_length.h"
+#include "encoder_dict.h"
+#include "find_match_length.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -74,10 +74,27 @@ static BROTLI_INLINE BROTLI_BOOL IsMatch(const BrotliDictionary* dictionary,
}
}
-BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
+/* Finds matches for a single static dictionary */
+static BROTLI_BOOL BrotliFindAllStaticDictionaryMatchesFor(
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
size_t min_length, size_t max_length, uint32_t* matches) {
BROTLI_BOOL has_found_match = BROTLI_FALSE;
+#if defined(BROTLI_EXPERIMENTAL)
+ if (dictionary->has_words_heavy) {
+ const BrotliTrieNode* node = &dictionary->trie.root;
+ size_t l = 0;
+ while (node && l < max_length) {
+ uint8_t c;
+ if (l >= min_length && node->len_) {
+ AddMatch(node->idx_, l, node->len_, matches);
+ has_found_match = BROTLI_TRUE;
+ }
+ c = data[l++];
+ node = BrotliTrieSub(&dictionary->trie, node, c);
+ }
+ return has_found_match;
+ }
+#endif /* BROTLI_EXPERIMENTAL */
{
size_t offset = dictionary->buckets[Hash(data)];
BROTLI_BOOL end = !offset;
@@ -481,6 +498,45 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
return has_found_match;
}
+/* Finds matches for one or more dictionaries, if multiple are present
+ in the contextual dictionary */
+BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
+ const BrotliEncoderDictionary* dictionary, const uint8_t* data,
+ size_t min_length, size_t max_length, uint32_t* matches) {
+ BROTLI_BOOL has_found_match =
+ BrotliFindAllStaticDictionaryMatchesFor(
+ dictionary, data, min_length, max_length, matches);
+
+ if (!!dictionary->parent && dictionary->parent->num_dictionaries > 1) {
+ uint32_t matches2[BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1];
+ int l;
+ const BrotliEncoderDictionary* dictionary2 = dictionary->parent->dict[0];
+ if (dictionary2 == dictionary) {
+ dictionary2 = dictionary->parent->dict[1];
+ }
+
+ for (l = 0; l < BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1; l++) {
+ matches2[l] = kInvalidMatch;
+ }
+
+ has_found_match |= BrotliFindAllStaticDictionaryMatchesFor(
+ dictionary2, data, min_length, max_length, matches2);
+
+ for (l = 0; l < BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1; l++) {
+ if (matches2[l] != kInvalidMatch) {
+ uint32_t dist = (uint32_t)(matches2[l] >> 5);
+ uint32_t len_code = matches2[l] & 31;
+ uint32_t skipdist = (uint32_t)((uint32_t)(1 << dictionary->words->
+ size_bits_by_length[len_code]) & ~1u) *
+ (uint32_t)dictionary->num_transforms;
+ /* TODO(lode): check for dist overflow */
+ dist += skipdist;
+ AddMatch(dist, (size_t)l, len_code, matches);
+ }
+ }
+ }
+ return has_found_match;
+}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/static_dict.h b/dviware/dvisvgm/libs/brotli/enc/static_dict.h
index 6b5d4eb0c9..ab832207d1 100644
--- a/dviware/dvisvgm/libs/brotli/enc/static_dict.h
+++ b/dviware/dvisvgm/libs/brotli/enc/static_dict.h
@@ -9,10 +9,11 @@
#ifndef BROTLI_ENC_STATIC_DICT_H_
#define BROTLI_ENC_STATIC_DICT_H_
+#include <brotli/types.h>
+
#include "../common/dictionary.h"
#include "../common/platform.h"
-#include <brotli/types.h>
-#include "./encoder_dict.h"
+#include "encoder_dict.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
diff --git a/dviware/dvisvgm/libs/brotli/enc/static_dict_lut.h b/dviware/dvisvgm/libs/brotli/enc/static_dict_lut.h
index e299cda6d8..a465ffde74 100644
--- a/dviware/dvisvgm/libs/brotli/enc/static_dict_lut.h
+++ b/dviware/dvisvgm/libs/brotli/enc/static_dict_lut.h
@@ -22,6 +22,7 @@ typedef struct DictWord {
uint16_t idx;
} DictWord;
+/* GENERATED CODE START */
static const int kDictNumBits = 15;
static const uint32_t kDictHashMul32 = 0x1E35A7BD;
@@ -5856,6 +5857,7 @@ static const DictWord kStaticDictionaryWords[31705] = {
,0,1735},{5,0,598},{7,0,791},{8,0,108},{9,0,123},{7,10,1570},{140,10,542},{142,
11,410},{9,11,660},{138,11,347}
};
+/* GENERATED CODE END */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
diff --git a/dviware/dvisvgm/libs/brotli/enc/utf8_util.c b/dviware/dvisvgm/libs/brotli/enc/utf8_util.c
index e802b6a751..65ec3f5c8d 100644
--- a/dviware/dvisvgm/libs/brotli/enc/utf8_util.c
+++ b/dviware/dvisvgm/libs/brotli/enc/utf8_util.c
@@ -6,7 +6,7 @@
/* Heuristics for deciding about the UTF8-ness of strings. */
-#include "./utf8_util.h"
+#include "utf8_util.h"
#include <brotli/types.h>
diff --git a/dviware/dvisvgm/libs/brotli/enc/utf8_util.h b/dviware/dvisvgm/libs/brotli/enc/utf8_util.h
index 8fda80c220..a38a95383c 100644
--- a/dviware/dvisvgm/libs/brotli/enc/utf8_util.h
+++ b/dviware/dvisvgm/libs/brotli/enc/utf8_util.h
@@ -9,9 +9,10 @@
#ifndef BROTLI_ENC_UTF8_UTIL_H_
#define BROTLI_ENC_UTF8_UTIL_H_
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
diff --git a/dviware/dvisvgm/libs/brotli/enc/write_bits.h b/dviware/dvisvgm/libs/brotli/enc/write_bits.h
index f6f88b45be..242754b0ee 100644
--- a/dviware/dvisvgm/libs/brotli/enc/write_bits.h
+++ b/dviware/dvisvgm/libs/brotli/enc/write_bits.h
@@ -9,9 +9,10 @@
#ifndef BROTLI_ENC_WRITE_BITS_H_
#define BROTLI_ENC_WRITE_BITS_H_
-#include "../common/platform.h"
#include <brotli/types.h>
+#include "../common/platform.h"
+
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
diff --git a/dviware/dvisvgm/libs/brotli/include/brotli/decode.h b/dviware/dvisvgm/libs/brotli/include/brotli/decode.h
index 0f5c8f9d11..af1aa23f60 100644
--- a/dviware/dvisvgm/libs/brotli/include/brotli/decode.h
+++ b/dviware/dvisvgm/libs/brotli/include/brotli/decode.h
@@ -13,6 +13,7 @@
#define BROTLI_DEC_DECODE_H_
#include <brotli/port.h>
+#include <brotli/shared_dictionary.h>
#include <brotli/types.h>
#if defined(__cplusplus) || defined(c_plusplus)
@@ -85,8 +86,9 @@ typedef enum {
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, DISTANCE, -16) SEPARATOR \
\
- /* -17..-18 codes are reserved */ \
+ /* -17 code is reserved */ \
\
+ BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \
\
@@ -155,6 +157,28 @@ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value);
/**
+ * Adds LZ77 prefix dictionary, adds or replaces built-in static dictionary and
+ * transforms.
+ *
+ * Attached dictionary ownership is not transferred.
+ * Data provided to this method should be kept accessible until
+ * decoding is finished and decoder instance is destroyed.
+ *
+ * @note Dictionaries can NOT be attached after actual decoding is started.
+ *
+ * @param state decoder instance
+ * @param type dictionary data format
+ * @param data_size length of memory region pointed by @p data
+ * @param data dictionary data in format corresponding to @p type
+ * @returns ::BROTLI_FALSE if dictionary is corrupted,
+ * or dictionary count limit is reached
+ * @returns ::BROTLI_TRUE if dictionary is accepted / attached
+ */
+BROTLI_DEC_API BROTLI_BOOL BrotliDecoderAttachDictionary(
+ BrotliDecoderState* state, BrotliSharedDictionaryType type,
+ size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)]);
+
+/**
* Creates an instance of ::BrotliDecoderState and initializes it.
*
* The instance can be used once for decoding and should then be destroyed with
@@ -333,10 +357,51 @@ BROTLI_DEC_API const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c);
/**
* Gets a decoder library version.
*
- * Look at BROTLI_VERSION for more information.
+ * Look at BROTLI_MAKE_HEX_VERSION for more information.
*/
BROTLI_DEC_API uint32_t BrotliDecoderVersion(void);
+/**
+ * Callback to fire on metadata block start.
+ *
+ * After this callback is fired, if @p size is not @c 0, it is followed by
+ * ::brotli_decoder_metadata_chunk_func as more metadata block contents become
+ * accessible.
+ *
+ * @param opaque callback handle
+ * @param size size of metadata block
+ */
+typedef void (*brotli_decoder_metadata_start_func)(void* opaque, size_t size);
+
+/**
+ * Callback to fire on metadata block chunk becomes available.
+ *
+ * This function can be invoked multiple times per metadata block; block should
+ * be considered finished when sum of @p size matches the announced metadata
+ * block size. Chunks contents pointed by @p data are transient and shouln not
+ * be accessed after leaving the callback.
+ *
+ * @param opaque callback handle
+ * @param data pointer to metadata contents
+ * @param size size of metadata block chunk, at least @c 1
+ */
+typedef void (*brotli_decoder_metadata_chunk_func)(void* opaque,
+ const uint8_t* data,
+ size_t size);
+
+/**
+ * Sets callback for receiving metadata blocks.
+ *
+ * @param state decoder instance
+ * @param start_func callback on metadata block start
+ * @param chunk_func callback on metadata block chunk
+ * @param opaque callback handle
+ */
+BROTLI_DEC_API void BrotliDecoderSetMetadataCallbacks(
+ BrotliDecoderState* state,
+ brotli_decoder_metadata_start_func start_func,
+ brotli_decoder_metadata_chunk_func chunk_func, void* opaque);
+
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
diff --git a/dviware/dvisvgm/libs/brotli/include/brotli/encode.h b/dviware/dvisvgm/libs/brotli/include/brotli/encode.h
index b2774cb631..dea9931ebb 100644
--- a/dviware/dvisvgm/libs/brotli/include/brotli/encode.h
+++ b/dviware/dvisvgm/libs/brotli/include/brotli/encode.h
@@ -13,6 +13,7 @@
#define BROTLI_ENC_ENCODE_H_
#include <brotli/port.h>
+#include <brotli/shared_dictionary.h>
#include <brotli/types.h>
#if defined(__cplusplus) || defined(c_plusplus)
@@ -269,6 +270,51 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
*/
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
+/* Opaque type for pointer to different possible internal structures containing
+ dictionary prepared for the encoder */
+typedef struct BrotliEncoderPreparedDictionaryStruct
+ BrotliEncoderPreparedDictionary;
+
+/**
+ * Prepares a shared dictionary from the given file format for the encoder.
+ *
+ * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
+ * case they are both zero, default memory allocators are used. @p opaque is
+ * passed to @p alloc_func and @p free_func when they are called. @p free_func
+ * has to return without doing anything when asked to free a NULL pointer.
+ *
+ * @param type type of dictionary stored in data
+ * @param data_size size of @p data buffer
+ * @param data pointer to the dictionary data
+ * @param quality the maximum Brotli quality to prepare the dictionary for,
+ * use BROTLI_MAX_QUALITY by default
+ * @param alloc_func custom memory allocation function
+ * @param free_func custom memory free function
+ * @param opaque custom memory manager handle
+ */
+BROTLI_ENC_API BrotliEncoderPreparedDictionary*
+BrotliEncoderPrepareDictionary(BrotliSharedDictionaryType type,
+ size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)],
+ int quality,
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
+
+BROTLI_ENC_API void BrotliEncoderDestroyPreparedDictionary(
+ BrotliEncoderPreparedDictionary* dictionary);
+
+/**
+ * Attaches a prepared dictionary of any type to the encoder. Can be used
+ * multiple times to attach multiple dictionaries. The dictionary type was
+ * determined by BrotliEncoderPrepareDictionary. Multiple raw prefix
+ * dictionaries and/or max 1 serialized dictionary with custom words can be
+ * attached.
+ *
+ * @returns ::BROTLI_FALSE in case of error
+ * @returns ::BROTLI_TRUE otherwise
+ */
+BROTLI_ENC_API BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(
+ BrotliEncoderState* state,
+ const BrotliEncoderPreparedDictionary* dictionary);
+
/**
* Calculates the output size bound for the given @p input_size.
*
@@ -407,7 +453,7 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
*
* This method is used to make language bindings easier and more efficient:
* -# push data to ::BrotliEncoderCompressStream,
- * until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE
+ * until ::BrotliEncoderHasMoreOutput returns BROTLI_TRUE
* -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
* entity
*
@@ -433,11 +479,18 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
BrotliEncoderState* state, size_t* size);
+/* Returns the estimated peak memory usage (in bytes) of the BrotliCompress()
+ function, not counting the memory needed for the input and output. */
+BROTLI_ENC_EXTRA_API size_t BrotliEncoderEstimatePeakMemoryUsage(
+ int quality, int lgwin, size_t input_size);
+/* Returns 0 if dictionary is not valid; otherwise returns allocation size. */
+BROTLI_ENC_EXTRA_API size_t BrotliEncoderGetPreparedDictionarySize(
+ const BrotliEncoderPreparedDictionary* dictionary);
/**
* Gets an encoder library version.
*
- * Look at BROTLI_VERSION for more information.
+ * Look at BROTLI_MAKE_HEX_VERSION for more information.
*/
BROTLI_ENC_API uint32_t BrotliEncoderVersion(void);
diff --git a/dviware/dvisvgm/libs/brotli/include/brotli/port.h b/dviware/dvisvgm/libs/brotli/include/brotli/port.h
index 825237a335..0d50019042 100644
--- a/dviware/dvisvgm/libs/brotli/include/brotli/port.h
+++ b/dviware/dvisvgm/libs/brotli/include/brotli/port.h
@@ -224,14 +224,6 @@
#define BROTLI_HAS_FEATURE(feature) (0)
#endif
-#if defined(ADDRESS_SANITIZER) || BROTLI_HAS_FEATURE(address_sanitizer) || \
- defined(THREAD_SANITIZER) || BROTLI_HAS_FEATURE(thread_sanitizer) || \
- defined(MEMORY_SANITIZER) || BROTLI_HAS_FEATURE(memory_sanitizer)
-#define BROTLI_SANITIZED 1
-#else
-#define BROTLI_SANITIZED 0
-#endif
-
#if defined(_WIN32) || defined(__CYGWIN__)
#define BROTLI_PUBLIC
#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
@@ -247,9 +239,28 @@
#define BROTLI_PUBLIC
#endif
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
- !defined(__STDC_NO_VLA__) && !defined(__cplusplus) && \
- !defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__)
+/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
+#if !defined(BROTLI_INTERNAL)
+#if defined(_WIN32) || defined(__CYGWIN__)
+#define BROTLI_INTERNAL
+#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
+ BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
+ BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
+ BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
+ BROTLI_IBM_VERSION_CHECK(13, 1, 0) || \
+ BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
+ (BROTLI_TI_VERSION_CHECK(7, 3, 0) && \
+ defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
+#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
+#else
+#define BROTLI_INTERNAL
+#endif
+#endif
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
+ !defined(__STDC_NO_VLA__) && !defined(__cplusplus) && \
+ !defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__) && \
+ !defined(__clang__)
#define BROTLI_ARRAY_PARAM(name) (name)
#else
#define BROTLI_ARRAY_PARAM(name)
@@ -285,4 +296,10 @@
#define BROTLI_ENC_API
#endif
+#if defined(BROTLI_BUILD_ENC_EXTRA_API)
+#define BROTLI_ENC_EXTRA_API BROTLI_ENC_API
+#else
+#define BROTLI_ENC_EXTRA_API BROTLI_INTERNAL
+#endif
+
#endif /* BROTLI_COMMON_PORT_H_ */
diff --git a/dviware/dvisvgm/libs/brotli/include/brotli/shared_dictionary.h b/dviware/dvisvgm/libs/brotli/include/brotli/shared_dictionary.h
new file mode 100644
index 0000000000..2970c2dcc9
--- /dev/null
+++ b/dviware/dvisvgm/libs/brotli/include/brotli/shared_dictionary.h
@@ -0,0 +1,100 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+ Distributed under MIT license.
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* (Opaque) Shared Dictionary definition and utilities. */
+
+#ifndef BROTLI_COMMON_SHARED_DICTIONARY_H_
+#define BROTLI_COMMON_SHARED_DICTIONARY_H_
+
+#include <brotli/port.h>
+#include <brotli/types.h>
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+#define SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
+#define SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH 31
+#define SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS 64
+#define SHARED_BROTLI_MAX_COMPOUND_DICTS 15
+
+/**
+ * Opaque structure that holds shared dictionary data.
+ *
+ * Allocated and initialized with ::BrotliSharedDictionaryCreateInstance.
+ * Cleaned up and deallocated with ::BrotliSharedDictionaryDestroyInstance.
+ */
+typedef struct BrotliSharedDictionaryStruct BrotliSharedDictionary;
+
+/**
+ * Input data type for ::BrotliSharedDictionaryAttach.
+ */
+typedef enum BrotliSharedDictionaryType {
+ /** Raw LZ77 prefix dictionary. */
+ BROTLI_SHARED_DICTIONARY_RAW = 0,
+ /** Serialized shared dictionary.
+ *
+ * DO NOT USE: methods accepting this value will fail.
+ */
+ BROTLI_SHARED_DICTIONARY_SERIALIZED = 1
+} BrotliSharedDictionaryType;
+
+/**
+ * Creates an instance of ::BrotliSharedDictionary.
+ *
+ * Fresh instance has default word dictionary and transforms
+ * and no LZ77 prefix dictionary.
+ *
+ * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
+ * case they are both zero, default memory allocators are used. @p opaque is
+ * passed to @p alloc_func and @p free_func when they are called. @p free_func
+ * has to return without doing anything when asked to free a NULL pointer.
+ *
+ * @param alloc_func custom memory allocation function
+ * @param free_func custom memory free function
+ * @param opaque custom memory manager handle
+ * @returns @c 0 if instance can not be allocated or initialized
+ * @returns pointer to initialized ::BrotliSharedDictionary otherwise
+ */
+BROTLI_COMMON_API BrotliSharedDictionary* BrotliSharedDictionaryCreateInstance(
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
+
+/**
+ * Deinitializes and frees ::BrotliSharedDictionary instance.
+ *
+ * @param dict shared dictionary instance to be cleaned up and deallocated
+ */
+BROTLI_COMMON_API void BrotliSharedDictionaryDestroyInstance(
+ BrotliSharedDictionary* dict);
+
+/**
+ * Attaches dictionary to a given instance of ::BrotliSharedDictionary.
+ *
+ * Dictionary to be attached is represented in a serialized format as a region
+ * of memory.
+ *
+ * Provided data it partially referenced by a resulting (compound) dictionary,
+ * and should be kept untouched, while at least one compound dictionary uses it.
+ * This way memory overhead is kept minimal by the cost of additional resource
+ * management.
+ *
+ * @param dict dictionary to extend
+ * @param type type of dictionary to attach
+ * @param data_size size of @p data
+ * @param data serialized dictionary of type @p type, with at least @p data_size
+ * addressable bytes
+ * @returns ::BROTLI_TRUE if provided dictionary is successfully attached
+ * @returns ::BROTLI_FALSE otherwise
+ */
+BROTLI_COMMON_API BROTLI_BOOL BrotliSharedDictionaryAttach(
+ BrotliSharedDictionary* dict, BrotliSharedDictionaryType type,
+ size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)]);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+} /* extern "C" */
+#endif
+
+#endif /* BROTLI_COMMON_SHARED_DICTIONARY_H_ */