diff options
author | Karl Berry <karl@freefriends.org> | 2014-04-19 22:09:10 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2014-04-19 22:09:10 +0000 |
commit | f63732007b6a055afad48521b678be0a9c0ed7ae (patch) | |
tree | 1b698adf2ec8387c8fdef044a32d321ae50e9633 /Build/source/extra/xz/doc | |
parent | f089b79c939f3448e2fa7fcc6687a6417b3107f8 (diff) |
messed up xz timestamps, better retrieved from original anyway
git-svn-id: svn://tug.org/texlive/trunk@33544 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/extra/xz/doc')
32 files changed, 0 insertions, 4412 deletions
diff --git a/Build/source/extra/xz/doc/examples/00_README.txt b/Build/source/extra/xz/doc/examples/00_README.txt deleted file mode 100644 index a3b9eaa1c97..00000000000 --- a/Build/source/extra/xz/doc/examples/00_README.txt +++ /dev/null @@ -1,27 +0,0 @@ - -liblzma example programs -======================== - -Introduction - - The examples are written so that the same comments aren't - repeated (much) in later files. - - On POSIX systems, the examples should build by just typing "make". - - The examples that use stdin or stdout don't set stdin and stdout - to binary mode. On systems where it matters (e.g. Windows) it is - possible that the examples won't work without modification. - - -List of examples - - 01_compress_easy.c Multi-call compression using - a compression preset - - 02_decompress.c Multi-call decompression - - 03_compress_custom.c Like 01_compress_easy.c but using - a custom filter chain - (x86 BCJ + LZMA2) - diff --git a/Build/source/extra/xz/doc/examples/01_compress_easy.c b/Build/source/extra/xz/doc/examples/01_compress_easy.c deleted file mode 100644 index e6dd2b0cda3..00000000000 --- a/Build/source/extra/xz/doc/examples/01_compress_easy.c +++ /dev/null @@ -1,297 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file 01_compress_easy.c -/// \brief Compress from stdin to stdout in multi-call mode -/// -/// Usage: ./01_compress_easy PRESET < INFILE > OUTFILE -/// -/// Example: ./01_compress_easy 6 < foo > foo.xz -// -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. -// -/////////////////////////////////////////////////////////////////////////////// - -#include <stdbool.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <errno.h> -#include <lzma.h> - - -static void -show_usage_and_exit(const char *argv0) -{ - fprintf(stderr, "Usage: %s PRESET < INFILE > OUTFILE\n" - "PRESET is a number 0-9 and can optionally be " - "followed by `e' to indicate extreme preset\n", - argv0); - exit(EXIT_FAILURE); -} - - -static uint32_t -get_preset(int argc, char **argv) -{ - // One argument whose first char must be 0-9. - if (argc != 2 || argv[1][0] < '0' || argv[1][0] > '9') - show_usage_and_exit(argv[0]); - - // Calculate the preste level 0-9. - uint32_t preset = argv[1][0] - '0'; - - // If there is a second char, it must be 'e'. It will set - // the LZMA_PRESET_EXTREME flag. - if (argv[1][1] != '\0') { - if (argv[1][1] != 'e' || argv[1][2] != '\0') - show_usage_and_exit(argv[0]); - - preset |= LZMA_PRESET_EXTREME; - } - - return preset; -} - - -static bool -init_encoder(lzma_stream *strm, uint32_t preset) -{ - // Initialize the encoder using a preset. Set the integrity to check - // to CRC64, which is the default in the xz command line tool. If - // the .xz file needs to be decompressed with XZ Embedded, use - // LZMA_CHECK_CRC32 instead. - lzma_ret ret = lzma_easy_encoder(strm, preset, LZMA_CHECK_CRC64); - - // Return successfully if the initialization went fine. - if (ret == LZMA_OK) - return true; - - // Something went wrong. The possible errors are documented in - // lzma/container.h (src/liblzma/api/lzma/container.h in the source - // package or e.g. /usr/include/lzma/container.h depending on the - // install prefix). - const char *msg; - switch (ret) { - case LZMA_MEM_ERROR: - msg = "Memory allocation failed"; - break; - - case LZMA_OPTIONS_ERROR: - msg = "Specified preset is not supported"; - break; - - case LZMA_UNSUPPORTED_CHECK: - msg = "Specified integrity check is not supported"; - break; - - default: - // This is most likely LZMA_PROG_ERROR indicating a bug in - // this program or in liblzma. It is inconvenient to have a - // separate error message for errors that should be impossible - // to occur, but knowing the error code is important for - // debugging. That's why it is good to print the error code - // at least when there is no good error message to show. - msg = "Unknown error, possibly a bug"; - break; - } - - fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n", - msg, ret); - return false; -} - - -static bool -compress(lzma_stream *strm, FILE *infile, FILE *outfile) -{ - // This will be LZMA_RUN until the end of the input file is reached. - // This tells lzma_code() when there will be no more input. - lzma_action action = LZMA_RUN; - - // Buffers to temporarily hold uncompressed input - // and compressed output. - uint8_t inbuf[BUFSIZ]; - uint8_t outbuf[BUFSIZ]; - - // Initialize the input and output pointers. Initializing next_in - // and avail_in isn't really necessary when we are going to encode - // just one file since LZMA_STREAM_INIT takes care of initializing - // those already. But it doesn't hurt much and it will be needed - // if encoding more than one file like we will in 02_decompress.c. - // - // While we don't care about strm->total_in or strm->total_out in this - // example, it is worth noting that initializing the encoder will - // always reset total_in and total_out to zero. But the encoder - // initialization doesn't touch next_in, avail_in, next_out, or - // avail_out. - strm->next_in = NULL; - strm->avail_in = 0; - strm->next_out = outbuf; - strm->avail_out = sizeof(outbuf); - - // Loop until the file has been successfully compressed or until - // an error occurs. - while (true) { - // Fill the input buffer if it is empty. - if (strm->avail_in == 0 && !feof(infile)) { - strm->next_in = inbuf; - strm->avail_in = fread(inbuf, 1, sizeof(inbuf), - infile); - - if (ferror(infile)) { - fprintf(stderr, "Read error: %s\n", - strerror(errno)); - return false; - } - - // Once the end of the input file has been reached, - // we need to tell lzma_code() that no more input - // will be coming and that it should finish the - // encoding. - if (feof(infile)) - action = LZMA_FINISH; - } - - // Tell liblzma do the actual encoding. - // - // This reads up to strm->avail_in bytes of input starting - // from strm->next_in. avail_in will be decremented and - // next_in incremented by an equal amount to match the - // number of input bytes consumed. - // - // Up to strm->avail_out bytes of compressed output will be - // written starting from strm->next_out. avail_out and next_out - // will be incremented by an equal amount to match the number - // of output bytes written. - // - // The encoder has to do internal buffering, which means that - // it may take quite a bit of input before the same data is - // available in compressed form in the output buffer. - lzma_ret ret = lzma_code(strm, action); - - // If the output buffer is full or if the compression finished - // successfully, write the data from the output bufffer to - // the output file. - if (strm->avail_out == 0 || ret == LZMA_STREAM_END) { - // When lzma_code() has returned LZMA_STREAM_END, - // the output buffer is likely to be only partially - // full. Calculate how much new data there is to - // be written to the output file. - size_t write_size = sizeof(outbuf) - strm->avail_out; - - if (fwrite(outbuf, 1, write_size, outfile) - != write_size) { - fprintf(stderr, "Write error: %s\n", - strerror(errno)); - return false; - } - - // Reset next_out and avail_out. - strm->next_out = outbuf; - strm->avail_out = sizeof(outbuf); - } - - // Normally the return value of lzma_code() will be LZMA_OK - // until everything has been encoded. - if (ret != LZMA_OK) { - // Once everything has been encoded successfully, the - // return value of lzma_code() will be LZMA_STREAM_END. - // - // It is important to check for LZMA_STREAM_END. Do not - // assume that getting ret != LZMA_OK would mean that - // everything has gone well. - if (ret == LZMA_STREAM_END) - return true; - - // It's not LZMA_OK nor LZMA_STREAM_END, - // so it must be an error code. See lzma/base.h - // (src/liblzma/api/lzma/base.h in the source package - // or e.g. /usr/include/lzma/base.h depending on the - // install prefix) for the list and documentation of - // possible values. Most values listen in lzma_ret - // enumeration aren't possible in this example. - const char *msg; - switch (ret) { - case LZMA_MEM_ERROR: - msg = "Memory allocation failed"; - break; - - case LZMA_DATA_ERROR: - // This error is returned if the compressed - // or uncompressed size get near 8 EiB - // (2^63 bytes) because that's where the .xz - // file format size limits currently are. - // That is, the possibility of this error - // is mostly theoretical unless you are doing - // something very unusual. - // - // Note that strm->total_in and strm->total_out - // have nothing to do with this error. Changing - // those variables won't increase or decrease - // the chance of getting this error. - msg = "File size limits exceeded"; - break; - - default: - // This is most likely LZMA_PROG_ERROR, but - // if this program is buggy (or liblzma has - // a bug), it may be e.g. LZMA_BUF_ERROR or - // LZMA_OPTIONS_ERROR too. - // - // It is inconvenient to have a separate - // error message for errors that should be - // impossible to occur, but knowing the error - // code is important for debugging. That's why - // it is good to print the error code at least - // when there is no good error message to show. - msg = "Unknown error, possibly a bug"; - break; - } - - fprintf(stderr, "Encoder error: %s (error code %u)\n", - msg, ret); - return false; - } - } -} - - -extern int -main(int argc, char **argv) -{ - // Get the preset number from the command line. - uint32_t preset = get_preset(argc, argv); - - // Initialize a lzma_stream structure. When it is allocated on stack, - // it is simplest to use LZMA_STREAM_INIT macro like below. When it - // is allocated on heap, using memset(strmptr, 0, sizeof(*strmptr)) - // works (as long as NULL pointers are represented with zero bits - // as they are on practically all computers today). - lzma_stream strm = LZMA_STREAM_INIT; - - // Initialize the encoder. If it succeeds, compress from - // stdin to stdout. - bool success = init_encoder(&strm, preset); - if (success) - success = compress(&strm, stdin, stdout); - - // Free the memory allocated for the encoder. If we were encoding - // multiple files, this would only need to be done after the last - // file. See 02_decompress.c for handling of multiple files. - // - // It is OK to call lzma_end() multiple times or when it hasn't been - // actually used except initialized with LZMA_STREAM_INIT. - lzma_end(&strm); - - // Close stdout to catch possible write errors that can occur - // when pending data is flushed from the stdio buffers. - if (fclose(stdout)) { - fprintf(stderr, "Write error: %s\n", strerror(errno)); - success = false; - } - - return success ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/Build/source/extra/xz/doc/examples/02_decompress.c b/Build/source/extra/xz/doc/examples/02_decompress.c deleted file mode 100644 index 4c0f37cbc57..00000000000 --- a/Build/source/extra/xz/doc/examples/02_decompress.c +++ /dev/null @@ -1,287 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file 02_decompress.c -/// \brief Decompress .xz files to stdout -/// -/// Usage: ./02_decompress INPUT_FILES... > OUTFILE -/// -/// Example: ./02_decompress foo.xz bar.xz > foobar -// -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. -// -/////////////////////////////////////////////////////////////////////////////// - -#include <stdbool.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <errno.h> -#include <lzma.h> - - -static bool -init_decoder(lzma_stream *strm) -{ - // Initialize a .xz decoder. The decoder supports a memory usage limit - // and a set of flags. - // - // The memory usage of the decompressor depends on the settings used - // to compress a .xz file. It can vary from less than a megabyte to - // a few gigabytes, but in practice (at least for now) it rarely - // exceeds 65 MiB because that's how much memory is required to - // decompress files created with "xz -9". Settings requiring more - // memory take extra effort to use and don't (at least for now) - // provide significantly better compression in most cases. - // - // Memory usage limit is useful if it is important that the - // decompressor won't consume gigabytes of memory. The need - // for limiting depends on the application. In this example, - // no memory usage limiting is used. This is done by setting - // the limit to UINT64_MAX. - // - // The .xz format allows concatenating compressed files as is: - // - // echo foo | xz > foobar.xz - // echo bar | xz >> foobar.xz - // - // When decompressing normal standalone .xz files, LZMA_CONCATENATED - // should always be used to support decompression of concatenated - // .xz files. If LZMA_CONCATENATED isn't used, the decoder will stop - // after the first .xz stream. This can be useful when .xz data has - // been embedded inside another file format. - // - // Flags other than LZMA_CONCATENATED are supported too, and can - // be combined with bitwise-or. See lzma/container.h - // (src/liblzma/api/lzma/container.h in the source package or e.g. - // /usr/include/lzma/container.h depending on the install prefix) - // for details. - lzma_ret ret = lzma_stream_decoder( - strm, UINT64_MAX, LZMA_CONCATENATED); - - // Return successfully if the initialization went fine. - if (ret == LZMA_OK) - return true; - - // Something went wrong. The possible errors are documented in - // lzma/container.h (src/liblzma/api/lzma/container.h in the source - // package or e.g. /usr/include/lzma/container.h depending on the - // install prefix). - // - // Note that LZMA_MEMLIMIT_ERROR is never possible here. If you - // specify a very tiny limit, the error will be delayed until - // the first headers have been parsed by a call to lzma_code(). - const char *msg; - switch (ret) { - case LZMA_MEM_ERROR: - msg = "Memory allocation failed"; - break; - - case LZMA_OPTIONS_ERROR: - msg = "Unsupported decompressor flags"; - break; - - default: - // This is most likely LZMA_PROG_ERROR indicating a bug in - // this program or in liblzma. It is inconvenient to have a - // separate error message for errors that should be impossible - // to occur, but knowing the error code is important for - // debugging. That's why it is good to print the error code - // at least when there is no good error message to show. - msg = "Unknown error, possibly a bug"; - break; - } - - fprintf(stderr, "Error initializing the decoder: %s (error code %u)\n", - msg, ret); - return false; -} - - -static bool -decompress(lzma_stream *strm, const char *inname, FILE *infile, FILE *outfile) -{ - // When LZMA_CONCATENATED flag was used when initializing the decoder, - // we need to tell lzma_code() when there will be no more input. - // This is done by setting action to LZMA_FINISH instead of LZMA_RUN - // in the same way as it is done when encoding. - // - // When LZMA_CONCATENATED isn't used, there is no need to use - // LZMA_FINISH to tell when all the input has been read, but it - // is still OK to use it if you want. When LZMA_CONCATENATED isn't - // used, the decoder will stop after the first .xz stream. In that - // case some unused data may be left in strm->next_in. - lzma_action action = LZMA_RUN; - - uint8_t inbuf[BUFSIZ]; - uint8_t outbuf[BUFSIZ]; - - strm->next_in = NULL; - strm->avail_in = 0; - strm->next_out = outbuf; - strm->avail_out = sizeof(outbuf); - - while (true) { - if (strm->avail_in == 0 && !feof(infile)) { - strm->next_in = inbuf; - strm->avail_in = fread(inbuf, 1, sizeof(inbuf), - infile); - - if (ferror(infile)) { - fprintf(stderr, "%s: Read error: %s\n", - inname, strerror(errno)); - return false; - } - - // Once the end of the input file has been reached, - // we need to tell lzma_code() that no more input - // will be coming. As said before, this isn't required - // if the LZMA_CONATENATED flag isn't used when - // initializing the decoder. - if (feof(infile)) - action = LZMA_FINISH; - } - - lzma_ret ret = lzma_code(strm, action); - - if (strm->avail_out == 0 || ret == LZMA_STREAM_END) { - size_t write_size = sizeof(outbuf) - strm->avail_out; - - if (fwrite(outbuf, 1, write_size, outfile) - != write_size) { - fprintf(stderr, "Write error: %s\n", - strerror(errno)); - return false; - } - - strm->next_out = outbuf; - strm->avail_out = sizeof(outbuf); - } - - if (ret != LZMA_OK) { - // Once everything has been decoded successfully, the - // return value of lzma_code() will be LZMA_STREAM_END. - // - // It is important to check for LZMA_STREAM_END. Do not - // assume that getting ret != LZMA_OK would mean that - // everything has gone well or that when you aren't - // getting more output it must have successfully - // decoded everything. - if (ret == LZMA_STREAM_END) - return true; - - // It's not LZMA_OK nor LZMA_STREAM_END, - // so it must be an error code. See lzma/base.h - // (src/liblzma/api/lzma/base.h in the source package - // or e.g. /usr/include/lzma/base.h depending on the - // install prefix) for the list and documentation of - // possible values. Many values listen in lzma_ret - // enumeration aren't possible in this example, but - // can be made possible by enabling memory usage limit - // or adding flags to the decoder initialization. - const char *msg; - switch (ret) { - case LZMA_MEM_ERROR: - msg = "Memory allocation failed"; - break; - - case LZMA_FORMAT_ERROR: - // .xz magic bytes weren't found. - msg = "The input is not in the .xz format"; - break; - - case LZMA_OPTIONS_ERROR: - // For example, the headers specify a filter - // that isn't supported by this liblzma - // version (or it hasn't been enabled when - // building liblzma, but no-one sane does - // that unless building liblzma for an - // embedded system). Upgrading to a newer - // liblzma might help. - // - // Note that it is unlikely that the file has - // accidentally became corrupt if you get this - // error. The integrity of the .xz headers is - // always verified with a CRC32, so - // unintentionally corrupt files can be - // distinguished from unsupported files. - msg = "Unsupported compression options"; - break; - - case LZMA_DATA_ERROR: - msg = "Compressed file is corrupt"; - break; - - case LZMA_BUF_ERROR: - // Typically this error means that a valid - // file has got truncated, but it might also - // be a damaged part in the file that makes - // the decoder think the file is truncated. - // If you prefer, you can use the same error - // message for this as for LZMA_DATA_ERROR. - msg = "Compressed file is truncated or " - "otherwise corrupt"; - break; - - default: - // This is most likely LZMA_PROG_ERROR. - msg = "Unknown error, possibly a bug"; - break; - } - - fprintf(stderr, "%s: Decoder error: " - "%s (error code %u)\n", - inname, msg, ret); - return false; - } - } -} - - -extern int -main(int argc, char **argv) -{ - if (argc <= 1) { - fprintf(stderr, "Usage: %s FILES...\n", argv[0]); - return EXIT_FAILURE; - } - - lzma_stream strm = LZMA_STREAM_INIT; - - bool success = true; - - // Try to decompress all files. - for (int i = 1; i < argc; ++i) { - if (!init_decoder(&strm)) { - // Decoder initialization failed. There's no point - // to retry it so we need to exit. - success = false; - break; - } - - FILE *infile = fopen(argv[i], "rb"); - - if (infile == NULL) { - fprintf(stderr, "%s: Error opening the " - "input file: %s\n", - argv[i], strerror(errno)); - success = false; - } else { - success &= decompress(&strm, argv[i], infile, stdout); - fclose(infile); - } - } - - // Free the memory allocated for the decoder. This only needs to be - // done after the last file. - lzma_end(&strm); - - if (fclose(stdout)) { - fprintf(stderr, "Write error: %s\n", strerror(errno)); - success = false; - } - - return success ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/Build/source/extra/xz/doc/examples/03_compress_custom.c b/Build/source/extra/xz/doc/examples/03_compress_custom.c deleted file mode 100644 index 51abbb171ef..00000000000 --- a/Build/source/extra/xz/doc/examples/03_compress_custom.c +++ /dev/null @@ -1,193 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file 03_compress_custom.c -/// \brief Compress in multi-call mode using x86 BCJ and LZMA2 -/// -/// Usage: ./03_compress_custom < INFILE > OUTFILE -/// -/// Example: ./03_compress_custom < foo > foo.xz -// -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. -// -/////////////////////////////////////////////////////////////////////////////// - -#include <stdbool.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <errno.h> -#include <lzma.h> - - -static bool -init_encoder(lzma_stream *strm) -{ - // Use the default preset (6) for LZMA2. - // - // The lzma_options_lzma structure and the lzma_lzma_preset() function - // are declared in lzma/lzma.h (src/liblzma/api/lzma/lzma.h in the - // source package or e.g. /usr/include/lzma/lzma.h depending on - // the install prefix). - lzma_options_lzma opt_lzma2; - if (lzma_lzma_preset(&opt_lzma2, LZMA_PRESET_DEFAULT)) { - // It should never fail because the default preset - // (and presets 0-9 optionally with LZMA_PRESET_EXTREME) - // are supported by all stable liblzma versions. - // - // (The encoder initialization later in this function may - // still fail due to unsupported preset *if* the features - // required by the preset have been disabled at build time, - // but no-one does such things except on embedded systems.) - fprintf(stderr, "Unsupported preset, possibly a bug\n"); - return false; - } - - // Now we could customize the LZMA2 options if we wanted. For example, - // we could set the the dictionary size (opt_lzma2.dict_size) to - // something else than the default (8 MiB) of the default preset. - // See lzma/lzma.h for details of all LZMA2 options. - // - // The x86 BCJ filter will try to modify the x86 instruction stream so - // that LZMA2 can compress it better. The x86 BCJ filter doesn't need - // any options so it will be set to NULL below. - // - // Construct the filter chain. The uncompressed data goes first to - // the first filter in the array, in this case the x86 BCJ filter. - // The array is always terminated by setting .id = LZMA_VLI_UNKNOWN. - // - // See lzma/filter.h for more information about the lzma_filter - // structure. - lzma_filter filters[] = { - { .id = LZMA_FILTER_X86, .options = NULL }, - { .id = LZMA_FILTER_LZMA2, .options = &opt_lzma2 }, - { .id = LZMA_VLI_UNKNOWN, .options = NULL }, - }; - - // Initialize the encoder using the custom filter chain. - lzma_ret ret = lzma_stream_encoder(strm, filters, LZMA_CHECK_CRC64); - - if (ret == LZMA_OK) - return true; - - const char *msg; - switch (ret) { - case LZMA_MEM_ERROR: - msg = "Memory allocation failed"; - break; - - case LZMA_OPTIONS_ERROR: - // We are no longer using a plain preset so this error - // message has been edited accordingly compared to - // 01_compress_easy.c. - msg = "Specified filter chain is not supported"; - break; - - case LZMA_UNSUPPORTED_CHECK: - msg = "Specified integrity check is not supported"; - break; - - default: - msg = "Unknown error, possibly a bug"; - break; - } - - fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n", - msg, ret); - return false; -} - - -// This function is identical to the one in 01_compress_easy.c. -static bool -compress(lzma_stream *strm, FILE *infile, FILE *outfile) -{ - lzma_action action = LZMA_RUN; - - uint8_t inbuf[BUFSIZ]; - uint8_t outbuf[BUFSIZ]; - - strm->next_in = NULL; - strm->avail_in = 0; - strm->next_out = outbuf; - strm->avail_out = sizeof(outbuf); - - while (true) { - if (strm->avail_in == 0 && !feof(infile)) { - strm->next_in = inbuf; - strm->avail_in = fread(inbuf, 1, sizeof(inbuf), - infile); - - if (ferror(infile)) { - fprintf(stderr, "Read error: %s\n", - strerror(errno)); - return false; - } - - if (feof(infile)) - action = LZMA_FINISH; - } - - lzma_ret ret = lzma_code(strm, action); - - if (strm->avail_out == 0 || ret == LZMA_STREAM_END) { - size_t write_size = sizeof(outbuf) - strm->avail_out; - - if (fwrite(outbuf, 1, write_size, outfile) - != write_size) { - fprintf(stderr, "Write error: %s\n", - strerror(errno)); - return false; - } - - strm->next_out = outbuf; - strm->avail_out = sizeof(outbuf); - } - - if (ret != LZMA_OK) { - if (ret == LZMA_STREAM_END) - return true; - - const char *msg; - switch (ret) { - case LZMA_MEM_ERROR: - msg = "Memory allocation failed"; - break; - - case LZMA_DATA_ERROR: - msg = "File size limits exceeded"; - break; - - default: - msg = "Unknown error, possibly a bug"; - break; - } - - fprintf(stderr, "Encoder error: %s (error code %u)\n", - msg, ret); - return false; - } - } -} - - -extern int -main(void) -{ - lzma_stream strm = LZMA_STREAM_INIT; - - bool success = init_encoder(&strm); - if (success) - success = compress(&strm, stdin, stdout); - - lzma_end(&strm); - - if (fclose(stdout)) { - fprintf(stderr, "Write error: %s\n", strerror(errno)); - success = false; - } - - return success ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/Build/source/extra/xz/doc/examples/Makefile b/Build/source/extra/xz/doc/examples/Makefile deleted file mode 100644 index 644dc32ee7b..00000000000 --- a/Build/source/extra/xz/doc/examples/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# -# Author: Lasse Collin -# -# This file has been put into the public domain. -# You can do whatever you want with this file. -# - -CC = c99 -CFLAGS = -g -LDFLAGS = -llzma - -PROGS = \ - 01_compress_easy \ - 02_decompress \ - 03_compress_custom - -all: $(PROGS) - -.c: - $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) - -clean: - -rm -f $(PROGS) diff --git a/Build/source/extra/xz/doc/examples_old/xz_pipe_comp.c b/Build/source/extra/xz/doc/examples_old/xz_pipe_comp.c deleted file mode 100644 index 9f9224bcad9..00000000000 --- a/Build/source/extra/xz/doc/examples_old/xz_pipe_comp.c +++ /dev/null @@ -1,127 +0,0 @@ -/* - * xz_pipe_comp.c - * A simple example of pipe-only xz compressor implementation. - * version: 2010-07-12 - by Daniel Mealha Cabrita - * Not copyrighted -- provided to the public domain. - * - * Compiling: - * Link with liblzma. GCC example: - * $ gcc -llzma xz_pipe_comp.c -o xz_pipe_comp - * - * Usage example: - * $ cat some_file | ./xz_pipe_comp > some_file.xz - */ - -#include <stdio.h> -#include <stdint.h> -#include <inttypes.h> -#include <stdbool.h> -#include <lzma.h> - - -/* COMPRESSION SETTINGS */ - -/* analogous to xz CLI options: -0 to -9 */ -#define COMPRESSION_LEVEL 6 - -/* boolean setting, analogous to xz CLI option: -e */ -#define COMPRESSION_EXTREME true - -/* see: /usr/include/lzma/check.h LZMA_CHECK_* */ -#define INTEGRITY_CHECK LZMA_CHECK_CRC64 - - -/* read/write buffer sizes */ -#define IN_BUF_MAX 4096 -#define OUT_BUF_MAX 4096 - -/* error codes */ -#define RET_OK 0 -#define RET_ERROR_INIT 1 -#define RET_ERROR_INPUT 2 -#define RET_ERROR_OUTPUT 3 -#define RET_ERROR_COMPRESSION 4 - - -/* note: in_file and out_file must be open already */ -int xz_compress (FILE *in_file, FILE *out_file) -{ - uint32_t preset = COMPRESSION_LEVEL | (COMPRESSION_EXTREME ? LZMA_PRESET_EXTREME : 0); - lzma_check check = INTEGRITY_CHECK; - lzma_stream strm = LZMA_STREAM_INIT; /* alloc and init lzma_stream struct */ - uint8_t in_buf [IN_BUF_MAX]; - uint8_t out_buf [OUT_BUF_MAX]; - size_t in_len; /* length of useful data in in_buf */ - size_t out_len; /* length of useful data in out_buf */ - bool in_finished = false; - bool out_finished = false; - lzma_action action; - lzma_ret ret_xz; - int ret; - - ret = RET_OK; - - /* initialize xz encoder */ - ret_xz = lzma_easy_encoder (&strm, preset, check); - if (ret_xz != LZMA_OK) { - fprintf (stderr, "lzma_easy_encoder error: %d\n", (int) ret_xz); - return RET_ERROR_INIT; - } - - while ((! in_finished) && (! out_finished)) { - /* read incoming data */ - in_len = fread (in_buf, 1, IN_BUF_MAX, in_file); - - if (feof (in_file)) { - in_finished = true; - } - if (ferror (in_file)) { - in_finished = true; - ret = RET_ERROR_INPUT; - } - - strm.next_in = in_buf; - strm.avail_in = in_len; - - /* if no more data from in_buf, flushes the - internal xz buffers and closes the xz data - with LZMA_FINISH */ - action = in_finished ? LZMA_FINISH : LZMA_RUN; - - /* loop until there's no pending compressed output */ - do { - /* out_buf is clean at this point */ - strm.next_out = out_buf; - strm.avail_out = OUT_BUF_MAX; - - /* compress data */ - ret_xz = lzma_code (&strm, action); - - if ((ret_xz != LZMA_OK) && (ret_xz != LZMA_STREAM_END)) { - fprintf (stderr, "lzma_code error: %d\n", (int) ret_xz); - out_finished = true; - ret = RET_ERROR_COMPRESSION; - } else { - /* write compressed data */ - out_len = OUT_BUF_MAX - strm.avail_out; - fwrite (out_buf, 1, out_len, out_file); - if (ferror (out_file)) { - out_finished = true; - ret = RET_ERROR_OUTPUT; - } - } - } while (strm.avail_out == 0); - } - - lzma_end (&strm); - return ret; -} - -int main () -{ - int ret; - - ret = xz_compress (stdin, stdout); - return ret; -} - diff --git a/Build/source/extra/xz/doc/examples_old/xz_pipe_decomp.c b/Build/source/extra/xz/doc/examples_old/xz_pipe_decomp.c deleted file mode 100644 index fb5ad89515d..00000000000 --- a/Build/source/extra/xz/doc/examples_old/xz_pipe_decomp.c +++ /dev/null @@ -1,123 +0,0 @@ -/* - * xz_pipe_decomp.c - * A simple example of pipe-only xz decompressor implementation. - * version: 2012-06-14 - by Daniel Mealha Cabrita - * Not copyrighted -- provided to the public domain. - * - * Compiling: - * Link with liblzma. GCC example: - * $ gcc -llzma xz_pipe_decomp.c -o xz_pipe_decomp - * - * Usage example: - * $ cat some_file.xz | ./xz_pipe_decomp > some_file - */ - -#include <stdio.h> -#include <stdint.h> -#include <inttypes.h> -#include <stdbool.h> -#include <lzma.h> - - -/* read/write buffer sizes */ -#define IN_BUF_MAX 4096 -#define OUT_BUF_MAX 4096 - -/* error codes */ -#define RET_OK 0 -#define RET_ERROR_INIT 1 -#define RET_ERROR_INPUT 2 -#define RET_ERROR_OUTPUT 3 -#define RET_ERROR_DECOMPRESSION 4 - - -/* note: in_file and out_file must be open already */ -int xz_decompress (FILE *in_file, FILE *out_file) -{ - lzma_stream strm = LZMA_STREAM_INIT; /* alloc and init lzma_stream struct */ - const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED; - const uint64_t memory_limit = UINT64_MAX; /* no memory limit */ - uint8_t in_buf [IN_BUF_MAX]; - uint8_t out_buf [OUT_BUF_MAX]; - size_t in_len; /* length of useful data in in_buf */ - size_t out_len; /* length of useful data in out_buf */ - bool in_finished = false; - bool out_finished = false; - lzma_action action; - lzma_ret ret_xz; - int ret; - - ret = RET_OK; - - /* initialize xz decoder */ - ret_xz = lzma_stream_decoder (&strm, memory_limit, flags); - if (ret_xz != LZMA_OK) { - fprintf (stderr, "lzma_stream_decoder error: %d\n", (int) ret_xz); - return RET_ERROR_INIT; - } - - while ((! in_finished) && (! out_finished)) { - /* read incoming data */ - in_len = fread (in_buf, 1, IN_BUF_MAX, in_file); - - if (feof (in_file)) { - in_finished = true; - } - if (ferror (in_file)) { - in_finished = true; - ret = RET_ERROR_INPUT; - } - - strm.next_in = in_buf; - strm.avail_in = in_len; - - /* if no more data from in_buf, flushes the - internal xz buffers and closes the decompressed data - with LZMA_FINISH */ - action = in_finished ? LZMA_FINISH : LZMA_RUN; - - /* loop until there's no pending decompressed output */ - do { - /* out_buf is clean at this point */ - strm.next_out = out_buf; - strm.avail_out = OUT_BUF_MAX; - - /* decompress data */ - ret_xz = lzma_code (&strm, action); - - if ((ret_xz != LZMA_OK) && (ret_xz != LZMA_STREAM_END)) { - fprintf (stderr, "lzma_code error: %d\n", (int) ret_xz); - out_finished = true; - ret = RET_ERROR_DECOMPRESSION; - } else { - /* write decompressed data */ - out_len = OUT_BUF_MAX - strm.avail_out; - fwrite (out_buf, 1, out_len, out_file); - if (ferror (out_file)) { - out_finished = true; - ret = RET_ERROR_OUTPUT; - } - } - } while (strm.avail_out == 0); - } - - /* Bug fix (2012-06-14): If no errors were detected, check - that the last lzma_code() call returned LZMA_STREAM_END. - If not, the file is probably truncated. */ - if ((ret == RET_OK) && (ret_xz != LZMA_STREAM_END)) { - fprintf (stderr, "Input truncated or corrupt\n"); - ret = RET_ERROR_DECOMPRESSION; - } - - lzma_end (&strm); - return ret; -} - -int main () -{ - int ret; - - ret = xz_decompress (stdin, stdout); - return ret; -} - diff --git a/Build/source/extra/xz/doc/faq.txt b/Build/source/extra/xz/doc/faq.txt deleted file mode 100644 index 333bee09c1c..00000000000 --- a/Build/source/extra/xz/doc/faq.txt +++ /dev/null @@ -1,224 +0,0 @@ - -XZ Utils FAQ -============ - -Q: What do the letters XZ mean? - -A: Nothing. They are just two letters, which come from the file format - suffix .xz. The .xz suffix was selected, because it seemed to be - pretty much unused. It has no deeper meaning. - - -Q: What are LZMA and LZMA2? - -A: LZMA stands for Lempel-Ziv-Markov chain-Algorithm. It is the name - of the compression algorithm designed by Igor Pavlov for 7-Zip. - LZMA is based on LZ77 and range encoding. - - LZMA2 is an updated version of the original LZMA to fix a couple of - practical issues. In context of XZ Utils, LZMA is called LZMA1 to - emphasize that LZMA is not the same thing as LZMA2. LZMA2 is the - primary compression algorithm in the .xz file format. - - -Q: There are many LZMA related projects. How does XZ Utils relate to them? - -A: 7-Zip and LZMA SDK are the original projects. LZMA SDK is roughly - a subset of the 7-Zip source tree. - - p7zip is 7-Zip's command-line tools ported to POSIX-like systems. - - LZMA Utils provide a gzip-like lzma tool for POSIX-like systems. - LZMA Utils are based on LZMA SDK. XZ Utils are the successor to - LZMA Utils. - - There are several other projects using LZMA. Most are more or less - based on LZMA SDK. See <http://7-zip.org/links.html>. - - -Q: Why is liblzma named liblzma if its primary file format is .xz? - Shouldn't it be e.g. libxz? - -A: When the designing of the .xz format began, the idea was to replace - the .lzma format and use the same .lzma suffix. It would have been - quite OK to reuse the suffix when there were very few .lzma files - around. However, the old .lzma format became popular before the - new format was finished. The new format was renamed to .xz but the - name of liblzma wasn't changed. - - -Q: Do XZ Utils support the .7z format? - -A: No. Use 7-Zip (Windows) or p7zip (POSIX-like systems) to handle .7z - files. - - -Q: I have many .tar.7z files. Can I convert them to .tar.xz without - spending hours recompressing the data? - -A: In the "extra" directory, there is a script named 7z2lzma.bash which - is able to convert some .7z files to the .lzma format (not .xz). It - needs the 7za (or 7z) command from p7zip. The script may silently - produce corrupt output if certain assumptions are not met, so - decompress the resulting .lzma file and compare it against the - original before deleting the original file! - - -Q: I have many .lzma files. Can I quickly convert them to the .xz format? - -A: For now, no. Since XZ Utils supports the .lzma format, it's usually - not too bad to keep the old files in the old format. If you want to - do the conversion anyway, you need to decompress the .lzma files and - then recompress to the .xz format. - - Technically, there is a way to make the conversion relatively fast - (roughly twice the time that normal decompression takes). Writing - such a tool would take quite a bit of time though, and would probably - be useful to only a few people. If you really want such a conversion - tool, contact Lasse Collin and offer some money. - - -Q: I have installed xz, but my tar doesn't recognize .tar.xz files. - How can I extract .tar.xz files? - -A: xz -dc foo.tar.xz | tar xf - - - -Q: Can I recover parts of a broken .xz file (e.g. a corrupted CD-R)? - -A: It may be possible if the file consists of multiple blocks, which - typically is not the case if the file was created in single-threaded - mode. There is no recovery program yet. - - -Q: Is (some part of) XZ Utils patented? - -A: Lasse Collin is not aware of any patents that could affect XZ Utils. - However, due to the nature of software patents, it's not possible to - guarantee that XZ Utils isn't affected by any third party patent(s). - - -Q: Where can I find documentation about the file format and algorithms? - -A: The .xz format is documented in xz-file-format.txt. It is a container - format only, and doesn't include descriptions of any non-trivial - filters. - - Documenting LZMA and LZMA2 is planned, but for now, there is no other - documentation than the source code. Before you begin, you should know - the basics of LZ77 and range-coding algorithms. LZMA is based on LZ77, - but LZMA is a lot more complex. Range coding is used to compress - the final bitstream like Huffman coding is used in Deflate. - - -Q: I cannot find BCJ and BCJ2 filters. Don't they exist in liblzma? - -A: BCJ filter is called "x86" in liblzma. BCJ2 is not included, - because it requires using more than one encoded output stream. - A streamable version of BCJ2-style filtering is planned. - - -Q: I need to use a script that runs "xz -9". On a system with 256 MiB - of RAM, xz says that it cannot allocate memory. Can I make the - script work without modifying it? - -A: Set a default memory usage limit for compression. You can do it e.g. - in a shell initialization script such as ~/.bashrc or /etc/profile: - - XZ_DEFAULTS=--memlimit-compress=150MiB - export XZ_DEFAULTS - - xz will then scale the compression settings down so that the given - memory usage limit is not reached. This way xz shouldn't run out - of memory. - - Check also that memory-related resource limits are high enough. - On most systems, "ulimit -a" will show the current resource limits. - - -Q: How do I create files that can be decompressed with XZ Embedded? - -A: See the documentation in XZ Embedded. In short, something like - this is a good start: - - xz --check=crc32 --lzma2=preset=6e,dict=64KiB - - Or if a BCJ filter is needed too, e.g. if compressing - a kernel image for PowerPC: - - xz --check=crc32 --powerpc --lzma2=preset=6e,dict=64KiB - - Adjust the dictionary size to get a good compromise between - compression ratio and decompressor memory usage. Note that - in single-call decompression mode of XZ Embedded, a big - dictionary doesn't increase memory usage. - - -Q: Will xz support threaded compression? - -A: It is planned and has been taken into account when designing - the .xz file format. Eventually there will probably be three types - of threading, each method having its own advantages and disadvantages. - - The simplest method is splitting the uncompressed data into blocks - and compressing them in parallel independent from each other. - Since the blocks are compressed independently, they can also be - decompressed independently. Together with the index feature in .xz, - this allows using threads to create .xz files for random-access - reading. This also makes threaded decompression possible, although - it is not clear if threaded decompression will ever be implemented. - - The independent blocks method has a couple of disadvantages too. It - will compress worse than a single-block method. Often the difference - is not too big (maybe 1-2 %) but sometimes it can be too big. Also, - the memory usage of the compressor increases linearly when adding - threads. - - Match finder parallelization is another threading method. It has - been in 7-Zip for ages. It doesn't affect compression ratio or - memory usage significantly. Among the three threading methods, only - this is useful when compressing small files (files that are not - significantly bigger than the dictionary). Unfortunately this method - scales only to about two CPU cores. - - The third method is pigz-style threading (I use that name, because - pigz <http://www.zlib.net/pigz/> uses that method). It doesn't - affect compression ratio significantly and scales to many cores. - The memory usage scales linearly when threads are added. This isn't - significant with pigz, because Deflate uses only a 32 KiB dictionary, - but with LZMA2 the memory usage will increase dramatically just like - with the independent-blocks method. There is also a constant - computational overhead, which may make pigz-method a bit dull on - dual-core compared to the parallel match finder method, but with more - cores the overhead is not a big deal anymore. - - Combining the threading methods will be possible and also useful. - E.g. combining match finder parallelization with pigz-style threading - can cut the memory usage by 50 %. - - It is possible that the single-threaded method will be modified to - create files identical to the pigz-style method. We'll see once - pigz-style threading has been implemented in liblzma. - - -Q: How do I build a program that needs liblzmadec (lzmadec.h)? - -A: liblzmadec is part of LZMA Utils. XZ Utils has liblzma, but no - liblzmadec. The code using liblzmadec should be ported to use - liblzma instead. If you cannot or don't want to do that, download - LZMA Utils from <http://tukaani.org/lzma/>. - - -Q: The default build of liblzma is too big. How can I make it smaller? - -A: Give --enable-small to the configure script. Use also appropriate - --enable or --disable options to include only those filter encoders - and decoders and integrity checks that you actually need. Use - CFLAGS=-Os (with GCC) or equivalent to tell your compiler to optimize - for size. See INSTALL for information about configure options. - - If the result is still too big, take a look at XZ Embedded. It is - a separate project, which provides a limited but significantly - smaller XZ decoder implementation than XZ Utils. You can find it - at <http://tukaani.org/xz/embedded.html>. - diff --git a/Build/source/extra/xz/doc/history.txt b/Build/source/extra/xz/doc/history.txt deleted file mode 100644 index 8545e232254..00000000000 --- a/Build/source/extra/xz/doc/history.txt +++ /dev/null @@ -1,150 +0,0 @@ - -History of LZMA Utils and XZ Utils -================================== - -Tukaani distribution - - In 2005, there was a small group working on the Tukaani distribution, - which was a Slackware fork. One of the project's goals was to fit the - distro on a single 700 MiB ISO-9660 image. Using LZMA instead of gzip - helped a lot. Roughly speaking, one could fit data that took 1000 MiB - in gzipped form into 700 MiB with LZMA. Naturally, the compression - ratio varied across packages, but this was what we got on average. - - Slackware packages have traditionally had .tgz as the filename suffix, - which is an abbreviation of .tar.gz. A logical naming for LZMA - compressed packages was .tlz, being an abbreviation of .tar.lzma. - - At the end of the year 2007, there was no distribution under the - Tukaani project anymore, but development of LZMA Utils was kept going. - Still, there were .tlz packages around, because at least Vector Linux - (a Slackware based distribution) used LZMA for its packages. - - First versions of the modified pkgtools used the LZMA_Alone tool from - Igor Pavlov's LZMA SDK as is. It was fine, because users wouldn't need - to interact with LZMA_Alone directly. But people soon wanted to use - LZMA for other files too, and the interface of LZMA_Alone wasn't - comfortable for those used to gzip and bzip2. - - -First steps of LZMA Utils - - The first version of LZMA Utils (4.22.0) included a shell script called - lzmash. It was a wrapper that had a gzip-like command-line interface. It - used the LZMA_Alone tool from LZMA SDK to do all the real work. zgrep, - zdiff, and related scripts from gzip were adapted to work with LZMA and - were part of the first LZMA Utils release too. - - LZMA Utils 4.22.0 included also lzmadec, which was a small (less than - 10 KiB) decoder-only command-line tool. It was written on top of the - decoder-only C code found from the LZMA SDK. lzmadec was convenient in - situations where LZMA_Alone (a few hundred KiB) would be too big. - - lzmash and lzmadec were written by Lasse Collin. - - -Second generation - - The lzmash script was an ugly and not very secure hack. The last - version of LZMA Utils to use lzmash was 4.27.1. - - LZMA Utils 4.32.0beta1 introduced a new lzma command-line tool written - by Ville Koskinen. It was written in C++, and used the encoder and - decoder from C++ LZMA SDK with some little modifications. This tool - replaced both the lzmash script and the LZMA_Alone command-line tool - in LZMA Utils. - - Introducing this new tool caused some temporary incompatibilities, - because the LZMA_Alone executable was simply named lzma like the new - command-line tool, but they had a completely different command-line - interface. The file format was still the same. - - Lasse wrote liblzmadec, which was a small decoder-only library based - on the C code found from LZMA SDK. liblzmadec had an API similar to - zlib, although there were some significant differences, which made it - non-trivial to use it in some applications designed for zlib and - libbzip2. - - The lzmadec command-line tool was converted to use liblzmadec. - - Alexandre Sauvé helped converting the build system to use GNU - Autotools. This made it easier to test for certain less portable - features needed by the new command-line tool. - - Since the new command-line tool never got completely finished (for - example, it didn't support the LZMA_OPT environment variable), the - intent was to not call 4.32.x stable. Similarly, liblzmadec wasn't - polished, but appeared to work well enough, so some people started - using it too. - - Because the development of the third generation of LZMA Utils was - delayed considerably (3-4 years), the 4.32.x branch had to be kept - maintained. It got some bug fixes now and then, and finally it was - decided to call it stable, although most of the missing features were - never added. - - -File format problems - - The file format used by LZMA_Alone was primitive. It was designed with - embedded systems in mind, and thus provided only a minimal set of - features. The two biggest problems for non-embedded use were the lack - of magic bytes and an integrity check. - - Igor and Lasse started developing a new file format with some help - from Ville Koskinen. Also Mark Adler, Mikko Pouru, H. Peter Anvin, - and Lars Wirzenius helped with some minor things at some point of the - development. Designing the new format took quite a long time (actually, - too long a time would be a more appropriate expression). It was mostly - because Lasse was quite slow at getting things done due to personal - reasons. - - Originally the new format was supposed to use the same .lzma suffix - that was already used by the old file format. Switching to the new - format wouldn't have caused much trouble when the old format wasn't - used by many people. But since the development of the new format took - such a long time, the old format got quite popular, and it was decided - that the new file format must use a different suffix. - - It was decided to use .xz as the suffix of the new file format. The - first stable .xz file format specification was finally released in - December 2008. In addition to fixing the most obvious problems of - the old .lzma format, the .xz format added some new features like - support for multiple filters (compression algorithms), filter chaining - (like piping on the command line), and limited random-access reading. - - Currently the primary compression algorithm used in .xz is LZMA2. - It is an extension on top of the original LZMA to fix some practical - problems: LZMA2 adds support for flushing the encoder, uncompressed - chunks, eases stateful decoder implementations, and improves support - for multithreading. Since LZMA2 is better than the original LZMA, the - original LZMA is not supported in .xz. - - -Transition to XZ Utils - - The early versions of XZ Utils were called LZMA Utils. The first - releases were 4.42.0alphas. They dropped the rest of the C++ LZMA SDK. - The code was still directly based on LZMA SDK but ported to C and - converted from a callback API to a stateful API. Later, Igor Pavlov - made a C version of the LZMA encoder too; these ports from C++ to C - were independent in LZMA SDK and LZMA Utils. - - The core of the new LZMA Utils was liblzma, a compression library with - a zlib-like API. liblzma supported both the old and new file format. - The gzip-like lzma command-line tool was rewritten to use liblzma. - - The new LZMA Utils code base was renamed to XZ Utils when the name - of the new file format had been decided. The liblzma compression - library retained its name though, because changing it would have - caused unnecessary breakage in applications already using the early - liblzma snapshots. - - The xz command-line tool can emulate the gzip-like lzma tool by - creating appropriate symlinks (e.g. lzma -> xz). Thus, practically - all scripts using the lzma tool from LZMA Utils will work as is with - XZ Utils (and will keep using the old .lzma format). Still, the .lzma - format is more or less deprecated. XZ Utils will keep supporting it, - but new applications should use the .xz format, and migrating old - applications to .xz is often a good idea too. - diff --git a/Build/source/extra/xz/doc/lzma-file-format.txt b/Build/source/extra/xz/doc/lzma-file-format.txt deleted file mode 100644 index 015b0fae628..00000000000 --- a/Build/source/extra/xz/doc/lzma-file-format.txt +++ /dev/null @@ -1,166 +0,0 @@ - -The .lzma File Format -===================== - - 0. Preface - 0.1. Notices and Acknowledgements - 0.2. Changes - 1. File Format - 1.1. Header - 1.1.1. Properties - 1.1.2. Dictionary Size - 1.1.3. Uncompressed Size - 1.2. LZMA Compressed Data - 2. References - - -0. Preface - - This document describes the .lzma file format, which is - sometimes also called LZMA_Alone format. It is a legacy file - format, which is being or has been replaced by the .xz format. - The MIME type of the .lzma format is `application/x-lzma'. - - The most commonly used software to handle .lzma files are - LZMA SDK, LZMA Utils, 7-Zip, and XZ Utils. This document - describes some of the differences between these implementations - and gives hints what subset of the .lzma format is the most - portable. - - -0.1. Notices and Acknowledgements - - This file format was designed by Igor Pavlov for use in - LZMA SDK. This document was written by Lasse Collin - <lasse.collin@tukaani.org> using the documentation found - from the LZMA SDK. - - This document has been put into the public domain. - - -0.2. Changes - - Last modified: 2011-04-12 11:55+0300 - - -1. File Format - - +-+-+-+-+-+-+-+-+-+-+-+-+-+==========================+ - | Header | LZMA Compressed Data | - +-+-+-+-+-+-+-+-+-+-+-+-+-+==========================+ - - The .lzma format file consist of 13-byte Header followed by - the LZMA Compressed Data. - - Unlike the .gz, .bz2, and .xz formats, it is not possible to - concatenate multiple .lzma files as is and expect the - decompression tool to decode the resulting file as if it were - a single .lzma file. - - For example, the command line tools from LZMA Utils and - LZMA SDK silently ignore all the data after the first .lzma - stream. In contrast, the command line tool from XZ Utils - considers the .lzma file to be corrupt if there is data after - the first .lzma stream. - - -1.1. Header - - +------------+----+----+----+----+--+--+--+--+--+--+--+--+ - | Properties | Dictionary Size | Uncompressed Size | - +------------+----+----+----+----+--+--+--+--+--+--+--+--+ - - -1.1.1. Properties - - The Properties field contains three properties. An abbreviation - is given in parentheses, followed by the value range of the - property. The field consists of - - 1) the number of literal context bits (lc, [0, 8]); - 2) the number of literal position bits (lp, [0, 4]); and - 3) the number of position bits (pb, [0, 4]). - - The properties are encoded using the following formula: - - Properties = (pb * 5 + lp) * 9 + lc - - The following C code illustrates a straightforward way to - decode the Properties field: - - uint8_t lc, lp, pb; - uint8_t prop = get_lzma_properties(); - if (prop > (4 * 5 + 4) * 9 + 8) - return LZMA_PROPERTIES_ERROR; - - pb = prop / (9 * 5); - prop -= pb * 9 * 5; - lp = prop / 9; - lc = prop - lp * 9; - - XZ Utils has an additional requirement: lc + lp <= 4. Files - which don't follow this requirement cannot be decompressed - with XZ Utils. Usually this isn't a problem since the most - common lc/lp/pb values are 3/0/2. It is the only lc/lp/pb - combination that the files created by LZMA Utils can have, - but LZMA Utils can decompress files with any lc/lp/pb. - - -1.1.2. Dictionary Size - - Dictionary Size is stored as an unsigned 32-bit little endian - integer. Any 32-bit value is possible, but for maximum - portability, only sizes of 2^n and 2^n + 2^(n-1) should be - used. - - LZMA Utils creates only files with dictionary size 2^n, - 16 <= n <= 25. LZMA Utils can decompress files with any - dictionary size. - - XZ Utils creates and decompresses .lzma files only with - dictionary sizes 2^n and 2^n + 2^(n-1). If some other - dictionary size is specified when compressing, the value - stored in the Dictionary Size field is a rounded up, but the - specified value is still used in the actual compression code. - - -1.1.3. Uncompressed Size - - Uncompressed Size is stored as unsigned 64-bit little endian - integer. A special value of 0xFFFF_FFFF_FFFF_FFFF indicates - that Uncompressed Size is unknown. End of Payload Marker (*) - is used if and only if Uncompressed Size is unknown. - - XZ Utils rejects files whose Uncompressed Size field specifies - a known size that is 256 GiB or more. This is to reject false - positives when trying to guess if the input file is in the - .lzma format. When Uncompressed Size is unknown, there is no - limit for the uncompressed size of the file. - - (*) Some tools use the term End of Stream (EOS) marker - instead of End of Payload Marker. - - -1.2. LZMA Compressed Data - - Detailed description of the format of this field is out of - scope of this document. - - -2. References - - LZMA SDK - The original LZMA implementation - http://7-zip.org/sdk.html - - 7-Zip - http://7-zip.org/ - - LZMA Utils - LZMA adapted to POSIX-like systems - http://tukaani.org/lzma/ - - XZ Utils - The next generation of LZMA Utils - http://tukaani.org/xz/ - - The .xz file format - The successor of the .lzma format - http://tukaani.org/xz/xz-file-format.txt - diff --git a/Build/source/extra/xz/doc/man/pdf-a4/lzmainfo-a4.pdf b/Build/source/extra/xz/doc/man/pdf-a4/lzmainfo-a4.pdf Binary files differdeleted file mode 100644 index d84ddedb6de..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-a4/lzmainfo-a4.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-a4/xz-a4.pdf b/Build/source/extra/xz/doc/man/pdf-a4/xz-a4.pdf Binary files differdeleted file mode 100644 index 5bb20742b2c..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-a4/xz-a4.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-a4/xzdec-a4.pdf b/Build/source/extra/xz/doc/man/pdf-a4/xzdec-a4.pdf Binary files differdeleted file mode 100644 index 8bc124b6fb8..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-a4/xzdec-a4.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-a4/xzdiff-a4.pdf b/Build/source/extra/xz/doc/man/pdf-a4/xzdiff-a4.pdf Binary files differdeleted file mode 100644 index 1a392cc9595..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-a4/xzdiff-a4.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-a4/xzgrep-a4.pdf b/Build/source/extra/xz/doc/man/pdf-a4/xzgrep-a4.pdf Binary files differdeleted file mode 100644 index 48969812363..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-a4/xzgrep-a4.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-a4/xzless-a4.pdf b/Build/source/extra/xz/doc/man/pdf-a4/xzless-a4.pdf Binary files differdeleted file mode 100644 index 9073d5f17f1..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-a4/xzless-a4.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-a4/xzmore-a4.pdf b/Build/source/extra/xz/doc/man/pdf-a4/xzmore-a4.pdf Binary files differdeleted file mode 100644 index 90dd4d642a3..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-a4/xzmore-a4.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-letter/lzmainfo-letter.pdf b/Build/source/extra/xz/doc/man/pdf-letter/lzmainfo-letter.pdf Binary files differdeleted file mode 100644 index 053594325ff..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-letter/lzmainfo-letter.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-letter/xz-letter.pdf b/Build/source/extra/xz/doc/man/pdf-letter/xz-letter.pdf Binary files differdeleted file mode 100644 index 9bba861effa..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-letter/xz-letter.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-letter/xzdec-letter.pdf b/Build/source/extra/xz/doc/man/pdf-letter/xzdec-letter.pdf Binary files differdeleted file mode 100644 index cc5ed1117eb..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-letter/xzdec-letter.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-letter/xzdiff-letter.pdf b/Build/source/extra/xz/doc/man/pdf-letter/xzdiff-letter.pdf Binary files differdeleted file mode 100644 index 3aeb20aab52..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-letter/xzdiff-letter.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-letter/xzgrep-letter.pdf b/Build/source/extra/xz/doc/man/pdf-letter/xzgrep-letter.pdf Binary files differdeleted file mode 100644 index 5b44f5bd873..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-letter/xzgrep-letter.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-letter/xzless-letter.pdf b/Build/source/extra/xz/doc/man/pdf-letter/xzless-letter.pdf Binary files differdeleted file mode 100644 index be64177b44b..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-letter/xzless-letter.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/pdf-letter/xzmore-letter.pdf b/Build/source/extra/xz/doc/man/pdf-letter/xzmore-letter.pdf Binary files differdeleted file mode 100644 index be375355cde..00000000000 --- a/Build/source/extra/xz/doc/man/pdf-letter/xzmore-letter.pdf +++ /dev/null diff --git a/Build/source/extra/xz/doc/man/txt/lzmainfo.txt b/Build/source/extra/xz/doc/man/txt/lzmainfo.txt deleted file mode 100644 index a47a68abd3b..00000000000 --- a/Build/source/extra/xz/doc/man/txt/lzmainfo.txt +++ /dev/null @@ -1,40 +0,0 @@ -LZMAINFO(1) XZ Utils LZMAINFO(1) - - - -NAME - lzmainfo - show information stored in the .lzma file header - -SYNOPSIS - lzmainfo [--help] [--version] [file...] - -DESCRIPTION - lzmainfo shows information stored in the .lzma file header. It reads - the first 13 bytes from the specified file, decodes the header, and - prints it to standard output in human readable format. If no files are - given or file is -, standard input is read. - - Usually the most interesting information is the uncompressed size and - the dictionary size. Uncompressed size can be shown only if the file - is in the non-streamed .lzma format variant. The amount of memory - required to decompress the file is a few dozen kilobytes plus the dic- - tionary size. - - lzmainfo is included in XZ Utils primarily for backward compatibility - with LZMA Utils. - -EXIT STATUS - 0 All is good. - - 1 An error occurred. - -BUGS - lzmainfo uses MB while the correct suffix would be MiB (2^20 bytes). - This is to keep the output compatible with LZMA Utils. - -SEE ALSO - xz(1) - - - -Tukaani 2013-06-30 LZMAINFO(1) diff --git a/Build/source/extra/xz/doc/man/txt/xz.txt b/Build/source/extra/xz/doc/man/txt/xz.txt deleted file mode 100644 index 90486f90311..00000000000 --- a/Build/source/extra/xz/doc/man/txt/xz.txt +++ /dev/null @@ -1,1377 +0,0 @@ -XZ(1) XZ Utils XZ(1) - - - -NAME - xz, unxz, xzcat, lzma, unlzma, lzcat - Compress or decompress .xz and - .lzma files - -SYNOPSIS - xz [option...] [file...] - -COMMAND ALIASES - unxz is equivalent to xz --decompress. - xzcat is equivalent to xz --decompress --stdout. - lzma is equivalent to xz --format=lzma. - unlzma is equivalent to xz --format=lzma --decompress. - lzcat is equivalent to xz --format=lzma --decompress --stdout. - - When writing scripts that need to decompress files, it is recommended - to always use the name xz with appropriate arguments (xz -d or xz -dc) - instead of the names unxz and xzcat. - -DESCRIPTION - xz is a general-purpose data compression tool with command line syntax - similar to gzip(1) and bzip2(1). The native file format is the .xz - format, but the legacy .lzma format used by LZMA Utils and raw com- - pressed streams with no container format headers are also supported. - - xz compresses or decompresses each file according to the selected oper- - ation mode. If no files are given or file is -, xz reads from standard - input and writes the processed data to standard output. xz will refuse - (display an error and skip the file) to write compressed data to stan- - dard output if it is a terminal. Similarly, xz will refuse to read - compressed data from standard input if it is a terminal. - - Unless --stdout is specified, files other than - are written to a new - file whose name is derived from the source file name: - - o When compressing, the suffix of the target file format (.xz or - .lzma) is appended to the source filename to get the target file- - name. - - o When decompressing, the .xz or .lzma suffix is removed from the - filename to get the target filename. xz also recognizes the suf- - fixes .txz and .tlz, and replaces them with the .tar suffix. - - If the target file already exists, an error is displayed and the file - is skipped. - - Unless writing to standard output, xz will display a warning and skip - the file if any of the following applies: - - o File is not a regular file. Symbolic links are not followed, and - thus they are not considered to be regular files. - - o File has more than one hard link. - - o File has setuid, setgid, or sticky bit set. - - o The operation mode is set to compress and the file already has a - suffix of the target file format (.xz or .txz when compressing to - the .xz format, and .lzma or .tlz when compressing to the .lzma for- - mat). - - o The operation mode is set to decompress and the file doesn't have a - suffix of any of the supported file formats (.xz, .txz, .lzma, or - .tlz). - - After successfully compressing or decompressing the file, xz copies the - owner, group, permissions, access time, and modification time from the - source file to the target file. If copying the group fails, the per- - missions are modified so that the target file doesn't become accessible - to users who didn't have permission to access the source file. xz - doesn't support copying other metadata like access control lists or - extended attributes yet. - - Once the target file has been successfully closed, the source file is - removed unless --keep was specified. The source file is never removed - if the output is written to standard output. - - Sending SIGINFO or SIGUSR1 to the xz process makes it print progress - information to standard error. This has only limited use since when - standard error is a terminal, using --verbose will display an automati- - cally updating progress indicator. - - Memory usage - The memory usage of xz varies from a few hundred kilobytes to several - gigabytes depending on the compression settings. The settings used - when compressing a file determine the memory requirements of the decom- - pressor. Typically the decompressor needs 5 % to 20 % of the amount of - memory that the compressor needed when creating the file. For example, - decompressing a file created with xz -9 currently requires 65 MiB of - memory. Still, it is possible to have .xz files that require several - gigabytes of memory to decompress. - - Especially users of older systems may find the possibility of very - large memory usage annoying. To prevent uncomfortable surprises, xz - has a built-in memory usage limiter, which is disabled by default. - While some operating systems provide ways to limit the memory usage of - processes, relying on it wasn't deemed to be flexible enough (e.g. - using ulimit(1) to limit virtual memory tends to cripple mmap(2)). - - The memory usage limiter can be enabled with the command line option - --memlimit=limit. Often it is more convenient to enable the limiter by - default by setting the environment variable XZ_DEFAULTS, e.g. - XZ_DEFAULTS=--memlimit=150MiB. It is possible to set the limits sepa- - rately for compression and decompression by using --memlimit-com- - press=limit and --memlimit-decompress=limit. Using these two options - outside XZ_DEFAULTS is rarely useful because a single run of xz cannot - do both compression and decompression and --memlimit=limit (or -M - limit) is shorter to type on the command line. - - If the specified memory usage limit is exceeded when decompressing, xz - will display an error and decompressing the file will fail. If the - limit is exceeded when compressing, xz will try to scale the settings - down so that the limit is no longer exceeded (except when using --for- - mat=raw or --no-adjust). This way the operation won't fail unless the - limit is very small. The scaling of the settings is done in steps that - don't match the compression level presets, e.g. if the limit is only - slightly less than the amount required for xz -9, the settings will be - scaled down only a little, not all the way down to xz -8. - - Concatenation and padding with .xz files - It is possible to concatenate .xz files as is. xz will decompress such - files as if they were a single .xz file. - - It is possible to insert padding between the concatenated parts or - after the last part. The padding must consist of null bytes and the - size of the padding must be a multiple of four bytes. This can be use- - ful e.g. if the .xz file is stored on a medium that measures file sizes - in 512-byte blocks. - - Concatenation and padding are not allowed with .lzma files or raw - streams. - -OPTIONS - Integer suffixes and special values - In most places where an integer argument is expected, an optional suf- - fix is supported to easily indicate large integers. There must be no - space between the integer and the suffix. - - KiB Multiply the integer by 1,024 (2^10). Ki, k, kB, K, and KB are - accepted as synonyms for KiB. - - MiB Multiply the integer by 1,048,576 (2^20). Mi, m, M, and MB are - accepted as synonyms for MiB. - - GiB Multiply the integer by 1,073,741,824 (2^30). Gi, g, G, and GB - are accepted as synonyms for GiB. - - The special value max can be used to indicate the maximum integer value - supported by the option. - - Operation mode - If multiple operation mode options are given, the last one takes - effect. - - -z, --compress - Compress. This is the default operation mode when no operation - mode option is specified and no other operation mode is implied - from the command name (for example, unxz implies --decompress). - - -d, --decompress, --uncompress - Decompress. - - -t, --test - Test the integrity of compressed files. This option is equiva- - lent to --decompress --stdout except that the decompressed data - is discarded instead of being written to standard output. No - files are created or removed. - - -l, --list - Print information about compressed files. No uncompressed out- - put is produced, and no files are created or removed. In list - mode, the program cannot read the compressed data from standard - input or from other unseekable sources. - - The default listing shows basic information about files, one - file per line. To get more detailed information, use also the - --verbose option. For even more information, use --verbose - twice, but note that this may be slow, because getting all the - extra information requires many seeks. The width of verbose - output exceeds 80 characters, so piping the output to e.g. - less -S may be convenient if the terminal isn't wide enough. - - The exact output may vary between xz versions and different - locales. For machine-readable output, --robot --list should be - used. - - Operation modifiers - -k, --keep - Don't delete the input files. - - -f, --force - This option has several effects: - - o If the target file already exists, delete it before compress- - ing or decompressing. - - o Compress or decompress even if the input is a symbolic link - to a regular file, has more than one hard link, or has the - setuid, setgid, or sticky bit set. The setuid, setgid, and - sticky bits are not copied to the target file. - - o When used with --decompress --stdout and xz cannot recognize - the type of the source file, copy the source file as is to - standard output. This allows xzcat --force to be used like - cat(1) for files that have not been compressed with xz. Note - that in future, xz might support new compressed file formats, - which may make xz decompress more types of files instead of - copying them as is to standard output. --format=format can - be used to restrict xz to decompress only a single file for- - mat. - - -c, --stdout, --to-stdout - Write the compressed or decompressed data to standard output - instead of a file. This implies --keep. - - --no-sparse - Disable creation of sparse files. By default, if decompressing - into a regular file, xz tries to make the file sparse if the - decompressed data contains long sequences of binary zeros. It - also works when writing to standard output as long as standard - output is connected to a regular file and certain additional - conditions are met to make it safe. Creating sparse files may - save disk space and speed up the decompression by reducing the - amount of disk I/O. - - -S .suf, --suffix=.suf - When compressing, use .suf as the suffix for the target file - instead of .xz or .lzma. If not writing to standard output and - the source file already has the suffix .suf, a warning is dis- - played and the file is skipped. - - When decompressing, recognize files with the suffix .suf in - addition to files with the .xz, .txz, .lzma, or .tlz suffix. If - the source file has the suffix .suf, the suffix is removed to - get the target filename. - - When compressing or decompressing raw streams (--format=raw), - the suffix must always be specified unless writing to standard - output, because there is no default suffix for raw streams. - - --files[=file] - Read the filenames to process from file; if file is omitted, - filenames are read from standard input. Filenames must be ter- - minated with the newline character. A dash (-) is taken as a - regular filename; it doesn't mean standard input. If filenames - are given also as command line arguments, they are processed - before the filenames read from file. - - --files0[=file] - This is identical to --files[=file] except that each filename - must be terminated with the null character. - - Basic file format and compression options - -F format, --format=format - Specify the file format to compress or decompress: - - auto This is the default. When compressing, auto is equiva- - lent to xz. When decompressing, the format of the input - file is automatically detected. Note that raw streams - (created with --format=raw) cannot be auto-detected. - - xz Compress to the .xz file format, or accept only .xz files - when decompressing. - - lzma, alone - Compress to the legacy .lzma file format, or accept only - .lzma files when decompressing. The alternative name - alone is provided for backwards compatibility with LZMA - Utils. - - raw Compress or uncompress a raw stream (no headers). This - is meant for advanced users only. To decode raw streams, - you need use --format=raw and explicitly specify the fil- - ter chain, which normally would have been stored in the - container headers. - - -C check, --check=check - Specify the type of the integrity check. The check is calcu- - lated from the uncompressed data and stored in the .xz file. - This option has an effect only when compressing into the .xz - format; the .lzma format doesn't support integrity checks. The - integrity check (if any) is verified when the .xz file is decom- - pressed. - - Supported check types: - - none Don't calculate an integrity check at all. This is usu- - ally a bad idea. This can be useful when integrity of - the data is verified by other means anyway. - - crc32 Calculate CRC32 using the polynomial from IEEE-802.3 - (Ethernet). - - crc64 Calculate CRC64 using the polynomial from ECMA-182. This - is the default, since it is slightly better than CRC32 at - detecting damaged files and the speed difference is neg- - ligible. - - sha256 Calculate SHA-256. This is somewhat slower than CRC32 - and CRC64. - - Integrity of the .xz headers is always verified with CRC32. It - is not possible to change or disable it. - - -0 ... -9 - Select a compression preset level. The default is -6. If mul- - tiple preset levels are specified, the last one takes effect. - If a custom filter chain was already specified, setting a com- - pression preset level clears the custom filter chain. - - The differences between the presets are more significant than - with gzip(1) and bzip2(1). The selected compression settings - determine the memory requirements of the decompressor, thus - using a too high preset level might make it painful to decom- - press the file on an old system with little RAM. Specifically, - it's not a good idea to blindly use -9 for everything like it - often is with gzip(1) and bzip2(1). - - -0 ... -3 - These are somewhat fast presets. -0 is sometimes faster - than gzip -9 while compressing much better. The higher - ones often have speed comparable to bzip2(1) with compa- - rable or better compression ratio, although the results - depend a lot on the type of data being compressed. - - -4 ... -6 - Good to very good compression while keeping decompressor - memory usage reasonable even for old systems. -6 is the - default, which is usually a good choice e.g. for dis- - tributing files that need to be decompressible even on - systems with only 16 MiB RAM. (-5e or -6e may be worth - considering too. See --extreme.) - - -7 ... -9 - These are like -6 but with higher compressor and decom- - pressor memory requirements. These are useful only when - compressing files bigger than 8 MiB, 16 MiB, and 32 MiB, - respectively. - - On the same hardware, the decompression speed is approximately a - constant number of bytes of compressed data per second. In - other words, the better the compression, the faster the decom- - pression will usually be. This also means that the amount of - uncompressed output produced per second can vary a lot. - - The following table summarises the features of the presets: - - Preset DictSize CompCPU CompMem DecMem - -0 256 KiB 0 3 MiB 1 MiB - -1 1 MiB 1 9 MiB 2 MiB - -2 2 MiB 2 17 MiB 3 MiB - -3 4 MiB 3 32 MiB 5 MiB - -4 4 MiB 4 48 MiB 5 MiB - -5 8 MiB 5 94 MiB 9 MiB - -6 8 MiB 6 94 MiB 9 MiB - -7 16 MiB 6 186 MiB 17 MiB - -8 32 MiB 6 370 MiB 33 MiB - -9 64 MiB 6 674 MiB 65 MiB - - Column descriptions: - - o DictSize is the LZMA2 dictionary size. It is waste of memory - to use a dictionary bigger than the size of the uncompressed - file. This is why it is good to avoid using the presets -7 - ... -9 when there's no real need for them. At -6 and lower, - the amount of memory wasted is usually low enough to not mat- - ter. - - o CompCPU is a simplified representation of the LZMA2 settings - that affect compression speed. The dictionary size affects - speed too, so while CompCPU is the same for levels -6 ... -9, - higher levels still tend to be a little slower. To get even - slower and thus possibly better compression, see --extreme. - - o CompMem contains the compressor memory requirements in the - single-threaded mode. It may vary slightly between xz ver- - sions. Memory requirements of some of the future multi- - threaded modes may be dramatically higher than that of the - single-threaded mode. - - o DecMem contains the decompressor memory requirements. That - is, the compression settings determine the memory require- - ments of the decompressor. The exact decompressor memory - usage is slightly more than the LZMA2 dictionary size, but - the values in the table have been rounded up to the next full - MiB. - - -e, --extreme - Use a slower variant of the selected compression preset level - (-0 ... -9) to hopefully get a little bit better compression - ratio, but with bad luck this can also make it worse. Decom- - pressor memory usage is not affected, but compressor memory - usage increases a little at preset levels -0 ... -3. - - Since there are two presets with dictionary sizes 4 MiB and - 8 MiB, the presets -3e and -5e use slightly faster settings - (lower CompCPU) than -4e and -6e, respectively. That way no two - presets are identical. - - Preset DictSize CompCPU CompMem DecMem - -0e 256 KiB 8 4 MiB 1 MiB - -1e 1 MiB 8 13 MiB 2 MiB - -2e 2 MiB 8 25 MiB 3 MiB - -3e 4 MiB 7 48 MiB 5 MiB - -4e 4 MiB 8 48 MiB 5 MiB - -5e 8 MiB 7 94 MiB 9 MiB - -6e 8 MiB 8 94 MiB 9 MiB - -7e 16 MiB 8 186 MiB 17 MiB - -8e 32 MiB 8 370 MiB 33 MiB - -9e 64 MiB 8 674 MiB 65 MiB - - For example, there are a total of four presets that use 8 MiB - dictionary, whose order from the fastest to the slowest is -5, - -6, -5e, and -6e. - - --fast - --best These are somewhat misleading aliases for -0 and -9, respec- - tively. These are provided only for backwards compatibility - with LZMA Utils. Avoid using these options. - - --memlimit-compress=limit - Set a memory usage limit for compression. If this option is - specified multiple times, the last one takes effect. - - If the compression settings exceed the limit, xz will adjust the - settings downwards so that the limit is no longer exceeded and - display a notice that automatic adjustment was done. Such - adjustments are not made when compressing with --format=raw or - if --no-adjust has been specified. In those cases, an error is - displayed and xz will exit with exit status 1. - - The limit can be specified in multiple ways: - - o The limit can be an absolute value in bytes. Using an inte- - ger suffix like MiB can be useful. Example: --memlimit-com- - press=80MiB - - o The limit can be specified as a percentage of total physical - memory (RAM). This can be useful especially when setting the - XZ_DEFAULTS environment variable in a shell initialization - script that is shared between different computers. That way - the limit is automatically bigger on systems with more mem- - ory. Example: --memlimit-compress=70% - - o The limit can be reset back to its default value by setting - it to 0. This is currently equivalent to setting the limit - to max (no memory usage limit). Once multithreading support - has been implemented, there may be a difference between 0 and - max for the multithreaded case, so it is recommended to use 0 - instead of max until the details have been decided. - - See also the section Memory usage. - - --memlimit-decompress=limit - Set a memory usage limit for decompression. This also affects - the --list mode. If the operation is not possible without - exceeding the limit, xz will display an error and decompressing - the file will fail. See --memlimit-compress=limit for possible - ways to specify the limit. - - -M limit, --memlimit=limit, --memory=limit - This is equivalent to specifying --memlimit-compress=limit - --memlimit-decompress=limit. - - --no-adjust - Display an error and exit if the compression settings exceed the - memory usage limit. The default is to adjust the settings down- - wards so that the memory usage limit is not exceeded. Automatic - adjusting is always disabled when creating raw streams (--for- - mat=raw). - - -T threads, --threads=threads - Specify the number of worker threads to use. The actual number - of threads can be less than threads if using more threads would - exceed the memory usage limit. - - Multithreaded compression and decompression are not implemented - yet, so this option has no effect for now. - - As of writing (2010-09-27), it hasn't been decided if threads - will be used by default on multicore systems once support for - threading has been implemented. Comments are welcome. The com- - plicating factor is that using many threads will increase the - memory usage dramatically. Note that if multithreading will be - the default, it will probably be done so that single-threaded - and multithreaded modes produce the same output, so compression - ratio won't be significantly affected if threading will be - enabled by default. - - Custom compressor filter chains - A custom filter chain allows specifying the compression settings in - detail instead of relying on the settings associated to the presets. - When a custom filter chain is specified, preset options (-0 ... -9 and - --extreme) earlier on the command line are forgotten. If a preset - option is specified after one or more custom filter chain options, the - new preset takes effect and the custom filter chain options specified - earlier are forgotten. - - A filter chain is comparable to piping on the command line. When com- - pressing, the uncompressed input goes to the first filter, whose output - goes to the next filter (if any). The output of the last filter gets - written to the compressed file. The maximum number of filters in the - chain is four, but typically a filter chain has only one or two fil- - ters. - - Many filters have limitations on where they can be in the filter chain: - some filters can work only as the last filter in the chain, some only - as a non-last filter, and some work in any position in the chain. - Depending on the filter, this limitation is either inherent to the fil- - ter design or exists to prevent security issues. - - A custom filter chain is specified by using one or more filter options - in the order they are wanted in the filter chain. That is, the order - of filter options is significant! When decoding raw streams (--for- - mat=raw), the filter chain is specified in the same order as it was - specified when compressing. - - Filters take filter-specific options as a comma-separated list. Extra - commas in options are ignored. Every option has a default value, so - you need to specify only those you want to change. - - To see the whole filter chain and options, use xz -vv (that is, use - --verbose twice). This works also for viewing the filter chain options - used by presets. - - --lzma1[=options] - --lzma2[=options] - Add LZMA1 or LZMA2 filter to the filter chain. These filters - can be used only as the last filter in the chain. - - LZMA1 is a legacy filter, which is supported almost solely due - to the legacy .lzma file format, which supports only LZMA1. - LZMA2 is an updated version of LZMA1 to fix some practical - issues of LZMA1. The .xz format uses LZMA2 and doesn't support - LZMA1 at all. Compression speed and ratios of LZMA1 and LZMA2 - are practically the same. - - LZMA1 and LZMA2 share the same set of options: - - preset=preset - Reset all LZMA1 or LZMA2 options to preset. Preset con- - sist of an integer, which may be followed by single-let- - ter preset modifiers. The integer can be from 0 to 9, - matching the command line options -0 ... -9. The only - supported modifier is currently e, which matches - --extreme. If no preset is specified, the default values - of LZMA1 or LZMA2 options are taken from the preset 6. - - dict=size - Dictionary (history buffer) size indicates how many bytes - of the recently processed uncompressed data is kept in - memory. The algorithm tries to find repeating byte - sequences (matches) in the uncompressed data, and replace - them with references to the data currently in the dictio- - nary. The bigger the dictionary, the higher is the - chance to find a match. Thus, increasing dictionary size - usually improves compression ratio, but a dictionary big- - ger than the uncompressed file is waste of memory. - - Typical dictionary size is from 64 KiB to 64 MiB. The - minimum is 4 KiB. The maximum for compression is cur- - rently 1.5 GiB (1536 MiB). The decompressor already sup- - ports dictionaries up to one byte less than 4 GiB, which - is the maximum for the LZMA1 and LZMA2 stream formats. - - Dictionary size and match finder (mf) together determine - the memory usage of the LZMA1 or LZMA2 encoder. The same - (or bigger) dictionary size is required for decompressing - that was used when compressing, thus the memory usage of - the decoder is determined by the dictionary size used - when compressing. The .xz headers store the dictionary - size either as 2^n or 2^n + 2^(n-1), so these sizes are - somewhat preferred for compression. Other sizes will get - rounded up when stored in the .xz headers. - - lc=lc Specify the number of literal context bits. The minimum - is 0 and the maximum is 4; the default is 3. In addi- - tion, the sum of lc and lp must not exceed 4. - - All bytes that cannot be encoded as matches are encoded - as literals. That is, literals are simply 8-bit bytes - that are encoded one at a time. - - The literal coding makes an assumption that the highest - lc bits of the previous uncompressed byte correlate with - the next byte. E.g. in typical English text, an upper- - case letter is often followed by a lower-case letter, and - a lower-case letter is usually followed by another lower- - case letter. In the US-ASCII character set, the highest - three bits are 010 for upper-case letters and 011 for - lower-case letters. When lc is at least 3, the literal - coding can take advantage of this property in the uncom- - pressed data. - - The default value (3) is usually good. If you want maxi- - mum compression, test lc=4. Sometimes it helps a little, - and sometimes it makes compression worse. If it makes it - worse, test e.g. lc=2 too. - - lp=lp Specify the number of literal position bits. The minimum - is 0 and the maximum is 4; the default is 0. - - Lp affects what kind of alignment in the uncompressed - data is assumed when encoding literals. See pb below for - more information about alignment. - - pb=pb Specify the number of position bits. The minimum is 0 - and the maximum is 4; the default is 2. - - Pb affects what kind of alignment in the uncompressed - data is assumed in general. The default means four-byte - alignment (2^pb=2^2=4), which is often a good choice when - there's no better guess. - - When the aligment is known, setting pb accordingly may - reduce the file size a little. E.g. with text files hav- - ing one-byte alignment (US-ASCII, ISO-8859-*, UTF-8), - setting pb=0 can improve compression slightly. For - UTF-16 text, pb=1 is a good choice. If the alignment is - an odd number like 3 bytes, pb=0 might be the best - choice. - - Even though the assumed alignment can be adjusted with pb - and lp, LZMA1 and LZMA2 still slightly favor 16-byte - alignment. It might be worth taking into account when - designing file formats that are likely to be often com- - pressed with LZMA1 or LZMA2. - - mf=mf Match finder has a major effect on encoder speed, memory - usage, and compression ratio. Usually Hash Chain match - finders are faster than Binary Tree match finders. The - default depends on the preset: 0 uses hc3, 1-3 use hc4, - and the rest use bt4. - - The following match finders are supported. The memory - usage formulas below are rough approximations, which are - closest to the reality when dict is a power of two. - - hc3 Hash Chain with 2- and 3-byte hashing - Minimum value for nice: 3 - Memory usage: - dict * 7.5 (if dict <= 16 MiB); - dict * 5.5 + 64 MiB (if dict > 16 MiB) - - hc4 Hash Chain with 2-, 3-, and 4-byte hashing - Minimum value for nice: 4 - Memory usage: - dict * 7.5 (if dict <= 32 MiB); - dict * 6.5 (if dict > 32 MiB) - - bt2 Binary Tree with 2-byte hashing - Minimum value for nice: 2 - Memory usage: dict * 9.5 - - bt3 Binary Tree with 2- and 3-byte hashing - Minimum value for nice: 3 - Memory usage: - dict * 11.5 (if dict <= 16 MiB); - dict * 9.5 + 64 MiB (if dict > 16 MiB) - - bt4 Binary Tree with 2-, 3-, and 4-byte hashing - Minimum value for nice: 4 - Memory usage: - dict * 11.5 (if dict <= 32 MiB); - dict * 10.5 (if dict > 32 MiB) - - mode=mode - Compression mode specifies the method to analyze the data - produced by the match finder. Supported modes are fast - and normal. The default is fast for presets 0-3 and nor- - mal for presets 4-9. - - Usually fast is used with Hash Chain match finders and - normal with Binary Tree match finders. This is also what - the presets do. - - nice=nice - Specify what is considered to be a nice length for a - match. Once a match of at least nice bytes is found, the - algorithm stops looking for possibly better matches. - - Nice can be 2-273 bytes. Higher values tend to give bet- - ter compression ratio at the expense of speed. The - default depends on the preset. - - depth=depth - Specify the maximum search depth in the match finder. - The default is the special value of 0, which makes the - compressor determine a reasonable depth from mf and nice. - - Reasonable depth for Hash Chains is 4-100 and 16-1000 for - Binary Trees. Using very high values for depth can make - the encoder extremely slow with some files. Avoid set- - ting the depth over 1000 unless you are prepared to - interrupt the compression in case it is taking far too - long. - - When decoding raw streams (--format=raw), LZMA2 needs only the - dictionary size. LZMA1 needs also lc, lp, and pb. - - --x86[=options] - --powerpc[=options] - --ia64[=options] - --arm[=options] - --armthumb[=options] - --sparc[=options] - Add a branch/call/jump (BCJ) filter to the filter chain. These - filters can be used only as a non-last filter in the filter - chain. - - A BCJ filter converts relative addresses in the machine code to - their absolute counterparts. This doesn't change the size of - the data, but it increases redundancy, which can help LZMA2 to - produce 0-15 % smaller .xz file. The BCJ filters are always - reversible, so using a BCJ filter for wrong type of data doesn't - cause any data loss, although it may make the compression ratio - slightly worse. - - It is fine to apply a BCJ filter on a whole executable; there's - no need to apply it only on the executable section. Applying a - BCJ filter on an archive that contains both executable and non- - executable files may or may not give good results, so it gener- - ally isn't good to blindly apply a BCJ filter when compressing - binary packages for distribution. - - These BCJ filters are very fast and use insignificant amount of - memory. If a BCJ filter improves compression ratio of a file, - it can improve decompression speed at the same time. This is - because, on the same hardware, the decompression speed of LZMA2 - is roughly a fixed number of bytes of compressed data per sec- - ond. - - These BCJ filters have known problems related to the compression - ratio: - - o Some types of files containing executable code (e.g. object - files, static libraries, and Linux kernel modules) have the - addresses in the instructions filled with filler values. - These BCJ filters will still do the address conversion, which - will make the compression worse with these files. - - o Applying a BCJ filter on an archive containing multiple simi- - lar executables can make the compression ratio worse than not - using a BCJ filter. This is because the BCJ filter doesn't - detect the boundaries of the executable files, and doesn't - reset the address conversion counter for each executable. - - Both of the above problems will be fixed in the future in a new - filter. The old BCJ filters will still be useful in embedded - systems, because the decoder of the new filter will be bigger - and use more memory. - - Different instruction sets have have different alignment: - - Filter Alignment Notes - x86 1 32-bit or 64-bit x86 - PowerPC 4 Big endian only - ARM 4 Little endian only - ARM-Thumb 2 Little endian only - IA-64 16 Big or little endian - SPARC 4 Big or little endian - - Since the BCJ-filtered data is usually compressed with LZMA2, - the compression ratio may be improved slightly if the LZMA2 - options are set to match the alignment of the selected BCJ fil- - ter. For example, with the IA-64 filter, it's good to set pb=4 - with LZMA2 (2^4=16). The x86 filter is an exception; it's usu- - ally good to stick to LZMA2's default four-byte alignment when - compressing x86 executables. - - All BCJ filters support the same options: - - start=offset - Specify the start offset that is used when converting - between relative and absolute addresses. The offset must - be a multiple of the alignment of the filter (see the ta- - ble above). The default is zero. In practice, the - default is good; specifying a custom offset is almost - never useful. - - --delta[=options] - Add the Delta filter to the filter chain. The Delta filter can - be only used as a non-last filter in the filter chain. - - Currently only simple byte-wise delta calculation is supported. - It can be useful when compressing e.g. uncompressed bitmap - images or uncompressed PCM audio. However, special purpose - algorithms may give significantly better results than Delta + - LZMA2. This is true especially with audio, which compresses - faster and better e.g. with flac(1). - - Supported options: - - dist=distance - Specify the distance of the delta calculation in bytes. - distance must be 1-256. The default is 1. - - For example, with dist=2 and eight-byte input A1 B1 A2 B3 - A3 B5 A4 B7, the output will be A1 B1 01 02 01 02 01 02. - - Other options - -q, --quiet - Suppress warnings and notices. Specify this twice to suppress - errors too. This option has no effect on the exit status. That - is, even if a warning was suppressed, the exit status to indi- - cate a warning is still used. - - -v, --verbose - Be verbose. If standard error is connected to a terminal, xz - will display a progress indicator. Specifying --verbose twice - will give even more verbose output. - - The progress indicator shows the following information: - - o Completion percentage is shown if the size of the input file - is known. That is, the percentage cannot be shown in pipes. - - o Amount of compressed data produced (compressing) or consumed - (decompressing). - - o Amount of uncompressed data consumed (compressing) or pro- - duced (decompressing). - - o Compression ratio, which is calculated by dividing the amount - of compressed data processed so far by the amount of uncom- - pressed data processed so far. - - o Compression or decompression speed. This is measured as the - amount of uncompressed data consumed (compression) or pro- - duced (decompression) per second. It is shown after a few - seconds have passed since xz started processing the file. - - o Elapsed time in the format M:SS or H:MM:SS. - - o Estimated remaining time is shown only when the size of the - input file is known and a couple of seconds have already - passed since xz started processing the file. The time is - shown in a less precise format which never has any colons, - e.g. 2 min 30 s. - - When standard error is not a terminal, --verbose will make xz - print the filename, compressed size, uncompressed size, compres- - sion ratio, and possibly also the speed and elapsed time on a - single line to standard error after compressing or decompressing - the file. The speed and elapsed time are included only when the - operation took at least a few seconds. If the operation didn't - finish, e.g. due to user interruption, also the completion per- - centage is printed if the size of the input file is known. - - -Q, --no-warn - Don't set the exit status to 2 even if a condition worth a warn- - ing was detected. This option doesn't affect the verbosity - level, thus both --quiet and --no-warn have to be used to not - display warnings and to not alter the exit status. - - --robot - Print messages in a machine-parsable format. This is intended - to ease writing frontends that want to use xz instead of - liblzma, which may be the case with various scripts. The output - with this option enabled is meant to be stable across xz - releases. See the section ROBOT MODE for details. - - --info-memory - Display, in human-readable format, how much physical memory - (RAM) xz thinks the system has and the memory usage limits for - compression and decompression, and exit successfully. - - -h, --help - Display a help message describing the most commonly used - options, and exit successfully. - - -H, --long-help - Display a help message describing all features of xz, and exit - successfully - - -V, --version - Display the version number of xz and liblzma in human readable - format. To get machine-parsable output, specify --robot before - --version. - -ROBOT MODE - The robot mode is activated with the --robot option. It makes the out- - put of xz easier to parse by other programs. Currently --robot is sup- - ported only together with --version, --info-memory, and --list. It - will be supported for compression and decompression in the future. - - Version - xz --robot --version will print the version number of xz and liblzma in - the following format: - - XZ_VERSION=XYYYZZZS - LIBLZMA_VERSION=XYYYZZZS - - X Major version. - - YYY Minor version. Even numbers are stable. Odd numbers are alpha - or beta versions. - - ZZZ Patch level for stable releases or just a counter for develop- - ment releases. - - S Stability. 0 is alpha, 1 is beta, and 2 is stable. S should be - always 2 when YYY is even. - - XYYYZZZS are the same on both lines if xz and liblzma are from the same - XZ Utils release. - - Examples: 4.999.9beta is 49990091 and 5.0.0 is 50000002. - - Memory limit information - xz --robot --info-memory prints a single line with three tab-separated - columns: - - 1. Total amount of physical memory (RAM) in bytes - - 2. Memory usage limit for compression in bytes. A special value of - zero indicates the default setting, which for single-threaded mode - is the same as no limit. - - 3. Memory usage limit for decompression in bytes. A special value of - zero indicates the default setting, which for single-threaded mode - is the same as no limit. - - In the future, the output of xz --robot --info-memory may have more - columns, but never more than a single line. - - List mode - xz --robot --list uses tab-separated output. The first column of every - line has a string that indicates the type of the information found on - that line: - - name This is always the first line when starting to list a file. The - second column on the line is the filename. - - file This line contains overall information about the .xz file. This - line is always printed after the name line. - - stream This line type is used only when --verbose was specified. There - are as many stream lines as there are streams in the .xz file. - - block This line type is used only when --verbose was specified. There - are as many block lines as there are blocks in the .xz file. - The block lines are shown after all the stream lines; different - line types are not interleaved. - - summary - This line type is used only when --verbose was specified twice. - This line is printed after all block lines. Like the file line, - the summary line contains overall information about the .xz - file. - - totals This line is always the very last line of the list output. It - shows the total counts and sizes. - - The columns of the file lines: - 2. Number of streams in the file - 3. Total number of blocks in the stream(s) - 4. Compressed size of the file - 5. Uncompressed size of the file - 6. Compression ratio, for example 0.123. If ratio is over - 9.999, three dashes (---) are displayed instead of the - ratio. - 7. Comma-separated list of integrity check names. The follow- - ing strings are used for the known check types: None, CRC32, - CRC64, and SHA-256. For unknown check types, Unknown-N is - used, where N is the Check ID as a decimal number (one or - two digits). - 8. Total size of stream padding in the file - - The columns of the stream lines: - 2. Stream number (the first stream is 1) - 3. Number of blocks in the stream - 4. Compressed start offset - 5. Uncompressed start offset - 6. Compressed size (does not include stream padding) - 7. Uncompressed size - 8. Compression ratio - 9. Name of the integrity check - 10. Size of stream padding - - The columns of the block lines: - 2. Number of the stream containing this block - 3. Block number relative to the beginning of the stream (the - first block is 1) - 4. Block number relative to the beginning of the file - 5. Compressed start offset relative to the beginning of the - file - 6. Uncompressed start offset relative to the beginning of the - file - 7. Total compressed size of the block (includes headers) - 8. Uncompressed size - 9. Compression ratio - 10. Name of the integrity check - - If --verbose was specified twice, additional columns are included on - the block lines. These are not displayed with a single --verbose, - because getting this information requires many seeks and can thus be - slow: - 11. Value of the integrity check in hexadecimal - 12. Block header size - 13. Block flags: c indicates that compressed size is present, - and u indicates that uncompressed size is present. If the - flag is not set, a dash (-) is shown instead to keep the - string length fixed. New flags may be added to the end of - the string in the future. - 14. Size of the actual compressed data in the block (this - excludes the block header, block padding, and check fields) - 15. Amount of memory (in bytes) required to decompress this - block with this xz version - 16. Filter chain. Note that most of the options used at com- - pression time cannot be known, because only the options that - are needed for decompression are stored in the .xz headers. - - The columns of the summary lines: - 2. Amount of memory (in bytes) required to decompress this file - with this xz version - 3. yes or no indicating if all block headers have both com- - pressed size and uncompressed size stored in them - - The columns of the totals line: - 2. Number of streams - 3. Number of blocks - 4. Compressed size - 5. Uncompressed size - 6. Average compression ratio - 7. Comma-separated list of integrity check names that were - present in the files - 8. Stream padding size - 9. Number of files. This is here to keep the order of the ear- - lier columns the same as on file lines. - - If --verbose was specified twice, additional columns are included on - the totals line: - 10. Maximum amount of memory (in bytes) required to decompress - the files with this xz version - 11. yes or no indicating if all block headers have both com- - pressed size and uncompressed size stored in them - - Future versions may add new line types and new columns can be added to - the existing line types, but the existing columns won't be changed. - -EXIT STATUS - 0 All is good. - - 1 An error occurred. - - 2 Something worth a warning occurred, but no actual errors - occurred. - - Notices (not warnings or errors) printed on standard error don't affect - the exit status. - -ENVIRONMENT - xz parses space-separated lists of options from the environment vari- - ables XZ_DEFAULTS and XZ_OPT, in this order, before parsing the options - from the command line. Note that only options are parsed from the - environment variables; all non-options are silently ignored. Parsing - is done with getopt_long(3) which is used also for the command line - arguments. - - XZ_DEFAULTS - User-specific or system-wide default options. Typically this is - set in a shell initialization script to enable xz's memory usage - limiter by default. Excluding shell initialization scripts and - similar special cases, scripts must never set or unset - XZ_DEFAULTS. - - XZ_OPT This is for passing options to xz when it is not possible to set - the options directly on the xz command line. This is the case - e.g. when xz is run by a script or tool, e.g. GNU tar(1): - - XZ_OPT=-2v tar caf foo.tar.xz foo - - Scripts may use XZ_OPT e.g. to set script-specific default com- - pression options. It is still recommended to allow users to - override XZ_OPT if that is reasonable, e.g. in sh(1) scripts one - may use something like this: - - XZ_OPT=${XZ_OPT-"-7e"} - export XZ_OPT - -LZMA UTILS COMPATIBILITY - The command line syntax of xz is practically a superset of lzma, - unlzma, and lzcat as found from LZMA Utils 4.32.x. In most cases, it - is possible to replace LZMA Utils with XZ Utils without breaking exist- - ing scripts. There are some incompatibilities though, which may some- - times cause problems. - - Compression preset levels - The numbering of the compression level presets is not identical in xz - and LZMA Utils. The most important difference is how dictionary sizes - are mapped to different presets. Dictionary size is roughly equal to - the decompressor memory usage. - - Level xz LZMA Utils - -0 256 KiB N/A - -1 1 MiB 64 KiB - -2 2 MiB 1 MiB - -3 4 MiB 512 KiB - -4 4 MiB 1 MiB - -5 8 MiB 2 MiB - -6 8 MiB 4 MiB - -7 16 MiB 8 MiB - -8 32 MiB 16 MiB - -9 64 MiB 32 MiB - - The dictionary size differences affect the compressor memory usage too, - but there are some other differences between LZMA Utils and XZ Utils, - which make the difference even bigger: - - Level xz LZMA Utils 4.32.x - -0 3 MiB N/A - -1 9 MiB 2 MiB - -2 17 MiB 12 MiB - -3 32 MiB 12 MiB - -4 48 MiB 16 MiB - - -5 94 MiB 26 MiB - -6 94 MiB 45 MiB - -7 186 MiB 83 MiB - -8 370 MiB 159 MiB - -9 674 MiB 311 MiB - - The default preset level in LZMA Utils is -7 while in XZ Utils it is - -6, so both use an 8 MiB dictionary by default. - - Streamed vs. non-streamed .lzma files - The uncompressed size of the file can be stored in the .lzma header. - LZMA Utils does that when compressing regular files. The alternative - is to mark that uncompressed size is unknown and use end-of-payload - marker to indicate where the decompressor should stop. LZMA Utils uses - this method when uncompressed size isn't known, which is the case for - example in pipes. - - xz supports decompressing .lzma files with or without end-of-payload - marker, but all .lzma files created by xz will use end-of-payload - marker and have uncompressed size marked as unknown in the .lzma - header. This may be a problem in some uncommon situations. For exam- - ple, a .lzma decompressor in an embedded device might work only with - files that have known uncompressed size. If you hit this problem, you - need to use LZMA Utils or LZMA SDK to create .lzma files with known - uncompressed size. - - Unsupported .lzma files - The .lzma format allows lc values up to 8, and lp values up to 4. LZMA - Utils can decompress files with any lc and lp, but always creates files - with lc=3 and lp=0. Creating files with other lc and lp is possible - with xz and with LZMA SDK. - - The implementation of the LZMA1 filter in liblzma requires that the sum - of lc and lp must not exceed 4. Thus, .lzma files, which exceed this - limitation, cannot be decompressed with xz. - - LZMA Utils creates only .lzma files which have a dictionary size of 2^n - (a power of 2) but accepts files with any dictionary size. liblzma - accepts only .lzma files which have a dictionary size of 2^n or 2^n + - 2^(n-1). This is to decrease false positives when detecting .lzma - files. - - These limitations shouldn't be a problem in practice, since practically - all .lzma files have been compressed with settings that liblzma will - accept. - - Trailing garbage - When decompressing, LZMA Utils silently ignore everything after the - first .lzma stream. In most situations, this is a bug. This also - means that LZMA Utils don't support decompressing concatenated .lzma - files. - - If there is data left after the first .lzma stream, xz considers the - file to be corrupt. This may break obscure scripts which have assumed - that trailing garbage is ignored. - -NOTES - Compressed output may vary - The exact compressed output produced from the same uncompressed input - file may vary between XZ Utils versions even if compression options are - identical. This is because the encoder can be improved (faster or bet- - ter compression) without affecting the file format. The output can - vary even between different builds of the same XZ Utils version, if - different build options are used. - - The above means that once --rsyncable has been implemented, the result- - ing files won't necessarily be rsyncable unless both old and new files - have been compressed with the same xz version. This problem can be - fixed if a part of the encoder implementation is frozen to keep rsynca- - ble output stable across xz versions. - - Embedded .xz decompressors - Embedded .xz decompressor implementations like XZ Embedded don't neces- - sarily support files created with integrity check types other than none - and crc32. Since the default is --check=crc64, you must use - --check=none or --check=crc32 when creating files for embedded systems. - - Outside embedded systems, all .xz format decompressors support all the - check types, or at least are able to decompress the file without veri- - fying the integrity check if the particular check is not supported. - - XZ Embedded supports BCJ filters, but only with the default start off- - set. - -EXAMPLES - Basics - Compress the file foo into foo.xz using the default compression level - (-6), and remove foo if compression is successful: - - xz foo - - Decompress bar.xz into bar and don't remove bar.xz even if decompres- - sion is successful: - - xz -dk bar.xz - - Create baz.tar.xz with the preset -4e (-4 --extreme), which is slower - than e.g. the default -6, but needs less memory for compression and - decompression (48 MiB and 5 MiB, respectively): - - tar cf - baz | xz -4e > baz.tar.xz - - A mix of compressed and uncompressed files can be decompressed to stan- - dard output with a single command: - - xz -dcf a.txt b.txt.xz c.txt d.txt.lzma > abcd.txt - - Parallel compression of many files - On GNU and *BSD, find(1) and xargs(1) can be used to parallelize com- - pression of many files: - - find . -type f \! -name '*.xz' -print0 \ - | xargs -0r -P4 -n16 xz -T1 - - The -P option to xargs(1) sets the number of parallel xz processes. - The best value for the -n option depends on how many files there are to - be compressed. If there are only a couple of files, the value should - probably be 1; with tens of thousands of files, 100 or even more may be - appropriate to reduce the number of xz processes that xargs(1) will - eventually create. - - The option -T1 for xz is there to force it to single-threaded mode, - because xargs(1) is used to control the amount of parallelization. - - Robot mode - Calculate how many bytes have been saved in total after compressing - multiple files: - - xz --robot --list *.xz | awk '/^totals/{print $5-$4}' - - A script may want to know that it is using new enough xz. The follow- - ing sh(1) script checks that the version number of the xz tool is at - least 5.0.0. This method is compatible with old beta versions, which - didn't support the --robot option: - - if ! eval "$(xz --robot --version 2> /dev/null)" || - [ "$XZ_VERSION" -lt 50000002 ]; then - echo "Your xz is too old." - fi - unset XZ_VERSION LIBLZMA_VERSION - - Set a memory usage limit for decompression using XZ_OPT, but if a limit - has already been set, don't increase it: - - NEWLIM=$((123 << 20)) # 123 MiB - OLDLIM=$(xz --robot --info-memory | cut -f3) - if [ $OLDLIM -eq 0 -o $OLDLIM -gt $NEWLIM ]; then - XZ_OPT="$XZ_OPT --memlimit-decompress=$NEWLIM" - export XZ_OPT - fi - - Custom compressor filter chains - The simplest use for custom filter chains is customizing a LZMA2 pre- - set. This can be useful, because the presets cover only a subset of - the potentially useful combinations of compression settings. - - The CompCPU columns of the tables from the descriptions of the options - -0 ... -9 and --extreme are useful when customizing LZMA2 presets. - Here are the relevant parts collected from those two tables: - - Preset CompCPU - -0 0 - -1 1 - -2 2 - -3 3 - -4 4 - -5 5 - -6 6 - -5e 7 - -6e 8 - - If you know that a file requires somewhat big dictionary (e.g. 32 MiB) - to compress well, but you want to compress it quicker than xz -8 would - do, a preset with a low CompCPU value (e.g. 1) can be modified to use a - bigger dictionary: - - xz --lzma2=preset=1,dict=32MiB foo.tar - - With certain files, the above command may be faster than xz -6 while - compressing significantly better. However, it must be emphasized that - only some files benefit from a big dictionary while keeping the CompCPU - value low. The most obvious situation, where a big dictionary can help - a lot, is an archive containing very similar files of at least a few - megabytes each. The dictionary size has to be significantly bigger - than any individual file to allow LZMA2 to take full advantage of the - similarities between consecutive files. - - If very high compressor and decompressor memory usage is fine, and the - file being compressed is at least several hundred megabytes, it may be - useful to use an even bigger dictionary than the 64 MiB that xz -9 - would use: - - xz -vv --lzma2=dict=192MiB big_foo.tar - - Using -vv (--verbose --verbose) like in the above example can be useful - to see the memory requirements of the compressor and decompressor. - Remember that using a dictionary bigger than the size of the uncom- - pressed file is waste of memory, so the above command isn't useful for - small files. - - Sometimes the compression time doesn't matter, but the decompressor - memory usage has to be kept low e.g. to make it possible to decompress - the file on an embedded system. The following command uses -6e (-6 - --extreme) as a base and sets the dictionary to only 64 KiB. The - resulting file can be decompressed with XZ Embedded (that's why there - is --check=crc32) using about 100 KiB of memory. - - xz --check=crc32 --lzma2=preset=6e,dict=64KiB foo - - If you want to squeeze out as many bytes as possible, adjusting the - number of literal context bits (lc) and number of position bits (pb) - can sometimes help. Adjusting the number of literal position bits (lp) - might help too, but usually lc and pb are more important. E.g. a - source code archive contains mostly US-ASCII text, so something like - the following might give slightly (like 0.1 %) smaller file than xz -6e - (try also without lc=4): - - xz --lzma2=preset=6e,pb=0,lc=4 source_code.tar - - Using another filter together with LZMA2 can improve compression with - certain file types. E.g. to compress a x86-32 or x86-64 shared library - using the x86 BCJ filter: - - xz --x86 --lzma2 libfoo.so - - Note that the order of the filter options is significant. If --x86 is - specified after --lzma2, xz will give an error, because there cannot be - any filter after LZMA2, and also because the x86 BCJ filter cannot be - used as the last filter in the chain. - - The Delta filter together with LZMA2 can give good results with bitmap - images. It should usually beat PNG, which has a few more advanced fil- - ters than simple delta but uses Deflate for the actual compression. - - The image has to be saved in uncompressed format, e.g. as uncompressed - TIFF. The distance parameter of the Delta filter is set to match the - number of bytes per pixel in the image. E.g. 24-bit RGB bitmap needs - dist=3, and it is also good to pass pb=0 to LZMA2 to accommodate the - three-byte alignment: - - xz --delta=dist=3 --lzma2=pb=0 foo.tiff - - If multiple images have been put into a single archive (e.g. .tar), the - Delta filter will work on that too as long as all images have the same - number of bytes per pixel. - -SEE ALSO - xzdec(1), xzdiff(1), xzgrep(1), xzless(1), xzmore(1), gzip(1), - bzip2(1), 7z(1) - - XZ Utils: <http://tukaani.org/xz/> - XZ Embedded: <http://tukaani.org/xz/embedded.html> - LZMA SDK: <http://7-zip.org/sdk.html> - - - -Tukaani 2013-06-21 XZ(1) diff --git a/Build/source/extra/xz/doc/man/txt/xzdec.txt b/Build/source/extra/xz/doc/man/txt/xzdec.txt deleted file mode 100644 index 3d69ff3f366..00000000000 --- a/Build/source/extra/xz/doc/man/txt/xzdec.txt +++ /dev/null @@ -1,80 +0,0 @@ -XZDEC(1) XZ Utils XZDEC(1) - - - -NAME - xzdec, lzmadec - Small .xz and .lzma decompressors - -SYNOPSIS - xzdec [option...] [file...] - lzmadec [option...] [file...] - -DESCRIPTION - xzdec is a liblzma-based decompression-only tool for .xz (and only .xz) - files. xzdec is intended to work as a drop-in replacement for xz(1) in - the most common situations where a script has been written to use xz - --decompress --stdout (and possibly a few other commonly used options) - to decompress .xz files. lzmadec is identical to xzdec except that - lzmadec supports .lzma files instead of .xz files. - - To reduce the size of the executable, xzdec doesn't support multi- - threading or localization, and doesn't read options from XZ_DEFAULTS - and XZ_OPT environment variables. xzdec doesn't support displaying - intermediate progress information: sending SIGINFO to xzdec does noth- - ing, but sending SIGUSR1 terminates the process instead of displaying - progress information. - -OPTIONS - -d, --decompress, --uncompress - Ignored for xz(1) compatibility. xzdec supports only decompres- - sion. - - -k, --keep - Ignored for xz(1) compatibility. xzdec never creates or removes - any files. - - -c, --stdout, --to-stdout - Ignored for xz(1) compatibility. xzdec always writes the decom- - pressed data to standard output. - - -q, --quiet - Specifying this once does nothing since xzdec never displays any - warnings or notices. Specify this twice to suppress errors. - - -Q, --no-warn - Ignored for xz(1) compatibility. xzdec never uses the exit sta- - tus 2. - - -h, --help - Display a help message and exit successfully. - - -V, --version - Display the version number of xzdec and liblzma. - -EXIT STATUS - 0 All was good. - - 1 An error occurred. - - xzdec doesn't have any warning messages like xz(1) has, thus the exit - status 2 is not used by xzdec. - -NOTES - Use xz(1) instead of xzdec or lzmadec for normal everyday use. xzdec - or lzmadec are meant only for situations where it is important to have - a smaller decompressor than the full-featured xz(1). - - xzdec and lzmadec are not really that small. The size can be reduced - further by dropping features from liblzma at compile time, but that - shouldn't usually be done for executables distributed in typical non- - embedded operating system distributions. If you need a truly small .xz - decompressor, consider using XZ Embedded. - -SEE ALSO - xz(1) - - XZ Embedded: <http://tukaani.org/xz/embedded.html> - - - -Tukaani 2013-06-30 XZDEC(1) diff --git a/Build/source/extra/xz/doc/man/txt/xzdiff.txt b/Build/source/extra/xz/doc/man/txt/xzdiff.txt deleted file mode 100644 index 99af443b2cc..00000000000 --- a/Build/source/extra/xz/doc/man/txt/xzdiff.txt +++ /dev/null @@ -1,36 +0,0 @@ -XZDIFF(1) XZ Utils XZDIFF(1) - - - -NAME - xzcmp, xzdiff, lzcmp, lzdiff - compare compressed files - -SYNOPSIS - xzcmp [cmp_options] file1 [file2] - xzdiff [diff_options] file1 [file2] - lzcmp [cmp_options] file1 [file2] - lzdiff [diff_options] file1 [file2] - -DESCRIPTION - xzcmp and xzdiff invoke cmp(1) or diff(1) on files compressed with - xz(1), lzma(1), gzip(1), or bzip2(1). All options specified are passed - directly to cmp(1) or diff(1). If only one file is specified, then the - files compared are file1 (which must have a suffix of a supported com- - pression format) and file1 from which the compression format suffix has - been stripped. If two files are specified, then they are uncompressed - if necessary and fed to cmp(1) or diff(1). The exit status from cmp(1) - or diff(1) is preserved. - - The names lzcmp and lzdiff are provided for backward compatibility with - LZMA Utils. - -SEE ALSO - cmp(1), diff(1), xz(1), gzip(1), bzip2(1), zdiff(1) - -BUGS - Messages from the cmp(1) or diff(1) programs refer to temporary file- - names instead of those specified. - - - -Tukaani 2010-09-27 XZDIFF(1) diff --git a/Build/source/extra/xz/doc/man/txt/xzgrep.txt b/Build/source/extra/xz/doc/man/txt/xzgrep.txt deleted file mode 100644 index cdffd0735cf..00000000000 --- a/Build/source/extra/xz/doc/man/txt/xzgrep.txt +++ /dev/null @@ -1,39 +0,0 @@ -XZGREP(1) XZ Utils XZGREP(1) - - - -NAME - xzgrep - search compressed files for a regular expression - -SYNOPSIS - xzgrep [grep_options] [-e] pattern file... - xzegrep ... - xzfgrep ... - lzgrep ... - lzegrep ... - lzfgrep ... - -DESCRIPTION - xzgrep invokes grep(1) on files which may be either uncompressed or - compressed with xz(1), lzma(1), gzip(1), or bzip2(1). All options - specified are passed directly to grep(1). - - If no file is specified, then standard input is decompressed if neces- - sary and fed to grep(1). When reading from standard input, gzip(1) and - bzip2(1) compressed files are not supported. - - If xzgrep is invoked as xzegrep or xzfgrep then egrep(1) or fgrep(1) is - used instead of grep(1). The same applies to names lzgrep, lzegrep, - and lzfgrep, which are provided for backward compatibility with LZMA - Utils. - -ENVIRONMENT - GREP If the GREP environment variable is set, xzgrep uses it instead - of grep(1), egrep(1), or fgrep(1). - -SEE ALSO - grep(1), xz(1), gzip(1), bzip2(1), zgrep(1) - - - -Tukaani 2010-09-27 XZGREP(1) diff --git a/Build/source/extra/xz/doc/man/txt/xzless.txt b/Build/source/extra/xz/doc/man/txt/xzless.txt deleted file mode 100644 index 5c14c80c235..00000000000 --- a/Build/source/extra/xz/doc/man/txt/xzless.txt +++ /dev/null @@ -1,39 +0,0 @@ -XZLESS(1) XZ Utils XZLESS(1) - - - -NAME - xzless, lzless - view xz or lzma compressed (text) files - -SYNOPSIS - xzless [file...] - lzless [file...] - -DESCRIPTION - xzless is a filter that displays text from compressed files to a termi- - nal. It works on files compressed with xz(1) or lzma(1). If no files - are given, xzless reads from standard input. - - xzless uses less(1) to present its output. Unlike xzmore, its choice - of pager cannot be altered by setting an environment variable. Com- - mands are based on both more(1) and vi(1) and allow back and forth - movement and searching. See the less(1) manual for more information. - - The command named lzless is provided for backward compatibility with - LZMA Utils. - -ENVIRONMENT - LESSMETACHARS - A list of characters special to the shell. Set by xzless unless - it is already set in the environment. - - LESSOPEN - Set to a command line to invoke the xz(1) decompressor for pre- - processing the input files to less(1). - -SEE ALSO - less(1), xz(1), xzmore(1), zless(1) - - - -Tukaani 2010-09-27 XZLESS(1) diff --git a/Build/source/extra/xz/doc/man/txt/xzmore.txt b/Build/source/extra/xz/doc/man/txt/xzmore.txt deleted file mode 100644 index 5a9d86c7853..00000000000 --- a/Build/source/extra/xz/doc/man/txt/xzmore.txt +++ /dev/null @@ -1,34 +0,0 @@ -XZMORE(1) XZ Utils XZMORE(1) - - - -NAME - xzmore, lzmore - view xz or lzma compressed (text) files - -SYNOPSIS - xzmore [file...] - lzmore [file...] - -DESCRIPTION - xzmore is a filter which allows examination of xz(1) or lzma(1) com- - pressed text files one screenful at a time on a soft-copy terminal. - - To use a pager other than the default more, set environment variable - PAGER to the name of the desired program. The name lzmore is provided - for backward compatibility with LZMA Utils. - - e or q When the prompt --More--(Next file: file) is printed, this com- - mand causes xzmore to exit. - - s When the prompt --More--(Next file: file) is printed, this com- - mand causes xzmore to skip the next file and continue. - - For list of keyboard commands supported while actually viewing the con- - tent of a file, refer to manual of the pager you use, usually more(1). - -SEE ALSO - more(1), xz(1), xzless(1), zmore(1) - - - -Tukaani 2013-06-30 XZMORE(1) diff --git a/Build/source/extra/xz/doc/xz-file-format.txt b/Build/source/extra/xz/doc/xz-file-format.txt deleted file mode 100644 index 4ed66506018..00000000000 --- a/Build/source/extra/xz/doc/xz-file-format.txt +++ /dev/null @@ -1,1150 +0,0 @@ - -The .xz File Format -=================== - -Version 1.0.4 (2009-08-27) - - - 0. Preface - 0.1. Notices and Acknowledgements - 0.2. Getting the Latest Version - 0.3. Version History - 1. Conventions - 1.1. Byte and Its Representation - 1.2. Multibyte Integers - 2. Overall Structure of .xz File - 2.1. Stream - 2.1.1. Stream Header - 2.1.1.1. Header Magic Bytes - 2.1.1.2. Stream Flags - 2.1.1.3. CRC32 - 2.1.2. Stream Footer - 2.1.2.1. CRC32 - 2.1.2.2. Backward Size - 2.1.2.3. Stream Flags - 2.1.2.4. Footer Magic Bytes - 2.2. Stream Padding - 3. Block - 3.1. Block Header - 3.1.1. Block Header Size - 3.1.2. Block Flags - 3.1.3. Compressed Size - 3.1.4. Uncompressed Size - 3.1.5. List of Filter Flags - 3.1.6. Header Padding - 3.1.7. CRC32 - 3.2. Compressed Data - 3.3. Block Padding - 3.4. Check - 4. Index - 4.1. Index Indicator - 4.2. Number of Records - 4.3. List of Records - 4.3.1. Unpadded Size - 4.3.2. Uncompressed Size - 4.4. Index Padding - 4.5. CRC32 - 5. Filter Chains - 5.1. Alignment - 5.2. Security - 5.3. Filters - 5.3.1. LZMA2 - 5.3.2. Branch/Call/Jump Filters for Executables - 5.3.3. Delta - 5.3.3.1. Format of the Encoded Output - 5.4. Custom Filter IDs - 5.4.1. Reserved Custom Filter ID Ranges - 6. Cyclic Redundancy Checks - 7. References - - -0. Preface - - This document describes the .xz file format (filename suffix - ".xz", MIME type "application/x-xz"). It is intended that this - this format replace the old .lzma format used by LZMA SDK and - LZMA Utils. - - -0.1. Notices and Acknowledgements - - This file format was designed by Lasse Collin - <lasse.collin@tukaani.org> and Igor Pavlov. - - Special thanks for helping with this document goes to - Ville Koskinen. Thanks for helping with this document goes to - Mark Adler, H. Peter Anvin, Mikko Pouru, and Lars Wirzenius. - - This document has been put into the public domain. - - -0.2. Getting the Latest Version - - The latest official version of this document can be downloaded - from <http://tukaani.org/xz/xz-file-format.txt>. - - Specific versions of this document have a filename - xz-file-format-X.Y.Z.txt where X.Y.Z is the version number. - For example, the version 1.0.0 of this document is available - at <http://tukaani.org/xz/xz-file-format-1.0.0.txt>. - - -0.3. Version History - - Version Date Description - - 1.0.4 2009-08-27 Language improvements in Sections 1.2, - 2.1.1.2, 3.1.1, 3.1.2, and 5.3.1 - - 1.0.3 2009-06-05 Spelling fixes in Sections 5.1 and 5.4 - - 1.0.2 2009-06-04 Typo fixes in Sections 4 and 5.3.1 - - 1.0.1 2009-06-01 Typo fix in Section 0.3 and minor - clarifications to Sections 2, 2.2, - 3.3, 4.4, and 5.3.2 - - 1.0.0 2009-01-14 The first official version - - -1. Conventions - - The key words "MUST", "MUST NOT", "REQUIRED", "SHOULD", - "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - document are to be interpreted as described in [RFC-2119]. - - Indicating a warning means displaying a message, returning - appropriate exit status, or doing something else to let the - user know that something worth warning occurred. The operation - SHOULD still finish if a warning is indicated. - - Indicating an error means displaying a message, returning - appropriate exit status, or doing something else to let the - user know that something prevented successfully finishing the - operation. The operation MUST be aborted once an error has - been indicated. - - -1.1. Byte and Its Representation - - In this document, byte is always 8 bits. - - A "null byte" has all bits unset. That is, the value of a null - byte is 0x00. - - To represent byte blocks, this document uses notation that - is similar to the notation used in [RFC-1952]: - - +-------+ - | Foo | One byte. - +-------+ - - +---+---+ - | Foo | Two bytes; that is, some of the vertical bars - +---+---+ can be missing. - - +=======+ - | Foo | Zero or more bytes. - +=======+ - - In this document, a boxed byte or a byte sequence declared - using this notation is called "a field". The example field - above would be called "the Foo field" or plain "Foo". - - If there are many fields, they may be split to multiple lines. - This is indicated with an arrow ("--->"): - - +=====+ - | Foo | - +=====+ - - +=====+ - ---> | Bar | - +=====+ - - The above is equivalent to this: - - +=====+=====+ - | Foo | Bar | - +=====+=====+ - - -1.2. Multibyte Integers - - Multibyte integers of static length, such as CRC values, - are stored in little endian byte order (least significant - byte first). - - When smaller values are more likely than bigger values (for - example file sizes), multibyte integers are encoded in a - variable-length representation: - - Numbers in the range [0, 127] are copied as is, and take - one byte of space. - - Bigger numbers will occupy two or more bytes. All but the - last byte of the multibyte representation have the highest - (eighth) bit set. - - For now, the value of the variable-length integers is limited - to 63 bits, which limits the encoded size of the integer to - nine bytes. These limits may be increased in the future if - needed. - - The following C code illustrates encoding and decoding of - variable-length integers. The functions return the number of - bytes occupied by the integer (1-9), or zero on error. - - #include <stddef.h> - #include <inttypes.h> - - size_t - encode(uint8_t buf[static 9], uint64_t num) - { - if (num > UINT64_MAX / 2) - return 0; - - size_t i = 0; - - while (num >= 0x80) { - buf[i++] = (uint8_t)(num) | 0x80; - num >>= 7; - } - - buf[i++] = (uint8_t)(num); - - return i; - } - - size_t - decode(const uint8_t buf[], size_t size_max, uint64_t *num) - { - if (size_max == 0) - return 0; - - if (size_max > 9) - size_max = 9; - - *num = buf[0] & 0x7F; - size_t i = 0; - - while (buf[i++] & 0x80) { - if (i >= size_max || buf[i] == 0x00) - return 0; - - *num |= (uint64_t)(buf[i] & 0x7F) << (i * 7); - } - - return i; - } - - -2. Overall Structure of .xz File - - A standalone .xz files consist of one or more Streams which may - have Stream Padding between or after them: - - +========+================+========+================+ - | Stream | Stream Padding | Stream | Stream Padding | ... - +========+================+========+================+ - - The sizes of Stream and Stream Padding are always multiples - of four bytes, thus the size of every valid .xz file MUST be - a multiple of four bytes. - - While a typical file contains only one Stream and no Stream - Padding, a decoder handling standalone .xz files SHOULD support - files that have more than one Stream or Stream Padding. - - In contrast to standalone .xz files, when the .xz file format - is used as an internal part of some other file format or - communication protocol, it usually is expected that the decoder - stops after the first Stream, and doesn't look for Stream - Padding or possibly other Streams. - - -2.1. Stream - - +-+-+-+-+-+-+-+-+-+-+-+-+=======+=======+ +=======+ - | Stream Header | Block | Block | ... | Block | - +-+-+-+-+-+-+-+-+-+-+-+-+=======+=======+ +=======+ - - +=======+-+-+-+-+-+-+-+-+-+-+-+-+ - ---> | Index | Stream Footer | - +=======+-+-+-+-+-+-+-+-+-+-+-+-+ - - All the above fields have a size that is a multiple of four. If - Stream is used as an internal part of another file format, it - is RECOMMENDED to make the Stream start at an offset that is - a multiple of four bytes. - - Stream Header, Index, and Stream Footer are always present in - a Stream. The maximum size of the Index field is 16 GiB (2^34). - - There are zero or more Blocks. The maximum number of Blocks is - limited only by the maximum size of the Index field. - - Total size of a Stream MUST be less than 8 EiB (2^63 bytes). - The same limit applies to the total amount of uncompressed - data stored in a Stream. - - If an implementation supports handling .xz files with multiple - concatenated Streams, it MAY apply the above limits to the file - as a whole instead of limiting per Stream basis. - - -2.1.1. Stream Header - - +---+---+---+---+---+---+-------+------+--+--+--+--+ - | Header Magic Bytes | Stream Flags | CRC32 | - +---+---+---+---+---+---+-------+------+--+--+--+--+ - - -2.1.1.1. Header Magic Bytes - - The first six (6) bytes of the Stream are so called Header - Magic Bytes. They can be used to identify the file type. - - Using a C array and ASCII: - const uint8_t HEADER_MAGIC[6] - = { 0xFD, '7', 'z', 'X', 'Z', 0x00 }; - - In plain hexadecimal: - FD 37 7A 58 5A 00 - - Notes: - - The first byte (0xFD) was chosen so that the files cannot - be erroneously detected as being in .lzma format, in which - the first byte is in the range [0x00, 0xE0]. - - The sixth byte (0x00) was chosen to prevent applications - from misdetecting the file as a text file. - - If the Header Magic Bytes don't match, the decoder MUST - indicate an error. - - -2.1.1.2. Stream Flags - - The first byte of Stream Flags is always a null byte. In the - future, this byte may be used to indicate a new Stream version - or other Stream properties. - - The second byte of Stream Flags is a bit field: - - Bit(s) Mask Description - 0-3 0x0F Type of Check (see Section 3.4): - ID Size Check name - 0x00 0 bytes None - 0x01 4 bytes CRC32 - 0x02 4 bytes (Reserved) - 0x03 4 bytes (Reserved) - 0x04 8 bytes CRC64 - 0x05 8 bytes (Reserved) - 0x06 8 bytes (Reserved) - 0x07 16 bytes (Reserved) - 0x08 16 bytes (Reserved) - 0x09 16 bytes (Reserved) - 0x0A 32 bytes SHA-256 - 0x0B 32 bytes (Reserved) - 0x0C 32 bytes (Reserved) - 0x0D 64 bytes (Reserved) - 0x0E 64 bytes (Reserved) - 0x0F 64 bytes (Reserved) - 4-7 0xF0 Reserved for future use; MUST be zero for now. - - Implementations SHOULD support at least the Check IDs 0x00 - (None) and 0x01 (CRC32). Supporting other Check IDs is - OPTIONAL. If an unsupported Check is used, the decoder SHOULD - indicate a warning or error. - - If any reserved bit is set, the decoder MUST indicate an error. - It is possible that there is a new field present which the - decoder is not aware of, and can thus parse the Stream Header - incorrectly. - - -2.1.1.3. CRC32 - - The CRC32 is calculated from the Stream Flags field. It is - stored as an unsigned 32-bit little endian integer. If the - calculated value does not match the stored one, the decoder - MUST indicate an error. - - The idea is that Stream Flags would always be two bytes, even - if new features are needed. This way old decoders will be able - to verify the CRC32 calculated from Stream Flags, and thus - distinguish between corrupt files (CRC32 doesn't match) and - files that the decoder doesn't support (CRC32 matches but - Stream Flags has reserved bits set). - - -2.1.2. Stream Footer - - +-+-+-+-+---+---+---+---+-------+------+----------+---------+ - | CRC32 | Backward Size | Stream Flags | Footer Magic Bytes | - +-+-+-+-+---+---+---+---+-------+------+----------+---------+ - - -2.1.2.1. CRC32 - - The CRC32 is calculated from the Backward Size and Stream Flags - fields. It is stored as an unsigned 32-bit little endian - integer. If the calculated value does not match the stored one, - the decoder MUST indicate an error. - - The reason to have the CRC32 field before the Backward Size and - Stream Flags fields is to keep the four-byte fields aligned to - a multiple of four bytes. - - -2.1.2.2. Backward Size - - Backward Size is stored as a 32-bit little endian integer, - which indicates the size of the Index field as multiple of - four bytes, minimum value being four bytes: - - real_backward_size = (stored_backward_size + 1) * 4; - - If the stored value does not match the real size of the Index - field, the decoder MUST indicate an error. - - Using a fixed-size integer to store Backward Size makes - it slightly simpler to parse the Stream Footer when the - application needs to parse the Stream backwards. - - -2.1.2.3. Stream Flags - - This is a copy of the Stream Flags field from the Stream - Header. The information stored to Stream Flags is needed - when parsing the Stream backwards. The decoder MUST compare - the Stream Flags fields in both Stream Header and Stream - Footer, and indicate an error if they are not identical. - - -2.1.2.4. Footer Magic Bytes - - As the last step of the decoding process, the decoder MUST - verify the existence of Footer Magic Bytes. If they don't - match, an error MUST be indicated. - - Using a C array and ASCII: - const uint8_t FOOTER_MAGIC[2] = { 'Y', 'Z' }; - - In hexadecimal: - 59 5A - - The primary reason to have Footer Magic Bytes is to make - it easier to detect incomplete files quickly, without - uncompressing. If the file does not end with Footer Magic Bytes - (excluding Stream Padding described in Section 2.2), it cannot - be undamaged, unless someone has intentionally appended garbage - after the end of the Stream. - - -2.2. Stream Padding - - Only the decoders that support decoding of concatenated Streams - MUST support Stream Padding. - - Stream Padding MUST contain only null bytes. To preserve the - four-byte alignment of consecutive Streams, the size of Stream - Padding MUST be a multiple of four bytes. Empty Stream Padding - is allowed. If these requirements are not met, the decoder MUST - indicate an error. - - Note that non-empty Stream Padding is allowed at the end of the - file; there doesn't need to be a new Stream after non-empty - Stream Padding. This can be convenient in certain situations - [GNU-tar]. - - The possibility of Stream Padding MUST be taken into account - when designing an application that parses Streams backwards, - and the application supports concatenated Streams. - - -3. Block - - +==============+=================+===============+=======+ - | Block Header | Compressed Data | Block Padding | Check | - +==============+=================+===============+=======+ - - -3.1. Block Header - - +-------------------+-------------+=================+ - | Block Header Size | Block Flags | Compressed Size | - +-------------------+-------------+=================+ - - +===================+======================+ - ---> | Uncompressed Size | List of Filter Flags | - +===================+======================+ - - +================+--+--+--+--+ - ---> | Header Padding | CRC32 | - +================+--+--+--+--+ - - -3.1.1. Block Header Size - - This field overlaps with the Index Indicator field (see - Section 4.1). - - This field contains the size of the Block Header field, - including the Block Header Size field itself. Valid values are - in the range [0x01, 0xFF], which indicate the size of the Block - Header as multiples of four bytes, minimum size being eight - bytes: - - real_header_size = (encoded_header_size + 1) * 4; - - If a Block Header bigger than 1024 bytes is needed in the - future, a new field can be added between the Block Header and - Compressed Data fields. The presence of this new field would - be indicated in the Block Header field. - - -3.1.2. Block Flags - - The Block Flags field is a bit field: - - Bit(s) Mask Description - 0-1 0x03 Number of filters (1-4) - 2-5 0x3C Reserved for future use; MUST be zero for now. - 6 0x40 The Compressed Size field is present. - 7 0x80 The Uncompressed Size field is present. - - If any reserved bit is set, the decoder MUST indicate an error. - It is possible that there is a new field present which the - decoder is not aware of, and can thus parse the Block Header - incorrectly. - - -3.1.3. Compressed Size - - This field is present only if the appropriate bit is set in - the Block Flags field (see Section 3.1.2). - - The Compressed Size field contains the size of the Compressed - Data field, which MUST be non-zero. Compressed Size is stored - using the encoding described in Section 1.2. If the Compressed - Size doesn't match the size of the Compressed Data field, the - decoder MUST indicate an error. - - -3.1.4. Uncompressed Size - - This field is present only if the appropriate bit is set in - the Block Flags field (see Section 3.1.2). - - The Uncompressed Size field contains the size of the Block - after uncompressing. Uncompressed Size is stored using the - encoding described in Section 1.2. If the Uncompressed Size - does not match the real uncompressed size, the decoder MUST - indicate an error. - - Storing the Compressed Size and Uncompressed Size fields serves - several purposes: - - The decoder knows how much memory it needs to allocate - for a temporary buffer in multithreaded mode. - - Simple error detection: wrong size indicates a broken file. - - Seeking forwards to a specific location in streamed mode. - - It should be noted that the only reliable way to determine - the real uncompressed size is to uncompress the Block, - because the Block Header and Index fields may contain - (intentionally or unintentionally) invalid information. - - -3.1.5. List of Filter Flags - - +================+================+ +================+ - | Filter 0 Flags | Filter 1 Flags | ... | Filter n Flags | - +================+================+ +================+ - - The number of Filter Flags fields is stored in the Block Flags - field (see Section 3.1.2). - - The format of each Filter Flags field is as follows: - - +===========+====================+===================+ - | Filter ID | Size of Properties | Filter Properties | - +===========+====================+===================+ - - Both Filter ID and Size of Properties are stored using the - encoding described in Section 1.2. Size of Properties indicates - the size of the Filter Properties field as bytes. The list of - officially defined Filter IDs and the formats of their Filter - Properties are described in Section 5.3. - - Filter IDs greater than or equal to 0x4000_0000_0000_0000 - (2^62) are reserved for implementation-specific internal use. - These Filter IDs MUST never be used in List of Filter Flags. - - -3.1.6. Header Padding - - This field contains as many null byte as it is needed to make - the Block Header have the size specified in Block Header Size. - If any of the bytes are not null bytes, the decoder MUST - indicate an error. It is possible that there is a new field - present which the decoder is not aware of, and can thus parse - the Block Header incorrectly. - - -3.1.7. CRC32 - - The CRC32 is calculated over everything in the Block Header - field except the CRC32 field itself. It is stored as an - unsigned 32-bit little endian integer. If the calculated - value does not match the stored one, the decoder MUST indicate - an error. - - By verifying the CRC32 of the Block Header before parsing the - actual contents allows the decoder to distinguish between - corrupt and unsupported files. - - -3.2. Compressed Data - - The format of Compressed Data depends on Block Flags and List - of Filter Flags. Excluding the descriptions of the simplest - filters in Section 5.3, the format of the filter-specific - encoded data is out of scope of this document. - - -3.3. Block Padding - - Block Padding MUST contain 0-3 null bytes to make the size of - the Block a multiple of four bytes. This can be needed when - the size of Compressed Data is not a multiple of four. If any - of the bytes in Block Padding are not null bytes, the decoder - MUST indicate an error. - - -3.4. Check - - The type and size of the Check field depends on which bits - are set in the Stream Flags field (see Section 2.1.1.2). - - The Check, when used, is calculated from the original - uncompressed data. If the calculated Check does not match the - stored one, the decoder MUST indicate an error. If the selected - type of Check is not supported by the decoder, it SHOULD - indicate a warning or error. - - -4. Index - - +-----------------+===================+ - | Index Indicator | Number of Records | - +-----------------+===================+ - - +=================+===============+-+-+-+-+ - ---> | List of Records | Index Padding | CRC32 | - +=================+===============+-+-+-+-+ - - Index serves several purposes. Using it, one can - - verify that all Blocks in a Stream have been processed; - - find out the uncompressed size of a Stream; and - - quickly access the beginning of any Block (random access). - - -4.1. Index Indicator - - This field overlaps with the Block Header Size field (see - Section 3.1.1). The value of Index Indicator is always 0x00. - - -4.2. Number of Records - - This field indicates how many Records there are in the List - of Records field, and thus how many Blocks there are in the - Stream. The value is stored using the encoding described in - Section 1.2. If the decoder has decoded all the Blocks of the - Stream, and then notices that the Number of Records doesn't - match the real number of Blocks, the decoder MUST indicate an - error. - - -4.3. List of Records - - List of Records consists of as many Records as indicated by the - Number of Records field: - - +========+========+ - | Record | Record | ... - +========+========+ - - Each Record contains information about one Block: - - +===============+===================+ - | Unpadded Size | Uncompressed Size | - +===============+===================+ - - If the decoder has decoded all the Blocks of the Stream, it - MUST verify that the contents of the Records match the real - Unpadded Size and Uncompressed Size of the respective Blocks. - - Implementation hint: It is possible to verify the Index with - constant memory usage by calculating for example SHA-256 of - both the real size values and the List of Records, then - comparing the hash values. Implementing this using - non-cryptographic hash like CRC32 SHOULD be avoided unless - small code size is important. - - If the decoder supports random-access reading, it MUST verify - that Unpadded Size and Uncompressed Size of every completely - decoded Block match the sizes stored in the Index. If only - partial Block is decoded, the decoder MUST verify that the - processed sizes don't exceed the sizes stored in the Index. - - -4.3.1. Unpadded Size - - This field indicates the size of the Block excluding the Block - Padding field. That is, Unpadded Size is the size of the Block - Header, Compressed Data, and Check fields. Unpadded Size is - stored using the encoding described in Section 1.2. The value - MUST never be zero; with the current structure of Blocks, the - actual minimum value for Unpadded Size is five. - - Implementation note: Because the size of the Block Padding - field is not included in Unpadded Size, calculating the total - size of a Stream or doing random-access reading requires - calculating the actual size of the Blocks by rounding Unpadded - Sizes up to the next multiple of four. - - The reason to exclude Block Padding from Unpadded Size is to - ease making a raw copy of Compressed Data without Block - Padding. This can be useful, for example, if someone wants - to convert Streams to some other file format quickly. - - -4.3.2. Uncompressed Size - - This field indicates the Uncompressed Size of the respective - Block as bytes. The value is stored using the encoding - described in Section 1.2. - - -4.4. Index Padding - - This field MUST contain 0-3 null bytes to pad the Index to - a multiple of four bytes. If any of the bytes are not null - bytes, the decoder MUST indicate an error. - - -4.5. CRC32 - - The CRC32 is calculated over everything in the Index field - except the CRC32 field itself. The CRC32 is stored as an - unsigned 32-bit little endian integer. If the calculated - value does not match the stored one, the decoder MUST indicate - an error. - - -5. Filter Chains - - The Block Flags field defines how many filters are used. When - more than one filter is used, the filters are chained; that is, - the output of one filter is the input of another filter. The - following figure illustrates the direction of data flow. - - v Uncompressed Data ^ - | Filter 0 | - Encoder | Filter 1 | Decoder - | Filter n | - v Compressed Data ^ - - -5.1. Alignment - - Alignment of uncompressed input data is usually the job of - the application producing the data. For example, to get the - best results, an archiver tool should make sure that all - PowerPC executable files in the archive stream start at - offsets that are multiples of four bytes. - - Some filters, for example LZMA2, can be configured to take - advantage of specified alignment of input data. Note that - taking advantage of aligned input can be beneficial also when - a filter is not the first filter in the chain. For example, - if you compress PowerPC executables, you may want to use the - PowerPC filter and chain that with the LZMA2 filter. Because - not only the input but also the output alignment of the PowerPC - filter is four bytes, it is now beneficial to set LZMA2 - settings so that the LZMA2 encoder can take advantage of its - four-byte-aligned input data. - - The output of the last filter in the chain is stored to the - Compressed Data field, which is is guaranteed to be aligned - to a multiple of four bytes relative to the beginning of the - Stream. This can increase - - speed, if the filtered data is handled multiple bytes at - a time by the filter-specific encoder and decoder, - because accessing aligned data in computer memory is - usually faster; and - - compression ratio, if the output data is later compressed - with an external compression tool. - - -5.2. Security - - If filters would be allowed to be chained freely, it would be - possible to create malicious files, that would be very slow to - decode. Such files could be used to create denial of service - attacks. - - Slow files could occur when multiple filters are chained: - - v Compressed input data - | Filter 1 decoder (last filter) - | Filter 0 decoder (non-last filter) - v Uncompressed output data - - The decoder of the last filter in the chain produces a lot of - output from little input. Another filter in the chain takes the - output of the last filter, and produces very little output - while consuming a lot of input. As a result, a lot of data is - moved inside the filter chain, but the filter chain as a whole - gets very little work done. - - To prevent this kind of slow files, there are restrictions on - how the filters can be chained. These restrictions MUST be - taken into account when designing new filters. - - The maximum number of filters in the chain has been limited to - four, thus there can be at maximum of three non-last filters. - Of these three non-last filters, only two are allowed to change - the size of the data. - - The non-last filters, that change the size of the data, MUST - have a limit how much the decoder can compress the data: the - decoder SHOULD produce at least n bytes of output when the - filter is given 2n bytes of input. This limit is not - absolute, but significant deviations MUST be avoided. - - The above limitations guarantee that if the last filter in the - chain produces 4n bytes of output, the chain as a whole will - produce at least n bytes of output. - - -5.3. Filters - -5.3.1. LZMA2 - - LZMA (Lempel-Ziv-Markov chain-Algorithm) is a general-purpose - compression algorithm with high compression ratio and fast - decompression. LZMA is based on LZ77 and range coding - algorithms. - - LZMA2 is an extension on top of the original LZMA. LZMA2 uses - LZMA internally, but adds support for flushing the encoder, - uncompressed chunks, eases stateful decoder implementations, - and improves support for multithreading. Thus, the plain LZMA - will not be supported in this file format. - - Filter ID: 0x21 - Size of Filter Properties: 1 byte - Changes size of data: Yes - Allow as a non-last filter: No - Allow as the last filter: Yes - - Preferred alignment: - Input data: Adjustable to 1/2/4/8/16 byte(s) - Output data: 1 byte - - The format of the one-byte Filter Properties field is as - follows: - - Bits Mask Description - 0-5 0x3F Dictionary Size - 6-7 0xC0 Reserved for future use; MUST be zero for now. - - Dictionary Size is encoded with one-bit mantissa and five-bit - exponent. The smallest dictionary size is 4 KiB and the biggest - is 4 GiB. - - Raw value Mantissa Exponent Dictionary size - 0 2 11 4 KiB - 1 3 11 6 KiB - 2 2 12 8 KiB - 3 3 12 12 KiB - 4 2 13 16 KiB - 5 3 13 24 KiB - 6 2 14 32 KiB - ... ... ... ... - 35 3 27 768 MiB - 36 2 28 1024 MiB - 37 3 29 1536 MiB - 38 2 30 2048 MiB - 39 3 30 3072 MiB - 40 2 31 4096 MiB - 1 B - - Instead of having a table in the decoder, the dictionary size - can be decoded using the following C code: - - const uint8_t bits = get_dictionary_flags() & 0x3F; - if (bits > 40) - return DICTIONARY_TOO_BIG; // Bigger than 4 GiB - - uint32_t dictionary_size; - if (bits == 40) { - dictionary_size = UINT32_MAX; - } else { - dictionary_size = 2 | (bits & 1); - dictionary_size <<= bits / 2 + 11; - } - - -5.3.2. Branch/Call/Jump Filters for Executables - - These filters convert relative branch, call, and jump - instructions to their absolute counterparts in executable - files. This conversion increases redundancy and thus - compression ratio. - - Size of Filter Properties: 0 or 4 bytes - Changes size of data: No - Allow as a non-last filter: Yes - Allow as the last filter: No - - Below is the list of filters in this category. The alignment - is the same for both input and output data. - - Filter ID Alignment Description - 0x04 1 byte x86 filter (BCJ) - 0x05 4 bytes PowerPC (big endian) filter - 0x06 16 bytes IA64 filter - 0x07 4 bytes ARM (little endian) filter - 0x08 2 bytes ARM Thumb (little endian) filter - 0x09 4 bytes SPARC filter - - If the size of Filter Properties is four bytes, the Filter - Properties field contains the start offset used for address - conversions. It is stored as an unsigned 32-bit little endian - integer. The start offset MUST be a multiple of the alignment - of the filter as listed in the table above; if it isn't, the - decoder MUST indicate an error. If the size of Filter - Properties is zero, the start offset is zero. - - Setting the start offset may be useful if an executable has - multiple sections, and there are many cross-section calls. - Taking advantage of this feature usually requires usage of - the Subblock filter, whose design is not complete yet. - - -5.3.3. Delta - - The Delta filter may increase compression ratio when the value - of the next byte correlates with the value of an earlier byte - at specified distance. - - Filter ID: 0x03 - Size of Filter Properties: 1 byte - Changes size of data: No - Allow as a non-last filter: Yes - Allow as the last filter: No - - Preferred alignment: - Input data: 1 byte - Output data: Same as the original input data - - The Properties byte indicates the delta distance, which can be - 1-256 bytes backwards from the current byte: 0x00 indicates - distance of 1 byte and 0xFF distance of 256 bytes. - - -5.3.3.1. Format of the Encoded Output - - The code below illustrates both encoding and decoding with - the Delta filter. - - // Distance is in the range [1, 256]. - const unsigned int distance = get_properties_byte() + 1; - uint8_t pos = 0; - uint8_t delta[256]; - - memset(delta, 0, sizeof(delta)); - - while (1) { - const int byte = read_byte(); - if (byte == EOF) - break; - - uint8_t tmp = delta[(uint8_t)(distance + pos)]; - if (is_encoder) { - tmp = (uint8_t)(byte) - tmp; - delta[pos] = (uint8_t)(byte); - } else { - tmp = (uint8_t)(byte) + tmp; - delta[pos] = tmp; - } - - write_byte(tmp); - --pos; - } - - -5.4. Custom Filter IDs - - If a developer wants to use custom Filter IDs, he has two - choices. The first choice is to contact Lasse Collin and ask - him to allocate a range of IDs for the developer. - - The second choice is to generate a 40-bit random integer, - which the developer can use as his personal Developer ID. - To minimize the risk of collisions, Developer ID has to be - a randomly generated integer, not manually selected "hex word". - The following command, which works on many free operating - systems, can be used to generate Developer ID: - - dd if=/dev/urandom bs=5 count=1 | hexdump - - The developer can then use his Developer ID to create unique - (well, hopefully unique) Filter IDs. - - Bits Mask Description - 0-15 0x0000_0000_0000_FFFF Filter ID - 16-55 0x00FF_FFFF_FFFF_0000 Developer ID - 56-62 0x3F00_0000_0000_0000 Static prefix: 0x3F - - The resulting 63-bit integer will use 9 bytes of space when - stored using the encoding described in Section 1.2. To get - a shorter ID, see the beginning of this Section how to - request a custom ID range. - - -5.4.1. Reserved Custom Filter ID Ranges - - Range Description - 0x0000_0300 - 0x0000_04FF Reserved to ease .7z compatibility - 0x0002_0000 - 0x0007_FFFF Reserved to ease .7z compatibility - 0x0200_0000 - 0x07FF_FFFF Reserved to ease .7z compatibility - - -6. Cyclic Redundancy Checks - - There are several incompatible variations to calculate CRC32 - and CRC64. For simplicity and clarity, complete examples are - provided to calculate the checks as they are used in this file - format. Implementations MAY use different code as long as it - gives identical results. - - The program below reads data from standard input, calculates - the CRC32 and CRC64 values, and prints the calculated values - as big endian hexadecimal strings to standard output. - - #include <stddef.h> - #include <inttypes.h> - #include <stdio.h> - - uint32_t crc32_table[256]; - uint64_t crc64_table[256]; - - void - init(void) - { - static const uint32_t poly32 = UINT32_C(0xEDB88320); - static const uint64_t poly64 - = UINT64_C(0xC96C5795D7870F42); - - for (size_t i = 0; i < 256; ++i) { - uint32_t crc32 = i; - uint64_t crc64 = i; - - for (size_t j = 0; j < 8; ++j) { - if (crc32 & 1) - crc32 = (crc32 >> 1) ^ poly32; - else - crc32 >>= 1; - - if (crc64 & 1) - crc64 = (crc64 >> 1) ^ poly64; - else - crc64 >>= 1; - } - - crc32_table[i] = crc32; - crc64_table[i] = crc64; - } - } - - uint32_t - crc32(const uint8_t *buf, size_t size, uint32_t crc) - { - crc = ~crc; - for (size_t i = 0; i < size; ++i) - crc = crc32_table[buf[i] ^ (crc & 0xFF)] - ^ (crc >> 8); - return ~crc; - } - - uint64_t - crc64(const uint8_t *buf, size_t size, uint64_t crc) - { - crc = ~crc; - for (size_t i = 0; i < size; ++i) - crc = crc64_table[buf[i] ^ (crc & 0xFF)] - ^ (crc >> 8); - return ~crc; - } - - int - main() - { - init(); - - uint32_t value32 = 0; - uint64_t value64 = 0; - uint64_t total_size = 0; - uint8_t buf[8192]; - - while (1) { - const size_t buf_size - = fread(buf, 1, sizeof(buf), stdin); - if (buf_size == 0) - break; - - total_size += buf_size; - value32 = crc32(buf, buf_size, value32); - value64 = crc64(buf, buf_size, value64); - } - - printf("Bytes: %" PRIu64 "\n", total_size); - printf("CRC-32: 0x%08" PRIX32 "\n", value32); - printf("CRC-64: 0x%016" PRIX64 "\n", value64); - - return 0; - } - - -7. References - - LZMA SDK - The original LZMA implementation - http://7-zip.org/sdk.html - - LZMA Utils - LZMA adapted to POSIX-like systems - http://tukaani.org/lzma/ - - XZ Utils - The next generation of LZMA Utils - http://tukaani.org/xz/ - - [RFC-1952] - GZIP file format specification version 4.3 - http://www.ietf.org/rfc/rfc1952.txt - - Notation of byte boxes in section "2.1. Overall conventions" - - [RFC-2119] - Key words for use in RFCs to Indicate Requirement Levels - http://www.ietf.org/rfc/rfc2119.txt - - [GNU-tar] - GNU tar 1.21 manual - http://www.gnu.org/software/tar/manual/html_node/Blocking-Factor.html - - Node 9.4.2 "Blocking Factor", paragraph that begins - "gzip will complain about trailing garbage" - - Note that this URL points to the latest version of the - manual, and may some day not contain the note which is in - 1.21. For the exact version of the manual, download GNU - tar 1.21: ftp://ftp.gnu.org/pub/gnu/tar/tar-1.21.tar.gz - |