diff options
author | Karl Berry <karl@freefriends.org> | 2011-04-03 23:53:28 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2011-04-03 23:53:28 +0000 |
commit | 1e665e264ead3dedcf83afc950c43a6764a79884 (patch) | |
tree | f2de017feac2b53a9fc3c15b31d9dba1541a7837 /Build/source/extra/xz/src | |
parent | 7566b1e9fc57db87642f5e344164e5bbfe209b50 (diff) |
xz 5.0.2, from http://tukaani.org/xz/xz-5.0.2.tar.xz
git-svn-id: svn://tug.org/texlive/trunk@21944 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/extra/xz/src')
73 files changed, 2815 insertions, 1647 deletions
diff --git a/Build/source/extra/xz/src/Makefile.am b/Build/source/extra/xz/src/Makefile.am index f03f5a3da30..9fb4caad669 100644 --- a/Build/source/extra/xz/src/Makefile.am +++ b/Build/source/extra/xz/src/Makefile.am @@ -5,5 +5,18 @@ ## You can do whatever you want with this file. ## -SUBDIRS = liblzma xz xzdec lzmainfo scripts +SUBDIRS = liblzma xzdec + +if COND_XZ +SUBDIRS += xz +endif + +if COND_LZMAINFO +SUBDIRS += lzmainfo +endif + +if COND_SCRIPTS +SUBDIRS += scripts +endif + EXTRA_DIST = common diff --git a/Build/source/extra/xz/src/Makefile.in b/Build/source/extra/xz/src/Makefile.in index e9b73534c92..1b01b7b9ed3 100644 --- a/Build/source/extra/xz/src/Makefile.in +++ b/Build/source/extra/xz/src/Makefile.in @@ -33,6 +33,9 @@ PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ +@COND_XZ_TRUE@am__append_1 = xz +@COND_LZMAINFO_TRUE@am__append_2 = lzmainfo +@COND_SCRIPTS_TRUE@am__append_3 = scripts subdir = src DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 @@ -47,6 +50,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/acx_pthread.m4 \ $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/tuklib_common.m4 \ $(top_srcdir)/m4/tuklib_cpucores.m4 \ $(top_srcdir)/m4/tuklib_integer.m4 \ + $(top_srcdir)/m4/tuklib_mbstr.m4 \ $(top_srcdir)/m4/tuklib_physmem.m4 \ $(top_srcdir)/m4/tuklib_progname.m4 \ $(top_srcdir)/m4/visibility.m4 $(top_srcdir)/configure.ac @@ -72,7 +76,7 @@ AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \ distdir ETAGS = etags CTAGS = ctags -DIST_SUBDIRS = $(SUBDIRS) +DIST_SUBDIRS = liblzma xzdec xz lzmainfo scripts DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) am__relativize = \ dir0=`pwd`; \ @@ -123,8 +127,6 @@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ -DYNAMIC_CPPFLAGS = @DYNAMIC_CPPFLAGS@ -DYNAMIC_LDFLAGS = @DYNAMIC_LDFLAGS@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ @@ -157,6 +159,7 @@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -186,8 +189,6 @@ RC = @RC@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ -STATIC_CPPFLAGS = @STATIC_CPPFLAGS@ -STATIC_LDFLAGS = @STATIC_LDFLAGS@ STRIP = @STRIP@ USE_NLS = @USE_NLS@ VERSION = @VERSION@ @@ -197,6 +198,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ acx_pthread_config = @acx_pthread_config@ @@ -246,7 +248,8 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ xz = @xz@ -SUBDIRS = liblzma xz xzdec lzmainfo scripts +SUBDIRS = liblzma xzdec $(am__append_1) $(am__append_2) \ + $(am__append_3) EXTRA_DIST = common all: all-recursive diff --git a/Build/source/extra/xz/src/common/common_w32res.rc b/Build/source/extra/xz/src/common/common_w32res.rc index ad9e165394a..fdb88d184ac 100644 --- a/Build/source/extra/xz/src/common/common_w32res.rc +++ b/Build/source/extra/xz/src/common/common_w32res.rc @@ -17,9 +17,8 @@ #define MY_VERSION LZMA_VERSION_MAJOR,LZMA_VERSION_MINOR,LZMA_VERSION_PATCH,MY_BUILD #define MY_FILENAME MY_NAME MY_SUFFIX -#define MY_COMPANY "Lasse Collin and Igor Pavlov" -#define MY_COPYRIGHT "Public Domain by " MY_COMPANY -#define MY_COMMENTS "http://tukaani.org/xz/" +#define MY_COMPANY "The Tukaani Project <http://tukaani.org/>" +#define MY_PRODUCT PACKAGE_NAME " <" PACKAGE_URL ">" LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US VS_VERSION_INFO VERSIONINFO @@ -35,14 +34,12 @@ BEGIN BEGIN BLOCK "040904b0" BEGIN - VALUE "Comments", MY_COMMENTS VALUE "CompanyName", MY_COMPANY VALUE "FileDescription", MY_DESC VALUE "FileVersion", LZMA_VERSION_STRING VALUE "InternalName", MY_NAME - VALUE "LegalCopyright", MY_COPYRIGHT VALUE "OriginalFilename", MY_FILENAME - VALUE "ProductName", PACKAGE_NAME + VALUE "ProductName", MY_PRODUCT VALUE "ProductVersion", LZMA_VERSION_STRING END END diff --git a/Build/source/extra/xz/src/common/sysdefs.h b/Build/source/extra/xz/src/common/sysdefs.h index 1e138b1c2e7..69370ba4210 100644 --- a/Build/source/extra/xz/src/common/sysdefs.h +++ b/Build/source/extra/xz/src/common/sysdefs.h @@ -24,6 +24,11 @@ # include <config.h> #endif +// Get standard-compliant stdio functions under MinGW and MinGW-w64. +#ifdef __MINGW32__ +# define __USE_MINGW_ANSI_STDIO 1 +#endif + // size_t and NULL #include <stddef.h> @@ -60,6 +65,9 @@ #ifndef PRIu32 # define PRIu32 "u" #endif +#ifndef PRIx32 +# define PRIx32 "x" +#endif #ifndef PRIX32 # define PRIX32 "X" #endif @@ -71,6 +79,9 @@ # ifndef PRIu64 # define PRIu64 "llu" # endif +# ifndef PRIx64 +# define PRIx64 "llx" +# endif # ifndef PRIX64 # define PRIX64 "llX" # endif @@ -81,6 +92,9 @@ # ifndef PRIu64 # define PRIu64 "lu" # endif +# ifndef PRIx64 +# define PRIx64 "lx" +# endif # ifndef PRIX64 # define PRIX64 "lX" # endif @@ -166,4 +180,10 @@ typedef unsigned char _Bool; # define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) #endif +#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4 +# define lzma_attr_alloc_size(x) __attribute__((__alloc_size__(x))) +#else +# define lzma_attr_alloc_size(x) +#endif + #endif diff --git a/Build/source/extra/xz/src/common/tuklib_gettext.h b/Build/source/extra/xz/src/common/tuklib_gettext.h index 248521347ab..ff189040712 100644 --- a/Build/source/extra/xz/src/common/tuklib_gettext.h +++ b/Build/source/extra/xz/src/common/tuklib_gettext.h @@ -33,12 +33,12 @@ textdomain(package); \ } while (0) # define _(msgid) gettext(msgid) -# define N_(msgid1, msgid2, n) ngettext(msgid1, msgid2, n) #else # define tuklib_gettext_init(package, localedir) \ setlocale(LC_ALL, "") # define _(msgid) (msgid) -# define N_(msgid1, msgid2, n) ((n) == 1 ? (msgid1) : (msgid2)) +# define ngettext(msgid1, msgid2, n) ((n) == 1 ? (msgid1) : (msgid2)) #endif +#define N_(msgid) msgid #endif diff --git a/Build/source/extra/xz/src/common/tuklib_mbstr.h b/Build/source/extra/xz/src/common/tuklib_mbstr.h new file mode 100644 index 00000000000..9f358355180 --- /dev/null +++ b/Build/source/extra/xz/src/common/tuklib_mbstr.h @@ -0,0 +1,66 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file tuklib_mstr.h +/// \brief Utility functions for handling multibyte strings +/// +/// If not enough multibyte string support is available in the C library, +/// these functions keep working with the assumption that all strings +/// are in a single-byte character set without combining characters, e.g. +/// US-ASCII or ISO-8859-*. +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef TUKLIB_MBSTR_H +#define TUKLIB_MBSTR_H + +#include "tuklib_common.h" +TUKLIB_DECLS_BEGIN + +#define tuklib_mbstr_width TUKLIB_SYMBOL(tuklib_mbstr_width) +extern size_t tuklib_mbstr_width(const char *str, size_t *bytes); +///< +/// \brief Get the number of columns needed for the multibyte string +/// +/// This is somewhat similar to wcswidth() but works on multibyte strings. +/// +/// \param str String whose width is to be calculated. If the +/// current locale uses a multibyte character set +/// that has shift states, the string must begin +/// and end in the initial shift state. +/// \param bytes If this is not NULL, *bytes is set to the +/// value returned by strlen(str) (even if an +/// error occurs when calculating the width). +/// +/// \return On success, the number of columns needed to display the +/// string e.g. in a terminal emulator is returned. On error, +/// (size_t)-1 is returned. Possible errors include invalid, +/// partial, or non-printable multibyte character in str, or +/// that str doesn't end in the initial shift state. + +#define tuklib_mbstr_fw TUKLIB_SYMBOL(tuklib_mbstr_fw) +extern int tuklib_mbstr_fw(const char *str, int columns_min); +///< +/// \brief Get the field width for printf() e.g. to align table columns +/// +/// Printing simple tables to a terminal can be done using the field field +/// feature in the printf() format string, but it works only with single-byte +/// character sets. To do the same with multibyte strings, tuklib_mbstr_fw() +/// can be used to calculate appropriate field width. +/// +/// The behavior of this function is undefined, if +/// - str is NULL or not terminated with '\0'; +/// - columns_min <= 0; or +/// - the calculated field width exceeds INT_MAX. +/// +/// \return If tuklib_mbstr_width(str, NULL) fails, -1 is returned. +/// If str needs more columns than columns_min, zero is returned. +/// Otherwise a positive integer is returned, which can be +/// used as the field width, e.g. printf("%*s", fw, str). + +TUKLIB_DECLS_END +#endif diff --git a/Build/source/extra/xz/src/common/tuklib_mbstr_fw.c b/Build/source/extra/xz/src/common/tuklib_mbstr_fw.c new file mode 100644 index 00000000000..978a3fe10d9 --- /dev/null +++ b/Build/source/extra/xz/src/common/tuklib_mbstr_fw.c @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file tuklib_mstr_fw.c +/// \brief Get the field width for printf() e.g. to align table columns +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "tuklib_mbstr.h" + + +extern int +tuklib_mbstr_fw(const char *str, int columns_min) +{ + size_t len; + const size_t width = tuklib_mbstr_width(str, &len); + if (width == (size_t)-1) + return -1; + + if (width > (size_t)columns_min) + return 0; + + if (width < (size_t)columns_min) + len += (size_t)columns_min - width; + + return len; +} diff --git a/Build/source/extra/xz/src/common/tuklib_mbstr_width.c b/Build/source/extra/xz/src/common/tuklib_mbstr_width.c new file mode 100644 index 00000000000..3c38990f460 --- /dev/null +++ b/Build/source/extra/xz/src/common/tuklib_mbstr_width.c @@ -0,0 +1,64 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file tuklib_mstr_width.c +/// \brief Calculate width of a multibyte string +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "tuklib_mbstr.h" + +#if defined(HAVE_MBRTOWC) && defined(HAVE_WCWIDTH) +# include <wchar.h> +#endif + + +extern size_t +tuklib_mbstr_width(const char *str, size_t *bytes) +{ + const size_t len = strlen(str); + if (bytes != NULL) + *bytes = len; + +#if !(defined(HAVE_MBRTOWC) && defined(HAVE_WCWIDTH)) + // In single-byte mode, the width of the string is the same + // as its length. + return len; + +#else + mbstate_t state; + memset(&state, 0, sizeof(state)); + + size_t width = 0; + size_t i = 0; + + // Convert one multibyte character at a time to wchar_t + // and get its width using wcwidth(). + while (i < len) { + wchar_t wc; + const size_t ret = mbrtowc(&wc, str + i, len - i, &state); + if (ret < 1 || ret > len) + return (size_t)-1; + + i += ret; + + const int wc_width = wcwidth(wc); + if (wc_width < 0) + return (size_t)-1; + + width += wc_width; + } + + // Require that the string ends in the initial shift state. + // This way the caller can be combine the string with other + // strings without needing to worry about the shift states. + if (!mbsinit(&state)) + return (size_t)-1; + + return width; +#endif +} diff --git a/Build/source/extra/xz/src/liblzma/Makefile.am b/Build/source/extra/xz/src/liblzma/Makefile.am index f6c572b2885..813dc406760 100644 --- a/Build/source/extra/xz/src/liblzma/Makefile.am +++ b/Build/source/extra/xz/src/liblzma/Makefile.am @@ -24,7 +24,7 @@ liblzma_la_CPPFLAGS = \ -I$(top_srcdir)/src/liblzma/simple \ -I$(top_srcdir)/src/common \ -DTUKLIB_SYMBOL_PREFIX=lzma_ -liblzma_la_LDFLAGS = -no-undefined -version-info 0:0:0 +liblzma_la_LDFLAGS = -no-undefined -version-info 5:2:0 include $(srcdir)/common/Makefile.inc include $(srcdir)/check/Makefile.inc diff --git a/Build/source/extra/xz/src/liblzma/Makefile.in b/Build/source/extra/xz/src/liblzma/Makefile.in index 4179c1b5467..7a9e268c452 100644 --- a/Build/source/extra/xz/src/liblzma/Makefile.in +++ b/Build/source/extra/xz/src/liblzma/Makefile.in @@ -192,6 +192,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/acx_pthread.m4 \ $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/tuklib_common.m4 \ $(top_srcdir)/m4/tuklib_cpucores.m4 \ $(top_srcdir)/m4/tuklib_integer.m4 \ + $(top_srcdir)/m4/tuklib_mbstr.m4 \ $(top_srcdir)/m4/tuklib_physmem.m4 \ $(top_srcdir)/m4/tuklib_progname.m4 \ $(top_srcdir)/m4/visibility.m4 $(top_srcdir)/configure.ac @@ -450,8 +451,6 @@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ -DYNAMIC_CPPFLAGS = @DYNAMIC_CPPFLAGS@ -DYNAMIC_LDFLAGS = @DYNAMIC_LDFLAGS@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ @@ -484,6 +483,7 @@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -513,8 +513,6 @@ RC = @RC@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ -STATIC_CPPFLAGS = @STATIC_CPPFLAGS@ -STATIC_LDFLAGS = @STATIC_LDFLAGS@ STRIP = @STRIP@ USE_NLS = @USE_NLS@ VERSION = @VERSION@ @@ -524,6 +522,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ acx_pthread_config = @acx_pthread_config@ @@ -610,7 +609,7 @@ liblzma_la_CPPFLAGS = \ -I$(top_srcdir)/src/common \ -DTUKLIB_SYMBOL_PREFIX=lzma_ -liblzma_la_LDFLAGS = -no-undefined -version-info 0:0:0 \ +liblzma_la_LDFLAGS = -no-undefined -version-info 5:2:0 \ $(am__append_38) pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = liblzma.pc diff --git a/Build/source/extra/xz/src/liblzma/api/Makefile.in b/Build/source/extra/xz/src/liblzma/api/Makefile.in index 186506cd876..82c8eb0b1ae 100644 --- a/Build/source/extra/xz/src/liblzma/api/Makefile.in +++ b/Build/source/extra/xz/src/liblzma/api/Makefile.in @@ -49,6 +49,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/acx_pthread.m4 \ $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/tuklib_common.m4 \ $(top_srcdir)/m4/tuklib_cpucores.m4 \ $(top_srcdir)/m4/tuklib_integer.m4 \ + $(top_srcdir)/m4/tuklib_mbstr.m4 \ $(top_srcdir)/m4/tuklib_physmem.m4 \ $(top_srcdir)/m4/tuklib_progname.m4 \ $(top_srcdir)/m4/visibility.m4 $(top_srcdir)/configure.ac @@ -110,8 +111,6 @@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ -DYNAMIC_CPPFLAGS = @DYNAMIC_CPPFLAGS@ -DYNAMIC_LDFLAGS = @DYNAMIC_LDFLAGS@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ @@ -144,6 +143,7 @@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -173,8 +173,6 @@ RC = @RC@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ -STATIC_CPPFLAGS = @STATIC_CPPFLAGS@ -STATIC_LDFLAGS = @STATIC_LDFLAGS@ STRIP = @STRIP@ USE_NLS = @USE_NLS@ VERSION = @VERSION@ @@ -184,6 +182,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ acx_pthread_config = @acx_pthread_config@ diff --git a/Build/source/extra/xz/src/liblzma/api/lzma.h b/Build/source/extra/xz/src/liblzma/api/lzma.h index fb593a35a07..fb874c3e137 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma.h @@ -60,8 +60,8 @@ * * Some could argue that liblzma API should provide all the required types, * for example lzma_uint64, LZMA_UINT64_C(n), and LZMA_UINT64_MAX. This was - * seen unnecessary mess, since most systems already provide all the necessary - * types and macros in the standard headers. + * seen as an unnecessary mess, since most systems already provide all the + * necessary types and macros in the standard headers. * * Note that liblzma API still has lzma_bool, because using stdbool.h would * break C89 and C++ programs on many systems. sizeof(bool) in C99 isn't diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/base.h b/Build/source/extra/xz/src/liblzma/api/lzma/base.h index 993626a4cdf..43dde8d60f3 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/base.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/base.h @@ -281,21 +281,21 @@ typedef enum { * Using LZMA_SYNC_FLUSH very often can dramatically reduce * the compression ratio. With some filters (for example, * LZMA2), fine-tuning the compression options may help - * mitigate this problem significantly. + * mitigate this problem significantly (for example, + * match finder with LZMA2). * * Decoders don't support LZMA_SYNC_FLUSH. */ LZMA_FULL_FLUSH = 2, /**< - * \brief Make all the input available at output + * \brief Finish encoding of the current Block * - * Finish encoding of the current Block. All the input - * data going to the current Block must have been given - * to the encoder (the last bytes can still be pending in - * next_in). Call lzma_code() with LZMA_FULL_FLUSH until - * it returns LZMA_STREAM_END. Then continue normally with - * LZMA_RUN or finish the Stream with LZMA_FINISH. + * All the input data going to the current Block must have + * been given to the encoder (the last bytes can still be + * pending in* next_in). Call lzma_code() with LZMA_FULL_FLUSH + * until it returns LZMA_STREAM_END. Then continue normally + * with LZMA_RUN or finish the Stream with LZMA_FINISH. * * This action is currently supported only by Stream encoder * and easy encoder (which uses Stream encoder). If there is @@ -306,12 +306,12 @@ typedef enum { /**< * \brief Finish the coding operation * - * Finishes the coding operation. All the input data must - * have been given to the encoder (the last bytes can still - * be pending in next_in). Call lzma_code() with LZMA_FINISH - * until it returns LZMA_STREAM_END. Once LZMA_FINISH has - * been used, the amount of input must no longer be changed - * by the application. + * All the input data must have been given to the encoder + * (the last bytes can still be pending in next_in). + * Call lzma_code() with LZMA_FINISH until it returns + * LZMA_STREAM_END. Once LZMA_FINISH has been used, + * the amount of input must no longer be changed by + * the application. * * When decoding, using LZMA_FINISH is optional unless the * LZMA_CONCATENATED flag was used when the decoder was @@ -478,8 +478,12 @@ typedef struct { */ void *reserved_ptr1; void *reserved_ptr2; + void *reserved_ptr3; + void *reserved_ptr4; uint64_t reserved_int1; uint64_t reserved_int2; + size_t reserved_int3; + size_t reserved_int4; lzma_reserved_enum reserved_enum1; lzma_reserved_enum reserved_enum2; @@ -506,7 +510,8 @@ typedef struct { */ #define LZMA_STREAM_INIT \ { NULL, 0, 0, NULL, 0, 0, NULL, NULL, \ - NULL, NULL, 0, 0, LZMA_RESERVED_ENUM, LZMA_RESERVED_ENUM } + NULL, NULL, NULL, NULL, 0, 0, 0, 0, \ + LZMA_RESERVED_ENUM, LZMA_RESERVED_ENUM } /** @@ -554,11 +559,11 @@ extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow; * this may give misleading information if decoding .xz Streams that have * multiple Blocks, because each Block can have different memory requirements. * - * \return Rough estimate of how much memory is currently allocated - * for the filter decoders. If no filter chain is currently - * allocated, some non-zero value is still returned, which is - * less than or equal to what any filter chain would indicate - * as its memory requirement. + * \return How much memory is currently allocated for the filter + * decoders. If no filter chain is currently allocated, + * some non-zero value is still returned, which is less than + * or equal to what any filter chain would indicate as its + * memory requirement. * * If this function isn't supported by *strm or some other error * occurs, zero is returned. diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/bcj.h b/Build/source/extra/xz/src/liblzma/api/lzma/bcj.h index 82e4a440b21..8e37538ad4a 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/bcj.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/bcj.h @@ -31,7 +31,7 @@ #define LZMA_FILTER_IA64 LZMA_VLI_C(0x06) /**< - * Filter for IA64 (Itanium) binaries. + * Filter for IA-64 (Itanium) binaries. */ #define LZMA_FILTER_ARM LZMA_VLI_C(0x07) @@ -41,7 +41,7 @@ #define LZMA_FILTER_ARMTHUMB LZMA_VLI_C(0x08) /**< - * Filter for ARMThumb binaries. + * Filter for ARM-Thumb binaries. */ #define LZMA_FILTER_SPARC LZMA_VLI_C(0x09) diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/block.h b/Build/source/extra/xz/src/liblzma/api/lzma/block.h index 8e681ed2811..3019bf916fe 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/block.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/block.h @@ -318,8 +318,8 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( * The size of the Block Header must have already been decoded with * lzma_block_header_size_decode() macro and stored to block->header_size. * - * block->filters must have been allocated, but not necessarily initialized. - * Possible existing filter options are _not_ freed. + * block->filters must have been allocated, but they don't need to be + * initialized (possible existing filter options are not freed). * * \param block Destination for Block options. * \param allocator lzma_allocator for custom allocator functions. diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/container.h b/Build/source/extra/xz/src/liblzma/api/lzma/container.h index 0d907650fc4..83e70b44f22 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/container.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/container.h @@ -55,13 +55,13 @@ * * This flag doesn't affect the memory usage requirements of the decoder (at * least not significantly). The memory usage of the encoder may be increased - * a little but only at the lowest preset levels (0-2). + * a little but only at the lowest preset levels (0-3). */ #define LZMA_PRESET_EXTREME (UINT32_C(1) << 31) /** - * \brief Calculate rough memory usage of easy encoder + * \brief Calculate approximate memory usage of easy encoder * * This function is a wrapper for lzma_raw_encoder_memusage(). * @@ -72,7 +72,7 @@ extern LZMA_API(uint64_t) lzma_easy_encoder_memusage(uint32_t preset) /** - * \brief Calculate rough decoder memory usage of a preset + * \brief Calculate approximate decoder memory usage of a preset * * This function is a wrapper for lzma_raw_decoder_memusage(). * @@ -93,16 +93,19 @@ extern LZMA_API(uint64_t) lzma_easy_decoder_memusage(uint32_t preset) * \param preset Compression preset to use. A preset consist of level * number and zero or more flags. Usually flags aren't * used, so preset is simply a number [0, 9] which match - * the options -0 .. -9 of the xz command line tool. + * the options -0 ... -9 of the xz command line tool. * Additional flags can be be set using bitwise-or with * the preset level number, e.g. 6 | LZMA_PRESET_EXTREME. * \param check Integrity check type to use. See check.h for available - * checks. If you are unsure, use LZMA_CHECK_CRC32. + * checks. The xz command line tool defaults to + * LZMA_CHECK_CRC64, which is a good choice if you are + * unsure. LZMA_CHECK_CRC32 is good too as long as the + * uncompressed file is not many gigabytes. * * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to * encode your data. * - LZMA_MEM_ERROR: Memory allocation failed. - * - LZMA_OPTIONS_ERROR: The given compression level is not + * - LZMA_OPTIONS_ERROR: The given compression preset is not * supported by this build of liblzma. * - LZMA_UNSUPPORTED_CHECK: The given check type is not * supported by this liblzma build. @@ -310,7 +313,8 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( * \brief Initialize .xz Stream decoder * * \param strm Pointer to properly prepared lzma_stream - * \param memlimit Rough memory usage limit as bytes + * \param memlimit Memory usage limit as bytes. Use UINT64_MAX + * to effectively disable the limiter. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_TELL_ANY_CHECK, LZMA_CONCATENATED @@ -318,6 +322,7 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( * \return - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_OPTIONS_ERROR: Unsupported flags + * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_decoder( lzma_stream *strm, uint64_t memlimit, uint32_t flags) @@ -332,12 +337,14 @@ extern LZMA_API(lzma_ret) lzma_stream_decoder( * of the input file has been detected. * * \param strm Pointer to properly prepared lzma_stream - * \param memlimit Rough memory usage limit as bytes + * \param memlimit Memory usage limit as bytes. Use UINT64_MAX + * to effectively disable the limiter. * \param flags Bitwise-or of flags, or zero for no flags. * * \return - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_OPTIONS_ERROR: Unsupported flags + * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_auto_decoder( lzma_stream *strm, uint64_t memlimit, uint32_t flags) @@ -353,6 +360,7 @@ extern LZMA_API(lzma_ret) lzma_auto_decoder( * * \return - LZMA_OK * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_alone_decoder( lzma_stream *strm, uint64_t memlimit) @@ -379,7 +387,7 @@ extern LZMA_API(lzma_ret) lzma_alone_decoder( * won't be read is in[in_size]. * \param out Beginning of the output buffer * \param out_pos The next byte will be written to out[*out_pos]. - * *out_pos is updated only if encoding succeeds. + * *out_pos is updated only if decoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/filter.h b/Build/source/extra/xz/src/liblzma/api/lzma/filter.h index 038a93398c6..efd036f7f04 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/filter.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/filter.h @@ -1,6 +1,6 @@ /** * \file lzma/filter.h - * \brief Common filter related types + * \brief Common filter related types and functions */ /* @@ -59,11 +59,6 @@ typedef struct { * If the filter doesn't need options, set this to NULL. If id is * set to LZMA_VLI_UNKNOWN, options is ignored, and thus * doesn't need be initialized. - * - * Some filters support changing the options in the middle of - * the encoding process. These filters store the pointer of the - * options structure and communicate with the application via - * modifications of the options structure. */ void *options; @@ -126,15 +121,16 @@ extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, /** - * \brief Calculate rough memory requirements for raw encoder + * \brief Calculate approximate memory requirements for raw encoder * - * Because the calculation is rough, this function can be used to calculate - * the memory requirements for Block and Stream encoders too. + * This function can be used to calculate the memory requirements for + * Block and Stream encoders too because Block and Stream encoders don't + * need significantly more memory than raw encoder. * * \param filters Array of filters terminated with * .id == LZMA_VLI_UNKNOWN. * - * \return Rough number of bytes of memory required for the given + * \return Number of bytes of memory required for the given * filter chain when encoding. */ extern LZMA_API(uint64_t) lzma_raw_encoder_memusage(const lzma_filter *filters) @@ -142,15 +138,16 @@ extern LZMA_API(uint64_t) lzma_raw_encoder_memusage(const lzma_filter *filters) /** - * \brief Calculate rough memory requirements for raw decoder + * \brief Calculate approximate memory requirements for raw decoder * - * Because the calculation is rough, this function can be used to calculate - * the memory requirements for Block and Stream decoders too. + * This function can be used to calculate the memory requirements for + * Block and Stream decoders too because Block and Stream decoders don't + * need significantly more memory than raw decoder. * * \param filters Array of filters terminated with * .id == LZMA_VLI_UNKNOWN. * - * \return Rough number of bytes of memory required for the given + * \return Number of bytes of memory required for the given * filter chain when decoding. */ extern LZMA_API(uint64_t) lzma_raw_decoder_memusage(const lzma_filter *filters) @@ -251,7 +248,8 @@ extern LZMA_API(lzma_ret) lzma_filters_update( * * \note There is no function to calculate how big output buffer * would surely be big enough. (lzma_stream_buffer_bound() - * works only for lzma_stream_buffer_encode().) + * works only for lzma_stream_buffer_encode(); raw encoder + * won't necessarily meet that bound.) */ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( const lzma_filter *filters, lzma_allocator *allocator, @@ -323,9 +321,10 @@ extern LZMA_API(lzma_ret) lzma_properties_size( * succeeds but using the same options to initialize the encoder * will fail. * - * \note It is OK to skip calling this function if - * lzma_properties_size() indicated that the size - * of the Filter Properties field is zero. + * \note If lzma_properties_size() indicated that the size + * of the Filter Properties field is zero, calling + * lzma_properties_encode() is not required, but it + * won't do any harm either. */ extern LZMA_API(lzma_ret) lzma_properties_encode( const lzma_filter *filter, uint8_t *props) lzma_nothrow; @@ -364,11 +363,11 @@ extern LZMA_API(lzma_ret) lzma_properties_decode( * memory to hold the encoded Filter Flags. * * \param size Pointer to integer to hold the calculated size - * \param filters Filter ID and associated options whose encoded + * \param filter Filter ID and associated options whose encoded * size is to be calculated * * \return - LZMA_OK: *size set successfully. Note that this doesn't - * guarantee that filters->options is valid, thus + * guarantee that filter->options is valid, thus * lzma_filter_flags_encode() may still fail. * - LZMA_OPTIONS_ERROR: Unknown Filter ID or unsupported options. * - LZMA_PROG_ERROR: Invalid options @@ -377,7 +376,7 @@ extern LZMA_API(lzma_ret) lzma_properties_decode( * you need to loop over every lzma_filter entry. */ extern LZMA_API(lzma_ret) lzma_filter_flags_size( - uint32_t *size, const lzma_filter *filters) + uint32_t *size, const lzma_filter *filter) lzma_nothrow lzma_attr_warn_unused_result; @@ -387,7 +386,7 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_size( * In contrast to some functions, this doesn't allocate the needed buffer. * This is due to how this function is used internally by liblzma. * - * \param filters Filter ID and options to be encoded + * \param filter Filter ID and options to be encoded * \param out Beginning of the output buffer * \param out_pos out[*out_pos] is the next write position. This * is updated by the encoder. @@ -399,7 +398,7 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_size( * buffer space (you should have checked it with * lzma_filter_flags_size()). */ -extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filters, +extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -407,8 +406,8 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filters, /** * \brief Decode Filter Flags from given buffer * - * The decoded result is stored into *filters. filters->options is - * initialized but the old value is NOT free()d. + * The decoded result is stored into *filter. The old value of + * filter->options is not free()d. * * \return - LZMA_OK * - LZMA_OPTIONS_ERROR @@ -416,6 +415,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filters, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filters, lzma_allocator *allocator, + lzma_filter *filter, lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/hardware.h b/Build/source/extra/xz/src/liblzma/api/lzma/hardware.h index f44cb602ff9..e7dd03c3e8d 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/hardware.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/hardware.h @@ -39,8 +39,7 @@ * * This function may be useful when determining a reasonable memory * usage limit for decompressing or how much memory it is OK to use - * for compressing. For example, the default limit used by the xz - * command line tool is 40 % of RAM. + * for compressing. * * \return On success, the total amount of physical memory in bytes * is returned. If the amount of RAM cannot be determined, diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/index.h b/Build/source/extra/xz/src/liblzma/api/lzma/index.h index 5ea12e359e5..16bacc287df 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/index.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/index.h @@ -136,6 +136,13 @@ typedef struct { * * This offset is relative to the beginning of the lzma_index * (i.e. usually the beginning of the .xz file). + * + * When doing random-access reading, it is possible that + * the target offset is not exactly at Block boundary. One + * will need to compare the target offset against + * uncompressed_file_offset or uncompressed_stream_offset, + * and possibly decode and throw away some amount of data + * before reaching the target offset. */ lzma_vli uncompressed_file_offset; @@ -166,14 +173,8 @@ typedef struct { * \brief Uncompressed size of this Block * * You should pass this to the Block decoder if you will - * decode this Block. - * - * When doing random-access reading, it is possible that - * the target offset is not exactly at Block boundary. One - * will need to compare the target offset against - * uncompressed_file_offset or uncompressed_stream_offset, - * and possibly decode and throw away some amount of data - * before reaching the target offset. + * decode this Block. It will allow the Block decoder to + * validate the uncompressed size. */ lzma_vli uncompressed_size; @@ -181,7 +182,8 @@ typedef struct { * \brief Unpadded size of this Block * * You should pass this to the Block decoder if you will - * decode this Block. + * decode this Block. It will allow the Block decoder to + * validate the unpadded size. */ lzma_vli unpadded_size; @@ -583,7 +585,9 @@ extern LZMA_API(lzma_index *) lzma_index_dup( * \param strm Pointer to properly prepared lzma_stream * \param i Pointer to lzma_index which should be encoded. * - * The only valid action value for lzma_code() is LZMA_RUN. + * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. + * It is enough to use only one of them (you can choose freely; use LZMA_RUN + * to support liblzma versions older than 5.0.0). * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR @@ -608,7 +612,9 @@ extern LZMA_API(lzma_ret) lzma_index_encoder( * \param memlimit How much memory the resulting lzma_index is * allowed to require. * - * The only valid action value for lzma_code() is LZMA_RUN. + * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. + * It is enough to use only one of them (you can choose freely; use LZMA_RUN + * to support liblzma versions older than 5.0.0). * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/index_hash.h b/Build/source/extra/xz/src/liblzma/api/lzma/index_hash.h index 94726e7beea..fa2e048d552 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/index_hash.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/index_hash.h @@ -1,6 +1,6 @@ /** * \file lzma/index_hash.h - * \brief Validates Index by using a hash function + * \brief Validate Index by using a hash function * * Hashing makes it possible to use constant amount of memory to validate * Index of arbitrary size. diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/lzma.h b/Build/source/extra/xz/src/liblzma/api/lzma/lzma.h index c17736d7eaa..3f8e095f70e 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/lzma.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/lzma.h @@ -46,10 +46,14 @@ * Match finder has major effect on both speed and compression ratio. * Usually hash chains are faster than binary trees. * + * If you will use LZMA_SYNC_FLUSH often, the hash chains may be a better + * choice, because binary trees get much higher compression ratio penalty + * with LZMA_SYNC_FLUSH. + * * The memory usage formulas are only rough estimates, which are closest to * reality when dict_size is a power of two. The formulas are more complex * in reality, and can also change a little between liblzma versions. Use - * lzma_memusage_encoder() to get more accurate estimate of memory usage. + * lzma_raw_encoder_memusage() to get more accurate estimate of memory usage. */ typedef enum { LZMA_MF_HC3 = 0x03, @@ -69,7 +73,9 @@ typedef enum { * * Minimum nice_len: 4 * - * Memory usage: dict_size * 7.5 + * Memory usage: + * - dict_size <= 32 MiB: dict_size * 7.5 + * - dict_size > 32 MiB: dict_size * 6.5 */ LZMA_MF_BT2 = 0x12, @@ -98,7 +104,9 @@ typedef enum { * * Minimum nice_len: 4 * - * Memory usage: dict_size * 11.5 + * Memory usage: + * - dict_size <= 32 MiB: dict_size * 11.5 + * - dict_size > 32 MiB: dict_size * 10.5 */ } lzma_match_finder; @@ -169,6 +177,7 @@ extern LZMA_API(lzma_bool) lzma_mode_is_supported(lzma_mode mode) * Since LZMA1 and LZMA2 share most of the code, it's simplest to share * the options structure too. For encoding, all but the reserved variables * need to be initialized unless specifically mentioned otherwise. + * lzma_lzma_preset() can be used to get a good starting point. * * For raw decoding, both LZMA1 and LZMA2 need dict_size, preset_dict, and * preset_dict_size (if preset_dict != NULL). LZMA1 needs also lc, lp, and pb. @@ -251,7 +260,13 @@ typedef struct { * eight-bit byte (also known as `literal') are taken into * account when predicting the bits of the next literal. * - * \todo Example + * E.g. in typical English text, an upper-case letter is + * often followed by a lower-case letter, and a lower-case + * letter is usually followed by another lower-case letter. + * In the US-ASCII character set, the highest three bits are 010 + * for upper-case letters and 011 for lower-case letters. + * When lc is at least 3, the literal coding can take advantage of + * this property in the uncompressed data. * * There is a limit that applies to literal context bits and literal * position bits together: lc + lp <= 4. Without this limit the @@ -271,12 +286,9 @@ typedef struct { /** * \brief Number of literal position bits * - * How many of the lowest bits of the current position (number - * of bytes from the beginning of the uncompressed data) in the - * uncompressed data is taken into account when predicting the - * bits of the next literal (a single eight-bit byte). - * - * \todo Example + * lp affects what kind of alignment in the uncompressed data is + * assumed when encoding literals. A literal is a single 8-bit byte. + * See pb below for more information about alignment. */ uint32_t lp; # define LZMA_LP_DEFAULT 0 @@ -284,14 +296,22 @@ typedef struct { /** * \brief Number of position bits * - * How many of the lowest bits of the current position in the - * uncompressed data is taken into account when estimating - * probabilities of matches. A match is a sequence of bytes for - * which a matching sequence is found from the dictionary and - * thus can be stored as distance-length pair. + * pb affects what kind of alignment in the uncompressed data is + * assumed in general. The default means four-byte alignment + * (2^ pb =2^2=4), which is often a good choice when there's + * no better guess. + * + * When the aligment is known, setting pb accordingly may reduce + * the file size a little. E.g. with text files having one-byte + * alignment (US-ASCII, ISO-8859-*, UTF-8), setting pb=0 can + * improve compression slightly. For UTF-16 text, pb=1 is a good + * choice. If the alignment is an odd number like 3 bytes, pb=0 + * might be the best choice. * - * Example: If most of the matches occur at byte positions of - * 8 * n + 3, that is, 3, 11, 19, ... set pb to 3, because 2**3 == 8. + * Even though the assumed alignment can be adjusted with pb and + * lp, LZMA1 and LZMA2 still slightly favor 16-byte alignment. + * It might be worth taking into account when designing file formats + * that are likely to be often compressed with LZMA1 or LZMA2. */ uint32_t pb; # define LZMA_PB_MIN 0 @@ -342,7 +362,7 @@ typedef struct { * * Setting depth to zero tells liblzma to use an automatic default * value, that depends on the selected match finder and nice_len. - * The default is in the range [10, 200] or so (it may vary between + * The default is in the range [4, 200] or so (it may vary between * liblzma versions). * * Using a bigger depth value than the default can increase @@ -361,8 +381,6 @@ typedef struct { * with the currently supported options, so it is safe to leave these * uninitialized. */ - void *reserved_ptr1; - void *reserved_ptr2; uint32_t reserved_int1; uint32_t reserved_int2; uint32_t reserved_int3; @@ -375,6 +393,8 @@ typedef struct { lzma_reserved_enum reserved_enum2; lzma_reserved_enum reserved_enum3; lzma_reserved_enum reserved_enum4; + void *reserved_ptr1; + void *reserved_ptr2; } lzma_options_lzma; @@ -392,6 +412,9 @@ typedef struct { * * This function is available only if LZMA1 or LZMA2 encoder has been enabled * when building liblzma. + * + * \return On success, false is returned. If the preset is not + * supported, true is returned. */ extern LZMA_API(lzma_bool) lzma_lzma_preset( lzma_options_lzma *options, uint32_t preset) lzma_nothrow; diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/stream_flags.h b/Build/source/extra/xz/src/liblzma/api/lzma/stream_flags.h index d255bdda15f..bbdd408263e 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/stream_flags.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/stream_flags.h @@ -91,8 +91,6 @@ typedef struct { lzma_reserved_enum reserved_enum2; lzma_reserved_enum reserved_enum3; lzma_reserved_enum reserved_enum4; - lzma_reserved_enum reserved_enum5; - lzma_reserved_enum reserved_enum6; lzma_bool reserved_bool1; lzma_bool reserved_bool2; lzma_bool reserved_bool3; @@ -103,8 +101,6 @@ typedef struct { lzma_bool reserved_bool8; uint32_t reserved_int1; uint32_t reserved_int2; - uint32_t reserved_int3; - uint32_t reserved_int4; } lzma_stream_flags; @@ -148,7 +144,7 @@ extern LZMA_API(lzma_ret) lzma_stream_footer_encode( /** * \brief Decode Stream Header * - * \param options Stream Header options to be encoded. + * \param options Target for the decoded Stream Header options. * \param in Beginning of the input buffer of * LZMA_STREAM_HEADER_SIZE bytes. * @@ -183,7 +179,7 @@ extern LZMA_API(lzma_ret) lzma_stream_header_decode( /** * \brief Decode Stream Footer * - * \param options Stream Header options to be encoded. + * \param options Target for the decoded Stream Header options. * \param in Beginning of the input buffer of * LZMA_STREAM_HEADER_SIZE bytes. * diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/version.h b/Build/source/extra/xz/src/liblzma/api/lzma/version.h index 0f7c2d3737d..4959f6e8dbe 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/version.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/version.h @@ -20,10 +20,10 @@ /* * Version number split into components */ -#define LZMA_VERSION_MAJOR 4 -#define LZMA_VERSION_MINOR 999 -#define LZMA_VERSION_PATCH 9 -#define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_BETA +#define LZMA_VERSION_MAJOR 5 +#define LZMA_VERSION_MINOR 0 +#define LZMA_VERSION_PATCH 2 +#define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE #ifndef LZMA_VERSION_COMMIT # define LZMA_VERSION_COMMIT "" diff --git a/Build/source/extra/xz/src/liblzma/api/lzma/vli.h b/Build/source/extra/xz/src/liblzma/api/lzma/vli.h index 57e3d3882b5..9ad13f2e2fc 100644 --- a/Build/source/extra/xz/src/liblzma/api/lzma/vli.h +++ b/Build/source/extra/xz/src/liblzma/api/lzma/vli.h @@ -29,7 +29,7 @@ /** - * \brief Maximum supported value of variable-length integer + * \brief Maximum supported value of a variable-length integer */ #define LZMA_VLI_MAX (UINT64_MAX / 2) @@ -39,11 +39,10 @@ #define LZMA_VLI_UNKNOWN UINT64_MAX /** - * \brief Maximum supported length of variable length integers + * \brief Maximum supported encoded length of variable length integers */ #define LZMA_VLI_BYTES_MAX 9 - /** * \brief VLI constant suffix */ @@ -53,19 +52,19 @@ /** * \brief Variable-length integer type * - * This will always be unsigned integer. Valid VLI values are in the range - * [0, LZMA_VLI_MAX]. Unknown value is indicated with LZMA_VLI_UNKNOWN, - * which is the maximum value of the underlaying integer type. + * Valid VLI values are in the range [0, LZMA_VLI_MAX]. Unknown value is + * indicated with LZMA_VLI_UNKNOWN, which is the maximum value of the + * underlaying integer type. * - * In future, even if lzma_vli is defined to be something other than uint64_t, - * it is guaranteed that 2 * LZMA_VLI_MAX will not overflow lzma_vli. - * This simplifies integer overflow detection. + * lzma_vli will be uint64_t for the foreseeable future. If a bigger size + * is needed in the future, it is guaranteed that 2 * LZMA_VLI_MAX will + * not overflow lzma_vli. This simplifies integer overflow detection. */ typedef uint64_t lzma_vli; /** - * \brief Simple macro to validate variable-length integer + * \brief Validate a variable-length integer * * This is useful to test that application has given acceptable values * for example in the uncompressed_size and compressed_size variables. @@ -88,9 +87,9 @@ typedef uint64_t lzma_vli; * * \param vli Integer to be encoded * \param vli_pos How many VLI-encoded bytes have already been written - * out. When starting to encode a new integer, *vli_pos - * must be set to zero. To use single-call encoding, - * set vli_pos to NULL. + * out. When starting to encode a new integer in + * multi-call mode, *vli_pos must be set to zero. + * To use single-call encoding, set vli_pos to NULL. * \param out Beginning of the output buffer * \param out_pos The next byte will be written to out[*out_pos]. * \param out_size Size of the out buffer; the first byte into @@ -126,9 +125,9 @@ extern LZMA_API(lzma_ret) lzma_vli_encode(lzma_vli vli, size_t *vli_pos, * initialize it to zero when *vli_pos == 0, so * application isn't required to initialize *vli. * \param vli_pos How many bytes have already been decoded. When - * starting to decode a new integer, *vli_pos must - * be initialized to zero. To use single-call decoding, - * set this to NULL. + * starting to decode a new integer in multi-call + * mode, *vli_pos must be initialized to zero. To + * use single-call decoding, set vli_pos to NULL. * \param in Beginning of the input buffer * \param in_pos The next byte will be read from in[*in_pos]. * \param in_size Size of the input buffer; the first byte that diff --git a/Build/source/extra/xz/src/liblzma/common/common.c b/Build/source/extra/xz/src/liblzma/common/common.c index 07b1d4763f6..0408e15328b 100644 --- a/Build/source/extra/xz/src/liblzma/common/common.c +++ b/Build/source/extra/xz/src/liblzma/common/common.c @@ -182,6 +182,20 @@ lzma_code(lzma_stream *strm, lzma_action action) || !strm->internal->supported_actions[action]) return LZMA_PROG_ERROR; + // Check if unsupported members have been set to non-zero or non-NULL, + // which would indicate that some new feature is wanted. + if (strm->reserved_ptr1 != NULL + || strm->reserved_ptr2 != NULL + || strm->reserved_ptr3 != NULL + || strm->reserved_ptr4 != NULL + || strm->reserved_int1 != 0 + || strm->reserved_int2 != 0 + || strm->reserved_int3 != 0 + || strm->reserved_int4 != 0 + || strm->reserved_enum1 != LZMA_RESERVED_ENUM + || strm->reserved_enum2 != LZMA_RESERVED_ENUM) + return LZMA_OPTIONS_ERROR; + switch (strm->internal->sequence) { case ISEQ_RUN: switch (action) { diff --git a/Build/source/extra/xz/src/liblzma/common/common.h b/Build/source/extra/xz/src/liblzma/common/common.h index 3a85168f13b..b819432371d 100644 --- a/Build/source/extra/xz/src/liblzma/common/common.h +++ b/Build/source/extra/xz/src/liblzma/common/common.h @@ -205,7 +205,7 @@ struct lzma_internal_s { /// Allocates memory extern void *lzma_alloc(size_t size, lzma_allocator *allocator) - lzma_attribute((malloc)); + lzma_attribute((malloc)) lzma_attr_alloc_size(1); /// Frees memory extern void lzma_free(void *ptr, lzma_allocator *allocator); diff --git a/Build/source/extra/xz/src/liblzma/common/filter_encoder.c b/Build/source/extra/xz/src/liblzma/common/filter_encoder.c index 436d2cc6e42..635d8122347 100644 --- a/Build/source/extra/xz/src/liblzma/common/filter_encoder.c +++ b/Build/source/extra/xz/src/liblzma/common/filter_encoder.c @@ -226,6 +226,7 @@ lzma_raw_encoder_memusage(const lzma_filter *filters) } +/* extern LZMA_API(lzma_vli) lzma_chunk_size(const lzma_filter *filters) { @@ -247,6 +248,7 @@ lzma_chunk_size(const lzma_filter *filters) return max; } +*/ extern LZMA_API(lzma_ret) diff --git a/Build/source/extra/xz/src/liblzma/common/filter_encoder.h b/Build/source/extra/xz/src/liblzma/common/filter_encoder.h index a978932def7..5bc137f6458 100644 --- a/Build/source/extra/xz/src/liblzma/common/filter_encoder.h +++ b/Build/source/extra/xz/src/liblzma/common/filter_encoder.h @@ -16,8 +16,8 @@ #include "common.h" -// FIXME !!! Public API -extern lzma_vli lzma_chunk_size(const lzma_filter *filters); +// FIXME: Might become a part of the public API once finished. +// extern lzma_vli lzma_chunk_size(const lzma_filter *filters); extern lzma_ret lzma_raw_encoder_init( diff --git a/Build/source/extra/xz/src/liblzma/common/index.c b/Build/source/extra/xz/src/liblzma/common/index.c index 3941e28ba72..ddb9d364362 100644 --- a/Build/source/extra/xz/src/liblzma/common/index.c +++ b/Build/source/extra/xz/src/liblzma/common/index.c @@ -18,7 +18,7 @@ /// /// This should be big enough to avoid making lots of tiny allocations /// but small enough to avoid too much unused memory at once. -#define INDEX_GROUP_SIZE 500 +#define INDEX_GROUP_SIZE 512 /// \brief How many Records can be allocated at once at maximum diff --git a/Build/source/extra/xz/src/liblzma/common/index_decoder.c b/Build/source/extra/xz/src/liblzma/common/index_decoder.c index 86a22971b06..a6bc650e046 100644 --- a/Build/source/extra/xz/src/liblzma/common/index_decoder.c +++ b/Build/source/extra/xz/src/liblzma/common/index_decoder.c @@ -291,6 +291,7 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) lzma_next_strm_init(index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/Build/source/extra/xz/src/liblzma/common/index_encoder.c b/Build/source/extra/xz/src/liblzma/common/index_encoder.c index 706f1fd7990..c10d7afa06b 100644 --- a/Build/source/extra/xz/src/liblzma/common/index_encoder.c +++ b/Build/source/extra/xz/src/liblzma/common/index_encoder.c @@ -209,6 +209,7 @@ lzma_index_encoder(lzma_stream *strm, const lzma_index *i) lzma_next_strm_init(lzma_index_encoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/Build/source/extra/xz/src/liblzma/lz/lz_encoder.c b/Build/source/extra/xz/src/liblzma/lz/lz_encoder.c index 9e980a2c71d..273f577b69b 100644 --- a/Build/source/extra/xz/src/liblzma/lz/lz_encoder.c +++ b/Build/source/extra/xz/src/liblzma/lz/lz_encoder.c @@ -349,9 +349,10 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // Maximum number of match finder cycles mf->depth = lz_options->depth; if (mf->depth == 0) { - mf->depth = 16 + (mf->nice_len / 2); - if (!is_bt) - mf->depth /= 2; + if (is_bt) + mf->depth = 16 + mf->nice_len / 2; + else + mf->depth = 4 + mf->nice_len / 4; } return false; diff --git a/Build/source/extra/xz/src/liblzma/lzma/lzma2_decoder.c b/Build/source/extra/xz/src/liblzma/lzma/lzma2_decoder.c index b4c2f2d5ba7..3e42575d5b8 100644 --- a/Build/source/extra/xz/src/liblzma/lzma/lzma2_decoder.c +++ b/Build/source/extra/xz/src/liblzma/lzma/lzma2_decoder.c @@ -67,6 +67,10 @@ lzma2_decode(lzma_coder *restrict coder, lzma_dict *restrict dict, const uint32_t control = in[*in_pos]; ++*in_pos; + // End marker + if (control == 0x00) + return LZMA_STREAM_END; + if (control >= 0xE0 || control == 1) { // Dictionary reset implies that next LZMA chunk has // to set new properties. @@ -104,10 +108,6 @@ lzma2_decode(lzma_coder *restrict coder, lzma_dict *restrict dict, &coder->options); } } else { - // End marker - if (control == 0x00) - return LZMA_STREAM_END; - // Invalid control values if (control > 2) return LZMA_DATA_ERROR; @@ -191,7 +191,6 @@ lzma2_decode(lzma_coder *restrict coder, lzma_dict *restrict dict, case SEQ_COPY: { // Copy from input to the dictionary as is. - // FIXME Can copy too much? dict_write(dict, in, in_pos, in_size, &coder->compressed_size); if (coder->compressed_size != 0) return LZMA_OK; diff --git a/Build/source/extra/xz/src/liblzma/lzma/lzma_decoder.c b/Build/source/extra/xz/src/liblzma/lzma/lzma_decoder.c index 4329e019927..9979bb4261b 100644 --- a/Build/source/extra/xz/src/liblzma/lzma/lzma_decoder.c +++ b/Build/source/extra/xz/src/liblzma/lzma/lzma_decoder.c @@ -656,7 +656,8 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr, } case SEQ_EOPM: - // TODO Comment + // LZMA1 stream with + // end-of-payload marker. rc_normalize(SEQ_EOPM); ret = LZMA_STREAM_END; goto out; @@ -856,7 +857,6 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) // NOTE: We assume that lc/lp/pb are valid since they were // successfully decoded with lzma_lzma_decode_properties(). - // FIXME? // Calculate pos_mask. We don't need pos_bits as is for anything. coder->pos_mask = (1U << options->pb) - 1; diff --git a/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder.c b/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder.c index 0fe992d510a..0b9ee9e1507 100644 --- a/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder.c +++ b/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder.c @@ -334,7 +334,7 @@ lzma_lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf, // With LZMA2 we need to take care that compressed size of // a chunk doesn't get too big. - // TODO + // FIXME? Check if this could be improved. if (limit != UINT32_MAX && (mf->read_pos - mf->read_ahead >= limit || *out_pos + rc_pending(&coder->rc) diff --git a/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder_presets.c b/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder_presets.c index c4c9c146f55..21e427a8d02 100644 --- a/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder_presets.c +++ b/Build/source/extra/xz/src/liblzma/lzma/lzma_encoder_presets.c @@ -23,9 +23,6 @@ lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) if (level > 9 || (flags & ~supported_flags)) return true; - const uint32_t dict_shift = level <= 1 ? 16 : level + 17; - options->dict_size = UINT32_C(1) << dict_shift; - options->preset_dict = NULL; options->preset_dict_size = 0; @@ -33,19 +30,31 @@ lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) options->lp = LZMA_LP_DEFAULT; options->pb = LZMA_PB_DEFAULT; - options->mode = level <= 2 ? LZMA_MODE_FAST : LZMA_MODE_NORMAL; + options->dict_size = UINT32_C(1) << (uint8_t []){ + 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }[level]; - options->nice_len = level == 0 ? 8 : level <= 5 ? 32 : 64; - options->mf = level <= 1 ? LZMA_MF_HC3 : level <= 2 ? LZMA_MF_HC4 - : LZMA_MF_BT4; - options->depth = 0; + if (level <= 3) { + options->mode = LZMA_MODE_FAST; + options->mf = level == 0 ? LZMA_MF_HC3 : LZMA_MF_HC4; + options->nice_len = level <= 1 ? 128 : 273; + options->depth = (uint8_t []){ 4, 8, 24, 48 }[level]; + } else { + options->mode = LZMA_MODE_NORMAL; + options->mf = LZMA_MF_BT4; + options->nice_len = level == 4 ? 16 : level == 5 ? 32 : 64; + options->depth = 0; + } if (flags & LZMA_PRESET_EXTREME) { - options->lc = 4; // FIXME? options->mode = LZMA_MODE_NORMAL; options->mf = LZMA_MF_BT4; - options->nice_len = 273; - options->depth = 512; + if (level == 3 || level == 5) { + options->nice_len = 192; + options->depth = 0; + } else { + options->nice_len = 273; + options->depth = 512; + } } return false; diff --git a/Build/source/extra/xz/src/lzmainfo/Makefile.am b/Build/source/extra/xz/src/lzmainfo/Makefile.am index cbc7e5c7edf..b90b3b3d642 100644 --- a/Build/source/extra/xz/src/lzmainfo/Makefile.am +++ b/Build/source/extra/xz/src/lzmainfo/Makefile.am @@ -12,14 +12,16 @@ lzmainfo_SOURCES = \ $(top_srcdir)/src/common/tuklib_progname.c \ $(top_srcdir)/src/common/tuklib_exit.c +if COND_W32 +lzmainfo_SOURCES += lzmainfo_w32res.rc +endif + lzmainfo_CPPFLAGS = \ -DLOCALEDIR=\"$(localedir)\" \ -I$(top_srcdir)/src/common \ -I$(top_srcdir)/src/liblzma/api \ - -I$(top_builddir)/lib \ - $(DYNAMIC_CPPFLAGS) + -I$(top_builddir)/lib -lzmainfo_LDFLAGS = $(DYNAMIC_LDFLAGS) lzmainfo_LDADD = $(top_builddir)/src/liblzma/liblzma.la if COND_GNULIB @@ -30,3 +32,8 @@ lzmainfo_LDADD += $(LTLIBINTL) dist_man_MANS = lzmainfo.1 + +# Windows resource compiler support +.rc.o: + $(RC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(lzmainfo_CPPFLAGS) $(CPPFLAGS) $(RCFLAGS) -i $< -o $@ diff --git a/Build/source/extra/xz/src/lzmainfo/Makefile.in b/Build/source/extra/xz/src/lzmainfo/Makefile.in index b5c08c1035b..a87db6d893b 100644 --- a/Build/source/extra/xz/src/lzmainfo/Makefile.in +++ b/Build/source/extra/xz/src/lzmainfo/Makefile.in @@ -35,7 +35,8 @@ POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ bin_PROGRAMS = lzmainfo$(EXEEXT) -@COND_GNULIB_TRUE@am__append_1 = $(top_builddir)/lib/libgnu.a +@COND_W32_TRUE@am__append_1 = lzmainfo_w32res.rc +@COND_GNULIB_TRUE@am__append_2 = $(top_builddir)/lib/libgnu.a subdir = src/lzmainfo DIST_COMMON = $(dist_man_MANS) $(srcdir)/Makefile.am \ $(srcdir)/Makefile.in @@ -51,6 +52,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/acx_pthread.m4 \ $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/tuklib_common.m4 \ $(top_srcdir)/m4/tuklib_cpucores.m4 \ $(top_srcdir)/m4/tuklib_integer.m4 \ + $(top_srcdir)/m4/tuklib_mbstr.m4 \ $(top_srcdir)/m4/tuklib_physmem.m4 \ $(top_srcdir)/m4/tuklib_progname.m4 \ $(top_srcdir)/m4/visibility.m4 $(top_srcdir)/configure.ac @@ -62,16 +64,17 @@ CONFIG_CLEAN_FILES = CONFIG_CLEAN_VPATH_FILES = am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)" PROGRAMS = $(bin_PROGRAMS) +am__lzmainfo_SOURCES_DIST = lzmainfo.c \ + $(top_srcdir)/src/common/tuklib_progname.c \ + $(top_srcdir)/src/common/tuklib_exit.c lzmainfo_w32res.rc +@COND_W32_TRUE@am__objects_1 = lzmainfo_w32res.$(OBJEXT) am_lzmainfo_OBJECTS = lzmainfo-lzmainfo.$(OBJEXT) \ lzmainfo-tuklib_progname.$(OBJEXT) \ - lzmainfo-tuklib_exit.$(OBJEXT) + lzmainfo-tuklib_exit.$(OBJEXT) $(am__objects_1) lzmainfo_OBJECTS = $(am_lzmainfo_OBJECTS) am__DEPENDENCIES_1 = lzmainfo_DEPENDENCIES = $(top_builddir)/src/liblzma/liblzma.la \ - $(am__append_1) $(am__DEPENDENCIES_1) -lzmainfo_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ - --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(lzmainfo_LDFLAGS) \ - $(LDFLAGS) -o $@ + $(am__append_2) $(am__DEPENDENCIES_1) DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/build-aux/depcomp am__depfiles_maybe = depfiles @@ -86,7 +89,7 @@ LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ $(LDFLAGS) -o $@ SOURCES = $(lzmainfo_SOURCES) -DIST_SOURCES = $(lzmainfo_SOURCES) +DIST_SOURCES = $(am__lzmainfo_SOURCES_DIST) am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; am__vpath_adj = case $$p in \ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ @@ -138,8 +141,6 @@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ -DYNAMIC_CPPFLAGS = @DYNAMIC_CPPFLAGS@ -DYNAMIC_LDFLAGS = @DYNAMIC_LDFLAGS@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ @@ -172,6 +173,7 @@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -201,8 +203,6 @@ RC = @RC@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ -STATIC_CPPFLAGS = @STATIC_CPPFLAGS@ -STATIC_LDFLAGS = @STATIC_LDFLAGS@ STRIP = @STRIP@ USE_NLS = @USE_NLS@ VERSION = @VERSION@ @@ -212,6 +212,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ acx_pthread_config = @acx_pthread_config@ @@ -261,26 +262,22 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ xz = @xz@ -lzmainfo_SOURCES = \ - lzmainfo.c \ +lzmainfo_SOURCES = lzmainfo.c \ $(top_srcdir)/src/common/tuklib_progname.c \ - $(top_srcdir)/src/common/tuklib_exit.c - + $(top_srcdir)/src/common/tuklib_exit.c $(am__append_1) lzmainfo_CPPFLAGS = \ -DLOCALEDIR=\"$(localedir)\" \ -I$(top_srcdir)/src/common \ -I$(top_srcdir)/src/liblzma/api \ - -I$(top_builddir)/lib \ - $(DYNAMIC_CPPFLAGS) + -I$(top_builddir)/lib -lzmainfo_LDFLAGS = $(DYNAMIC_LDFLAGS) lzmainfo_LDADD = $(top_builddir)/src/liblzma/liblzma.la \ - $(am__append_1) $(LTLIBINTL) + $(am__append_2) $(LTLIBINTL) dist_man_MANS = lzmainfo.1 all: all-am .SUFFIXES: -.SUFFIXES: .c .lo .o .obj +.SUFFIXES: .c .lo .o .obj .rc $(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) @for dep in $?; do \ case '$(am__configure_deps)' in \ @@ -356,7 +353,7 @@ clean-binPROGRAMS: rm -f $$list lzmainfo$(EXEEXT): $(lzmainfo_OBJECTS) $(lzmainfo_DEPENDENCIES) @rm -f lzmainfo$(EXEEXT) - $(lzmainfo_LINK) $(lzmainfo_OBJECTS) $(lzmainfo_LDADD) $(LIBS) + $(LINK) $(lzmainfo_OBJECTS) $(lzmainfo_LDADD) $(LIBS) mostlyclean-compile: -rm -f *.$(OBJEXT) @@ -692,6 +689,11 @@ uninstall-man: uninstall-man1 uninstall-man1 +# Windows resource compiler support +.rc.o: + $(RC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(lzmainfo_CPPFLAGS) $(CPPFLAGS) $(RCFLAGS) -i $< -o $@ + # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: diff --git a/Build/source/extra/xz/src/lzmainfo/lzmainfo.1 b/Build/source/extra/xz/src/lzmainfo/lzmainfo.1 index ef736a6c662..f2b93b4c643 100644 --- a/Build/source/extra/xz/src/lzmainfo/lzmainfo.1 +++ b/Build/source/extra/xz/src/lzmainfo/lzmainfo.1 @@ -4,9 +4,9 @@ .\" This file has been put into the public domain. .\" You can do whatever you want with this file. .\" -.TH LZMAINFO 1 "2009-08-13" "Tukaani" "XZ Utils" +.TH LZMAINFO 1 "2010-09-27" "Tukaani" "XZ Utils" .SH NAME -lzmainfo \- show infomation stored in the .lzma file header +lzmainfo \- show information stored in the .lzma file header .SH SYNOPSIS .B lzmainfo .RB [ \-\-help ] @@ -16,10 +16,12 @@ lzmainfo \- show infomation stored in the .lzma file header .B lzmainfo shows information stored in the .B .lzma -file header. It reads the first 13 bytes from the specified +file header. +It reads the first 13 bytes from the specified .IR file , decodes the header, and prints it to standard output in human -readable format. If no +readable format. +If no .I files are given or .I file @@ -27,16 +29,19 @@ is .BR \- , standard input is read. .PP -Usually the most interesting information is the uncompressed size and -the dictionary size. Uncompressed size can be shown only if the file is -in the non-streamed +Usually the most interesting information is +the uncompressed size and the dictionary size. +Uncompressed size can be shown only if +the file is in the non-streamed .B .lzma -format variant. The amount of memory required to decompress the file is +format variant. +The amount of memory required to decompress the file is a few dozen kilobytes plus the dictionary size. .PP .B lzmainfo -is included in XZ Utils primarily for backward compatibility with LZMA Utils. -.SH EXIT STATUS +is included in XZ Utils primarily for +backward compatibility with LZMA Utils. +.SH "EXIT STATUS" .TP .B 0 All is good. @@ -51,5 +56,5 @@ while the correct suffix would be .B MiB (2^20 bytes). This is to keep the output compatible with LZMA Utils. -.SH SEE ALSO +.SH "SEE ALSO" .BR xz (1) diff --git a/Build/source/extra/xz/src/lzmainfo/lzmainfo.c b/Build/source/extra/xz/src/lzmainfo/lzmainfo.c index af8e66cc8f9..3100dc631af 100644 --- a/Build/source/extra/xz/src/lzmainfo/lzmainfo.c +++ b/Build/source/extra/xz/src/lzmainfo/lzmainfo.c @@ -20,6 +20,11 @@ #include "tuklib_progname.h" #include "tuklib_exit.h" +#ifdef TUKLIB_DOSLIKE +# include <fcntl.h> +# include <io.h> +#endif + static void lzma_attribute((noreturn)) help(void) @@ -43,7 +48,7 @@ _("Usage: %s [--help] [--version] [FILE]...\n" static void lzma_attribute((noreturn)) version(void) { - puts("lzmainfo (" PACKAGE_NAME ") " PACKAGE_VERSION); + puts("lzmainfo (" PACKAGE_NAME ") " LZMA_VERSION_STRING); tuklib_exit(EXIT_SUCCESS, EXIT_FAILURE, true); } @@ -150,7 +155,7 @@ lzmainfo(const char *name, FILE *f) lzma_options_lzma *opt = filter.options; printf("\nDictionary size: " - "%u MB (2^%u bytes)\n" + "%" PRIu32 " MB (2^%" PRIu32 " bytes)\n" "Literal context bits (lc): %" PRIu32 "\n" "Literal pos bits (lp): %" PRIu32 "\n" "Number of pos bits (pb): %" PRIu32 "\n", @@ -171,6 +176,10 @@ main(int argc, char **argv) parse_args(argc, argv); +#ifdef TUKLIB_DOSLIKE + setmode(fileno(stdin), O_BINARY); +#endif + int ret = EXIT_SUCCESS; // We print empty lines around the output only when reading from diff --git a/Build/source/extra/xz/src/lzmainfo/lzmainfo_w32res.rc b/Build/source/extra/xz/src/lzmainfo/lzmainfo_w32res.rc new file mode 100644 index 00000000000..378fbe6958f --- /dev/null +++ b/Build/source/extra/xz/src/lzmainfo/lzmainfo_w32res.rc @@ -0,0 +1,12 @@ +/* + * Author: Lasse Collin + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +#define MY_TYPE VFT_APP +#define MY_NAME "lzmainfo" +#define MY_SUFFIX ".exe" +#define MY_DESC "lzmainfo shows information about .lzma files" +#include "common_w32res.rc" diff --git a/Build/source/extra/xz/src/scripts/Makefile.am b/Build/source/extra/xz/src/scripts/Makefile.am index 86f273884f6..29bdbcd76de 100644 --- a/Build/source/extra/xz/src/scripts/Makefile.am +++ b/Build/source/extra/xz/src/scripts/Makefile.am @@ -8,58 +8,48 @@ nodist_bin_SCRIPTS = xzdiff xzgrep xzmore xzless dist_man_MANS = xzdiff.1 xzgrep.1 xzmore.1 xzless.1 +links = \ + xzdiff-xzcmp \ + xzgrep-xzegrep \ + xzgrep-xzfgrep + +if COND_LZMALINKS +links += \ + xzdiff-lzdiff \ + xzdiff-lzcmp \ + xzgrep-lzgrep \ + xzgrep-lzegrep \ + xzgrep-lzfgrep \ + xzmore-lzmore \ + xzless-lzless +endif + install-exec-hook: cd $(DESTDIR)$(bindir) && \ - target=`echo xzdiff | sed '$(transform)'` && \ - for name in xzcmp lzdiff lzcmp; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link && \ - $(LN_S) $$target $$link; \ - done && \ - target=`echo xzgrep | sed '$(transform)'` && \ - for name in xzegrep xzfgrep lzgrep lzegrep lzfgrep; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link && \ - $(LN_S) $$target $$link; \ - done && \ - for name in xzmore xzless; do \ - target=`echo $$name | sed '$(transform)'` && \ - link=`echo $$name | sed 's/xz/lz/;$(transform)'` && \ + for pair in $(links); do \ + target=`echo $$pair | sed 's/-.*$$//' | sed '$(transform)'` && \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link && \ $(LN_S) $$target $$link; \ done install-data-hook: cd $(DESTDIR)$(mandir)/man1 && \ - target=`echo xzdiff | sed '$(transform)'` && \ - for name in xzcmp lzdiff lzcmp; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link.1 && \ - $(LN_S) $$target.1 $$link.1; \ - done && \ - target=`echo xzgrep | sed '$(transform)'` && \ - for name in xzegrep xzfgrep lzgrep lzegrep lzfgrep; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link.1 && \ - $(LN_S) $$target.1 $$link.1; \ - done && \ - for name in xzmore xzless; do \ - target=`echo $$name | sed '$(transform)'` && \ - link=`echo $$name | sed 's/xz/lz/;$(transform)'` && \ + for pair in $(links); do \ + target=`echo $$pair | sed 's/-.*$$//' | sed '$(transform)'` && \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link.1 && \ $(LN_S) $$target.1 $$link.1; \ done uninstall-hook: cd $(DESTDIR)$(bindir) && \ - for name in xzcmp lzdiff lzcmp xzegrep xzfgrep \ - lzgrep lzegrep lzfgrep lzmore lzless; do \ - link=`echo $$name | sed '$(transform)'` && \ + for pair in $(links); do \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link; \ done cd $(DESTDIR)$(mandir)/man1 && \ - for name in xzcmp lzdiff lzcmp xzegrep xzfgrep \ - lzgrep lzegrep lzfgrep lzmore lzless; do \ - link=`echo $$name | sed '$(transform)'` && \ + for pair in $(links); do \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link.1; \ done diff --git a/Build/source/extra/xz/src/scripts/Makefile.in b/Build/source/extra/xz/src/scripts/Makefile.in index bd6ff405be1..a76870f3bed 100644 --- a/Build/source/extra/xz/src/scripts/Makefile.in +++ b/Build/source/extra/xz/src/scripts/Makefile.in @@ -34,6 +34,15 @@ PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ +@COND_LZMALINKS_TRUE@am__append_1 = \ +@COND_LZMALINKS_TRUE@ xzdiff-lzdiff \ +@COND_LZMALINKS_TRUE@ xzdiff-lzcmp \ +@COND_LZMALINKS_TRUE@ xzgrep-lzgrep \ +@COND_LZMALINKS_TRUE@ xzgrep-lzegrep \ +@COND_LZMALINKS_TRUE@ xzgrep-lzfgrep \ +@COND_LZMALINKS_TRUE@ xzmore-lzmore \ +@COND_LZMALINKS_TRUE@ xzless-lzless + subdir = src/scripts DIST_COMMON = $(dist_man_MANS) $(srcdir)/Makefile.am \ $(srcdir)/Makefile.in $(srcdir)/xzdiff.in $(srcdir)/xzgrep.in \ @@ -50,6 +59,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/acx_pthread.m4 \ $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/tuklib_common.m4 \ $(top_srcdir)/m4/tuklib_cpucores.m4 \ $(top_srcdir)/m4/tuklib_integer.m4 \ + $(top_srcdir)/m4/tuklib_mbstr.m4 \ $(top_srcdir)/m4/tuklib_physmem.m4 \ $(top_srcdir)/m4/tuklib_progname.m4 \ $(top_srcdir)/m4/visibility.m4 $(top_srcdir)/configure.ac @@ -112,8 +122,6 @@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ -DYNAMIC_CPPFLAGS = @DYNAMIC_CPPFLAGS@ -DYNAMIC_LDFLAGS = @DYNAMIC_LDFLAGS@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ @@ -146,6 +154,7 @@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -175,8 +184,6 @@ RC = @RC@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ -STATIC_CPPFLAGS = @STATIC_CPPFLAGS@ -STATIC_LDFLAGS = @STATIC_LDFLAGS@ STRIP = @STRIP@ USE_NLS = @USE_NLS@ VERSION = @VERSION@ @@ -186,6 +193,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ acx_pthread_config = @acx_pthread_config@ @@ -237,6 +245,7 @@ top_srcdir = @top_srcdir@ xz = @xz@ nodist_bin_SCRIPTS = xzdiff xzgrep xzmore xzless dist_man_MANS = xzdiff.1 xzgrep.1 xzmore.1 xzless.1 +links = xzdiff-xzcmp xzgrep-xzegrep xzgrep-xzfgrep $(am__append_1) all: all-am .SUFFIXES: @@ -529,57 +538,31 @@ uninstall-man: uninstall-man1 install-exec-hook: cd $(DESTDIR)$(bindir) && \ - target=`echo xzdiff | sed '$(transform)'` && \ - for name in xzcmp lzdiff lzcmp; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link && \ - $(LN_S) $$target $$link; \ - done && \ - target=`echo xzgrep | sed '$(transform)'` && \ - for name in xzegrep xzfgrep lzgrep lzegrep lzfgrep; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link && \ - $(LN_S) $$target $$link; \ - done && \ - for name in xzmore xzless; do \ - target=`echo $$name | sed '$(transform)'` && \ - link=`echo $$name | sed 's/xz/lz/;$(transform)'` && \ + for pair in $(links); do \ + target=`echo $$pair | sed 's/-.*$$//' | sed '$(transform)'` && \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link && \ $(LN_S) $$target $$link; \ done install-data-hook: cd $(DESTDIR)$(mandir)/man1 && \ - target=`echo xzdiff | sed '$(transform)'` && \ - for name in xzcmp lzdiff lzcmp; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link.1 && \ - $(LN_S) $$target.1 $$link.1; \ - done && \ - target=`echo xzgrep | sed '$(transform)'` && \ - for name in xzegrep xzfgrep lzgrep lzegrep lzfgrep; do \ - link=`echo $$name | sed '$(transform)'` && \ - rm -f $$link.1 && \ - $(LN_S) $$target.1 $$link.1; \ - done && \ - for name in xzmore xzless; do \ - target=`echo $$name | sed '$(transform)'` && \ - link=`echo $$name | sed 's/xz/lz/;$(transform)'` && \ + for pair in $(links); do \ + target=`echo $$pair | sed 's/-.*$$//' | sed '$(transform)'` && \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link.1 && \ $(LN_S) $$target.1 $$link.1; \ done uninstall-hook: cd $(DESTDIR)$(bindir) && \ - for name in xzcmp lzdiff lzcmp xzegrep xzfgrep \ - lzgrep lzegrep lzfgrep lzmore lzless; do \ - link=`echo $$name | sed '$(transform)'` && \ + for pair in $(links); do \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link; \ done cd $(DESTDIR)$(mandir)/man1 && \ - for name in xzcmp lzdiff lzcmp xzegrep xzfgrep \ - lzgrep lzegrep lzfgrep lzmore lzless; do \ - link=`echo $$name | sed '$(transform)'` && \ + for pair in $(links); do \ + link=`echo $$pair | sed 's/^.*-//' | sed '$(transform)'` && \ rm -f $$link.1; \ done diff --git a/Build/source/extra/xz/src/scripts/xzdiff.1 b/Build/source/extra/xz/src/scripts/xzdiff.1 index 318d06fb04d..d97f3cb8106 100644 --- a/Build/source/extra/xz/src/scripts/xzdiff.1 +++ b/Build/source/extra/xz/src/scripts/xzdiff.1 @@ -6,7 +6,7 @@ .\" .\" License: GNU GPLv2+ .\" -.TH XZDIFF 1 "2009-07-05" "Tukaani" "XZ Utils" +.TH XZDIFF 1 "2010-09-27" "Tukaani" "XZ Utils" .SH NAME xzcmp, xzdiff, lzcmp, lzdiff \- compare compressed files .SH SYNOPSIS @@ -22,7 +22,7 @@ xzcmp, xzdiff, lzcmp, lzdiff \- compare compressed files .B lzdiff .RI [ diff_options "] " file1 " [" file2 ] .SH DESCRIPTION -.B xzcmp +.B xzcmp and .B xzdiff invoke @@ -36,22 +36,23 @@ on files compressed with or .BR bzip2 (1). All options specified are passed directly to -.B cmp +.BR cmp (1) or -.BR diff . +.BR diff (1). If only one file is specified, then the files compared are .I file1 (which must have a suffix of a supported compression format) and .I file1 from which the compression format suffix has been stripped. -If two files are specified, then they are uncompressed if necessary and fed to +If two files are specified, +then they are uncompressed if necessary and fed to .BR cmp (1) or .BR diff (1). The exit status from -.B cmp +.BR cmp (1) or -.B diff +.BR diff (1) is preserved. .PP The names diff --git a/Build/source/extra/xz/src/scripts/xzdiff.in b/Build/source/extra/xz/src/scripts/xzdiff.in index 2d6e5da4760..45633e0a860 100644 --- a/Build/source/extra/xz/src/scripts/xzdiff.in +++ b/Build/source/extra/xz/src/scripts/xzdiff.in @@ -74,9 +74,9 @@ if test $# -eq 1; then *[-.]xz | *[-.]lzma | *.t[lx]z) ;; *[-.]bz2 | *.tbz | *.tbz2) - xz1=$bzip2;; + xz1=bzip2;; *[-.][zZ] | *_z | *[-.]gz | *.t[ag]z) - xz1=$gzip;; + xz1=gzip;; *) echo >&2 "$0: $1: Unknown compressed file name suffix" exit 2;; @@ -95,12 +95,12 @@ if test $# -eq 1; then ) elif test $# -eq 2; then case $1 in - *[-.]bz2 | *.tbz | *.tbz2) xz1=$bzip2;; - *[-.][zZ] | *_z | *[-.]gz | *.t[ag]z) xz1=$gzip;; + *[-.]bz2 | *.tbz | *.tbz2) xz1=bzip2;; + *[-.][zZ] | *_z | *[-.]gz | *.t[ag]z) xz1=gzip;; esac case $2 in - *[-.]bz2 | *.tbz | *.tbz2) xz2=$bzip2;; - *[-.][zZ] | *_z | *[-.]gz | *.t[ag]z) xz2=$gzip;; + *[-.]bz2 | *.tbz | *.tbz2) xz2=bzip2;; + *[-.][zZ] | *_z | *[-.]gz | *.t[ag]z) xz2=gzip;; esac case $1 in *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *.t[abglx]z | *.tbz2 | -) diff --git a/Build/source/extra/xz/src/scripts/xzgrep.1 b/Build/source/extra/xz/src/scripts/xzgrep.1 index 996d64aee8b..a96f1b89db8 100644 --- a/Build/source/extra/xz/src/scripts/xzgrep.1 +++ b/Build/source/extra/xz/src/scripts/xzgrep.1 @@ -6,15 +6,15 @@ .\" .\" License: GNU GPLv2+ .\" -.TH XZGREP 1 "2009-07-05" "Tukaani" "XZ Utils" +.TH XZGREP 1 "2010-09-27" "Tukaani" "XZ Utils" .SH NAME xzgrep \- search compressed files for a regular expression .SH SYNOPSIS .B xzgrep .RI [ grep_options ] -.RB [ \-e ] +.RB [ \-e ] .I pattern -.IR file ".\|.\|." +.IR file "..." .br .B xzegrep .RB ... @@ -31,7 +31,7 @@ xzgrep \- search compressed files for a regular expression .B lzfgrep .RB ... .SH DESCRIPTION -.B xzgrep +.B xzgrep invokes .BR grep (1) on @@ -47,7 +47,8 @@ All options specified are passed directly to .PP If no .I file -is specified, then the standard input is decompressed if necessary and fed to +is specified, then standard input is decompressed if necessary +and fed to .BR grep (1). When reading from standard input, .BR gzip (1) diff --git a/Build/source/extra/xz/src/scripts/xzgrep.in b/Build/source/extra/xz/src/scripts/xzgrep.in index bfa9fb536a4..a8b8117a0c1 100644 --- a/Build/source/extra/xz/src/scripts/xzgrep.in +++ b/Build/source/extra/xz/src/scripts/xzgrep.in @@ -126,6 +126,10 @@ while test $# -ne 0; do grep="$grep $option$optarg" done +if test $files_with_matches -eq 1 || test $files_without_matches -eq 1; then + grep="$grep -q" +fi + eval "set -- $operands "'${1+"$@"}' if test $have_pat -eq 0; then @@ -156,9 +160,9 @@ for i; do exec 5>&1 ($uncompress -- "$i" 5>&-; echo $? >&5) 3>&- | if test $files_with_matches -eq 1; then - eval "$grep" -q && { printf '%s\n' "$i" || exit 2; } + eval "$grep" && { printf '%s\n' "$i" || exit 2; } elif test $files_without_matches -eq 1; then - eval "$grep" -q || { + eval "$grep" || { r=$? if test $r -eq 1; then printf '%s\n' "$i" || r=2 diff --git a/Build/source/extra/xz/src/scripts/xzless.1 b/Build/source/extra/xz/src/scripts/xzless.1 index 299806fdfa6..2d05459d00c 100644 --- a/Build/source/extra/xz/src/scripts/xzless.1 +++ b/Build/source/extra/xz/src/scripts/xzless.1 @@ -7,7 +7,7 @@ .\" .\" (Note that this file is not based on gzip's zless.1.) .\" -.TH XZLESS 1 "2009-07-05" "Tukaani" "XZ Utils" +.TH XZLESS 1 "2010-09-27" "Tukaani" "XZ Utils" .SH NAME xzless, lzless \- view xz or lzma compressed (text) files .SH SYNOPSIS @@ -17,7 +17,7 @@ xzless, lzless \- view xz or lzma compressed (text) files .B lzless .RI [ file ...] .SH DESCRIPTION -.B xzless +.B xzless is a filter that displays text from compressed files to a terminal. It works on files compressed with .BR xz (1) @@ -32,9 +32,11 @@ reads from standard input. .B xzless uses .BR less (1) -to present its output. Unlike +to present its output. +Unlike .BR xzmore , -its choice of pager cannot be altered by setting an environment variable. +its choice of pager cannot be altered by +setting an environment variable. Commands are based on both .BR more (1) and @@ -50,7 +52,8 @@ is provided for backward compatibility with LZMA Utils. .SH ENVIRONMENT .TP .B LESSMETACHARS -A list of characters special to the shell. Set by +A list of characters special to the shell. +Set by .B xzless unless it is already set in the environment. .TP diff --git a/Build/source/extra/xz/src/scripts/xzmore.1 b/Build/source/extra/xz/src/scripts/xzmore.1 index 42542bfe8c2..30dad6871af 100644 --- a/Build/source/extra/xz/src/scripts/xzmore.1 +++ b/Build/source/extra/xz/src/scripts/xzmore.1 @@ -4,22 +4,23 @@ .\" .\" License: GNU GPLv2+ .\" -.TH XZMORE 1 "2009-07-05" "Tukaani" "XZ Utils" +.TH XZMORE 1 "2010-09-27" "Tukaani" "XZ Utils" .SH NAME xzmore, lzmore \- view xz or lzma compressed (text) files .SH SYNOPSIS .B xzmore -.RI [ "filename ..." ] +.RI [ "filename ..." ] .br .B lzmore -.RI [ "filename ..." ] +.RI [ "filename ..." ] .SH DESCRIPTION .B xzmore is a filter which allows examination of .BR xz (1) or .BR lzma (1) -compressed text files one screenful at a time on a soft-copy terminal. +compressed text files one screenful at a time +on a soft-copy terminal. .PP To use a pager other than the default .B more, diff --git a/Build/source/extra/xz/src/xz/Makefile.am b/Build/source/extra/xz/src/xz/Makefile.am index 4dbe0f2604e..10ceee75a65 100644 --- a/Build/source/extra/xz/src/xz/Makefile.am +++ b/Build/source/extra/xz/src/xz/Makefile.am @@ -34,7 +34,9 @@ xz_SOURCES = \ $(top_srcdir)/src/common/tuklib_open_stdxxx.c \ $(top_srcdir)/src/common/tuklib_progname.c \ $(top_srcdir)/src/common/tuklib_exit.c \ - $(top_srcdir)/src/common/tuklib_cpucores.c + $(top_srcdir)/src/common/tuklib_cpucores.c \ + $(top_srcdir)/src/common/tuklib_mbstr_width.c \ + $(top_srcdir)/src/common/tuklib_mbstr_fw.c if COND_W32 xz_SOURCES += xz_w32res.rc @@ -44,10 +46,8 @@ xz_CPPFLAGS = \ -DLOCALEDIR=\"$(localedir)\" \ -I$(top_srcdir)/src/common \ -I$(top_srcdir)/src/liblzma/api \ - -I$(top_builddir)/lib \ - $(STATIC_CPPFLAGS) + -I$(top_builddir)/lib -xz_LDFLAGS = $(STATIC_LDFLAGS) xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la if COND_GNULIB @@ -69,7 +69,11 @@ dist_man_MANS = xz.1 ## Create symlinks for unxz and xzcat for convenience. Create symlinks also ## for lzma, unlzma, and lzcat for compatibility with LZMA Utils 4.32.x. -xzlinks = unxz xzcat lzma unlzma lzcat +xzlinks = unxz xzcat + +if COND_LZMALINKS +xzlinks += lzma unlzma lzcat +endif install-exec-hook: cd $(DESTDIR)$(bindir) && \ diff --git a/Build/source/extra/xz/src/xz/Makefile.in b/Build/source/extra/xz/src/xz/Makefile.in index 8ff44cb0c8a..922425f839f 100644 --- a/Build/source/extra/xz/src/xz/Makefile.in +++ b/Build/source/extra/xz/src/xz/Makefile.in @@ -37,6 +37,7 @@ host_triplet = @host@ bin_PROGRAMS = xz$(EXEEXT) @COND_W32_TRUE@am__append_1 = xz_w32res.rc @COND_GNULIB_TRUE@am__append_2 = $(top_builddir)/lib/libgnu.a +@COND_LZMALINKS_TRUE@am__append_3 = lzma unlzma lzcat subdir = src/xz DIST_COMMON = $(dist_man_MANS) $(srcdir)/Makefile.am \ $(srcdir)/Makefile.in @@ -52,6 +53,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/acx_pthread.m4 \ $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/tuklib_common.m4 \ $(top_srcdir)/m4/tuklib_cpucores.m4 \ $(top_srcdir)/m4/tuklib_integer.m4 \ + $(top_srcdir)/m4/tuklib_mbstr.m4 \ $(top_srcdir)/m4/tuklib_physmem.m4 \ $(top_srcdir)/m4/tuklib_progname.m4 \ $(top_srcdir)/m4/visibility.m4 $(top_srcdir)/configure.ac @@ -70,7 +72,9 @@ am__xz_SOURCES_DIST = args.c args.h coder.c coder.h file_io.c \ $(top_srcdir)/src/common/tuklib_open_stdxxx.c \ $(top_srcdir)/src/common/tuklib_progname.c \ $(top_srcdir)/src/common/tuklib_exit.c \ - $(top_srcdir)/src/common/tuklib_cpucores.c xz_w32res.rc + $(top_srcdir)/src/common/tuklib_cpucores.c \ + $(top_srcdir)/src/common/tuklib_mbstr_width.c \ + $(top_srcdir)/src/common/tuklib_mbstr_fw.c xz_w32res.rc @COND_W32_TRUE@am__objects_1 = xz_w32res.$(OBJEXT) am_xz_OBJECTS = xz-args.$(OBJEXT) xz-coder.$(OBJEXT) \ xz-file_io.$(OBJEXT) xz-hardware.$(OBJEXT) xz-list.$(OBJEXT) \ @@ -78,14 +82,12 @@ am_xz_OBJECTS = xz-args.$(OBJEXT) xz-coder.$(OBJEXT) \ xz-signals.$(OBJEXT) xz-suffix.$(OBJEXT) xz-util.$(OBJEXT) \ xz-tuklib_open_stdxxx.$(OBJEXT) xz-tuklib_progname.$(OBJEXT) \ xz-tuklib_exit.$(OBJEXT) xz-tuklib_cpucores.$(OBJEXT) \ + xz-tuklib_mbstr_width.$(OBJEXT) xz-tuklib_mbstr_fw.$(OBJEXT) \ $(am__objects_1) xz_OBJECTS = $(am_xz_OBJECTS) am__DEPENDENCIES_1 = xz_DEPENDENCIES = $(top_builddir)/src/liblzma/liblzma.la \ $(am__append_2) $(am__DEPENDENCIES_1) -xz_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ - --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(xz_LDFLAGS) \ - $(LDFLAGS) -o $@ DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/build-aux/depcomp am__depfiles_maybe = depfiles @@ -152,8 +154,6 @@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ -DYNAMIC_CPPFLAGS = @DYNAMIC_CPPFLAGS@ -DYNAMIC_LDFLAGS = @DYNAMIC_LDFLAGS@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ @@ -186,6 +186,7 @@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -215,8 +216,6 @@ RC = @RC@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ -STATIC_CPPFLAGS = @STATIC_CPPFLAGS@ -STATIC_LDFLAGS = @STATIC_LDFLAGS@ STRIP = @STRIP@ USE_NLS = @USE_NLS@ VERSION = @VERSION@ @@ -226,6 +225,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ acx_pthread_config = @acx_pthread_config@ @@ -282,21 +282,21 @@ xz_SOURCES = args.c args.h coder.c coder.h file_io.c file_io.h \ $(top_srcdir)/src/common/tuklib_open_stdxxx.c \ $(top_srcdir)/src/common/tuklib_progname.c \ $(top_srcdir)/src/common/tuklib_exit.c \ - $(top_srcdir)/src/common/tuklib_cpucores.c $(am__append_1) + $(top_srcdir)/src/common/tuklib_cpucores.c \ + $(top_srcdir)/src/common/tuklib_mbstr_width.c \ + $(top_srcdir)/src/common/tuklib_mbstr_fw.c $(am__append_1) xz_CPPFLAGS = \ -DLOCALEDIR=\"$(localedir)\" \ -I$(top_srcdir)/src/common \ -I$(top_srcdir)/src/liblzma/api \ - -I$(top_builddir)/lib \ - $(STATIC_CPPFLAGS) + -I$(top_builddir)/lib -xz_LDFLAGS = $(STATIC_LDFLAGS) # libgnu.a may need these libs, so this must be after libgnu.a. xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(am__append_2) \ $(LTLIBINTL) dist_man_MANS = xz.1 -xzlinks = unxz xzcat lzma unlzma lzcat +xzlinks = unxz xzcat $(am__append_3) all: all-am .SUFFIXES: @@ -376,7 +376,7 @@ clean-binPROGRAMS: rm -f $$list xz$(EXEEXT): $(xz_OBJECTS) $(xz_DEPENDENCIES) @rm -f xz$(EXEEXT) - $(xz_LINK) $(xz_OBJECTS) $(xz_LDADD) $(LIBS) + $(LINK) $(xz_OBJECTS) $(xz_LDADD) $(LIBS) mostlyclean-compile: -rm -f *.$(OBJEXT) @@ -396,6 +396,8 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-suffix.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_cpucores.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_exit.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_mbstr_fw.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_mbstr_width.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_open_stdxxx.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_progname.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-util.Po@am__quote@ @@ -631,6 +633,34 @@ xz-tuklib_cpucores.obj: $(top_srcdir)/src/common/tuklib_cpucores.c @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-tuklib_cpucores.obj `if test -f '$(top_srcdir)/src/common/tuklib_cpucores.c'; then $(CYGPATH_W) '$(top_srcdir)/src/common/tuklib_cpucores.c'; else $(CYGPATH_W) '$(srcdir)/$(top_srcdir)/src/common/tuklib_cpucores.c'; fi` +xz-tuklib_mbstr_width.o: $(top_srcdir)/src/common/tuklib_mbstr_width.c +@am__fastdepCC_TRUE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-tuklib_mbstr_width.o -MD -MP -MF $(DEPDIR)/xz-tuklib_mbstr_width.Tpo -c -o xz-tuklib_mbstr_width.o `test -f '$(top_srcdir)/src/common/tuklib_mbstr_width.c' || echo '$(srcdir)/'`$(top_srcdir)/src/common/tuklib_mbstr_width.c +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/xz-tuklib_mbstr_width.Tpo $(DEPDIR)/xz-tuklib_mbstr_width.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$(top_srcdir)/src/common/tuklib_mbstr_width.c' object='xz-tuklib_mbstr_width.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-tuklib_mbstr_width.o `test -f '$(top_srcdir)/src/common/tuklib_mbstr_width.c' || echo '$(srcdir)/'`$(top_srcdir)/src/common/tuklib_mbstr_width.c + +xz-tuklib_mbstr_width.obj: $(top_srcdir)/src/common/tuklib_mbstr_width.c +@am__fastdepCC_TRUE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-tuklib_mbstr_width.obj -MD -MP -MF $(DEPDIR)/xz-tuklib_mbstr_width.Tpo -c -o xz-tuklib_mbstr_width.obj `if test -f '$(top_srcdir)/src/common/tuklib_mbstr_width.c'; then $(CYGPATH_W) '$(top_srcdir)/src/common/tuklib_mbstr_width.c'; else $(CYGPATH_W) '$(srcdir)/$(top_srcdir)/src/common/tuklib_mbstr_width.c'; fi` +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/xz-tuklib_mbstr_width.Tpo $(DEPDIR)/xz-tuklib_mbstr_width.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$(top_srcdir)/src/common/tuklib_mbstr_width.c' object='xz-tuklib_mbstr_width.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-tuklib_mbstr_width.obj `if test -f '$(top_srcdir)/src/common/tuklib_mbstr_width.c'; then $(CYGPATH_W) '$(top_srcdir)/src/common/tuklib_mbstr_width.c'; else $(CYGPATH_W) '$(srcdir)/$(top_srcdir)/src/common/tuklib_mbstr_width.c'; fi` + +xz-tuklib_mbstr_fw.o: $(top_srcdir)/src/common/tuklib_mbstr_fw.c +@am__fastdepCC_TRUE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-tuklib_mbstr_fw.o -MD -MP -MF $(DEPDIR)/xz-tuklib_mbstr_fw.Tpo -c -o xz-tuklib_mbstr_fw.o `test -f '$(top_srcdir)/src/common/tuklib_mbstr_fw.c' || echo '$(srcdir)/'`$(top_srcdir)/src/common/tuklib_mbstr_fw.c +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/xz-tuklib_mbstr_fw.Tpo $(DEPDIR)/xz-tuklib_mbstr_fw.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$(top_srcdir)/src/common/tuklib_mbstr_fw.c' object='xz-tuklib_mbstr_fw.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-tuklib_mbstr_fw.o `test -f '$(top_srcdir)/src/common/tuklib_mbstr_fw.c' || echo '$(srcdir)/'`$(top_srcdir)/src/common/tuklib_mbstr_fw.c + +xz-tuklib_mbstr_fw.obj: $(top_srcdir)/src/common/tuklib_mbstr_fw.c +@am__fastdepCC_TRUE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-tuklib_mbstr_fw.obj -MD -MP -MF $(DEPDIR)/xz-tuklib_mbstr_fw.Tpo -c -o xz-tuklib_mbstr_fw.obj `if test -f '$(top_srcdir)/src/common/tuklib_mbstr_fw.c'; then $(CYGPATH_W) '$(top_srcdir)/src/common/tuklib_mbstr_fw.c'; else $(CYGPATH_W) '$(srcdir)/$(top_srcdir)/src/common/tuklib_mbstr_fw.c'; fi` +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/xz-tuklib_mbstr_fw.Tpo $(DEPDIR)/xz-tuklib_mbstr_fw.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$(top_srcdir)/src/common/tuklib_mbstr_fw.c' object='xz-tuklib_mbstr_fw.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-tuklib_mbstr_fw.obj `if test -f '$(top_srcdir)/src/common/tuklib_mbstr_fw.c'; then $(CYGPATH_W) '$(top_srcdir)/src/common/tuklib_mbstr_fw.c'; else $(CYGPATH_W) '$(srcdir)/$(top_srcdir)/src/common/tuklib_mbstr_fw.c'; fi` + mostlyclean-libtool: -rm -f *.lo diff --git a/Build/source/extra/xz/src/xz/args.c b/Build/source/extra/xz/src/xz/args.c index 7b12529c06d..4bd84a0c141 100644 --- a/Build/source/extra/xz/src/xz/args.c +++ b/Build/source/extra/xz/src/xz/args.c @@ -25,7 +25,33 @@ bool opt_robot = false; // We don't modify or free() this, but we need to assign it in some // non-const pointers. -const char *stdin_filename = "(stdin)"; +const char stdin_filename[] = "(stdin)"; + + +/// Parse and set the memory usage limit for compression and/or decompression. +static void +parse_memlimit(const char *name, const char *name_percentage, char *str, + bool set_compress, bool set_decompress) +{ + bool is_percentage = false; + uint64_t value; + + const size_t len = strlen(str); + if (len > 0 && str[len - 1] == '%') { + str[len - 1] = '\0'; + is_percentage = true; + value = str_to_uint64(name_percentage, str, 1, 100); + } else { + // On 32-bit systems, SIZE_MAX would make more sense than + // UINT64_MAX. But use UINT64_MAX still so that scripts + // that assume > 4 GiB values don't break. + value = str_to_uint64(name, str, 0, UINT64_MAX); + } + + hardware_memlimit_set( + value, set_compress, set_decompress, is_percentage); + return; +} static void @@ -45,6 +71,9 @@ parse_real(args_info *args, int argc, char **argv) OPT_NO_SPARSE, OPT_FILES, OPT_FILES0, + OPT_MEM_COMPRESS, + OPT_MEM_DECOMPRESS, + OPT_NO_ADJUST, OPT_INFO_MEMORY, OPT_ROBOT, }; @@ -74,7 +103,11 @@ parse_real(args_info *args, int argc, char **argv) // Basic compression settings { "format", required_argument, NULL, 'F' }, { "check", required_argument, NULL, 'C' }, - { "memory", required_argument, NULL, 'M' }, + { "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS }, + { "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS }, + { "memlimit", required_argument, NULL, 'M' }, + { "memory", required_argument, NULL, 'M' }, // Old alias + { "no-adjust", no_argument, NULL, OPT_NO_ADJUST }, { "threads", required_argument, NULL, 'T' }, { "extreme", no_argument, NULL, 'e' }, @@ -102,7 +135,7 @@ parse_real(args_info *args, int argc, char **argv) { "long-help", no_argument, NULL, 'H' }, { "version", no_argument, NULL, 'V' }, - { NULL, 0, NULL, 0 } + { NULL, 0, NULL, 0 } }; int c; @@ -116,28 +149,25 @@ parse_real(args_info *args, int argc, char **argv) coder_set_preset(c - '0'); break; - // --memory - case 'M': { - // Support specifying the limit as a percentage of - // installed physical RAM. - size_t len = strlen(optarg); - if (len > 0 && optarg[len - 1] == '%') { - optarg[len - 1] = '\0'; - hardware_memlimit_set_percentage( - str_to_uint64( - "memory%", optarg, 1, 100)); - } else { - // On 32-bit systems, SIZE_MAX would make more - // sense than UINT64_MAX. But use UINT64_MAX - // still so that scripts that assume > 4 GiB - // values don't break. - hardware_memlimit_set(str_to_uint64( - "memory", optarg, - 0, UINT64_MAX)); - } + // --memlimit-compress + case OPT_MEM_COMPRESS: + parse_memlimit("memlimit-compress", + "memlimit-compress%", optarg, + true, false); + break; + // --memlimit-decompress + case OPT_MEM_DECOMPRESS: + parse_memlimit("memlimit-decompress", + "memlimit-decompress%", optarg, + false, true); + break; + + // --memlimit + case 'M': + parse_memlimit("memlimit", "memlimit%", optarg, + true, true); break; - } // --suffix case 'S': @@ -177,7 +207,7 @@ parse_real(args_info *args, int argc, char **argv) // --info-memory case OPT_INFO_MEMORY: // This doesn't return. - message_memlimit(); + hardware_memlimit_show(); // --help case 'h': @@ -367,6 +397,10 @@ parse_real(args_info *args, int argc, char **argv) break; + case OPT_NO_ADJUST: + opt_auto_adjust = false; + break; + default: message_try_help(); tuklib_exit(E_ERROR, E_ERROR, false); @@ -378,9 +412,9 @@ parse_real(args_info *args, int argc, char **argv) static void -parse_environment(args_info *args, char *argv0) +parse_environment(args_info *args, char *argv0, const char *varname) { - char *env = getenv("XZ_OPT"); + char *env = getenv(varname); if (env == NULL) return; @@ -409,8 +443,8 @@ parse_environment(args_info *args, char *argv0) if (++argc == my_min( INT_MAX, SIZE_MAX / sizeof(char *))) message_fatal(_("The environment variable " - "XZ_OPT contains too many " - "arguments")); + "%s contains too many " + "arguments"), varname); } } @@ -498,8 +532,9 @@ args_parse(args_info *args, int argc, char **argv) } } - // First the flags from environment - parse_environment(args, argv[0]); + // First the flags from the environment + parse_environment(args, argv[0], "XZ_DEFAULTS"); + parse_environment(args, argv[0], "XZ_OPT"); // Then from the command line parse_real(args, argc, argv); diff --git a/Build/source/extra/xz/src/xz/args.h b/Build/source/extra/xz/src/xz/args.h index ac5959b52ba..b23f4ef12c9 100644 --- a/Build/source/extra/xz/src/xz/args.h +++ b/Build/source/extra/xz/src/xz/args.h @@ -37,6 +37,6 @@ extern bool opt_keep_original; // extern bool opt_recursive; extern bool opt_robot; -extern const char *stdin_filename; +extern const char stdin_filename[]; extern void args_parse(args_info *args, int argc, char **argv); diff --git a/Build/source/extra/xz/src/xz/coder.c b/Build/source/extra/xz/src/xz/coder.c index 8d882d26871..5182dddc3ff 100644 --- a/Build/source/extra/xz/src/xz/coder.c +++ b/Build/source/extra/xz/src/xz/coder.c @@ -22,8 +22,8 @@ enum coder_init_ret { enum operation_mode opt_mode = MODE_COMPRESS; - enum format_type opt_format = FORMAT_AUTO; +bool opt_auto_adjust = true; /// Stream used to communicate with liblzma @@ -42,14 +42,6 @@ static size_t filters_count = 0; /// Number of the preset (0-9) static size_t preset_number = 6; -/// True if we should auto-adjust the compression settings to use less memory -/// if memory usage limit is too low for the original settings. -static bool auto_adjust = true; - -/// Indicate if no preset has been explicitly given. In that case, if we need -/// to auto-adjust for lower memory usage, we won't print a warning. -static bool preset_default = true; - /// If a preset is used (no custom filter chain) and preset_extreme is true, /// a significantly slower compression is used to achieve slightly better /// compression ratio. @@ -75,7 +67,15 @@ extern void coder_set_preset(size_t new_preset) { preset_number = new_preset; - preset_default = false; + + // Setting a preset makes us forget a possibly defined custom + // filter chain. + while (filters_count > 0) { + --filters_count; + free(filters[filters_count].options); + filters[filters_count].options = NULL; + } + return; } @@ -145,8 +145,6 @@ coder_set_compression_settings(void) ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2; filters[0].options = &opt_lzma; filters_count = 1; - } else { - preset_default = false; } // Terminate the filter options array. @@ -173,7 +171,7 @@ coder_set_compression_settings(void) // If using --format=raw, we can be decoding. The memusage function // also validates the filter chain and the options used for the // filters. - const uint64_t memory_limit = hardware_memlimit_get(); + const uint64_t memory_limit = hardware_memlimit_get(opt_mode); uint64_t memory_usage; if (opt_mode == MODE_COMPRESS) memory_usage = lzma_raw_encoder_memusage(filters); @@ -186,12 +184,19 @@ coder_set_compression_settings(void) // Print memory usage info before possible dictionary // size auto-adjusting. message_mem_needed(V_DEBUG, memory_usage); + if (opt_mode == MODE_COMPRESS) { + const uint64_t decmem = lzma_raw_decoder_memusage(filters); + if (decmem != UINT64_MAX) + message(V_DEBUG, _("Decompression will need " + "%s MiB of memory."), uint64_to_str( + round_up_to_mib(decmem), 0)); + } if (memory_usage > memory_limit) { // If --no-auto-adjust was used or we didn't find LZMA1 or // LZMA2 as the last filter, give an error immediately. // --format=raw implies --no-auto-adjust. - if (!auto_adjust || opt_format == FORMAT_RAW) + if (!opt_auto_adjust || opt_format == FORMAT_RAW) memlimit_too_small(memory_usage); assert(opt_mode == MODE_COMPRESS); @@ -239,18 +244,15 @@ coder_set_compression_settings(void) } // Tell the user that we decreased the dictionary size. - // However, omit the message if no preset or custom chain - // was given. FIXME: Always warn? - if (!preset_default) - message(V_WARNING, _("Adjusted LZMA%c dictionary size " - "from %s MiB to %s MiB to not exceed " - "the memory usage limit of %s MiB"), - filters[i].id == LZMA_FILTER_LZMA2 - ? '2' : '1', - uint64_to_str(orig_dict_size >> 20, 0), - uint64_to_str(opt->dict_size >> 20, 1), - uint64_to_str(round_up_to_mib( - memory_limit), 2)); + message(V_WARNING, _("Adjusted LZMA%c dictionary size " + "from %s MiB to %s MiB to not exceed " + "the memory usage limit of %s MiB"), + filters[i].id == LZMA_FILTER_LZMA2 + ? '2' : '1', + uint64_to_str(orig_dict_size >> 20, 0), + uint64_to_str(opt->dict_size >> 20, 1), + uint64_to_str(round_up_to_mib( + memory_limit), 2)); } /* @@ -410,12 +412,14 @@ coder_init(file_pair *pair) case FORMAT_XZ: ret = lzma_stream_decoder(&strm, - hardware_memlimit_get(), flags); + hardware_memlimit_get( + MODE_DECOMPRESS), flags); break; case FORMAT_LZMA: ret = lzma_alone_decoder(&strm, - hardware_memlimit_get()); + hardware_memlimit_get( + MODE_DECOMPRESS)); break; case FORMAT_RAW: diff --git a/Build/source/extra/xz/src/xz/coder.h b/Build/source/extra/xz/src/xz/coder.h index 0d3af6ad449..4626466f39c 100644 --- a/Build/source/extra/xz/src/xz/coder.h +++ b/Build/source/extra/xz/src/xz/coder.h @@ -37,6 +37,10 @@ extern enum operation_mode opt_mode; /// This is set in args.c. extern enum format_type opt_format; +/// If true, the compression settings are automatically adjusted down if +/// they exceed the memory usage limit. +extern bool opt_auto_adjust; + /// Set the integrity check type used when compressing extern void coder_set_check(lzma_check check); diff --git a/Build/source/extra/xz/src/xz/file_io.c b/Build/source/extra/xz/src/xz/file_io.c index a78002eba55..09edcca69eb 100644 --- a/Build/source/extra/xz/src/xz/file_io.c +++ b/Build/source/extra/xz/src/xz/file_io.c @@ -370,15 +370,16 @@ io_open_src_real(file_pair *pair) } #ifdef O_NOFOLLOW - // Give an understandable error message in if reason + // Give an understandable error message if the reason // for failing was that the file was a symbolic link. // // Note that at least Linux, OpenBSD, Solaris, and Darwin - // use ELOOP to indicate if O_NOFOLLOW was the reason + // use ELOOP to indicate that O_NOFOLLOW was the reason // that open() failed. Because there may be // directories in the pathname, ELOOP may occur also // because of a symlink loop in the directory part. - // So ELOOP doesn't tell us what actually went wrong. + // So ELOOP doesn't tell us what actually went wrong, + // and this stupidity went into POSIX-1.2008 too. // // FreeBSD associates EMLINK with O_NOFOLLOW and // Tru64 uses ENOTSUP. We use these directly here @@ -396,10 +397,10 @@ io_open_src_real(file_pair *pair) was_symlink = true; # elif defined(__NetBSD__) - // FIXME? As of 2008-11-20, NetBSD doesn't document what - // errno is used with O_NOFOLLOW. It seems to be EFTYPE, - // but since it isn't documented, it may be wrong to rely - // on it here. + // As of 2010-09-05, NetBSD doesn't document what errno is + // used with O_NOFOLLOW. It is EFTYPE though, and I + // understood that is very unlikely to change even though + // it is undocumented. if (errno == EFTYPE) was_symlink = true; @@ -456,15 +457,14 @@ io_open_src_real(file_pair *pair) goto error; } - if (reg_files_only) { - if (!S_ISREG(pair->src_st.st_mode)) { - message_warning(_("%s: Not a regular file, " - "skipping"), pair->src_name); - goto error; - } + if (reg_files_only && !S_ISREG(pair->src_st.st_mode)) { + message_warning(_("%s: Not a regular file, skipping"), + pair->src_name); + goto error; + } - // These are meaningless on Windows. #ifndef TUKLIB_DOSLIKE + if (reg_files_only && !opt_force) { if (pair->src_st.st_mode & (S_ISUID | S_ISGID)) { // gzip rejects setuid and setgid files even // when --force was used. bzip2 doesn't check @@ -494,8 +494,8 @@ io_open_src_real(file_pair *pair) "skipping"), pair->src_name); goto error; } -#endif } +#endif return false; diff --git a/Build/source/extra/xz/src/xz/hardware.c b/Build/source/extra/xz/src/xz/hardware.c index 74742fcec87..a4733c27e11 100644 --- a/Build/source/extra/xz/src/xz/hardware.c +++ b/Build/source/extra/xz/src/xz/hardware.c @@ -18,8 +18,11 @@ /// the --threads=NUM command line option. static uint32_t threadlimit; -/// Memory usage limit -static uint64_t memlimit; +/// Memory usage limit for compression +static uint64_t memlimit_compress; + +/// Memory usage limit for decompression +static uint64_t memlimit_decompress; /// Total amount of physical RAM static uint64_t total_ram; @@ -49,50 +52,79 @@ hardware_threadlimit_get(void) extern void -hardware_memlimit_set(uint64_t new_memlimit) +hardware_memlimit_set(uint64_t new_memlimit, + bool set_compress, bool set_decompress, bool is_percentage) { - if (new_memlimit != 0) { - memlimit = new_memlimit; - } else { - // The default depends on the amount of RAM but so that - // on "low-memory" systems the relative limit is higher - // to make it more likely that files created with "xz -9" - // will still decompress without overriding the limit - // manually. - // - // If 40 % of RAM is 80 MiB or more, use 40 % of RAM as - // the limit. - memlimit = 40 * total_ram / 100; - if (memlimit < UINT64_C(80) * 1024 * 1024) { - // If 80 % of RAM is less than 80 MiB, - // use 80 % of RAM as the limit. - memlimit = 80 * total_ram / 100; - if (memlimit > UINT64_C(80) * 1024 * 1024) { - // Otherwise use 80 MiB as the limit. - memlimit = UINT64_C(80) * 1024 * 1024; - } - } + if (is_percentage) { + assert(new_memlimit > 0); + assert(new_memlimit <= 100); + new_memlimit = (uint32_t)new_memlimit * total_ram / 100; } + if (set_compress) + memlimit_compress = new_memlimit; + + if (set_decompress) + memlimit_decompress = new_memlimit; + return; } -extern void -hardware_memlimit_set_percentage(uint32_t percentage) +extern uint64_t +hardware_memlimit_get(enum operation_mode mode) { - assert(percentage > 0); - assert(percentage <= 100); + // Zero is a special value that indicates the default. Currently + // the default simply disables the limit. Once there is threading + // support, this might be a little more complex, because there will + // probably be a special case where a user asks for "optimal" number + // of threads instead of a specific number (this might even become + // the default mode). Each thread may use a significant amount of + // memory. When there are no memory usage limits set, we need some + // default soft limit for calculating the "optimal" number of + // threads. + const uint64_t memlimit = mode == MODE_COMPRESS + ? memlimit_compress : memlimit_decompress; + return memlimit != 0 ? memlimit : UINT64_MAX; +} + + +/// Helper for hardware_memlimit_show() to print one human-readable info line. +static void +memlimit_show(const char *str, uint64_t value) +{ + // The memory usage limit is considered to be disabled if value + // is 0 or UINT64_MAX. This might get a bit more complex once there + // is threading support. See the comment in hardware_memlimit_get(). + if (value == 0 || value == UINT64_MAX) + printf("%s %s\n", str, _("Disabled")); + else + printf("%s %s MiB (%s B)\n", str, + uint64_to_str(round_up_to_mib(value), 0), + uint64_to_str(value, 1)); - memlimit = percentage * total_ram / 100; return; } -extern uint64_t -hardware_memlimit_get(void) +extern void +hardware_memlimit_show(void) { - return memlimit; + if (opt_robot) { + printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n", total_ram, + memlimit_compress, memlimit_decompress); + } else { + // TRANSLATORS: Test with "xz --info-memory" to see if + // the alignment looks nice. + memlimit_show(_("Total amount of physical memory (RAM): "), + total_ram); + memlimit_show(_("Memory usage limit for compression: "), + memlimit_compress); + memlimit_show(_("Memory usage limit for decompression: "), + memlimit_decompress); + } + + tuklib_exit(E_SUCCESS, E_ERROR, message_verbosity_get() != V_SILENT); } @@ -106,7 +138,7 @@ hardware_init(void) total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024; // Set the defaults. - hardware_memlimit_set(0); + hardware_memlimit_set(0, true, true, false); hardware_threadlimit_set(0); return; } diff --git a/Build/source/extra/xz/src/xz/hardware.h b/Build/source/extra/xz/src/xz/hardware.h index b2cf34cbc8c..bed952b07d2 100644 --- a/Build/source/extra/xz/src/xz/hardware.h +++ b/Build/source/extra/xz/src/xz/hardware.h @@ -23,13 +23,16 @@ extern void hardware_threadlimit_set(uint32_t threadlimit); extern uint32_t hardware_threadlimit_get(void); -/// Set custom memory usage limit. This is used for both encoding and -/// decoding. Zero indicates resetting the limit back to defaults. -extern void hardware_memlimit_set(uint64_t memlimit); - -/// Set custom memory usage limit as a percentage of installed RAM. -/// The percentage must be in the range [1, 100]. -extern void hardware_memlimit_set_percentage(uint32_t percentage); - -/// Get the current memory usage limit. -extern uint64_t hardware_memlimit_get(void); +/// Set the memory usage limit. There are separate limits for compression +/// and decompression (the latter includes also --list), one or both can +/// be set with a single call to this function. Zero indicates resetting +/// the limit back to the defaults. The limit can also be set as a percentage +/// of installed RAM; the percentage must be in the range [1, 100]. +extern void hardware_memlimit_set(uint64_t new_memlimit, + bool set_compress, bool set_decompress, bool is_percentage); + +/// Get the current memory usage limit for compression or decompression. +extern uint64_t hardware_memlimit_get(enum operation_mode mode); + +/// Display the amount of RAM and memory usage limits and exit. +extern void hardware_memlimit_show(void) lzma_attribute((noreturn)); diff --git a/Build/source/extra/xz/src/xz/list.c b/Build/source/extra/xz/src/xz/list.c index dda7c9bd782..1c93718b138 100644 --- a/Build/source/extra/xz/src/xz/list.c +++ b/Build/source/extra/xz/src/xz/list.c @@ -49,31 +49,43 @@ typedef struct { uint64_t memusage; /// The filter chain of this Block in human-readable form - const char *filter_chain; + char filter_chain[FILTERS_STR_SIZE]; } block_header_info; /// Check ID to string mapping static const char check_names[LZMA_CHECK_ID_MAX + 1][12] = { - "None", + // TRANSLATORS: Indicates that there is no integrity check. + // This string is used in tables, so the width must not + // exceed ten columns with a fixed-width font. + N_("None"), "CRC32", - "Unknown-2", - "Unknown-3", + // TRANSLATORS: Indicates that integrity check name is not known, + // but the Check ID is known (here 2). This and other "Unknown-N" + // strings are used in tables, so the width must not exceed ten + // columns with a fixed-width font. It's OK to omit the dash if + // you need space for one extra letter, but don't use spaces. + N_("Unknown-2"), + N_("Unknown-3"), "CRC64", - "Unknown-5", - "Unknown-6", - "Unknown-7", - "Unknown-8", - "Unknown-9", + N_("Unknown-5"), + N_("Unknown-6"), + N_("Unknown-7"), + N_("Unknown-8"), + N_("Unknown-9"), "SHA-256", - "Unknown-11", - "Unknown-12", - "Unknown-13", - "Unknown-14", - "Unknown-15", + N_("Unknown-11"), + N_("Unknown-12"), + N_("Unknown-13"), + N_("Unknown-14"), + N_("Unknown-15"), }; +/// Buffer size for get_check_names(). This may be a bit ridiculous, +/// but at least it's enough if some language needs many multibyte chars. +#define CHECKS_STR_SIZE 1024 + /// Value of the Check field as hexadecimal string. /// This is set by parse_check_value(). @@ -203,7 +215,7 @@ parse_indexes(xz_file_info *xfi, file_pair *pair) pos -= index_size; // See how much memory we can use for decoding this Index. - uint64_t memlimit = hardware_memlimit_get(); + uint64_t memlimit = hardware_memlimit_get(MODE_LIST); uint64_t memused = 0; if (combined_index != NULL) { memused = lzma_index_memused(combined_index); @@ -442,7 +454,7 @@ parse_block_header(file_pair *pair, const lzma_index_iter *iter, xfi->memusage_max = bhi->memusage; // Convert the filter chain to human readable form. - bhi->filter_chain = message_filters_to_str(filters, false); + message_filters_to_str(bhi->filter_chain, filters, false); // Free the memory allocated by lzma_block_header_decode(). for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) @@ -533,7 +545,7 @@ parse_details(file_pair *pair, const lzma_index_iter *iter, /// \brief Get the compression ratio /// -/// This has slightly different format than that is used by in message.c. +/// This has slightly different format than that is used in message.c. static const char * get_ratio(uint64_t compressed_size, uint64_t uncompressed_size) { @@ -545,7 +557,7 @@ get_ratio(uint64_t compressed_size, uint64_t uncompressed_size) if (ratio > 9.999) return "---"; - static char buf[6]; + static char buf[16]; snprintf(buf, sizeof(buf), "%.3f", ratio); return buf; } @@ -553,19 +565,22 @@ get_ratio(uint64_t compressed_size, uint64_t uncompressed_size) /// \brief Get a comma-separated list of Check names /// +/// The check names are translated with gettext except when in robot mode. +/// +/// \param buf Buffer to hold the resulting string /// \param checks Bit mask of Checks to print /// \param space_after_comma /// It's better to not use spaces in table-like listings, /// but in more verbose formats a space after a comma /// is good for readability. -static const char * -get_check_names(uint32_t checks, bool space_after_comma) +static void +get_check_names(char buf[CHECKS_STR_SIZE], + uint32_t checks, bool space_after_comma) { assert(checks != 0); - static char buf[sizeof(check_names)]; char *pos = buf; - size_t left = sizeof(buf); + size_t left = CHECKS_STR_SIZE; const char *sep = space_after_comma ? ", " : ","; bool comma = false; @@ -573,12 +588,14 @@ get_check_names(uint32_t checks, bool space_after_comma) for (size_t i = 0; i <= LZMA_CHECK_ID_MAX; ++i) { if (checks & (UINT32_C(1) << i)) { my_snprintf(&pos, &left, "%s%s", - comma ? sep : "", check_names[i]); + comma ? sep : "", + opt_robot ? check_names[i] + : _(check_names[i])); comma = true; } } - return buf; + return; } @@ -588,26 +605,37 @@ print_info_basic(const xz_file_info *xfi, file_pair *pair) static bool headings_displayed = false; if (!headings_displayed) { headings_displayed = true; - // TRANSLATORS: These are column titles. From Strms (Streams) + // TRANSLATORS: These are column headings. From Strms (Streams) // to Ratio, the columns are right aligned. Check and Filename // are left aligned. If you need longer words, it's OK to - // use two lines here. Test with xz --list. + // use two lines here. Test with "xz -l foo.xz". puts(_("Strms Blocks Compressed Uncompressed Ratio " "Check Filename")); } - printf("%5s %7s %11s %11s %5s %-7s %s\n", - uint64_to_str(lzma_index_stream_count(xfi->idx), 0), - uint64_to_str(lzma_index_block_count(xfi->idx), 1), - uint64_to_nicestr(lzma_index_file_size(xfi->idx), - NICESTR_B, NICESTR_TIB, false, 2), - uint64_to_nicestr( - lzma_index_uncompressed_size(xfi->idx), - NICESTR_B, NICESTR_TIB, false, 3), - get_ratio(lzma_index_file_size(xfi->idx), - lzma_index_uncompressed_size(xfi->idx)), - get_check_names(lzma_index_checks(xfi->idx), false), - pair->src_name); + char checks[CHECKS_STR_SIZE]; + get_check_names(checks, lzma_index_checks(xfi->idx), false); + + const char *cols[7] = { + uint64_to_str(lzma_index_stream_count(xfi->idx), 0), + uint64_to_str(lzma_index_block_count(xfi->idx), 1), + uint64_to_nicestr(lzma_index_file_size(xfi->idx), + NICESTR_B, NICESTR_TIB, false, 2), + uint64_to_nicestr(lzma_index_uncompressed_size(xfi->idx), + NICESTR_B, NICESTR_TIB, false, 3), + get_ratio(lzma_index_file_size(xfi->idx), + lzma_index_uncompressed_size(xfi->idx)), + checks, + pair->src_name, + }; + printf("%*s %*s %*s %*s %*s %-*s %s\n", + tuklib_mbstr_fw(cols[0], 5), cols[0], + tuklib_mbstr_fw(cols[1], 7), cols[1], + tuklib_mbstr_fw(cols[2], 11), cols[2], + tuklib_mbstr_fw(cols[3], 11), cols[3], + tuklib_mbstr_fw(cols[4], 5), cols[4], + tuklib_mbstr_fw(cols[5], 7), cols[5], + cols[6]); return false; } @@ -618,6 +646,9 @@ print_adv_helper(uint64_t stream_count, uint64_t block_count, uint64_t compressed_size, uint64_t uncompressed_size, uint32_t checks, uint64_t stream_padding) { + char checks_str[CHECKS_STR_SIZE]; + get_check_names(checks_str, checks, true); + printf(_(" Streams: %s\n"), uint64_to_str(stream_count, 0)); printf(_(" Blocks: %s\n"), @@ -630,8 +661,7 @@ print_adv_helper(uint64_t stream_count, uint64_t block_count, NICESTR_B, NICESTR_TIB, true, 0)); printf(_(" Ratio: %s\n"), get_ratio(compressed_size, uncompressed_size)); - printf(_(" Check: %s\n"), - get_check_names(checks, true)); + printf(_(" Check: %s\n"), checks_str); printf(_(" Stream padding: %s\n"), uint64_to_nicestr(stream_padding, NICESTR_B, NICESTR_TIB, true, 0)); @@ -656,6 +686,10 @@ print_info_adv(xz_file_info *xfi, file_pair *pair) uint32_t check_max = 0; // Print information about the Streams. + // + // TRANSLATORS: The second line is column headings. All except + // Check are right aligned; Check is left aligned. Test with + // "xz -lv foo.xz". puts(_(" Streams:\n Stream Blocks" " CompOffset UncompOffset" " CompSize UncompSize Ratio" @@ -665,21 +699,32 @@ print_info_adv(xz_file_info *xfi, file_pair *pair) lzma_index_iter_init(&iter, xfi->idx); while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) { - printf(" %6s %9s %15s %15s ", - uint64_to_str(iter.stream.number, 0), - uint64_to_str(iter.stream.block_count, 1), - uint64_to_str( - iter.stream.compressed_offset, 2), - uint64_to_str( - iter.stream.uncompressed_offset, 3)); - printf("%15s %15s %5s %-10s %7s\n", - uint64_to_str(iter.stream.compressed_size, 0), - uint64_to_str( - iter.stream.uncompressed_size, 1), - get_ratio(iter.stream.compressed_size, - iter.stream.uncompressed_size), - check_names[iter.stream.flags->check], - uint64_to_str(iter.stream.padding, 2)); + const char *cols1[4] = { + uint64_to_str(iter.stream.number, 0), + uint64_to_str(iter.stream.block_count, 1), + uint64_to_str(iter.stream.compressed_offset, 2), + uint64_to_str(iter.stream.uncompressed_offset, 3), + }; + printf(" %*s %*s %*s %*s ", + tuklib_mbstr_fw(cols1[0], 6), cols1[0], + tuklib_mbstr_fw(cols1[1], 9), cols1[1], + tuklib_mbstr_fw(cols1[2], 15), cols1[2], + tuklib_mbstr_fw(cols1[3], 15), cols1[3]); + + const char *cols2[5] = { + uint64_to_str(iter.stream.compressed_size, 0), + uint64_to_str(iter.stream.uncompressed_size, 1), + get_ratio(iter.stream.compressed_size, + iter.stream.uncompressed_size), + _(check_names[iter.stream.flags->check]), + uint64_to_str(iter.stream.padding, 2), + }; + printf("%*s %*s %*s %-*s %*s\n", + tuklib_mbstr_fw(cols2[0], 15), cols2[0], + tuklib_mbstr_fw(cols2[1], 15), cols2[1], + tuklib_mbstr_fw(cols2[2], 5), cols2[2], + tuklib_mbstr_fw(cols2[3], 10), cols2[3], + tuklib_mbstr_fw(cols2[4], 7), cols2[4]); // Update the maximum Check size. if (lzma_check_size(iter.stream.flags->check) > check_max) @@ -698,15 +743,24 @@ print_info_adv(xz_file_info *xfi, file_pair *pair) // Calculate the width of the CheckVal field. const int checkval_width = my_max(8, 2 * check_max); - // Print the headings. + // TRANSLATORS: The second line is column headings. All + // except Check are right aligned; Check is left aligned. printf(_(" Blocks:\n Stream Block" " CompOffset UncompOffset" " TotalSize UncompSize Ratio Check")); - if (detailed) - printf(_(" %-*s Header Flags CompSize" - " MemUsage Filters"), - checkval_width, _("CheckVal")); + if (detailed) { + // TRANSLATORS: These are additional column headings + // for the most verbose listing mode. CheckVal + // (Check value), Flags, and Filters are left aligned. + // Header (Block Header Size), CompSize, and MemUsage + // are right aligned. %*s is replaced with 0-120 + // spaces to make the CheckVal column wide enough. + // Test with "xz -lvv foo.xz". + printf(_(" CheckVal %*s Header Flags " + "CompSize MemUsage Filters"), + checkval_width - 8, ""); + } putchar('\n'); @@ -717,41 +771,63 @@ print_info_adv(xz_file_info *xfi, file_pair *pair) if (detailed && parse_details(pair, &iter, &bhi, xfi)) return true; - printf(" %6s %9s %15s %15s ", + const char *cols1[4] = { uint64_to_str(iter.stream.number, 0), uint64_to_str( iter.block.number_in_stream, 1), uint64_to_str( iter.block.compressed_file_offset, 2), uint64_to_str( - iter.block.uncompressed_file_offset, - 3)); - printf("%15s %15s %5s %-*s", + iter.block.uncompressed_file_offset, 3) + }; + printf(" %*s %*s %*s %*s ", + tuklib_mbstr_fw(cols1[0], 6), cols1[0], + tuklib_mbstr_fw(cols1[1], 9), cols1[1], + tuklib_mbstr_fw(cols1[2], 15), cols1[2], + tuklib_mbstr_fw(cols1[3], 15), cols1[3]); + + const char *cols2[4] = { uint64_to_str(iter.block.total_size, 0), uint64_to_str(iter.block.uncompressed_size, 1), get_ratio(iter.block.total_size, iter.block.uncompressed_size), - detailed ? 11 : 1, - check_names[iter.stream.flags->check]); + _(check_names[iter.stream.flags->check]) + }; + printf("%*s %*s %*s %-*s", + tuklib_mbstr_fw(cols2[0], 15), cols2[0], + tuklib_mbstr_fw(cols2[1], 15), cols2[1], + tuklib_mbstr_fw(cols2[2], 5), cols2[2], + tuklib_mbstr_fw(cols2[3], detailed ? 11 : 1), + cols2[3]); if (detailed) { - // Show MiB for memory usage, because it - // is the only size which is not in bytes. const lzma_vli compressed_size = iter.block.unpadded_size - bhi.header_size - lzma_check_size( iter.stream.flags->check); - printf("%-*s %6s %-5s %15s %7s MiB %s", - checkval_width, check_value, + + const char *cols3[6] = { + check_value, uint64_to_str(bhi.header_size, 0), bhi.flags, uint64_to_str(compressed_size, 1), uint64_to_str( round_up_to_mib(bhi.memusage), 2), - bhi.filter_chain); + bhi.filter_chain + }; + // Show MiB for memory usage, because it + // is the only size which is not in bytes. + printf("%-*s %*s %-5s %*s %*s MiB %s", + checkval_width, cols3[0], + tuklib_mbstr_fw(cols3[1], 6), cols3[1], + cols3[2], + tuklib_mbstr_fw(cols3[3], 15), + cols3[3], + tuklib_mbstr_fw(cols3[4], 7), cols3[4], + cols3[5]); } putchar('\n'); @@ -772,6 +848,9 @@ print_info_adv(xz_file_info *xfi, file_pair *pair) static bool print_info_robot(xz_file_info *xfi, file_pair *pair) { + char checks[CHECKS_STR_SIZE]; + get_check_names(checks, lzma_index_checks(xfi->idx), false); + printf("name\t%s\n", pair->src_name); printf("file\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 @@ -782,7 +861,7 @@ print_info_robot(xz_file_info *xfi, file_pair *pair) lzma_index_uncompressed_size(xfi->idx), get_ratio(lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx)), - get_check_names(lzma_index_checks(xfi->idx), false), + checks, xfi->stream_padding); if (message_verbosity_get() >= V_VERBOSE) { @@ -880,6 +959,10 @@ print_totals_basic(void) line[sizeof(line) - 1] = '\0'; puts(line); + // Get the check names. + char checks[CHECKS_STR_SIZE]; + get_check_names(checks, totals.checks, false); + // Print the totals except the file count, which needs // special handling. printf("%5s %7s %11s %11s %5s %-7s ", @@ -891,16 +974,16 @@ print_totals_basic(void) NICESTR_B, NICESTR_TIB, false, 3), get_ratio(totals.compressed_size, totals.uncompressed_size), - get_check_names(totals.checks, false)); + checks); // Since we print totals only when there are at least two files, // the English message will always use "%s files". But some other // languages need different forms for different plurals so we - // have to translate this string still. + // have to translate this with ngettext(). // - // TRANSLATORS: This simply indicates the number of files shown - // by --list even though the format string uses %s. - printf(N_("%s file", "%s files\n", + // TRANSLATORS: %s is an integer. Only the plural form of this + // message is used (e.g. "2 files"). Test with "xz -l foo.xz bar.xz". + printf(ngettext("%s file\n", "%s files\n", totals.files <= ULONG_MAX ? totals.files : (totals.files % 1000000) + 1000000), uint64_to_str(totals.files, 0)); @@ -934,6 +1017,9 @@ print_totals_adv(void) static void print_totals_robot(void) { + char checks[CHECKS_STR_SIZE]; + get_check_names(checks, totals.checks, false); + printf("totals\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s\t%" PRIu64 "\t%" PRIu64, totals.streams, @@ -942,7 +1028,7 @@ print_totals_robot(void) totals.uncompressed_size, get_ratio(totals.compressed_size, totals.uncompressed_size), - get_check_names(totals.checks, false), + checks, totals.stream_padding, totals.files); diff --git a/Build/source/extra/xz/src/xz/message.c b/Build/source/extra/xz/src/xz/message.c index 6eed5f7e690..38cce4a1b0f 100644 --- a/Build/source/extra/xz/src/xz/message.c +++ b/Build/source/extra/xz/src/xz/message.c @@ -77,6 +77,17 @@ static uint64_t start_time; // gettimeofday(). #ifdef SIGALRM +const int message_progress_sigs[] = { + SIGALRM, +#ifdef SIGINFO + SIGINFO, +#endif +#ifdef SIGUSR1 + SIGUSR1, +#endif + 0 +}; + /// The signal handler for SIGALRM sets this to true. It is set back to false /// once the progress message has been updated. static volatile sig_atomic_t progress_needs_updating = false; @@ -142,34 +153,15 @@ message_init(void) */ #ifdef SIGALRM - // DJGPP lacks SA_RESTART, but it shouldn't give EINTR - // in most places either. -# if defined(__DJGPP__) && !defined(SA_RESTART) -# define SA_RESTART 0 -# endif - // Establish the signal handlers which set a flag to tell us that - // progress info should be updated. Since these signals don't - // require any quick action, we set SA_RESTART. That way we don't - // need to block them either in signals_block() to keep stdio - // functions from getting EINTR. - static const int sigs[] = { - SIGALRM, -#ifdef SIGINFO - SIGINFO, -#endif -#ifdef SIGUSR1 - SIGUSR1, -#endif - }; - + // progress info should be updated. struct sigaction sa; sigemptyset(&sa.sa_mask); - sa.sa_flags = SA_RESTART; + sa.sa_flags = 0; sa.sa_handler = &progress_signal_handler; - for (size_t i = 0; i < ARRAY_SIZE(sigs); ++i) - if (sigaction(sigs[i], &sa, NULL)) + for (size_t i = 0; message_progress_sigs[i] != 0; ++i) + if (sigaction(message_progress_sigs[i], &sa, NULL)) message_signal_handler(); #endif @@ -321,7 +313,8 @@ progress_percentage(uint64_t in_pos) double percentage = (double)(in_pos) / (double)(expected_in_size) * 99.9; - static char buf[sizeof("99.9 %")]; + // Use big enough buffer to hold e.g. a multibyte decimal point. + static char buf[16]; snprintf(buf, sizeof(buf), "%.1f %%", percentage); return buf; @@ -333,12 +326,8 @@ progress_percentage(uint64_t in_pos) static const char * progress_sizes(uint64_t compressed_pos, uint64_t uncompressed_pos, bool final) { - // This is enough to hold sizes up to about 99 TiB if thousand - // separator is used, or about 1 PiB without thousand separator. - // After that the progress indicator will look a bit silly, since - // the compression ratio no longer fits with three decimal places. - static char buf[36]; - + // Use big enough buffer to hold e.g. a multibyte thousand separators. + static char buf[128]; char *pos = buf; size_t left = sizeof(buf); @@ -402,7 +391,8 @@ progress_speed(uint64_t uncompressed_pos, uint64_t elapsed) // - 9.9 KiB/s // - 99 KiB/s // - 999 KiB/s - static char buf[sizeof("999 GiB/s")]; + // Use big enough buffer to hold e.g. a multibyte decimal point. + static char buf[16]; snprintf(buf, sizeof(buf), "%.*f %s", speed > 9.9 ? 0 : 1, speed, unit[unit_index]); return buf; @@ -588,12 +578,19 @@ message_progress_update(void) // Print the actual progress message. The idea is that there is at // least three spaces between the fields in typical situations, but // even in rare situations there is at least one space. - fprintf(stderr, "\r %6s %35s %9s %10s %10s\r", + const char *cols[5] = { progress_percentage(in_pos), progress_sizes(compressed_pos, uncompressed_pos, false), progress_speed(uncompressed_pos, elapsed), progress_time(elapsed), - progress_remaining(in_pos, elapsed)); + progress_remaining(in_pos, elapsed), + }; + fprintf(stderr, "\r %*s %*s %*s %10s %10s\r", + tuklib_mbstr_fw(cols[0], 6), cols[0], + tuklib_mbstr_fw(cols[1], 35), cols[1], + tuklib_mbstr_fw(cols[2], 9), cols[2], + cols[3], + cols[4]); #ifdef SIGALRM // Updating the progress info was finished. Reset @@ -663,12 +660,19 @@ progress_flush(bool finished) // statistics are printed in the same format as the progress // indicator itself. if (progress_automatic) { - fprintf(stderr, "\r %6s %35s %9s %10s %10s\n", + const char *cols[5] = { finished ? "100 %" : progress_percentage(in_pos), progress_sizes(compressed_pos, uncompressed_pos, true), progress_speed(uncompressed_pos, elapsed), progress_time(elapsed), - finished ? "" : progress_remaining(in_pos, elapsed)); + finished ? "" : progress_remaining(in_pos, elapsed), + }; + fprintf(stderr, "\r %*s %*s %*s %10s %10s\n", + tuklib_mbstr_fw(cols[0], 6), cols[0], + tuklib_mbstr_fw(cols[1], 35), cols[1], + tuklib_mbstr_fw(cols[2], 9), cols[2], + cols[3], + cols[4]); } else { // The filename is always printed. fprintf(stderr, "%s: ", filename); @@ -829,10 +833,13 @@ message_strm(lzma_ret code) case LZMA_STREAM_END: case LZMA_GET_CHECK: case LZMA_PROG_ERROR: - return _("Internal error (bug)"); + // Without "default", compiler will warn if new constants + // are added to lzma_ret, it is not too easy to forget to + // add the new constants to this function. + break; } - return NULL; + return _("Internal error (bug)"); } @@ -848,13 +855,15 @@ message_mem_needed(enum message_verbosity v, uint64_t memusage) // the user might need to +1 MiB to get high enough limit.) memusage = round_up_to_mib(memusage); + // With US-ASCII: // 2^64 with thousand separators + " MiB" suffix + '\0' = 26 + 4 + 1 - char memlimitstr[32]; + // But there may be multibyte chars so reserve enough space. + char memlimitstr[128]; // Show the memory usage limit as MiB unless it is less than 1 MiB. // This way it's easy to notice errors where one has typed // --memory=123 instead of --memory=123MiB. - uint64_t memlimit = hardware_memlimit_get(); + uint64_t memlimit = hardware_memlimit_get(opt_mode); if (memlimit < (UINT32_C(1) << 20)) { snprintf(memlimitstr, sizeof(memlimitstr), "%s B", uint64_to_str(memlimit, 1)); @@ -895,13 +904,12 @@ uint32_to_optstr(uint32_t num) } -extern const char * -message_filters_to_str(const lzma_filter *filters, bool all_known) +extern void +message_filters_to_str(char buf[FILTERS_STR_SIZE], + const lzma_filter *filters, bool all_known) { - static char buf[512]; - char *pos = buf; - size_t left = sizeof(buf); + size_t left = FILTERS_STR_SIZE; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { // Add the dashes for the filter option. A space is @@ -1025,7 +1033,7 @@ message_filters_to_str(const lzma_filter *filters, bool all_known) } } - return buf; + return; } @@ -1035,8 +1043,9 @@ message_filters_show(enum message_verbosity v, const lzma_filter *filters) if (v > verbosity) return; - fprintf(stderr, _("%s: Filter chain: %s\n"), progname, - message_filters_to_str(filters, true)); + char buf[FILTERS_STR_SIZE]; + message_filters_to_str(buf, filters, true); + fprintf(stderr, _("%s: Filter chain: %s\n"), progname, buf); return; } @@ -1053,27 +1062,12 @@ message_try_help(void) extern void -message_memlimit(void) -{ - if (opt_robot) - printf("%" PRIu64 "\n", hardware_memlimit_get()); - else - printf(_("%s MiB (%s bytes)\n"), - uint64_to_str( - round_up_to_mib(hardware_memlimit_get()), 0), - uint64_to_str(hardware_memlimit_get(), 1)); - - tuklib_exit(E_SUCCESS, E_ERROR, verbosity != V_SILENT); -} - - -extern void message_version(void) { // It is possible that liblzma version is different than the command // line tool version, so print both. if (opt_robot) { - printf("XZ_VERSION=%d\nLIBLZMA_VERSION=%d\n", + printf("XZ_VERSION=%" PRIu32 "\nLIBLZMA_VERSION=%" PRIu32 "\n", LZMA_VERSION, lzma_version_number()); } else { printf("xz (" PACKAGE_NAME ") " LZMA_VERSION_STRING "\n"); @@ -1091,8 +1085,11 @@ message_help(bool long_help) "Compress or decompress FILEs in the .xz format.\n\n"), progname); - puts(_("Mandatory arguments to long options are mandatory for " - "short options too.\n")); + // NOTE: The short help doesn't currently have options that + // take arguments. + if (long_help) + puts(_("Mandatory arguments to long options are mandatory " + "for short options too.\n")); if (long_help) puts(_(" Operation mode:\n")); @@ -1101,7 +1098,7 @@ message_help(bool long_help) " -z, --compress force compression\n" " -d, --decompress force decompression\n" " -t, --test test compressed file integrity\n" -" -l, --list list information about files")); +" -l, --list list information about .xz files")); if (long_help) puts(_("\n Operation modifiers:\n")); @@ -1115,32 +1112,40 @@ message_help(bool long_help) puts(_( " --no-sparse do not create sparse files when decompressing\n" " -S, --suffix=.SUF use the suffix `.SUF' on compressed files\n" -" --files=[FILE] read filenames to process from FILE; if FILE is\n" +" --files[=FILE] read filenames to process from FILE; if FILE is\n" " omitted, filenames are read from the standard input;\n" " filenames must be terminated with the newline character\n" -" --files0=[FILE] like --files but use the null character as terminator")); +" --files0[=FILE] like --files but use the null character as terminator")); if (long_help) { puts(_("\n Basic file format and compression options:\n")); puts(_( " -F, --format=FMT file format to encode or decode; possible values are\n" " `auto' (default), `xz', `lzma', and `raw'\n" -" -C, --check=CHECK integrity check type: `crc32', `crc64' (default),\n" -" `sha256', or `none' (use with caution)")); +" -C, --check=CHECK integrity check type: `none' (use with caution),\n" +" `crc32', `crc64' (default), or `sha256'")); } puts(_( -" -0 .. -9 compression preset; 0-2 fast compression, 3-5 good\n" -" compression, 6-9 excellent compression; default is 6")); +" -0 ... -9 compression preset; default is 6; take compressor *and*\n" +" decompressor memory usage into account before using 7-9!")); puts(_( -" -e, --extreme use more CPU time when encoding to increase compression\n" -" ratio without increasing memory usage of the decoder")); +" -e, --extreme try to improve compression ratio by using more CPU time;\n" +" does not affect decompressor memory requirements")); - if (long_help) + if (long_help) { puts(_( // xgettext:no-c-format -" -M, --memory=NUM use roughly NUM bytes of memory at maximum; 0 indicates\n" -" the default setting, which is 40 % of total RAM")); +" --memlimit-compress=LIMIT\n" +" --memlimit-decompress=LIMIT\n" +" -M, --memlimit=LIMIT\n" +" set memory usage limit for compression, decompression,\n" +" or both; LIMIT is in bytes, % of RAM, or 0 for defaults")); + + puts(_( +" --no-adjust if compression settings exceed the memory usage limit,\n" +" give an error instead of adjusting the settings downwards")); + } if (long_help) { puts(_( @@ -1148,11 +1153,15 @@ message_help(bool long_help) #if defined(HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) \ || defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) + // TRANSLATORS: The word "literal" in "literal context bits" + // means how many "context bits" to use when encoding + // literals. A literal is a single 8-bit byte. It doesn't + // mean "literally" here. puts(_( "\n" " --lzma1[=OPTS] LZMA1 or LZMA2; OPTS is a comma-separated list of zero or\n" " --lzma2[=OPTS] more of the following options (valid values; default):\n" -" preset=NUM reset options to preset number NUM (0-9)\n" +" preset=PRE reset options to a preset (0-9[e])\n" " dict=NUM dictionary size (4KiB - 1536MiB; 8MiB)\n" " lc=NUM number of literal context bits (0-4; 3)\n" " lp=NUM number of literal position bits (0-4; 0)\n" @@ -1165,9 +1174,9 @@ message_help(bool long_help) puts(_( "\n" -" --x86[=OPTS] x86 BCJ filter\n" +" --x86[=OPTS] x86 BCJ filter (32-bit and 64-bit)\n" " --powerpc[=OPTS] PowerPC BCJ filter (big endian only)\n" -" --ia64[=OPTS] IA64 (Itanium) BCJ filter\n" +" --ia64[=OPTS] IA-64 (Itanium) BCJ filter\n" " --arm[=OPTS] ARM BCJ filter (little endian only)\n" " --armthumb[=OPTS] ARM-Thumb BCJ filter (little endian only)\n" " --sparc[=OPTS] SPARC BCJ filter\n" @@ -1197,7 +1206,8 @@ message_help(bool long_help) " --robot use machine-parsable messages (useful for scripts)")); puts(""); puts(_( -" --info-memory display the memory usage limit and exit")); +" --info-memory display the total amount of RAM and the currently active\n" +" memory usage limits, and exit")); puts(_( " -h, --help display the short help (lists only the basic options)\n" " -H, --long-help display this long help and exit")); @@ -1212,15 +1222,6 @@ message_help(bool long_help) puts(_("\nWith no FILE, or when FILE is -, read standard input.\n")); - if (long_help) { - printf(_( -"On this system and configuration, this program will use a maximum of roughly\n" -"%s MiB RAM and "), uint64_to_str(round_up_to_mib(hardware_memlimit_get()), 0)); - printf(N_("one thread.\n\n", "%s threads.\n\n", - hardware_threadlimit_get()), - uint64_to_str(hardware_threadlimit_get(), 0)); - } - // TRANSLATORS: This message indicates the bug reporting address // for this package. Please add _another line_ saying // "Report translation bugs to <...>\n" with the email or WWW diff --git a/Build/source/extra/xz/src/xz/message.h b/Build/source/extra/xz/src/xz/message.h index aea4fdfd667..e3fca3cc42c 100644 --- a/Build/source/extra/xz/src/xz/message.h +++ b/Build/source/extra/xz/src/xz/message.h @@ -16,10 +16,14 @@ enum message_verbosity { V_ERROR, ///< Only error messages V_WARNING, ///< Errors and warnings V_VERBOSE, ///< Errors, warnings, and verbose statistics - V_DEBUG, ///< Debugging, FIXME remove? + V_DEBUG, ///< Very verbose }; +/// \brief Signals used for progress message handling +extern const int message_progress_sigs[]; + + /// \brief Initializes the message functions /// /// If an error occurs, this function doesn't return. @@ -86,15 +90,19 @@ extern const char *message_strm(lzma_ret code); extern void message_mem_needed(enum message_verbosity v, uint64_t memusage); +/// Buffer size for message_filters_to_str() +#define FILTERS_STR_SIZE 512 + + /// \brief Get the filter chain as a string /// +/// \param buf Pointer to caller allocated buffer to hold +/// the filter chain string /// \param filters Pointer to the filter chain /// \param all_known If true, all filter options are printed. /// If false, only the options that get stored /// into .xz headers are printed. -/// -/// \return Pointer to a statically allocated buffer. -extern const char *message_filters_to_str( +extern void message_filters_to_str(char buf[FILTERS_STR_SIZE], const lzma_filter *filters, bool all_known); @@ -107,10 +115,6 @@ extern void message_filters_show( extern void message_try_help(void); -/// Print the memory usage limit and exit. -extern void message_memlimit(void) lzma_attribute((noreturn)); - - /// Prints the version number to stdout and exits with exit status SUCCESS. extern void message_version(void) lzma_attribute((noreturn)); diff --git a/Build/source/extra/xz/src/xz/options.c b/Build/source/extra/xz/src/xz/options.c index 7186f24c49d..379a2e46dc0 100644 --- a/Build/source/extra/xz/src/xz/options.c +++ b/Build/source/extra/xz/src/xz/options.c @@ -346,24 +346,13 @@ options_lzma(const char *str) }; lzma_options_lzma *options = xmalloc(sizeof(lzma_options_lzma)); - *options = (lzma_options_lzma){ - .dict_size = LZMA_DICT_SIZE_DEFAULT, - .preset_dict = NULL, - .preset_dict_size = 0, - .lc = LZMA_LC_DEFAULT, - .lp = LZMA_LP_DEFAULT, - .pb = LZMA_PB_DEFAULT, - .mode = LZMA_MODE_NORMAL, - .nice_len = 64, - .mf = LZMA_MF_BT4, - .depth = 0, - }; + if (lzma_lzma_preset(options, LZMA_PRESET_DEFAULT)) + message_bug(); parse_options(str, opts, &set_lzma, options); if (options->lc + options->lp > LZMA_LCLP_MAX) - message_fatal(_("The sum of lc and lp must be at " - "maximum of 4")); + message_fatal(_("The sum of lc and lp must not exceed 4")); const uint32_t nice_len_min = options->mf & 0x0F; if (options->nice_len < nice_len_min) diff --git a/Build/source/extra/xz/src/xz/private.h b/Build/source/extra/xz/src/xz/private.h index 15136bfe878..6b01e51354e 100644 --- a/Build/source/extra/xz/src/xz/private.h +++ b/Build/source/extra/xz/src/xz/private.h @@ -25,6 +25,7 @@ #include "tuklib_gettext.h" #include "tuklib_progname.h" #include "tuklib_exit.h" +#include "tuklib_mbstr.h" #if defined(_WIN32) && !defined(__CYGWIN__) # define WIN32_LEAN_AND_MEAN diff --git a/Build/source/extra/xz/src/xz/signals.c b/Build/source/extra/xz/src/xz/signals.c index 66d65373322..4d6a9da3162 100644 --- a/Build/source/extra/xz/src/xz/signals.c +++ b/Build/source/extra/xz/src/xz/signals.c @@ -71,6 +71,12 @@ signals_init(void) for (size_t i = 0; i < ARRAY_SIZE(sigs); ++i) sigaddset(&hooked_signals, sigs[i]); +#ifdef SIGALRM + // Add also the signals from message.c to hooked_signals. + for (size_t i = 0; message_progress_sigs[i] != 0; ++i) + sigaddset(&hooked_signals, message_progress_sigs[i]); +#endif + struct sigaction sa; // All the signals that we handle we also blocked while the signal @@ -142,12 +148,19 @@ signals_exit(void) const int sig = exit_signal; if (sig != 0) { +#if defined(TUKLIB_DOSLIKE) || defined(__VMS) + // Don't raise(), set only exit status. This avoids + // printing unwanted message about SIGINT when the user + // presses C-c. + set_exit_status(E_ERROR); +#else struct sigaction sa; sa.sa_handler = SIG_DFL; sigfillset(&sa.sa_mask); sa.sa_flags = 0; sigaction(sig, &sa, NULL); raise(exit_signal); +#endif } return; diff --git a/Build/source/extra/xz/src/xz/suffix.c b/Build/source/extra/xz/src/xz/suffix.c index f2a2da2749b..c89f67fe853 100644 --- a/Build/source/extra/xz/src/xz/suffix.c +++ b/Build/source/extra/xz/src/xz/suffix.c @@ -21,10 +21,28 @@ static char *custom_suffix = NULL; -struct suffix_pair { - const char *compressed; - const char *uncompressed; -}; +/// \brief Test if the char is a directory separator +static bool +is_dir_sep(char c) +{ +#ifdef TUKLIB_DOSLIKE + return c == '/' || c == '\\' || c == ':'; +#else + return c == '/'; +#endif +} + + +/// \brief Test if the string contains a directory separator +static bool +has_dir_sep(const char *str) +{ +#ifdef TUKLIB_DOSLIKE + return strpbrk(str, "/\\:") != NULL; +#else + return strchr(str, '/') != NULL; +#endif +} /// \brief Checks if src_name has given compressed_suffix @@ -44,7 +62,8 @@ test_suffix(const char *suffix, const char *src_name, size_t src_len) // The filename must have at least one character in addition to // the suffix. src_name may contain path to the filename, so we // need to check for directory separator too. - if (src_len <= suffix_len || src_name[src_len - suffix_len - 1] == '/') + if (src_len <= suffix_len + || is_dir_sep(src_name[src_len - suffix_len - 1])) return 0; if (strcmp(suffix, src_name + src_len - suffix_len) == 0) @@ -61,7 +80,10 @@ test_suffix(const char *suffix, const char *src_name, size_t src_len) static char * uncompressed_name(const char *src_name, const size_t src_len) { - static const struct suffix_pair suffixes[] = { + static const struct { + const char *compressed; + const char *uncompressed; + } suffixes[] = { { ".xz", "" }, { ".txz", ".tar" }, // .txz abbreviation for .txt.gz is rare. { ".lzma", "" }, @@ -120,25 +142,25 @@ static char * compressed_name(const char *src_name, const size_t src_len) { // The order of these must match the order in args.h. - static const struct suffix_pair all_suffixes[][3] = { + static const char *const all_suffixes[][3] = { { - { ".xz", "" }, - { ".txz", ".tar" }, - { NULL, NULL } + ".xz", + ".txz", + NULL }, { - { ".lzma", "" }, - { ".tlz", ".tar" }, - { NULL, NULL } + ".lzma", + ".tlz", + NULL /* }, { - { ".gz", "" }, - { ".tgz", ".tar" }, - { NULL, NULL } + ".gz", + ".tgz", + NULL */ }, { // --format=raw requires specifying the suffix // manually or using stdout. - { NULL, NULL } + NULL } }; @@ -146,14 +168,22 @@ compressed_name(const char *src_name, const size_t src_len) assert(opt_format != FORMAT_AUTO); const size_t format = opt_format - 1; - const struct suffix_pair *const suffixes = all_suffixes[format]; + const char *const *suffixes = all_suffixes[format]; + + for (size_t i = 0; suffixes[i] != NULL; ++i) { + if (test_suffix(suffixes[i], src_name, src_len) != 0) { + message_warning(_("%s: File already has `%s' " + "suffix, skipping"), src_name, + suffixes[i]); + return NULL; + } + } - for (size_t i = 0; suffixes[i].compressed != NULL; ++i) { - if (test_suffix(suffixes[i].compressed, src_name, src_len) - != 0) { + if (custom_suffix != NULL) { + if (test_suffix(custom_suffix, src_name, src_len) != 0) { message_warning(_("%s: File already has `%s' " "suffix, skipping"), src_name, - suffixes[i].compressed); + custom_suffix); return NULL; } } @@ -168,7 +198,7 @@ compressed_name(const char *src_name, const size_t src_len) } const char *suffix = custom_suffix != NULL - ? custom_suffix : suffixes[0].compressed; + ? custom_suffix : suffixes[0]; const size_t suffix_len = strlen(suffix); char *dest_name = xmalloc(src_len + suffix_len + 1); @@ -199,9 +229,9 @@ suffix_get_dest_name(const char *src_name) extern void suffix_set(const char *suffix) { - // Empty suffix and suffixes having a slash are rejected. Such - // suffixes would break things later. - if (suffix[0] == '\0' || strchr(suffix, '/') != NULL) + // Empty suffix and suffixes having a directory separator are + // rejected. Such suffixes would break things later. + if (suffix[0] == '\0' || has_dir_sep(suffix)) message_fatal(_("%s: Invalid filename suffix"), optarg); // Replace the old custom_suffix (if any) with the new suffix. diff --git a/Build/source/extra/xz/src/xz/util.c b/Build/source/extra/xz/src/xz/util.c index deb5dcc2fd3..987b4430253 100644 --- a/Build/source/extra/xz/src/xz/util.c +++ b/Build/source/extra/xz/src/xz/util.c @@ -14,6 +14,13 @@ #include <stdarg.h> +/// Buffers for uint64_to_str() and uint64_to_nicestr() +static char bufs[4][128]; + +/// Thousand separator support in uint64_to_str() and uint64_to_nicestr() +static enum { UNKNOWN, WORKS, BROKEN } thousand = UNKNOWN; + + extern void * xrealloc(void *ptr, size_t size) { @@ -56,11 +63,17 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max) do { // Don't overflow. - if (result > (UINT64_MAX - 9) / 10) + if (result > UINT64_MAX / 10) goto error; result *= 10; - result += *value - '0'; + + // Another overflow check + const uint32_t add = *value - '0'; + if (UINT64_MAX - add < result) + goto error; + + result += add; ++value; } while (*value >= '0' && *value <= '9'); @@ -119,22 +132,28 @@ round_up_to_mib(uint64_t n) } -extern const char * -uint64_to_str(uint64_t value, uint32_t slot) +/// Check if thousand separator is supported. Run-time checking is easiest, +/// because it seems to be sometimes lacking even on POSIXish system. +static void +check_thousand_sep(uint32_t slot) { - // 2^64 with thousand separators is 26 bytes plus trailing '\0'. - static char bufs[4][32]; - - assert(slot < ARRAY_SIZE(bufs)); - - static enum { UNKNOWN, WORKS, BROKEN } thousand = UNKNOWN; if (thousand == UNKNOWN) { bufs[slot][0] = '\0'; - snprintf(bufs[slot], sizeof(bufs[slot]), "%'" PRIu64, - UINT64_C(1)); + snprintf(bufs[slot], sizeof(bufs[slot]), "%'u", 1U); thousand = bufs[slot][0] == '1' ? WORKS : BROKEN; } + return; +} + + +extern const char * +uint64_to_str(uint64_t value, uint32_t slot) +{ + assert(slot < ARRAY_SIZE(bufs)); + + check_thousand_sep(slot); + if (thousand == WORKS) snprintf(bufs[slot], sizeof(bufs[slot]), "%'" PRIu64, value); else @@ -151,14 +170,21 @@ uint64_to_nicestr(uint64_t value, enum nicestr_unit unit_min, { assert(unit_min <= unit_max); assert(unit_max <= NICESTR_TIB); + assert(slot < ARRAY_SIZE(bufs)); + + check_thousand_sep(slot); enum nicestr_unit unit = NICESTR_B; - const char *str; + char *pos = bufs[slot]; + size_t left = sizeof(bufs[slot]); if ((unit_min == NICESTR_B && value < 10000) || unit_max == NICESTR_B) { // The value is shown as bytes. - str = uint64_to_str(value, slot); + if (thousand == WORKS) + my_snprintf(&pos, &left, "%'u", (unsigned int)value); + else + my_snprintf(&pos, &left, "%u", (unsigned int)value); } else { // Scale the value to a nicer unit. Unless unit_min and // unit_max limit us, we will show at most five significant @@ -169,49 +195,23 @@ uint64_to_nicestr(uint64_t value, enum nicestr_unit unit_min, ++unit; } while (unit < unit_min || (d > 9999.9 && unit < unit_max)); - str = double_to_str(d); + if (thousand == WORKS) + my_snprintf(&pos, &left, "%'.1f", d); + else + my_snprintf(&pos, &left, "%.1f", d); } static const char suffix[5][4] = { "B", "KiB", "MiB", "GiB", "TiB" }; + my_snprintf(&pos, &left, " %s", suffix[unit]); - // Minimum buffer size: - // 26 2^64 with thousand separators - // 4 " KiB" - // 2 " (" - // 26 2^64 with thousand separators - // 3 " B)" - // 1 '\0' - // 62 Total - static char buf[4][64]; - char *pos = buf[slot]; - size_t left = sizeof(buf[slot]); - my_snprintf(&pos, &left, "%s %s", str, suffix[unit]); - - if (always_also_bytes && value >= 10000) - snprintf(pos, left, " (%s B)", uint64_to_str(value, slot)); - - return buf[slot]; -} - - -extern const char * -double_to_str(double value) -{ - static char buf[64]; - - static enum { UNKNOWN, WORKS, BROKEN } thousand = UNKNOWN; - if (thousand == UNKNOWN) { - buf[0] = '\0'; - snprintf(buf, sizeof(buf), "%'.1f", 2.0); - thousand = buf[0] == '2' ? WORKS : BROKEN; + if (always_also_bytes && value >= 10000) { + if (thousand == WORKS) + snprintf(pos, left, " (%'" PRIu64 " B)", value); + else + snprintf(pos, left, " (%" PRIu64 " B)", value); } - if (thousand == WORKS) - snprintf(buf, sizeof(buf), "%'.1f", value); - else - snprintf(buf, sizeof(buf), "%.1f", value); - - return buf; + return bufs[slot]; } @@ -225,7 +225,10 @@ my_snprintf(char **pos, size_t *left, const char *fmt, ...) // If an error occurred, we want the caller to think that the whole // buffer was used. This way no more data will be written to the - // buffer. We don't need better error handling here. + // buffer. We don't need better error handling here, although it + // is possible that the result looks garbage on the terminal if + // e.g. an UTF-8 character gets split. That shouldn't (easily) + // happen though, because the buffers used have some extra room. if (len < 0 || (size_t)(len) >= *left) { *left = 0; } else { @@ -237,45 +240,6 @@ my_snprintf(char **pos, size_t *left, const char *fmt, ...) } -/* -/// \brief Simple quoting to get rid of ASCII control characters -/// -/// This is not so cool and locale-dependent, but should be good enough -/// At least we don't print any control characters on the terminal. -/// -extern char * -str_quote(const char *str) -{ - size_t dest_len = 0; - bool has_ctrl = false; - - while (str[dest_len] != '\0') - if (*(unsigned char *)(str + dest_len++) < 0x20) - has_ctrl = true; - - char *dest = malloc(dest_len + 1); - if (dest != NULL) { - if (has_ctrl) { - for (size_t i = 0; i < dest_len; ++i) - if (*(unsigned char *)(str + i) < 0x20) - dest[i] = '?'; - else - dest[i] = str[i]; - - dest[dest_len] = '\0'; - - } else { - // Usually there are no control characters, - // so we can optimize. - memcpy(dest, str, dest_len + 1); - } - } - - return dest; -} -*/ - - extern bool is_empty_filename(const char *filename) { diff --git a/Build/source/extra/xz/src/xz/util.h b/Build/source/extra/xz/src/xz/util.h index 2e08b4a868e..4b2d3e2f573 100644 --- a/Build/source/extra/xz/src/xz/util.h +++ b/Build/source/extra/xz/src/xz/util.h @@ -19,11 +19,12 @@ /// \brief Safe realloc() that never returns NULL -extern void *xrealloc(void *ptr, size_t size); +extern void *xrealloc(void *ptr, size_t size) + lzma_attribute((malloc)) lzma_attr_alloc_size(2); /// \brief Safe strdup() that never returns NULL -extern char *xstrdup(const char *src); +extern char *xstrdup(const char *src) lzma_attribute((malloc)); /// \brief Fancy version of strtoull() @@ -96,13 +97,6 @@ extern const char *uint64_to_nicestr(uint64_t value, bool always_also_bytes, uint32_t slot); -/// \brief Convert double to a string with one decimal place -/// -/// This is like uint64_to_str() except that this converts a double and -/// uses exactly one decimal place. -extern const char *double_to_str(double value); - - /// \brief Wrapper for snprintf() to help constructing a string in pieces /// /// A maximum of *left bytes is written starting from *pos. *pos and *left diff --git a/Build/source/extra/xz/src/xz/xz.1 b/Build/source/extra/xz/src/xz/xz.1 index 1f2fd9c27bf..cba1d077c2d 100644 --- a/Build/source/extra/xz/src/xz/xz.1 +++ b/Build/source/extra/xz/src/xz/xz.1 @@ -5,9 +5,11 @@ .\" This file has been put into the public domain. .\" You can do whatever you want with this file. .\" -.TH XZ 1 "2010-06-01" "Tukaani" "XZ Utils" +.TH XZ 1 "2010-10-04" "Tukaani" "XZ Utils" +. .SH NAME xz, unxz, xzcat, lzma, unlzma, lzcat \- Compress or decompress .xz and .lzma files +. .SH SYNOPSIS .B xz .RI [ option ]... @@ -33,8 +35,8 @@ is equivalent to is equivalent to .BR "xz \-\-format=lzma \-\-decompress \-\-stdout" . .PP -When writing scripts that need to decompress files, it is recommended to -always use the name +When writing scripts that need to decompress files, +it is recommended to always use the name .B xz with appropriate arguments .RB ( "xz \-d" @@ -43,19 +45,22 @@ or instead of the names .B unxz and -.BR xzcat. +.BR xzcat . +. .SH DESCRIPTION .B xz -is a general-purpose data compression tool with command line syntax similar to +is a general-purpose data compression tool with +command line syntax similar to .BR gzip (1) and .BR bzip2 (1). The native file format is the .B .xz -format, but also the legacy +format, but the legacy .B .lzma -format and raw compressed streams with no container format headers -are supported. +format used by LZMA Utils and +raw compressed streams with no container format headers +are also supported. .PP .B xz compresses or decompresses each @@ -68,13 +73,16 @@ are given or is .BR \- , .B xz -reads from standard input and writes the processed data to standard output. +reads from standard input and writes the processed data +to standard output. .B xz will refuse (display an error and skip the .IR file ) -to write compressed data to standard output if it is a terminal. Similarly, +to write compressed data to standard output if it is a terminal. +Similarly, .B xz -will refuse to read compressed data from standard input if it is a terminal. +will refuse to read compressed data +from standard input if it is a terminal. .PP Unless .B \-\-stdout @@ -117,8 +125,9 @@ will display a warning and skip the if any of the following applies: .IP \(bu 3 .I File -is not a regular file. Symbolic links are not followed, thus they -are not considered to be regular files. +is not a regular file. +Symbolic links are not followed, +and thus they are not considered to be regular files. .IP \(bu 3 .I File has more than one hard link. @@ -126,7 +135,7 @@ has more than one hard link. .I File has setuid, setgid, or sticky bit set. .IP \(bu 3 -The operation mode is set to compress, and the +The operation mode is set to compress and the .I file already has a suffix of the target file format .RB ( .xz @@ -142,7 +151,7 @@ when compressing to the .B .lzma format). .IP \(bu 3 -The operation mode is set to decompress, and the +The operation mode is set to decompress and the .I file doesn't have a suffix of any of the supported file formats .RB ( .xz , @@ -154,12 +163,13 @@ or After successfully compressing or decompressing the .IR file , .B xz -copies the owner, group, permissions, access time, and modification time -from the source +copies the owner, group, permissions, access time, +and modification time from the source .I file -to the target file. If copying the group fails, the permissions are modified -so that the target file doesn't become accessible to users who didn't have -permission to access the source +to the target file. +If copying the group fails, the permissions are modified +so that the target file doesn't become accessible to users +who didn't have permission to access the source .IR file . .B xz doesn't support copying other metadata like access control lists @@ -169,7 +179,8 @@ Once the target file has been successfully closed, the source .I file is removed unless .B \-\-keep -was specified. The source +was specified. +The source .I file is never removed if the output is written to standard output. .PP @@ -180,59 +191,78 @@ or to the .B xz process makes it print progress information to standard error. -This has only limited use since when standard error is a terminal, using +This has only limited use since when standard error +is a terminal, using .B \-\-verbose will display an automatically updating progress indicator. +. .SS "Memory usage" The memory usage of .B xz -varies from a few hundred kilobytes to several gigabytes depending on -the compression settings. The settings used when compressing a file -affect also the memory usage of the decompressor. Typically the decompressor -needs only 5\ % to 20\ % of the amount of RAM that the compressor needed when -creating the file. Still, the worst-case memory usage of the decompressor -is several gigabytes. -.PP -To prevent uncomfortable surprises caused by huge memory usage, +varies from a few hundred kilobytes to several gigabytes +depending on the compression settings. +The settings used when compressing a file determine +the memory requirements of the decompressor. +Typically the decompressor needs 5\ % to 20\ % of +the amount of memory that the compressor needed when +creating the file. +For example, decompressing a file created with +.B xz \-9 +currently requires 65\ MiB of memory. +Still, it is possible to have +.B .xz +files that require several gigabytes of memory to decompress. +.PP +Especially users of older systems may find +the possibility of very large memory usage annoying. +To prevent uncomfortable surprises, .B xz -has a built-in memory usage limiter. While some operating systems provide -ways to limit the memory usage of processes, relying on it wasn't deemed -to be flexible enough. The default limit depends on the total amount of -physical RAM: -.IP \(bu 3 -If 40\ % of RAM is at least 80 MiB, 40\ % of RAM is used as the limit. -.IP \(bu 3 -If 80\ % of RAM is over 80 MiB, 80 MiB is used as the limit. -.IP \(bu 3 -Otherwise 80\ % of RAM is used as the limit. -.PP -When compressing, if the selected compression settings exceed the memory -usage limit, the settings are automatically adjusted downwards and a notice -about this is displayed. As an exception, if the memory usage limit is -exceeded when compressing with -.BR \-\-format=raw , -an error is displayed and +has a built-in memory usage limiter, which is disabled by default. +While some operating systems provide ways to limit +the memory usage of processes, relying on it +wasn't deemed to be flexible enough (e.g. using +.BR ulimit (1) +to limit virtual memory tends to cripple +.BR mmap (2)). +.PP +The memory usage limiter can be enabled with +the command line option \fB\-\-memlimit=\fIlimit\fR. +Often it is more convenient to enable the limiter +by default by setting the environment variable +.BR XZ_DEFAULTS , +e.g.\& +.BR XZ_DEFAULTS=\-\-memlimit=150MiB . +It is possible to set the limits separately +for compression and decompression +by using \fB\-\-memlimit\-compress=\fIlimit\fR and +\fB\-\-memlimit\-decompress=\fIlimit\fR. +Using these two options outside +.B XZ_DEFAULTS +is rarely useful because a single run of .B xz -will exit with exit status -.BR 1 . +cannot do both compression and decompression and +.BI \-\-memlimit= limit +(or \fB\-M\fR \fIlimit\fR) +is shorter to type on the command line. .PP -If source -.I file -cannot be decompressed without exceeding the memory usage limit, an error -message is displayed and the file is skipped. Note that compressed files -may contain many blocks, which may have been compressed with different -settings. Typically all blocks will have roughly the same memory requirements, -but it is possible that a block later in the file will exceed the memory usage -limit, and an error about too low memory usage limit gets displayed after some -data has already been decompressed. -.PP -The absolute value of the active memory usage limit can be seen with -.B \-\-info-memory -or near the bottom of the output of -.BR \-\-long\-help . -The default limit can be overridden with -\fB\-\-memory=\fIlimit\fR. -.SS Concatenation and padding with .xz files +If the specified memory usage limit is exceeded when decompressing, +.B xz +will display an error and decompressing the file will fail. +If the limit is exceeded when compressing, +.B xz +will try to scale the settings down so that the limit +is no longer exceeded (except when using \fB\-\-format=raw\fR +or \fB\-\-no\-adjust\fR). +This way the operation won't fail unless the limit is very small. +The scaling of the settings is done in steps that don't +match the compression level presets, e.g. if the limit is +only slightly less than the amount required for +.BR "xz \-9" , +the settings will be scaled down only a little, +not all the way down to +.BR "xz \-8" . +. +.SS "Concatenation and padding with .xz files" It is possible to concatenate .B .xz files as is. @@ -241,23 +271,28 @@ will decompress such files as if they were a single .B .xz file. .PP -It is possible to insert padding between the concenated parts -or after the last part. The padding must be null bytes and the size -of the padding must be a multiple of four bytes. This can be useful -if the .xz file is stored on a medium that stores file sizes -e.g. as 512-byte blocks. +It is possible to insert padding between the concatenated parts +or after the last part. +The padding must consist of null bytes and the size +of the padding must be a multiple of four bytes. +This can be useful e.g. if the +.B .xz +file is stored on a medium that measures file sizes +in 512-byte blocks. .PP Concatenation and padding are not allowed with .B .lzma files or raw streams. +. .SH OPTIONS +. .SS "Integer suffixes and special values" -In most places where an integer argument is expected, an optional suffix -is supported to easily indicate large integers. There must be no space -between the integer and the suffix. +In most places where an integer argument is expected, +an optional suffix is supported to easily indicate large integers. +There must be no space between the integer and the suffix. .TP .B KiB -The integer is multiplied by 1,024 (2^10). Also +Multiply the integer by 1,024 (2^10). .BR Ki , .BR k , .BR kB , @@ -268,7 +303,7 @@ are accepted as synonyms for .BR KiB . .TP .B MiB -The integer is multiplied by 1,048,576 (2^20). Also +Multiply the integer by 1,048,576 (2^20). .BR Mi , .BR m , .BR M , @@ -278,7 +313,7 @@ are accepted as synonyms for .BR MiB . .TP .B GiB -The integer is multiplied by 1,073,741,824 (2^30). Also +Multiply the integer by 1,073,741,824 (2^30). .BR Gi , .BR g , .BR G , @@ -287,16 +322,20 @@ and are accepted as synonyms for .BR GiB . .PP -A special value +The special value .B max -can be used to indicate the maximum integer value supported by the option. +can be used to indicate the maximum integer value +supported by the option. +. .SS "Operation mode" -If multiple operation mode options are given, the last one takes effect. +If multiple operation mode options are given, +the last one takes effect. .TP .BR \-z ", " \-\-compress -Compress. This is the default operation mode when no operation mode option -is specified, and no other operation mode is implied from the command name -(for example, +Compress. +This is the default operation mode when no operation mode option +is specified and no other operation mode is implied from +the command name (for example, .B unxz implies .BR \-\-decompress ). @@ -307,62 +346,73 @@ Decompress. .BR \-t ", " \-\-test Test the integrity of compressed .IR files . -No files are created or removed. This option is equivalent to +This option is equivalent to .B "\-\-decompress \-\-stdout" except that the decompressed data is discarded instead of being written to standard output. +No files are created or removed. .TP .BR \-l ", " \-\-list -List information about compressed +Print information about compressed .IR files . -No uncompressed output is produced, and no files are created or removed. -In list mode, the program cannot read the compressed data from standard +No uncompressed output is produced, +and no files are created or removed. +In list mode, the program cannot read +the compressed data from standard input or from other unseekable sources. -.IP +.IP "" The default listing shows basic information about .IR files , -one file per line. To get more detailed information, use also the +one file per line. +To get more detailed information, use also the .B \-\-verbose -option. For even more information, use +option. +For even more information, use .B \-\-verbose -twice, but note that it may be slow, because getting all the extra -information requires many seeks. The width of verbose output exceeds -80 characters, so piping the output to e.g. +twice, but note that this may be slow, because getting all the extra +information requires many seeks. +The width of verbose output exceeds +80 characters, so piping the output to e.g.\& .B "less\ \-S" may be convenient if the terminal isn't wide enough. -.IP +.IP "" The exact output may vary between .B xz -versions and different locales. To get machine-readable output, +versions and different locales. +For machine-readable output, .B \-\-robot \-\-list should be used. +. .SS "Operation modifiers" .TP .BR \-k ", " \-\-keep -Keep (don't delete) the input files. +Don't delete the input files. .TP .BR \-f ", " \-\-force This option has several effects: .RS .IP \(bu 3 -If the target file already exists, delete it before compressing or -decompressing. +If the target file already exists, +delete it before compressing or decompressing. .IP \(bu 3 -Compress or decompress even if the input is a symbolic link to a regular file, -has more than one hard link, or has setuid, setgid, or sticky bit set. -The setuid, setgid, and sticky bits are not copied to the target file. +Compress or decompress even if the input is +a symbolic link to a regular file, +has more than one hard link, +or has the setuid, setgid, or sticky bit set. +The setuid, setgid, and sticky bits are not copied +to the target file. .IP \(bu 3 -If combined with +When used with .B \-\-decompress .BR \-\-stdout and .B xz -doesn't recognize the type of the source file, -.B xz -will copy the source file as is to standard output. This allows using +cannot recognize the type of the source file, +copy the source file as is to standard output. +This allows .B xzcat -.B \--force -like +.B \-\-force +to be used like .BR cat (1) for files that have not been compressed with .BR xz . @@ -378,21 +428,23 @@ can be used to restrict to decompress only a single file format. .RE .TP -.BR \-c ", " \-\-stdout ", " \-\-to-stdout -Write the compressed or decompressed data to standard output instead of -a file. This implies +.BR \-c ", " \-\-stdout ", " \-\-to\-stdout +Write the compressed or decompressed data to +standard output instead of a file. +This implies .BR \-\-keep . .TP .B \-\-no\-sparse -Disable creation of sparse files. By default, if decompressing into -a regular file, +Disable creation of sparse files. +By default, if decompressing into a regular file, .B xz -tries to make the file sparse if the decompressed data contains long -sequences of binary zeros. It works also when writing to standard output -as long as standard output is connected to a regular file, and certain -additional conditions are met to make it safe. Creating sparse files may -save disk space and speed up the decompression by reducing the amount of -disk I/O. +tries to make the file sparse if the decompressed data contains +long sequences of binary zeros. +It also works when writing to standard output +as long as standard output is connected to a regular file +and certain additional conditions are met to make it safe. +Creating sparse files may save disk space and speed up +the decompression by reducing the amount of disk I/O. .TP \fB\-S\fR \fI.suf\fR, \fB\-\-suffix=\fI.suf When compressing, use @@ -401,11 +453,12 @@ as the suffix for the target file instead of .B .xz or .BR .lzma . -If not writing to standard output and the source file already has the suffix +If not writing to standard output and +the source file already has the suffix .IR .suf , a warning is displayed and the file is skipped. -.IP -When decompressing, recognize also files with the suffix +.IP "" +When decompressing, recognize files with the suffix .I .suf in addition to files with the .BR .xz , @@ -413,13 +466,15 @@ in addition to files with the .BR .lzma , or .B .tlz -suffix. If the source file has the suffix +suffix. +If the source file has the suffix .IR .suf , the suffix is removed to get the target filename. -.IP +.IP "" When compressing or decompressing raw streams .RB ( \-\-format=raw ), -the suffix must always be specified unless writing to standard output, +the suffix must always be specified unless +writing to standard output, because there is no default suffix for raw streams. .TP \fB\-\-files\fR[\fB=\fIfile\fR] @@ -427,8 +482,9 @@ Read the filenames to process from .IR file ; if .I file -is omitted, filenames are read from standard input. Filenames must be -terminated with the newline character. A dash +is omitted, filenames are read from standard input. +Filenames must be terminated with the newline character. +A dash .RB ( \- ) is taken as a regular filename; it doesn't mean standard input. If filenames are given also as command line arguments, they are @@ -436,53 +492,62 @@ processed before the filenames read from .IR file . .TP \fB\-\-files0\fR[\fB=\fIfile\fR] -This is identical to \fB\-\-files\fR[\fB=\fIfile\fR] except that the -filenames must be terminated with the null character. +This is identical to \fB\-\-files\fR[\fB=\fIfile\fR] except +that each filename must be terminated with the null character. +. .SS "Basic file format and compression options" .TP \fB\-F\fR \fIformat\fR, \fB\-\-format=\fIformat -Specify the file format to compress or decompress: +Specify the file +.I format +to compress or decompress: .RS -.IP \(bu 3 -.BR auto : -This is the default. When compressing, +.TP +.B auto +This is the default. +When compressing, .B auto is equivalent to .BR xz . -When decompressing, the format of the input file is automatically detected. +When decompressing, +the format of the input file is automatically detected. Note that raw streams (created with .BR \-\-format=raw ) cannot be auto-detected. -.IP \(bu 3 -.BR xz : +.TP +.B xz Compress to the .B .xz file format, or accept only .B .xz files when decompressing. -.IP \(bu 3 -.B lzma -or -.BR alone : +.TP +.BR lzma ", " alone Compress to the legacy .B .lzma file format, or accept only .B .lzma -files when decompressing. The alternative name +files when decompressing. +The alternative name .B alone is provided for backwards compatibility with LZMA Utils. -.IP \(bu 3 -.BR raw : -Compress or uncompress a raw stream (no headers). This is meant for advanced -users only. To decode raw streams, you need to set not only +.TP +.B raw +Compress or uncompress a raw stream (no headers). +This is meant for advanced users only. +To decode raw streams, you need use .B \-\-format=raw -but also specify the filter chain, which would normally be stored in the -container format headers. +and explicitly specify the filter chain, +which normally would have been stored in the container headers. .RE .TP \fB\-C\fR \fIcheck\fR, \fB\-\-check=\fIcheck -Specify the type of the integrity check, which is calculated from the -uncompressed data. This option has an effect only when compressing into the +Specify the type of the integrity check. +The check is calculated from the uncompressed data and +stored in the +.B .xz +file. +This option has an effect only when compressing into the .B .xz format; the .B .lzma @@ -490,235 +555,406 @@ format doesn't support integrity checks. The integrity check (if any) is verified when the .B .xz file is decompressed. -.IP +.IP "" Supported .I check types: .RS -.IP \(bu 3 -.BR none : -Don't calculate an integrity check at all. This is usually a bad idea. This -can be useful when integrity of the data is verified by other means anyway. -.IP \(bu 3 -.BR crc32 : +.TP +.B none +Don't calculate an integrity check at all. +This is usually a bad idea. +This can be useful when integrity of the data is verified +by other means anyway. +.TP +.B crc32 Calculate CRC32 using the polynomial from IEEE-802.3 (Ethernet). -.IP \(bu 3 -.BR crc64 : -Calculate CRC64 using the polynomial from ECMA-182. This is the default, since -it is slightly better than CRC32 at detecting damaged files and the speed -difference is negligible. -.IP \(bu 3 -.BR sha256 : -Calculate SHA-256. This is somewhat slower than CRC32 and CRC64. +.TP +.B crc64 +Calculate CRC64 using the polynomial from ECMA-182. +This is the default, since it is slightly better than CRC32 +at detecting damaged files and the speed difference is negligible. +.TP +.B sha256 +Calculate SHA-256. +This is somewhat slower than CRC32 and CRC64. .RE -.IP +.IP "" Integrity of the .B .xz -headers is always verified with CRC32. It is not possible to change or -disable it. +headers is always verified with CRC32. +It is not possible to change or disable it. .TP .BR \-0 " ... " \-9 -Select compression preset. If a preset level is specified multiple times, +Select a compression preset level. +The default is +.BR \-6 . +If multiple preset levels are specified, the last one takes effect. -.IP -The compression preset levels can be categorised roughly into three -categories: -.RS -.IP "\fB\-0\fR ... \fB\-2" -Fast presets with relatively low memory usage. -.B \-1 +If a custom filter chain was already specified, setting +a compression preset level clears the custom filter chain. +.IP "" +The differences between the presets are more significant than with +.BR gzip (1) and -.B \-2 -should give compression speed and ratios comparable to -.B "bzip2 \-1" +.BR bzip2 (1). +The selected compression settings determine +the memory requirements of the decompressor, +thus using a too high preset level might make it painful +to decompress the file on an old system with little RAM. +Specifically, +.B "it's not a good idea to blindly use \-9 for everything" +like it often is with +.BR gzip (1) and -.BR "bzip2 \-9" , -respectively. -Currently -.B \-0 -is not very good (not much faster than -.B \-1 -but much worse compression). In future, +.BR bzip2 (1). +.RS +.TP +.BR "\-0" " ... " "\-3" +These are somewhat fast presets. .B \-0 -may be indicate some fast algorithm instead of LZMA2. -.IP "\fB\-3\fR ... \fB\-5" -Good compression ratio with low to medium memory usage. -These are significantly slower than levels 0\-2. -.IP "\fB\-6\fR ... \fB\-9" -Excellent compression with medium to high memory usage. These are also -slower than the lower preset levels. The default is -.BR \-6 . -Unless you want to maximize the compression ratio, you probably don't want -a higher preset level than -.B \-7 -due to speed and memory usage. +is sometimes faster than +.B "gzip \-9" +while compressing much better. +The higher ones often have speed comparable to +.BR bzip2 (1) +with comparable or better compression ratio, +although the results +depend a lot on the type of data being compressed. +.TP +.BR "\-4" " ... " "\-6" +Good to very good compression while keeping +decompressor memory usage reasonable even for old systems. +.B \-6 +is the default, which is usually a good choice +e.g. for distributing files that need to be decompressible +even on systems with only 16\ MiB RAM. +.RB ( \-5e +or +.B \-6e +may be worth considering too. +See +.BR \-\-extreme .) +.TP +.B "\-7 ... \-9" +These are like +.B \-6 +but with higher compressor and decompressor memory requirements. +These are useful only when compressing files bigger than +8\ MiB, 16\ MiB, and 32\ MiB, respectively. .RE -.IP -The exact compression settings (filter chain) used by each preset may -vary between -.B xz -versions. The settings may also vary between files being compressed, if -.B xz -determines that modified settings will probably give better compression -ratio without significantly affecting compression time or memory usage. -.IP -Because the settings may vary, the memory usage may vary too. The following -table lists the maximum memory usage of each preset level, which won't be -exceeded even in future versions of -.BR xz . -.IP -.B "FIXME: The table below is just a rough idea." +.IP "" +On the same hardware, the decompression speed is approximately +a constant number of bytes of compressed data per second. +In other words, the better the compression, +the faster the decompression will usually be. +This also means that the amount of uncompressed output +produced per second can vary a lot. +.IP "" +The following table summarises the features of the presets: .RS .RS +.PP .TS tab(;); -c c c -n n n. -Preset;Compression;Decompression -\-0;6 MiB;1 MiB -\-1;6 MiB;1 MiB -\-2;10 MiB;1 MiB -\-3;20 MiB;2 MiB -\-4;30 MiB;3 MiB -\-5;60 MiB;6 MiB -\-6;100 MiB;10 MiB -\-7;200 MiB;20 MiB -\-8;400 MiB;40 MiB -\-9;800 MiB;80 MiB +c c c c c +n n n n n. +Preset;DictSize;CompCPU;CompMem;DecMem +\-0;256 KiB;0;3 MiB;1 MiB +\-1;1 MiB;1;9 MiB;2 MiB +\-2;2 MiB;2;17 MiB;3 MiB +\-3;4 MiB;3;32 MiB;5 MiB +\-4;4 MiB;4;48 MiB;5 MiB +\-5;8 MiB;5;94 MiB;9 MiB +\-6;8 MiB;6;94 MiB;9 MiB +\-7;16 MiB;6;186 MiB;17 MiB +\-8;32 MiB;6;370 MiB;33 MiB +\-9;64 MiB;6;674 MiB;65 MiB .TE .RE .RE -.IP -When compressing, +.IP "" +Column descriptions: +.RS +.IP \(bu 3 +DictSize is the LZMA2 dictionary size. +It is waste of memory to use a dictionary bigger than +the size of the uncompressed file. +This is why it is good to avoid using the presets +.BR \-7 " ... " \-9 +when there's no real need for them. +At +.B \-6 +and lower, the amount of memory wasted is +usually low enough to not matter. +.IP \(bu 3 +CompCPU is a simplified representation of the LZMA2 settings +that affect compression speed. +The dictionary size affects speed too, +so while CompCPU is the same for levels +.BR \-6 " ... " \-9 , +higher levels still tend to be a little slower. +To get even slower and thus possibly better compression, see +.BR \-\-extreme . +.IP \(bu 3 +CompMem contains the compressor memory requirements +in the single-threaded mode. +It may vary slightly between .B xz -automatically adjusts the compression settings downwards if -the memory usage limit would be exceeded, so it is safe to specify -a high preset level even on systems that don't have lots of RAM. +versions. +Memory requirements of some of the future multithreaded modes may +be dramatically higher than that of the single-threaded mode. +.IP \(bu 3 +DecMem contains the decompressor memory requirements. +That is, the compression settings determine +the memory requirements of the decompressor. +The exact decompressor memory usage is slighly more than +the LZMA2 dictionary size, but the values in the table +have been rounded up to the next full MiB. +.RE +.TP +.BR \-e ", " \-\-extreme +Use a slower variant of the selected compression preset level +.RB ( \-0 " ... " \-9 ) +to hopefully get a little bit better compression ratio, +but with bad luck this can also make it worse. +Decompressor memory usage is not affected, +but compressor memory usage increases a little at preset levels +.BR \-0 " ... " \-3 . +.IP "" +Since there are two presets with dictionary sizes +4\ MiB and 8\ MiB, the presets +.B \-3e +and +.B \-5e +use slightly faster settings (lower CompCPU) than +.B \-4e +and +.BR \-6e , +respectively. +That way no two presets are identical. +.RS +.RS +.PP +.TS +tab(;); +c c c c c +n n n n n. +Preset;DictSize;CompCPU;CompMem;DecMem +\-0e;256 KiB;8;4 MiB;1 MiB +\-1e;1 MiB;8;13 MiB;2 MiB +\-2e;2 MiB;8;25 MiB;3 MiB +\-3e;4 MiB;7;48 MiB;5 MiB +\-4e;4 MiB;8;48 MiB;5 MiB +\-5e;8 MiB;7;94 MiB;9 MiB +\-6e;8 MiB;8;94 MiB;9 MiB +\-7e;16 MiB;8;186 MiB;17 MiB +\-8e;32 MiB;8;370 MiB;33 MiB +\-9e;64 MiB;8;674 MiB;65 MiB +.TE +.RE +.RE +.IP "" +For example, there are a total of four presets that use +8\ MiB dictionary, whose order from the fastest to the slowest is +.BR \-5 , +.BR \-6 , +.BR \-5e , +and +.BR \-6e . +.TP +.B \-\-fast +.PD 0 .TP -.BR \-\-fast " and " \-\-best +.B \-\-best +.PD These are somewhat misleading aliases for .B \-0 and .BR \-9 , respectively. -These are provided only for backwards compatibility with LZMA Utils. +These are provided only for backwards compatibility +with LZMA Utils. Avoid using these options. -.IP -Especially the name of -.B \-\-best -is misleading, because the definition of best depends on the input data, -and that usually people don't want the very best compression ratio anyway, -because it would be very slow. .TP -.BR \-e ", " \-\-extreme -Modify the compression preset (\fB\-0\fR ... \fB\-9\fR) so that a little bit -better compression ratio can be achieved without increasing memory usage -of the compressor or decompressor (exception: compressor memory usage may -increase a little with presets \fB\-0\fR ... \fB\-2\fR). The downside is that -the compression time will increase dramatically (it can easily double). -.TP -\fB\-M\fR \fIlimit\fR, \fB\-\-memory=\fIlimit -Set the memory usage limit. If this option is specified multiple times, -the last one takes effect. The +.BI \-\-memlimit\-compress= limit +Set a memory usage limit for compression. +If this option is specified multiple times, +the last one takes effect. +.IP "" +If the compression settings exceed the +.IR limit , +.B xz +will adjust the settings downwards so that +the limit is no longer exceeded and display a notice that +automatic adjustment was done. +Such adjustments are not made when compressing with +.B \-\-format=raw +or if +.B \-\-no\-adjust +has been specified. +In those cases, an error is displayed and +.B xz +will exit with exit status 1. +.IP "" +The .I limit can be specified in multiple ways: .RS .IP \(bu 3 The .I limit -can be an absolute value in bytes. Using an integer suffix like +can be an absolute value in bytes. +Using an integer suffix like .B MiB -can be useful. Example: -.B "\-\-memory=80MiB" +can be useful. +Example: +.B "\-\-memlimit\-compress=80MiB" .IP \(bu 3 The .I limit -can be specified as a percentage of physical RAM. Example: -.B "\-\-memory=70%" +can be specified as a percentage of total physical memory (RAM). +This can be useful especially when setting the +.B XZ_DEFAULTS +environment variable in a shell initialization script +that is shared between different computers. +That way the limit is automatically bigger +on systems with more memory. +Example: +.B "\-\-memlimit\-compress=70%" .IP \(bu 3 The .I limit can be reset back to its default value by setting it to .BR 0 . -See the section -.B "Memory usage" -for how the default limit is defined. -.IP \(bu 3 -The memory usage limiting can be effectively disabled by setting +This is currently equivalent to setting the .I limit to -.BR max . -This isn't recommended. It's usually better to use, for example, -.BR \-\-memory=90% . +.B max +(no memory usage limit). +Once multithreading support has been implemented, +there may be a difference between +.B 0 +and +.B max +for the multithreaded case, so it is recommended to use +.B 0 +instead of +.B max +until the details have been decided. .RE -.IP -The current -.I limit -can be seen near the bottom of the output of the -.B \-\-long-help -option. +.IP "" +See also the section +.BR "Memory usage" . +.TP +.BI \-\-memlimit\-decompress= limit +Set a memory usage limit for decompression. +This also affects the +.B \-\-list +mode. +If the operation is not possible without exceeding the +.IR limit , +.B xz +will display an error and decompressing the file will fail. +See +.BI \-\-memlimit\-compress= limit +for possible ways to specify the +.IR limit . +.TP +\fB\-M\fR \fIlimit\fR, \fB\-\-memlimit=\fIlimit\fR, \fB\-\-memory=\fIlimit +This is equivalent to specifying \fB\-\-memlimit\-compress=\fIlimit +\fB\-\-memlimit\-decompress=\fIlimit\fR. +.TP +.B \-\-no\-adjust +Display an error and exit if the compression settings exceed the +the memory usage limit. +The default is to adjust the settings downwards so +that the memory usage limit is not exceeded. +Automatic adjusting is always disabled when creating raw streams +.RB ( \-\-format=raw ). .TP \fB\-T\fR \fIthreads\fR, \fB\-\-threads=\fIthreads -Specify the maximum number of worker threads to use. The default is -the number of available CPU cores. You can see the current value of -.I threads -near the end of the output of the -.B \-\-long\-help -option. -.IP -The actual number of worker threads can be less than +Specify the number of worker threads to use. +The actual number of threads can be less than .I threads if using more threads would exceed the memory usage limit. -In addition to CPU-intensive worker threads, -.B xz -may use a few auxiliary threads, which don't use a lot of CPU time. -.IP -.B "Multithreaded compression and decompression are not implemented yet," -.B "so this option has no effect for now." -.SS Custom compressor filter chains -A custom filter chain allows specifying the compression settings in detail -instead of relying on the settings associated to the preset levels. -When a custom filter chain is specified, the compression preset level options -(\fB\-0\fR ... \fB\-9\fR and \fB\-\-extreme\fR) are silently ignored. -.PP -A filter chain is comparable to piping on the UN*X command line. -When compressing, the uncompressed input goes to the first filter, whose -output goes to the next filter (if any). The output of the last filter -gets written to the compressed file. The maximum number of filters in -the chain is four, but typically a filter chain has only one or two filters. -.PP -Many filters have limitations where they can be in the filter chain: -some filters can work only as the last filter in the chain, some only -as a non-last filter, and some work in any position in the chain. Depending -on the filter, this limitation is either inherent to the filter design or -exists to prevent security issues. -.PP -A custom filter chain is specified by using one or more filter options in -the order they are wanted in the filter chain. That is, the order of filter -options is significant! When decoding raw streams +.IP "" +.B "Multithreaded compression and decompression are not" +.B "implemented yet, so this option has no effect for now." +.IP "" +.B "As of writing (2010-09-27), it hasn't been decided" +.B "if threads will be used by default on multicore systems" +.B "once support for threading has been implemented." +.B "Comments are welcome." +The complicating factor is that using many threads +will increase the memory usage dramatically. +Note that if multithreading will be the default, +it will probably be done so that single-threaded and +multithreaded modes produce the same output, +so compression ratio won't be significantly affected +if threading will be enabled by default. +. +.SS "Custom compressor filter chains" +A custom filter chain allows specifying +the compression settings in detail instead of relying on +the settings associated to the preset levels. +When a custom filter chain is specified, +the compression preset level options +(\fB\-0\fR ... \fB\-9\fR and \fB\-\-extreme\fR) are +silently ignored. +.PP +A filter chain is comparable to piping on the command line. +When compressing, the uncompressed input goes to the first filter, +whose output goes to the next filter (if any). +The output of the last filter gets written to the compressed file. +The maximum number of filters in the chain is four, +but typically a filter chain has only one or two filters. +.PP +Many filters have limitations on where they can be +in the filter chain: +some filters can work only as the last filter in the chain, +some only as a non-last filter, and some work in any position +in the chain. +Depending on the filter, this limitation is either inherent to +the filter design or exists to prevent security issues. +.PP +A custom filter chain is specified by using one or more +filter options in the order they are wanted in the filter chain. +That is, the order of filter options is significant! +When decoding raw streams .RB ( \-\-format=raw ), -the filter chain is specified in the same order as it was specified when -compressing. +the filter chain is specified in the same order as +it was specified when compressing. .PP Filters take filter-specific .I options -as a comma-separated list. Extra commas in +as a comma-separated list. +Extra commas in .I options -are ignored. Every option has a default value, so you need to +are ignored. +Every option has a default value, so you need to specify only those you want to change. .TP -\fB\-\-lzma1\fR[\fB=\fIoptions\fR], \fB\-\-lzma2\fR[\fB=\fIoptions\fR] -Add LZMA1 or LZMA2 filter to the filter chain. These filter can be used -only as the last filter in the chain. -.IP -LZMA1 is a legacy filter, which is supported almost solely due to the legacy +\fB\-\-lzma1\fR[\fB=\fIoptions\fR] +.PD 0 +.TP +\fB\-\-lzma2\fR[\fB=\fIoptions\fR] +.PD +Add LZMA1 or LZMA2 filter to the filter chain. +These filters can be used only as the last filter in the chain. +.IP "" +LZMA1 is a legacy filter, +which is supported almost solely due to the legacy .B .lzma -file format, which supports only LZMA1. LZMA2 is an updated -version of LZMA1 to fix some practical issues of LZMA1. The +file format, which supports only LZMA1. +LZMA2 is an updated +version of LZMA1 to fix some practical issues of LZMA1. +The .B .xz -format uses LZMA2, and doesn't support LZMA1 at all. Compression speed and -ratios of LZMA1 and LZMA2 are practically the same. -.IP +format uses LZMA2 and doesn't support LZMA1 at all. +Compression speed and ratios of LZMA1 and LZMA2 +are practically the same. +.IP "" LZMA1 and LZMA2 share the same set of .IR options : .RS @@ -729,8 +965,9 @@ Reset all LZMA1 or LZMA2 to .IR preset . .I Preset -consist of an integer, which may be followed by single-letter preset -modifiers. The integer can be from +consist of an integer, which may be followed by single-letter +preset modifiers. +The integer can be from .B 0 to .BR 9 , @@ -739,7 +976,6 @@ The only supported modifier is currently .BR e , which matches .BR \-\-extreme . -.IP The default .I preset is @@ -749,84 +985,155 @@ from which the default values for the rest of the LZMA1 or LZMA2 are taken. .TP .BI dict= size -Dictionary (history buffer) size indicates how many bytes of the recently -processed uncompressed data is kept in memory. One method to reduce size of -the uncompressed data is to store distance-length pairs, which -indicate what data to repeat from the dictionary buffer. The bigger -the dictionary, the better the compression ratio usually is, -but dictionaries bigger than the uncompressed data are waste of RAM. -.IP -Typical dictionary size is from 64 KiB to 64 MiB. The minimum is 4 KiB. -The maximum for compression is currently 1.5 GiB. The decompressor already -supports dictionaries up to one byte less than 4 GiB, which is the -maximum for LZMA1 and LZMA2 stream formats. -.IP -Dictionary size has the biggest effect on compression ratio. -Dictionary size and match finder together determine the memory usage of -the LZMA1 or LZMA2 encoder. The same dictionary size is required -for decompressing that was used when compressing, thus the memory usage of -the decoder is determined by the dictionary size used when compressing. +Dictionary (history buffer) +.I size +indicates how many bytes of the recently processed +uncompressed data is kept in memory. +The algorithm tries to find repeating byte sequences (matches) in +the uncompressed data, and replace them with references +to the data currently in the dictionary. +The bigger the dictionary, the higher is the chance +to find a match. +Thus, increasing dictionary +.I size +usually improves compression ratio, but +a dictionary bigger than the uncompressed file is waste of memory. +.IP "" +Typical dictionary +.I size +is from 64\ KiB to 64\ MiB. +The minimum is 4\ KiB. +The maximum for compression is currently 1.5\ GiB (1536\ MiB). +The decompressor already supports dictionaries up to +one byte less than 4\ GiB, which is the maximum for +the LZMA1 and LZMA2 stream formats. +.IP "" +Dictionary +.I size +and match finder +.RI ( mf ) +together determine the memory usage of the LZMA1 or LZMA2 encoder. +The same (or bigger) dictionary +.I size +is required for decompressing that was used when compressing, +thus the memory usage of the decoder is determined +by the dictionary size used when compressing. +The +.B .xz +headers store the dictionary +.I size +either as +.RI "2^" n +or +.RI "2^" n " + 2^(" n "\-1)," +so these +.I sizes +are somewhat preferred for compression. +Other +.I sizes +will get rounded up when stored in the +.B .xz +headers. .TP .BI lc= lc -Specify the number of literal context bits. The minimum is -.B 0 -and the maximum is -.BR 4 ; -the default is -.BR 3 . +Specify the number of literal context bits. +The minimum is 0 and the maximum is 4; the default is 3. In addition, the sum of .I lc and .I lp -must not exceed -.BR 4 . +must not exceed 4. +.IP "" +All bytes that cannot be encoded as matches +are encoded as literals. +That is, literals are simply 8-bit bytes +that are encoded one at a time. +.IP "" +The literal coding makes an assumption that the highest +.I lc +bits of the previous uncompressed byte correlate +with the next byte. +E.g. in typical English text, an upper-case letter is +often followed by a lower-case letter, and a lower-case +letter is usually followed by another lower-case letter. +In the US-ASCII character set, the highest three bits are 010 +for upper-case letters and 011 for lower-case letters. +When +.I lc +is at least 3, the literal coding can take advantage of +this property in the uncompressed data. +.IP "" +The default value (3) is usually good. +If you want maximum compression, test +.BR lc=4 . +Sometimes it helps a little, and +sometimes it makes compression worse. +If it makes it worse, test e.g.\& +.B lc=2 +too. .TP .BI lp= lp -Specify the number of literal position bits. The minimum is -.B 0 -and the maximum is -.BR 4 ; -the default is -.BR 0 . +Specify the number of literal position bits. +The minimum is 0 and the maximum is 4; the default is 0. +.IP "" +.I Lp +affects what kind of alignment in the uncompressed data is +assumed when encoding literals. +See +.I pb +below for more information about alignment. .TP .BI pb= pb -Specify the number of position bits. The minimum is -.B 0 -and the maximum is -.BR 4 ; -the default is -.BR 2 . -.TP -.BI mode= mode -Compression -.I mode -specifies the function used to analyze the data produced by the match finder. -Supported -.I modes -are -.B fast +Specify the number of position bits. +The minimum is 0 and the maximum is 4; the default is 2. +.IP "" +.I Pb +affects what kind of alignment in the uncompressed data is +assumed in general. +The default means four-byte alignment +.RI (2^ pb =2^2=4), +which is often a good choice when there's no better guess. +.IP "" +When the aligment is known, setting +.I pb +accordingly may reduce the file size a little. +E.g. with text files having one-byte +alignment (US-ASCII, ISO-8859-*, UTF-8), setting +.B pb=0 +can improve compression slightly. +For UTF-16 text, +.B pb=1 +is a good choice. +If the alignment is an odd number like 3 bytes, +.B pb=0 +might be the best choice. +.IP "" +Even though the assumed alignment can be adjusted with +.I pb and -.BR normal . -The default is -.B fast -for -.I presets -.BR 0 \- 2 -and -.B normal -for -.I presets -.BR 3 \- 9 . +.IR lp , +LZMA1 and LZMA2 still slightly favor 16-byte alignment. +It might be worth taking into account when designing file formats +that are likely to be often compressed with LZMA1 or LZMA2. .TP .BI mf= mf -Match finder has a major effect on encoder speed, memory usage, and -compression ratio. Usually Hash Chain match finders are faster than -Binary Tree match finders. Hash Chains are usually used together with -.B mode=fast -and Binary Trees with -.BR mode=normal . -The memory usage formulas are only rough estimates, -which are closest to reality when +Match finder has a major effect on encoder speed, +memory usage, and compression ratio. +Usually Hash Chain match finders are faster than Binary Tree +match finders. +The default depends on the +.IR preset : +0 uses +.BR hc3 , +1\-3 +use +.BR hc4 , +and the rest use +.BR bt4 . +.IP "" +The following match finders are supported. +The memory usage formulas below are rough approximations, +which are closest to the reality when .I dict is a power of two. .RS @@ -839,6 +1146,7 @@ Minimum value for 3 .br Memory usage: +.br .I dict * 7.5 (if .I dict @@ -857,8 +1165,16 @@ Minimum value for 4 .br Memory usage: +.br +.I dict +* 7.5 (if .I dict -* 7.5 +<= 32 MiB); +.br +.I dict +* 6.5 (if +.I dict +> 32 MiB) .TP .B bt2 Binary Tree with 2-byte hashing @@ -879,6 +1195,7 @@ Minimum value for 3 .br Memory usage: +.br .I dict * 11.5 (if .I dict @@ -897,53 +1214,96 @@ Minimum value for 4 .br Memory usage: +.br +.I dict +* 11.5 (if +.I dict +<= 32 MiB); +.br .I dict -* 11.5 +* 10.5 (if +.I dict +> 32 MiB) .RE .TP +.BI mode= mode +Compression +.I mode +specifies the method to analyze +the data produced by the match finder. +Supported +.I modes +are +.B fast +and +.BR normal . +The default is +.B fast +for +.I presets +0\-3 and +.B normal +for +.I presets +4\-9. +.IP "" +Usually +.B fast +is used with Hash Chain match finders and +.B normal +with Binary Tree match finders. +This is also what the +.I presets +do. +.TP .BI nice= nice -Specify what is considered to be a nice length for a match. Once a match -of at least -.I nice -bytes is found, the algorithm stops looking for possibly better matches. -.IP +Specify what is considered to be a nice length for a match. +Once a match of at least .I nice -can be 2\-273 bytes. Higher values tend to give better compression ratio -at expense of speed. The default depends on the -.I preset -level. +bytes is found, the algorithm stops +looking for possibly better matches. +.IP "" +.I Nice +can be 2\-273 bytes. +Higher values tend to give better compression ratio +at the expense of speed. +The default depends on the +.IR preset . .TP .BI depth= depth -Specify the maximum search depth in the match finder. The default is the -special value -.BR 0 , +Specify the maximum search depth in the match finder. +The default is the special value of 0, which makes the compressor determine a reasonable .I depth from .I mf and .IR nice . -.IP +.IP "" +Reasonable +.I depth +for Hash Chains is 4\-100 and 16\-1000 for Binary Trees. Using very high values for .I depth -can make the encoder extremely slow with carefully crafted files. +can make the encoder extremely slow with some files. Avoid setting the .I depth -over 1000 unless you are prepared to interrupt the compression in case it -is taking too long. +over 1000 unless you are prepared to interrupt +the compression in case it is taking far too long. .RE -.IP +.IP "" When decoding raw streams .RB ( \-\-format=raw ), -LZMA2 needs only the value of -.BR dict . +LZMA2 needs only the dictionary +.IR size . LZMA1 needs also -.BR lc , -.BR lp , +.IR lc , +.IR lp , and -.BR pb. +.IR pb . .TP \fB\-\-x86\fR[\fB=\fIoptions\fR] +.PD 0 .TP \fB\-\-powerpc\fR[\fB=\fIoptions\fR] .TP @@ -954,28 +1314,72 @@ and \fB\-\-armthumb\fR[\fB=\fIoptions\fR] .TP \fB\-\-sparc\fR[\fB=\fIoptions\fR] -Add a branch/call/jump (BCJ) filter to the filter chain. These filters -can be used only as non-last filter in the filter chain. -.IP -A BCJ filter converts relative addresses in the machine code to their -absolute counterparts. This doesn't change the size of the data, but -it increases redundancy, which allows e.g. LZMA2 to get better -compression ratio. -.IP -The BCJ filters are always reversible, so using a BCJ filter for wrong -type of data doesn't cause any data loss. However, applying a BCJ filter -for wrong type of data is a bad idea, because it tends to make the -compression ratio worse. -.IP +.PD +Add a branch/call/jump (BCJ) filter to the filter chain. +These filters can be used only as a non-last filter +in the filter chain. +.IP "" +A BCJ filter converts relative addresses in +the machine code to their absolute counterparts. +This doesn't change the size of the data, +but it increases redundancy, +which can help LZMA2 to produce 0\-15\ % smaller +.B .xz +file. +The BCJ filters are always reversible, +so using a BCJ filter for wrong type of data +doesn't cause any data loss, although it may make +the compression ratio slightly worse. +.IP "" +It is fine to apply a BCJ filter on a whole executable; +there's no need to apply it only on the executable section. +Applying a BCJ filter on an archive that contains both executable +and non-executable files may or may not give good results, +so it generally isn't good to blindly apply a BCJ filter when +compressing binary packages for distribution. +.IP "" +These BCJ filters are very fast and +use insignificant amount of memory. +If a BCJ filter improves compression ratio of a file, +it can improve decompression speed at the same time. +This is because, on the same hardware, +the decompression speed of LZMA2 is roughly +a fixed number of bytes of compressed data per second. +.IP "" +These BCJ filters have known problems related to +the compression ratio: +.RS +.IP \(bu 3 +Some types of files containing executable code +(e.g. object files, static libraries, and Linux kernel modules) +have the addresses in the instructions filled with filler values. +These BCJ filters will still do the address conversion, +which will make the compression worse with these files. +.IP \(bu 3 +Applying a BCJ filter on an archive containing multiple similar +executables can make the compression ratio worse than not using +a BCJ filter. +This is because the BCJ filter doesn't detect the boundaries +of the executable files, and doesn't reset +the address conversion counter for each executable. +.RE +.IP "" +Both of the above problems will be fixed +in the future in a new filter. +The old BCJ filters will still be useful in embedded systems, +because the decoder of the new filter will be bigger +and use more memory. +.IP "" Different instruction sets have have different alignment: .RS .RS +.PP .TS tab(;); l n l l n l. Filter;Alignment;Notes -x86;1;32-bit and 64-bit x86 +x86;1;32-bit or 64-bit x86 PowerPC;4;Big endian only ARM;4;Little endian only ARM-Thumb;2;Little endian only @@ -984,15 +1388,18 @@ SPARC;4;Big or little endian .TE .RE .RE -.IP -Since the BCJ-filtered data is usually compressed with LZMA2, the compression -ratio may be improved slightly if the LZMA2 options are set to match the -alignment of the selected BCJ filter. For example, with the IA-64 filter, -it's good to set +.IP "" +Since the BCJ-filtered data is usually compressed with LZMA2, +the compression ratio may be improved slightly if +the LZMA2 options are set to match the +alignment of the selected BCJ filter. +For example, with the IA-64 filter, it's good to set .B pb=4 -with LZMA2 (2^4=16). The x86 filter is an exception; it's usually good to -stick to LZMA2's default four-byte alignment when compressing x86 executables. -.IP +with LZMA2 (2^4=16). +The x86 filter is an exception; +it's usually good to stick to LZMA2's default +four-byte alignment when compressing x86 executables. +.IP "" All BCJ filters support the same .IR options : .RS @@ -1000,36 +1407,32 @@ All BCJ filters support the same .BI start= offset Specify the start .I offset -that is used when converting between relative and absolute addresses. +that is used when converting between relative +and absolute addresses. The .I offset -must be a multiple of the alignment of the filter (see the table above). -The default is zero. In practice, the default is good; specifying -a custom +must be a multiple of the alignment of the filter +(see the table above). +The default is zero. +In practice, the default is good; specifying a custom .I offset is almost never useful. -.IP -Specifying a non-zero start -.I offset -is probably useful only if the executable has multiple sections, and there -are many cross-section jumps or calls. Applying a BCJ filter separately for -each section with proper start offset and then compressing the result as -a single chunk may give some improvement in compression ratio compared -to applying the BCJ filter with the default -.I offset -for the whole executable. .RE .TP \fB\-\-delta\fR[\fB=\fIoptions\fR] -Add Delta filter to the filter chain. The Delta filter -can be used only as non-last filter in the filter chain. -.IP -Currently only simple byte-wise delta calculation is supported. It can -be useful when compressing e.g. uncompressed bitmap images or uncompressed -PCM audio. However, special purpose algorithms may give significantly better -results than Delta + LZMA2. This is true especially with audio, which -compresses faster and better e.g. with FLAC. -.IP +Add the Delta filter to the filter chain. +The Delta filter can be only used as a non-last filter +in the filter chain. +.IP "" +Currently only simple byte-wise delta calculation is supported. +It can be useful when compressing e.g. uncompressed bitmap images +or uncompressed PCM audio. +However, special purpose algorithms may give significantly better +results than Delta + LZMA2. +This is true especially with audio, +which compresses faster and better e.g. with +.BR flac (1). +.IP "" Supported .IR options : .RS @@ -1037,99 +1440,111 @@ Supported .BI dist= distance Specify the .I distance -of the delta calculation as bytes. +of the delta calculation in bytes. .I distance -must be 1\-256. The default is 1. -.IP +must be 1\-256. +The default is 1. +.IP "" For example, with .B dist=2 and eight-byte input A1 B1 A2 B3 A3 B5 A4 B7, the output will be A1 B1 01 02 01 02 01 02. .RE +. .SS "Other options" .TP .BR \-q ", " \-\-quiet -Suppress warnings and notices. Specify this twice to suppress errors too. -This option has no effect on the exit status. That is, even if a warning -was suppressed, the exit status to indicate a warning is still used. +Suppress warnings and notices. +Specify this twice to suppress errors too. +This option has no effect on the exit status. +That is, even if a warning was suppressed, +the exit status to indicate a warning is still used. .TP .BR \-v ", " \-\-verbose -Be verbose. If standard error is connected to a terminal, +Be verbose. +If standard error is connected to a terminal, .B xz will display a progress indicator. Specifying .B \-\-verbose -twice will give even more verbose output (useful mostly for debugging). -.IP +twice will give even more verbose output. +.IP "" The progress indicator shows the following information: .RS .IP \(bu 3 -Completion percentage is shown if the size of the input file is known. -That is, percentage cannot be shown in pipes. +Completion percentage is shown +if the size of the input file is known. +That is, the percentage cannot be shown in pipes. .IP \(bu 3 -Amount of compressed data produced (compressing) or consumed (decompressing). +Amount of compressed data produced (compressing) +or consumed (decompressing). .IP \(bu 3 -Amount of uncompressed data consumed (compressing) or produced -(decompressing). +Amount of uncompressed data consumed (compressing) +or produced (decompressing). .IP \(bu 3 -Compression ratio, which is calculated by dividing the amount of -compressed data processed so far by the amount of uncompressed data -processed so far. +Compression ratio, which is calculated by dividing +the amount of compressed data processed so far by +the amount of uncompressed data processed so far. .IP \(bu 3 -Compression or decompression speed. This is measured as the amount of -uncompressed data consumed (compression) or produced (decompression) -per second. It is shown once a few seconds have passed since +Compression or decompression speed. +This is measured as the amount of uncompressed data consumed +(compression) or produced (decompression) per second. +It is shown after a few seconds have passed since .B xz started processing the file. .IP \(bu 3 -Elapsed time or estimated time remaining. -Elapsed time is displayed in the format M:SS or H:MM:SS. -The estimated remaining time is displayed in a less precise format -which never has colons, for example, 2 min 30 s. The estimate can -be shown only when the size of the input file is known and a couple of -seconds have already passed since +Elapsed time in the format M:SS or H:MM:SS. +.IP \(bu 3 +Estimated remaining time is shown +only when the size of the input file is +known and a couple of seconds have already passed since .B xz started processing the file. +The time is shown in a less precise format which +never has any colons, e.g. 2 min 30 s. .RE -.IP +.IP "" When standard error is not a terminal, .B \-\-verbose will make .B xz -print the filename, compressed size, uncompressed size, compression ratio, -speed, and elapsed time on a single line to standard error after -compressing or decompressing the file. If operating took at least a few -seconds, also the speed and elapsed time are printed. If the operation -didn't finish, for example due to user interruption, also the completion -percentage is printed if the size of the input file is known. +print the filename, compressed size, uncompressed size, +compression ratio, and possibly also the speed and elapsed time +on a single line to standard error after compressing or +decompressing the file. +The speed and elapsed time are included only when +the operation took at least a few seconds. +If the operation didn't finish, e.g. due to user interruption, +also the completion percentage is printed +if the size of the input file is known. .TP .BR \-Q ", " \-\-no\-warn -Don't set the exit status to -.B 2 -even if a condition worth a warning was detected. This option doesn't affect -the verbosity level, thus both +Don't set the exit status to 2 +even if a condition worth a warning was detected. +This option doesn't affect the verbosity level, thus both .B \-\-quiet and .B \-\-no\-warn -have to be used to not display warnings and to not alter the exit status. +have to be used to not display warnings and +to not alter the exit status. .TP .B \-\-robot -Print messages in a machine-parsable format. This is intended to ease -writing frontends that want to use +Print messages in a machine-parsable format. +This is intended to ease writing frontends that want to use .B xz -instead of liblzma, which may be the case with various scripts. The output -with this option enabled is meant to be stable across +instead of liblzma, which may be the case with various scripts. +The output with this option enabled is meant to be stable across .B xz -releases. See the section +releases. +See the section .B "ROBOT MODE" for details. .TP -.BR \-\-info-memory -Display the current memory usage limit in human-readable format on -a single line, and exit successfully. To see how much RAM +.BR \-\-info\-memory +Display, in human-readable format, how much physical memory (RAM) .B xz -thinks your system has, use -.BR "\-\-memory=100% \-\-info\-memory" . +thinks the system has and the memory usage limits for compression +and decompression, and exit successfully. .TP .BR \-h ", " \-\-help Display a help message describing the most commonly used options, @@ -1143,24 +1558,29 @@ and exit successfully .BR \-V ", " \-\-version Display the version number of .B xz -and liblzma in human readable format. To get machine-parsable output, specify +and liblzma in human readable format. +To get machine-parsable output, specify .B \-\-robot before .BR \-\-version . -.SH ROBOT MODE +. +.SH "ROBOT MODE" The robot mode is activated with the .B \-\-robot -option. It makes the output of +option. +It makes the output of .B xz -easier to parse by other programs. Currently +easier to parse by other programs. +Currently .B \-\-robot is supported only together with .BR \-\-version , -.BR \-\-info-memory , +.BR \-\-info\-memory , and .BR \-\-list . -It will be supported for normal compression and decompression in the future. -.PP +It will be supported for normal compression and +decompression in the future. +. .SS Version .B "xz \-\-robot \-\-version" will print the version number of @@ -1175,24 +1595,19 @@ and liblzma in the following format: Major version. .TP .I YYY -Minor version. Even numbers are stable. +Minor version. +Even numbers are stable. Odd numbers are alpha or beta versions. .TP .I ZZZ -Patch level for stable releases or just a counter for development releases. +Patch level for stable releases or +just a counter for development releases. .TP .I S Stability. -.B 0 -is alpha, -.B 1 -is beta, and -.B 2 -is stable. +0 is alpha, 1 is beta, and 2 is stable. .I S -should be always -.B 2 -when +should be always 2 when .I YYY is even. .PP @@ -1206,31 +1621,48 @@ Examples: 4.999.9beta is and 5.0.0 is .BR 50000002 . -.SS Memory limit information -.B "xz \-\-robot \-\-info-memory" -prints the current memory usage limit as bytes on a single line. -To get the total amount of installed RAM, use -.BR "xz \-\-robot \-\-memory=100% \-\-info-memory" . -.SS List mode +. +.SS "Memory limit information" +.B "xz \-\-robot \-\-info\-memory" +prints a single line with three tab-separated columns: +.IP 1. 4 +Total amount of physical memory (RAM) in bytes +.IP 2. 4 +Memory usage limit for compression in bytes. +A special value of zero indicates the default setting, +which for single-threaded mode is the same as no limit. +.IP 3. 4 +Memory usage limit for decompression in bytes. +A special value of zero indicates the default setting, +which for single-threaded mode is the same as no limit. +.PP +In the future, the output of +.B "xz \-\-robot \-\-info\-memory" +may have more columns, but never more than a single line. +. +.SS "List mode" .B "xz \-\-robot \-\-list" -uses tab-separated output. The first column of every line has a string +uses tab-separated output. +The first column of every line has a string that indicates the type of the information found on that line: .TP .B name -This is always the first line when starting to list a file. The second -column on the line is the filename. +This is always the first line when starting to list a file. +The second column on the line is the filename. .TP .B file This line contains overall information about the .B .xz -file. This line is always printed after the +file. +This line is always printed after the .B name line. .TP .B stream This line type is used only when .B \-\-verbose -was specified. There are as many +was specified. +There are as many .B stream lines as there are streams in the .B .xz @@ -1239,11 +1671,13 @@ file. .B block This line type is used only when .B \-\-verbose -was specified. There are as many +was specified. +There are as many .B block lines as there are blocks in the .B .xz -file. The +file. +The .B block lines are shown after all the .B stream @@ -1252,9 +1686,11 @@ lines; different line types are not interleaved. .B summary This line type is used only when .B \-\-verbose -was specified twice. This line is printed after all +was specified twice. +This line is printed after all .B block -lines. Like the +lines. +Like the .B file line, the .B summary @@ -1263,12 +1699,13 @@ line contains overall information about the file. .TP .B totals -This line is always the very last line of the list output. It shows -the total counts and sizes. +This line is always the very last line of the list output. +It shows the total counts and sizes. .PP The columns of the .B file lines: +.PD 0 .RS .IP 2. 4 Number of streams in the file @@ -1285,8 +1722,8 @@ If ratio is over 9.999, three dashes .RB ( \-\-\- ) are displayed instead of the ratio. .IP 7. 4 -Comma-separated list of integrity check names. The following strings are -used for the known check types: +Comma-separated list of integrity check names. +The following strings are used for the known check types: .BR None , .BR CRC32 , .BR CRC64 , @@ -1300,10 +1737,12 @@ is the Check ID as a decimal number (one or two digits). .IP 8. 4 Total size of stream padding in the file .RE +.PD .PP The columns of the .B stream lines: +.PD 0 .RS .IP 2. 4 Stream number (the first stream is 1) @@ -1324,15 +1763,18 @@ Name of the integrity check .IP 10. 4 Size of stream padding .RE +.PD .PP The columns of the .B block lines: +.PD 0 .RS .IP 2. 4 Number of the stream containing this block .IP 3. 4 -Block number relative to the beginning of the stream (the first block is 1) +Block number relative to the beginning of the stream +(the first block is 1) .IP 4. 4 Block number relative to the beginning of the file .IP 5. 4 @@ -1348,14 +1790,18 @@ Compression ratio .IP 10. 4 Name of the integrity check .RE +.PD .PP If .B \-\-verbose was specified twice, additional columns are included on the .B block -lines. These are not displayed with a single +lines. +These are not displayed with a single .BR \-\-verbose , -because getting this information requires many seeks and can thus be slow: +because getting this information requires many seeks +and can thus be slow: +.PD 0 .RS .IP 11. 4 Value of the integrity check in hexadecimal @@ -1369,26 +1815,30 @@ indicates that compressed size is present, and indicates that uncompressed size is present. If the flag is not set, a dash .RB ( \- ) -is shown instead to keep the string length fixed. New flags may be added -to the end of the string in the future. +is shown instead to keep the string length fixed. +New flags may be added to the end of the string in the future. .IP 14. 4 Size of the actual compressed data in the block (this excludes the block header, block padding, and check fields) .IP 15. 4 -Amount of memory (as bytes) required to decompress this block with this +Amount of memory (in bytes) required to decompress +this block with this .B xz version .IP 16. 4 -Filter chain. Note that most of the options used at compression time cannot -be known, because only the options that are needed for decompression are -stored in the +Filter chain. +Note that most of the options used at compression time +cannot be known, because only the options +that are needed for decompression are stored in the .B .xz headers. .RE +.PD .PP The columns of the .B totals line: +.PD 0 .RS .IP 2. 4 Number of streams @@ -1401,14 +1851,17 @@ Uncompressed size .IP 6. 4 Average compression ratio .IP 7. 4 -Comma-separated list of integrity check names that were present in the files +Comma-separated list of integrity check names +that were present in the files .IP 8. 4 Stream padding size .IP 9. 4 -Number of files. This is here to keep the order of the earlier columns -the same as on +Number of files. +This is here to +keep the order of the earlier columns the same as on .B file lines. +.PD .RE .PP If @@ -1416,10 +1869,11 @@ If was specified twice, additional columns are included on the .B totals line: +.PD 0 .RS .IP 10. 4 -Maximum amount of memory (as bytes) required to decompress the files -with this +Maximum amount of memory (in bytes) required to decompress +the files with this .B xz version .IP 11. 4 @@ -1429,9 +1883,12 @@ or indicating if all block headers have both compressed size and uncompressed size stored in them .RE +.PD .PP -Future versions may add new line types and new columns can be added to -the existing line types, but the existing columns won't be changed. +Future versions may add new line types and +new columns can be added to the existing line types, +but the existing columns won't be changed. +. .SH "EXIT STATUS" .TP .B 0 @@ -1441,21 +1898,76 @@ All is good. An error occurred. .TP .B 2 -Something worth a warning occurred, but no actual errors occurred. +Something worth a warning occurred, +but no actual errors occurred. .PP -Notices (not warnings or errors) printed on standard error don't affect -the exit status. +Notices (not warnings or errors) printed on standard error +don't affect the exit status. +. .SH ENVIRONMENT +.B xz +parses space-separated lists of options +from the environment variables +.B XZ_DEFAULTS +and +.BR XZ_OPT , +in this order, before parsing the options from the command line. +Note that only options are parsed from the environment variables; +all non-options are silently ignored. +Parsing is done with +.BR getopt_long (3) +which is used also for the command line arguments. +.TP +.B XZ_DEFAULTS +User-specific or system-wide default options. +Typically this is set in a shell initialization script to enable +.BR xz 's +memory usage limiter by default. +Excluding shell initialization scripts +and similar special cases, scripts must never set or unset +.BR XZ_DEFAULTS . .TP .B XZ_OPT -A space-separated list of options is parsed from +This is for passing options to +.B xz +when it is not possible to set the options directly on the +.B xz +command line. +This is the case e.g. when +.B xz +is run by a script or tool, e.g. GNU +.BR tar (1): +.RS +.RS +.PP +.nf +.ft CW +XZ_OPT=\-2v tar caf foo.tar.xz foo +.ft R +.fi +.RE +.RE +.IP "" +Scripts may use .B XZ_OPT -before parsing the options given on the command line. Note that only -options are parsed from -.BR XZ_OPT ; -all non-options are silently ignored. Parsing is done with -.BR getopt_long (3) -which is used also for the command line arguments. +e.g. to set script-specific default compression options. +It is still recommended to allow users to override +.B XZ_OPT +if that is reasonable, e.g. in +.BR sh (1) +scripts one may use something like this: +.RS +.RS +.PP +.nf +.ft CW +XZ_OPT=${XZ_OPT\-"\-7e"} +export XZ_OPT +.ft R +.fi +.RE +.RE +. .SH "LZMA UTILS COMPATIBILITY" The command line syntax of .B xz @@ -1464,26 +1976,32 @@ is practically a superset of .BR unlzma , and .BR lzcat -as found from LZMA Utils 4.32.x. In most cases, it is possible to replace -LZMA Utils with XZ Utils without breaking existing scripts. There are some -incompatibilities though, which may sometimes cause problems. +as found from LZMA Utils 4.32.x. +In most cases, it is possible to replace +LZMA Utils with XZ Utils without breaking existing scripts. +There are some incompatibilities though, +which may sometimes cause problems. +. .SS "Compression preset levels" The numbering of the compression level presets is not identical in .B xz and LZMA Utils. -The most important difference is how dictionary sizes are mapped to different -presets. Dictionary size is roughly equal to the decompressor memory usage. +The most important difference is how dictionary sizes +are mapped to different presets. +Dictionary size is roughly equal to the decompressor memory usage. .RS +.PP .TS tab(;); c c c c n n. Level;xz;LZMA Utils -\-1;64 KiB;64 KiB -\-2;512 KiB;1 MiB -\-3;1 MiB;512 KiB -\-4;2 MiB;1 MiB -\-5;4 MiB;2 MiB +\-0;256 KiB;N/A +\-1;1 MiB;64 KiB +\-2;2 MiB;1 MiB +\-3;4 MiB;512 KiB +\-4;4 MiB;1 MiB +\-5;8 MiB;2 MiB \-6;8 MiB;4 MiB \-7;16 MiB;8 MiB \-8;32 MiB;16 MiB @@ -1491,20 +2009,24 @@ Level;xz;LZMA Utils .TE .RE .PP -The dictionary size differences affect the compressor memory usage too, -but there are some other differences between LZMA Utils and XZ Utils, which +The dictionary size differences affect +the compressor memory usage too, +but there are some other differences between +LZMA Utils and XZ Utils, which make the difference even bigger: .RS +.PP .TS tab(;); c c c c n n. Level;xz;LZMA Utils 4.32.x -\-1;2 MiB;2 MiB -\-2;5 MiB;12 MiB -\-3;13 MiB;12 MiB -\-4;25 MiB;16 MiB -\-5;48 MiB;26 MiB +\-0;3 MiB;N/A +\-1;9 MiB;2 MiB +\-2;17 MiB;12 MiB +\-3;32 MiB;12 MiB +\-4;48 MiB;16 MiB +\-5;94 MiB;26 MiB \-6;94 MiB;45 MiB \-7;186 MiB;83 MiB \-8;370 MiB;159 MiB @@ -1516,33 +2038,40 @@ The default preset level in LZMA Utils is .B \-7 while in XZ Utils it is .BR \-6 , -so both use 8 MiB dictionary by default. +so both use an 8 MiB dictionary by default. +. .SS "Streamed vs. non-streamed .lzma files" -Uncompressed size of the file can be stored in the +The uncompressed size of the file can be stored in the .B .lzma -header. LZMA Utils does that when compressing regular files. -The alternative is to mark that uncompressed size is unknown and -use end of payload marker to indicate where the decompressor should stop. -LZMA Utils uses this method when uncompressed size isn't known, which is -the case for example in pipes. +header. +LZMA Utils does that when compressing regular files. +The alternative is to mark that uncompressed size is unknown +and use end-of-payload marker to indicate +where the decompressor should stop. +LZMA Utils uses this method when uncompressed size isn't known, +which is the case for example in pipes. .PP .B xz supports decompressing .B .lzma -files with or without end of payload marker, but all +files with or without end-of-payload marker, but all .B .lzma files created by .B xz -will use end of payload marker and have uncompressed size marked as unknown -in the +will use end-of-payload marker and have uncompressed size +marked as unknown in the .B .lzma -header. This may be a problem in some (uncommon) situations. For example, a +header. +This may be a problem in some uncommon situations. +For example, a .B .lzma -decompressor in an embedded device might work only with files that have known -uncompressed size. If you hit this problem, you need to use LZMA Utils or -LZMA SDK to create +decompressor in an embedded device might work +only with files that have known uncompressed size. +If you hit this problem, you need to use LZMA Utils +or LZMA SDK to create .B .lzma files with known uncompressed size. +. .SS "Unsupported .lzma files" The .B .lzma @@ -1550,7 +2079,8 @@ format allows .I lc values up to 8, and .I lp -values up to 4. LZMA Utils can decompress files with any +values up to 4. +LZMA Utils can decompress files with any .I lc and .IR lp , @@ -1566,24 +2096,25 @@ is possible with .B xz and with LZMA SDK. .PP -The implementation of the LZMA1 filter in liblzma requires -that the sum of +The implementation of the LZMA1 filter in liblzma +requires that the sum of .I lc and .I lp -must not exceed 4. Thus, +must not exceed 4. +Thus, .B .lzma -files which exceed this limitation, cannot be decompressed with +files, which exceed this limitation, cannot be decompressed with .BR xz . .PP LZMA Utils creates only .B .lzma -files which have dictionary size of +files which have a dictionary size of .RI "2^" n -(a power of 2), but accepts files with any dictionary size. +(a power of 2) but accepts files with any dictionary size. liblzma accepts only .B .lzma -files which have dictionary size of +files which have a dictionary size of .RI "2^" n or .RI "2^" n " + 2^(" n "\-1)." @@ -1591,13 +2122,18 @@ This is to decrease false positives when detecting .B .lzma files. .PP -These limitations shouldn't be a problem in practice, since practically all +These limitations shouldn't be a problem in practice, +since practically all .B .lzma files have been compressed with settings that liblzma will accept. +. .SS "Trailing garbage" -When decompressing, LZMA Utils silently ignore everything after the first +When decompressing, +LZMA Utils silently ignore everything after the first .B .lzma -stream. In most situations, this is a bug. This also means that LZMA Utils +stream. +In most situations, this is a bug. +This also means that LZMA Utils don't support decompressing concatenated .B .lzma files. @@ -1606,34 +2142,46 @@ If there is data left after the first .B .lzma stream, .B xz -considers the file to be corrupt. This may break obscure scripts which have +considers the file to be corrupt. +This may break obscure scripts which have assumed that trailing garbage is ignored. +. .SH NOTES -.SS Compressed output may vary -The exact compressed output produced from the same uncompressed input file -may vary between XZ Utils versions even if compression options are identical. -This is because the encoder can be improved (faster or better compression) -without affecting the file format. The output can vary even between different -builds of the same XZ Utils version, if different build options are used. +. +.SS "Compressed output may vary" +The exact compressed output produced from +the same uncompressed input file +may vary between XZ Utils versions even if +compression options are identical. +This is because the encoder can be improved +(faster or better compression) +without affecting the file format. +The output can vary even between different +builds of the same XZ Utils version, +if different build options are used. .PP The above means that implementing .B \-\-rsyncable to create rsyncable .B .xz -files is not going to happen without freezing a part of the encoder +files is not going to happen without +freezing a part of the encoder implementation, which can then be used with .BR \-\-rsyncable . -.SS Embedded .xz decompressors +. +.SS "Embedded .xz decompressors" Embedded .B .xz -decompressor implementations like XZ Embedded don't necessarily support files -created with +decompressor implementations like XZ Embedded don't necessarily +support files created with integrity .I check types other than .B none and .BR crc32 . -Since the default is \fB\-\-check=\fIcrc64\fR, you must use +Since the default is +.BR \-\-check=crc64 , +you must use .B \-\-check=none or .B \-\-check=crc32 @@ -1643,53 +2191,374 @@ Outside embedded systems, all .B .xz format decompressors support all the .I check -types, or at least are able to decompress the file without verifying the +types, or at least are able to decompress +the file without verifying the integrity check if the particular .I check is not supported. .PP -XZ Embedded supports BCJ filters, but only with the default start offset. +XZ Embedded supports BCJ filters, +but only with the default start offset. +. .SH EXAMPLES +. .SS Basics +Compress the file +.I foo +into +.I foo.xz +using the default compression level +.RB ( \-6 ), +and remove +.I foo +if compression is successful: +.RS +.PP +.nf +.ft CW +xz foo +.ft R +.fi +.RE +.PP +Decompress +.I bar.xz +into +.I bar +and don't remove +.I bar.xz +even if decompression is successful: +.RS +.PP +.nf +.ft CW +xz \-dk bar.xz +.ft R +.fi +.RE +.PP +Create +.I baz.tar.xz +with the preset +.B \-4e +.RB ( "\-4 \-\-extreme" ), +which is slower than e.g. the default +.BR \-6 , +but needs less memory for compression and decompression (48\ MiB +and 5\ MiB, respectively): +.RS +.PP +.nf +.ft CW +tar cf \- baz | xz \-4e > baz.tar.xz +.ft R +.fi +.RE +.PP A mix of compressed and uncompressed files can be decompressed to standard output with a single command: -.IP -.B "xz -dcf a.txt b.txt.xz c.txt d.txt.xz > abcd.txt" -.SS Parallel compression of many files +.RS +.PP +.nf +.ft CW +xz \-dcf a.txt b.txt.xz c.txt d.txt.lzma > abcd.txt +.ft R +.fi +.RE +. +.SS "Parallel compression of many files" On GNU and *BSD, .BR find (1) and .BR xargs (1) -can be used to parallellize compression of many files: +can be used to parallelize compression of many files: +.RS .PP -.IP -.B "find . \-type f \e! \-name '*.xz' \-print0 | xargs \-0r \-P4 \-n16 xz" +.nf +.ft CW +find . \-type f \e! \-name '*.xz' \-print0 \e + | xargs \-0r \-P4 \-n16 xz \-T1 +.ft R +.fi +.RE .PP The .B \-P -option sets the number of parallel +option to +.BR xargs (1) +sets the number of parallel .B xz -processes. The best value for the +processes. +The best value for the .B \-n option depends on how many files there are to be compressed. -If there are only a couple of files, the value should probably be -.BR 1 ; +If there are only a couple of files, +the value should probably be 1; with tens of thousands of files, -.B 100 -or even more may be appropriate to reduce the number of +100 or even more may be appropriate to reduce the number of .B xz processes that .BR xargs (1) will eventually create. -.SS Robot mode examples -Calculating how many bytes have been saved in total after compressing -multiple files: -.IP -.B "xz --robot --list *.xz | awk '/^totals/{print $5\-$4}'" +.PP +The option +.B \-T1 +for +.B xz +is there to force it to single-threaded mode, because +.BR xargs (1) +is used to control the amount of parallelization. +. +.SS "Robot mode" +Calculate how many bytes have been saved in total +after compressing multiple files: +.RS +.PP +.nf +.ft CW +xz \-\-robot \-\-list *.xz | awk '/^totals/{print $5\-$4}' +.ft R +.fi +.RE +.PP +A script may want to know that it is using new enough +.BR xz . +The following +.BR sh (1) +script checks that the version number of the +.B xz +tool is at least 5.0.0. +This method is compatible with old beta versions, +which didn't support the +.B \-\-robot +option: +.RS +.PP +.nf +.ft CW +if ! eval "$(xz \-\-robot \-\-version 2> /dev/null)" || + [ "$XZ_VERSION" \-lt 50000002 ]; then + echo "Your xz is too old." +fi +unset XZ_VERSION LIBLZMA_VERSION +.ft R +.fi +.RE +.PP +Set a memory usage limit for decompression using +.BR XZ_OPT , +but if a limit has already been set, don't increase it: +.RS +.PP +.nf +.ft CW +NEWLIM=$((123 << 20)) # 123 MiB +OLDLIM=$(xz \-\-robot \-\-info\-memory | cut \-f3) +if [ $OLDLIM \-eq 0 \-o $OLDLIM \-gt $NEWLIM ]; then + XZ_OPT="$XZ_OPT \-\-memlimit\-decompress=$NEWLIM" + export XZ_OPT +fi +.ft R +.fi +.RE +. +.SS "Custom compressor filter chains" +The simplest use for custom filter chains is +customizing a LZMA2 preset. +This can be useful, +because the presets cover only a subset of the +potentially useful combinations of compression settings. +.PP +The CompCPU columns of the tables +from the descriptions of the options +.BR "\-0" " ... " "\-9" +and +.B \-\-extreme +are useful when customizing LZMA2 presets. +Here are the relevant parts collected from those two tables: +.RS +.PP +.TS +tab(;); +c c +n n. +Preset;CompCPU +\-0;0 +\-1;1 +\-2;2 +\-3;3 +\-4;4 +\-5;5 +\-6;6 +\-5e;7 +\-6e;8 +.TE +.RE +.PP +If you know that a file requires +somewhat big dictionary (e.g. 32 MiB) to compress well, +but you want to compress it quicker than +.B "xz \-8" +would do, a preset with a low CompCPU value (e.g. 1) +can be modified to use a bigger dictionary: +.RS +.PP +.nf +.ft CW +xz \-\-lzma2=preset=1,dict=32MiB foo.tar +.ft R +.fi +.RE +.PP +With certain files, the above command may be faster than +.B "xz \-6" +while compressing significantly better. +However, it must be emphasized that only some files benefit from +a big dictionary while keeping the CompCPU value low. +The most obvious situation, +where a big dictionary can help a lot, +is an archive containing very similar files +of at least a few megabytes each. +The dictionary size has to be significantly bigger +than any individual file to allow LZMA2 to take +full advantage of the similarities between consecutive files. +.PP +If very high compressor and decompressor memory usage is fine, +and the file being compressed is +at least several hundred megabytes, it may be useful +to use an even bigger dictionary than the 64 MiB that +.B "xz \-9" +would use: +.RS +.PP +.nf +.ft CW +xz \-vv \-\-lzma2=dict=192MiB big_foo.tar +.ft R +.fi +.RE +.PP +Using +.B \-vv +.RB ( "\-\-verbose \-\-verbose" ) +like in the above example can be useful +to see the memory requirements +of the compressor and decompressor. +Remember that using a dictionary bigger than +the size of the uncompressed file is waste of memory, +so the above command isn't useful for small files. +.PP +Sometimes the compression time doesn't matter, +but the decompressor memory usage has to be kept low +e.g. to make it possible to decompress the file on +an embedded system. +The following command uses +.B \-6e +.RB ( "\-6 \-\-extreme" ) +as a base and sets the dictionary to only 64\ KiB. +The resulting file can be decompressed with XZ Embedded +(that's why there is +.BR \-\-check=crc32 ) +using about 100\ KiB of memory. +.RS +.PP +.nf +.ft CW +xz \-\-check=crc32 \-\-lzma2=preset=6e,dict=64KiB foo +.ft R +.fi +.RE +.PP +If you want to squeeze out as many bytes as possible, +adjusting the number of literal context bits +.RI ( lc ) +and number of position bits +.RI ( pb ) +can sometimes help. +Adjusting the number of literal position bits +.RI ( lp ) +might help too, but usually +.I lc +and +.I pb +are more important. +E.g. a source code archive contains mostly US-ASCII text, +so something like the following might give +slightly (like 0.1\ %) smaller file than +.B "xz \-6e" +(try also without +.BR lc=4 ): +.RS +.PP +.nf +.ft CW +xz \-\-lzma2=preset=6e,pb=0,lc=4 source_code.tar +.ft R +.fi +.RE +.PP +Using another filter together with LZMA2 can improve +compression with certain file types. +E.g. to compress a x86-32 or x86-64 shared library +using the x86 BCJ filter: +.RS +.PP +.nf +.ft CW +xz \-\-x86 \-\-lzma2 libfoo.so +.ft R +.fi +.RE +.PP +Note that the order of the filter options is significant. +If +.B \-\-x86 +is specified after +.BR \-\-lzma2 , +.B xz +will give an error, +because there cannot be any filter after LZMA2, +and also because the x86 BCJ filter cannot be used +as the last filter in the chain. +.PP +The Delta filter together with LZMA2 +can give good results with bitmap images. +It should usually beat PNG, +which has a few more advanced filters than simple +delta but uses Deflate for the actual compression. +.PP +The image has to be saved in uncompressed format, +e.g. as uncompressed TIFF. +The distance parameter of the Delta filter is set +to match the number of bytes per pixel in the image. +E.g. 24-bit RGB bitmap needs +.BR dist=3 , +and it is also good to pass +.B pb=0 +to LZMA2 to accommodate the three-byte alignment: +.RS +.PP +.nf +.ft CW +xz \-\-delta=dist=3 \-\-lzma2=pb=0 foo.tiff +.ft R +.fi +.RE +.PP +If multiple images have been put into a single archive (e.g.\& +.BR .tar ), +the Delta filter will work on that too as long as all images +have the same number of bytes per pixel. +. .SH "SEE ALSO" .BR xzdec (1), +.BR xzdiff (1), +.BR xzgrep (1), +.BR xzless (1), +.BR xzmore (1), .BR gzip (1), -.BR bzip2 (1) +.BR bzip2 (1), +.BR 7z (1) .PP XZ Utils: <http://tukaani.org/xz/> .br diff --git a/Build/source/extra/xz/src/xzdec/Makefile.am b/Build/source/extra/xz/src/xzdec/Makefile.am index ad487721338..2348e4e6ba6 100644 --- a/Build/source/extra/xz/src/xzdec/Makefile.am +++ b/Build/source/extra/xz/src/xzdec/Makefile.am @@ -12,8 +12,6 @@ $(xzdec_CPPFLAGS) $(CPPFLAGS) $(RCFLAGS) -i $< -o $@ -bin_PROGRAMS = xzdec lzmadec - xzdec_SOURCES = \ xzdec.c \ $(top_srcdir)/src/common/tuklib_progname.c \ @@ -27,9 +25,7 @@ xzdec_CPPFLAGS = \ -DTUKLIB_GETTEXT=0 \ -I$(top_srcdir)/src/common \ -I$(top_srcdir)/src/liblzma/api \ - -I$(top_builddir)/lib \ - $(STATIC_CPPFLAGS) -xzdec_LDFLAGS = $(STATIC_LDFLAGS) + -I$(top_builddir)/lib xzdec_LDADD = $(top_builddir)/src/liblzma/liblzma.la if COND_GNULIB @@ -53,8 +49,17 @@ lzmadec_LDFLAGS = $(xzdec_LDFLAGS) lzmadec_LDADD = $(xzdec_LDADD) +bin_PROGRAMS = + +if COND_XZDEC +bin_PROGRAMS += xzdec dist_man_MANS = xzdec.1 +endif +if COND_LZMADEC +bin_PROGRAMS += lzmadec + +# FIXME: If xzdec is disabled, this will create a dangling symlink. install-data-hook: cd $(DESTDIR)$(mandir)/man1 && \ target=`echo xzdec | sed '$(transform)'` && \ @@ -66,3 +71,4 @@ uninstall-hook: cd $(DESTDIR)$(mandir)/man1 && \ link=`echo lzmadec | sed '$(transform)'` && \ rm -f $$link.1 +endif diff --git a/Build/source/extra/xz/src/xzdec/Makefile.in b/Build/source/extra/xz/src/xzdec/Makefile.in index 058e92d1713..f6e129733ed 100644 --- a/Build/source/extra/xz/src/xzdec/Makefile.in +++ b/Build/source/extra/xz/src/xzdec/Makefile.in @@ -34,10 +34,12 @@ PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ -bin_PROGRAMS = xzdec$(EXEEXT) lzmadec$(EXEEXT) @COND_W32_TRUE@am__append_1 = xzdec_w32res.rc @COND_GNULIB_TRUE@am__append_2 = $(top_builddir)/lib/libgnu.a @COND_W32_TRUE@am__append_3 = lzmadec_w32res.rc +bin_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) +@COND_XZDEC_TRUE@am__append_4 = xzdec +@COND_LZMADEC_TRUE@am__append_5 = lzmadec subdir = src/xzdec DIST_COMMON = $(dist_man_MANS) $(srcdir)/Makefile.am \ $(srcdir)/Makefile.in @@ -53,6 +55,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/acx_pthread.m4 \ $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/tuklib_common.m4 \ $(top_srcdir)/m4/tuklib_cpucores.m4 \ $(top_srcdir)/m4/tuklib_integer.m4 \ + $(top_srcdir)/m4/tuklib_mbstr.m4 \ $(top_srcdir)/m4/tuklib_physmem.m4 \ $(top_srcdir)/m4/tuklib_progname.m4 \ $(top_srcdir)/m4/visibility.m4 $(top_srcdir)/configure.ac @@ -62,6 +65,8 @@ mkinstalldirs = $(install_sh) -d CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = CONFIG_CLEAN_VPATH_FILES = +@COND_XZDEC_TRUE@am__EXEEXT_1 = xzdec$(EXEEXT) +@COND_LZMADEC_TRUE@am__EXEEXT_2 = lzmadec$(EXEEXT) am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)" PROGRAMS = $(bin_PROGRAMS) am__lzmadec_SOURCES_DIST = xzdec.c \ @@ -89,9 +94,6 @@ am_xzdec_OBJECTS = xzdec-xzdec.$(OBJEXT) \ xzdec_OBJECTS = $(am_xzdec_OBJECTS) xzdec_DEPENDENCIES = $(top_builddir)/src/liblzma/liblzma.la \ $(am__append_2) $(am__DEPENDENCIES_1) -xzdec_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ - --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(xzdec_LDFLAGS) \ - $(LDFLAGS) -o $@ DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/build-aux/depcomp am__depfiles_maybe = depfiles @@ -158,8 +160,6 @@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ -DYNAMIC_CPPFLAGS = @DYNAMIC_CPPFLAGS@ -DYNAMIC_LDFLAGS = @DYNAMIC_LDFLAGS@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ @@ -192,6 +192,7 @@ LTLIBICONV = @LTLIBICONV@ LTLIBINTL = @LTLIBINTL@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -221,8 +222,6 @@ RC = @RC@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ -STATIC_CPPFLAGS = @STATIC_CPPFLAGS@ -STATIC_LDFLAGS = @STATIC_LDFLAGS@ STRIP = @STRIP@ USE_NLS = @USE_NLS@ VERSION = @VERSION@ @@ -232,6 +231,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ acx_pthread_config = @acx_pthread_config@ @@ -287,10 +287,8 @@ xzdec_CPPFLAGS = \ -DTUKLIB_GETTEXT=0 \ -I$(top_srcdir)/src/common \ -I$(top_srcdir)/src/liblzma/api \ - -I$(top_builddir)/lib \ - $(STATIC_CPPFLAGS) + -I$(top_builddir)/lib -xzdec_LDFLAGS = $(STATIC_LDFLAGS) xzdec_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(am__append_2) \ $(LTLIBINTL) lzmadec_SOURCES = xzdec.c $(top_srcdir)/src/common/tuklib_progname.c \ @@ -298,7 +296,7 @@ lzmadec_SOURCES = xzdec.c $(top_srcdir)/src/common/tuklib_progname.c \ lzmadec_CPPFLAGS = $(xzdec_CPPFLAGS) -DLZMADEC lzmadec_LDFLAGS = $(xzdec_LDFLAGS) lzmadec_LDADD = $(xzdec_LDADD) -dist_man_MANS = xzdec.1 +@COND_XZDEC_TRUE@dist_man_MANS = xzdec.1 all: all-am .SUFFIXES: @@ -381,7 +379,7 @@ lzmadec$(EXEEXT): $(lzmadec_OBJECTS) $(lzmadec_DEPENDENCIES) $(lzmadec_LINK) $(lzmadec_OBJECTS) $(lzmadec_LDADD) $(LIBS) xzdec$(EXEEXT): $(xzdec_OBJECTS) $(xzdec_DEPENDENCIES) @rm -f xzdec$(EXEEXT) - $(xzdec_LINK) $(xzdec_OBJECTS) $(xzdec_LDADD) $(LIBS) + $(LINK) $(xzdec_OBJECTS) $(xzdec_LDADD) $(LIBS) mostlyclean-compile: -rm -f *.$(OBJEXT) @@ -672,6 +670,8 @@ distclean-generic: maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." +@COND_LZMADEC_FALSE@install-data-hook: +@COND_LZMADEC_FALSE@uninstall-hook: clean: clean-am clean-am: clean-binPROGRAMS clean-generic clean-libtool mostlyclean-am @@ -770,17 +770,18 @@ uninstall-man: uninstall-man1 $(RC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ $(xzdec_CPPFLAGS) $(CPPFLAGS) $(RCFLAGS) -i $< -o $@ -install-data-hook: - cd $(DESTDIR)$(mandir)/man1 && \ - target=`echo xzdec | sed '$(transform)'` && \ - link=`echo lzmadec | sed '$(transform)'` && \ - rm -f $$link.1 && \ - $(LN_S) $$target.1 $$link.1 - -uninstall-hook: - cd $(DESTDIR)$(mandir)/man1 && \ - link=`echo lzmadec | sed '$(transform)'` && \ - rm -f $$link.1 +# FIXME: If xzdec is disabled, this will create a dangling symlink. +@COND_LZMADEC_TRUE@install-data-hook: +@COND_LZMADEC_TRUE@ cd $(DESTDIR)$(mandir)/man1 && \ +@COND_LZMADEC_TRUE@ target=`echo xzdec | sed '$(transform)'` && \ +@COND_LZMADEC_TRUE@ link=`echo lzmadec | sed '$(transform)'` && \ +@COND_LZMADEC_TRUE@ rm -f $$link.1 && \ +@COND_LZMADEC_TRUE@ $(LN_S) $$target.1 $$link.1 + +@COND_LZMADEC_TRUE@uninstall-hook: +@COND_LZMADEC_TRUE@ cd $(DESTDIR)$(mandir)/man1 && \ +@COND_LZMADEC_TRUE@ link=`echo lzmadec | sed '$(transform)'` && \ +@COND_LZMADEC_TRUE@ rm -f $$link.1 # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. diff --git a/Build/source/extra/xz/src/xzdec/lzmadec_w32res.rc b/Build/source/extra/xz/src/xzdec/lzmadec_w32res.rc index 2c71a51ac31..b85962df241 100644 --- a/Build/source/extra/xz/src/xzdec/lzmadec_w32res.rc +++ b/Build/source/extra/xz/src/xzdec/lzmadec_w32res.rc @@ -1,3 +1,10 @@ +/* + * Author: Lasse Collin + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + #define MY_TYPE VFT_APP #define MY_NAME "lzmadec" #define MY_SUFFIX ".exe" diff --git a/Build/source/extra/xz/src/xzdec/xzdec.1 b/Build/source/extra/xz/src/xzdec/xzdec.1 index 3057c586d25..7cc9be5de08 100644 --- a/Build/source/extra/xz/src/xzdec/xzdec.1 +++ b/Build/source/extra/xz/src/xzdec/xzdec.1 @@ -4,7 +4,7 @@ .\" This file has been put into the public domain. .\" You can do whatever you want with this file. .\" -.TH XZDEC 1 "2010-03-07" "Tukaani" "XZ Utils" +.TH XZDEC 1 "2010-09-27" "Tukaani" "XZ Utils" .SH NAME xzdec, lzmadec \- Small .xz and .lzma decompressors .SH SYNOPSIS @@ -25,7 +25,8 @@ files. .B xzdec is intended to work as a drop-in replacement for .BR xz (1) -in the most common situations where a script has been written to use +in the most common situations where a script +has been written to use .B "xz \-\-decompress \-\-stdout" (and possibly a few other commonly used options) to decompress .B .xz @@ -43,9 +44,12 @@ files. .PP To reduce the size of the executable, .B xzdec -doesn't support multithreading or localization, and doesn't read options from +doesn't support multithreading or localization, +and doesn't read options from +.B XZ_DEFAULTS +and .B XZ_OPT -environment variable. +environment variables. .B xzdec doesn't support displaying intermediate progress information: sending .B SIGINFO @@ -77,45 +81,6 @@ compatibility. .B xzdec always writes the decompressed data to standard output. .TP -\fB\-M\fR \fIlimit\fR, \fB\-\-memory=\fIlimit -Set the memory usage -.IR limit . -If this option is specified multiple times, the last one takes effect. The -.I limit -can be specified in multiple ways: -.RS -.IP \(bu 3 -The -.I limit -can be an absolute value in bytes. Using an integer suffix like -.B MiB -can be useful. Example: -.B "\-\-memory=80MiB" -.IP \(bu 3 -The -.I limit -can be specified as a percentage of physical RAM. Example: -.B "\-\-memory=70%" -.IP \(bu 3 -The -.I limit -can be reset back to its default value by setting it to -.BR 0 . -.IP \(bu 3 -The memory usage limiting can be effectively disabled by setting -.I limit -to -.BR max . -This isn't recommended. It's usually better to use, for example, -.BR \-\-memory=90% . -.RE -.IP -The current -.I limit -can be seen near the bottom of the output of the -.B \-\-help -option. -.TP .BR \-q ", " \-\-quiet Specifying this once does nothing since .B xzdec @@ -127,8 +92,7 @@ Ignored for .BR xz (1) compatibility. .B xzdec -never uses the exit status -.BR "2" . +never uses the exit status 2. .TP .BR \-h ", " \-\-help Display a help message and exit successfully. @@ -148,18 +112,32 @@ An error occurred. .B xzdec doesn't have any warning messages like .BR xz (1) -has, thus the exit status -.B 2 -is not used by +has, thus the exit status 2 is not used by .BR xzdec . .SH NOTES +Use +.BR xz (1) +instead of +.B xzdec +or +.B lzmadec +for normal everyday use. +.B xzdec +or +.B lzmadec +are meant only for situations where it is important to have +a smaller decompressor than the full-featured +.BR xz (1). +.PP .B xzdec and .B lzmadec -are not really that small. The size can be reduced further by dropping -features from liblzma at compile time, but that shouldn't usually be done -for executables distributed in typical non-embedded operating system -distributions. If you need a truly small +are not really that small. +The size can be reduced further by dropping +features from liblzma at compile time, +but that shouldn't usually be done for executables distributed +in typical non-embedded operating system distributions. +If you need a truly small .B .xz decompressor, consider using XZ Embedded. .SH "SEE ALSO" diff --git a/Build/source/extra/xz/src/xzdec/xzdec.c b/Build/source/extra/xz/src/xzdec/xzdec.c index 8518d362a2d..fd015076de9 100644 --- a/Build/source/extra/xz/src/xzdec/xzdec.c +++ b/Build/source/extra/xz/src/xzdec/xzdec.c @@ -35,12 +35,6 @@ #endif -/// Number of bytes to use memory at maximum -static uint64_t memlimit; - -/// Total amount of physical RAM -static uint64_t total_ram; - /// Error messages are suppressed if this is zero, which is the case when /// --quiet has been given at least twice. static unsigned int display_errors = 2; @@ -66,10 +60,6 @@ my_errorf(const char *fmt, ...) static void lzma_attribute((noreturn)) help(void) { - // Round up to the next MiB and do it correctly also with UINT64_MAX. - const uint64_t mem_mib = (memlimit >> 20) - + ((memlimit & ((UINT32_C(1) << 20) - 1)) != 0); - printf( "Usage: %s [OPTION]... [FILE]...\n" "Uncompress files in the ." TOOL_FORMAT " format to the standard output.\n" @@ -77,7 +67,6 @@ help(void) " -c, --stdout (ignored)\n" " -d, --decompress (ignored)\n" " -k, --keep (ignored)\n" -" -M, --memory=NUM use NUM bytes of memory at maximum (0 means default)\n" " -q, --quiet specify *twice* to suppress errors\n" " -Q, --no-warn (ignored)\n" " -h, --help display this help and exit\n" @@ -85,11 +74,9 @@ help(void) "\n" "With no FILE, or when FILE is -, read standard input.\n" "\n" -"On this system and configuration, this program will use a maximum of roughly\n" -"%" PRIu64 " MiB RAM.\n" -"\n" "Report bugs to <" PACKAGE_BUGREPORT "> (in English or Finnish).\n" -PACKAGE_NAME " home page: <" PACKAGE_URL ">\n", progname, mem_mib); +PACKAGE_NAME " home page: <" PACKAGE_URL ">\n", progname); + tuklib_exit(EXIT_SUCCESS, EXIT_FAILURE, display_errors); } @@ -104,117 +91,6 @@ version(void) } -/// Find out the amount of physical memory (RAM) in the system, and set -/// the memory usage limit to the given percentage of RAM. -static void -memlimit_set_percentage(uint32_t percentage) -{ - memlimit = percentage * total_ram / 100; - return; -} - - -/// Set the memory usage limit to give number of bytes. Zero is a special -/// value to indicate the default limit. -static void -memlimit_set(uint64_t new_memlimit) -{ - if (new_memlimit != 0) { - memlimit = new_memlimit; - } else { - memlimit = 40 * total_ram / 100; - if (memlimit < UINT64_C(80) * 1024 * 1024) { - memlimit = 80 * total_ram / 100; - if (memlimit > UINT64_C(80) * 1024 * 1024) - memlimit = UINT64_C(80) * 1024 * 1024; - } - } - - return; -} - - -/// Get the total amount of physical RAM and set the memory usage limit -/// to the default value. -static void -memlimit_init(void) -{ - // If we cannot determine the amount of RAM, use the assumption - // defined by the configure script. - total_ram = lzma_physmem(); - if (total_ram == 0) - total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024; - - memlimit_set(0); - return; -} - - -/// \brief Convert a string to uint64_t -/// -/// This is rudely copied from src/xz/util.c and modified a little. :-( -/// -/// \param max Return value when the string "max" was specified. -/// -static uint64_t -str_to_uint64(const char *value, uint64_t max) -{ - uint64_t result = 0; - - // Accept special value "max". - if (strcmp(value, "max") == 0) - return max; - - if (*value < '0' || *value > '9') { - my_errorf("%s: Value is not a non-negative decimal integer", - value); - exit(EXIT_FAILURE); - } - - do { - // Don't overflow. - if (result > (UINT64_MAX - 9) / 10) - return UINT64_MAX; - - result *= 10; - result += *value - '0'; - ++value; - } while (*value >= '0' && *value <= '9'); - - if (*value != '\0') { - // Look for suffix. - uint64_t multiplier = 0; - if (*value == 'k' || *value == 'K') - multiplier = UINT64_C(1) << 10; - else if (*value == 'm' || *value == 'M') - multiplier = UINT64_C(1) << 20; - else if (*value == 'g' || *value == 'G') - multiplier = UINT64_C(1) << 30; - - ++value; - - // Allow also e.g. Ki, KiB, and KB. - if (*value != '\0' && strcmp(value, "i") != 0 - && strcmp(value, "iB") != 0 - && strcmp(value, "B") != 0) - multiplier = 0; - - if (multiplier == 0) { - my_errorf("%s: Invalid suffix", value - 1); - exit(EXIT_FAILURE); - } - - // Don't overflow here either. - if (result > UINT64_MAX / multiplier) - result = UINT64_MAX; - else - result *= multiplier; - } - - return result; -} - - /// Parses command line options. static void parse_options(int argc, char **argv) @@ -226,7 +102,6 @@ parse_options(int argc, char **argv) { "decompress", no_argument, NULL, 'd' }, { "uncompress", no_argument, NULL, 'd' }, { "keep", no_argument, NULL, 'k' }, - { "memory", required_argument, NULL, 'M' }, { "quiet", no_argument, NULL, 'q' }, { "no-warn", no_argument, NULL, 'Q' }, { "help", no_argument, NULL, 'h' }, @@ -245,31 +120,6 @@ parse_options(int argc, char **argv) case 'Q': break; - case 'M': { - // Support specifying the limit as a percentage of - // installed physical RAM. - const size_t len = strlen(optarg); - if (len > 0 && optarg[len - 1] == '%') { - // Memory limit is a percentage of total - // installed RAM. - optarg[len - 1] = '\0'; - const uint64_t percentage - = str_to_uint64(optarg, 100); - if (percentage < 1 || percentage > 100) { - my_errorf("Percentage must be in " - "the range [1, 100]"); - exit(EXIT_FAILURE); - } - - memlimit_set_percentage(percentage); - } else { - memlimit_set(str_to_uint64( - optarg, UINT64_MAX)); - } - - break; - } - case 'q': if (display_errors > 0) --display_errors; @@ -298,13 +148,12 @@ uncompress(lzma_stream *strm, FILE *file, const char *filename) // Initialize the decoder #ifdef LZMADEC - ret = lzma_alone_decoder(strm, memlimit); + ret = lzma_alone_decoder(strm, UINT64_MAX); #else - ret = lzma_stream_decoder(strm, memlimit, LZMA_CONCATENATED); + ret = lzma_stream_decoder(strm, UINT64_MAX, LZMA_CONCATENATED); #endif // The only reasonable error here is LZMA_MEM_ERROR. - // FIXME: Maybe also LZMA_MEMLIMIT_ERROR in future? if (ret != LZMA_OK) { my_errorf("%s", ret == LZMA_MEM_ERROR ? strerror(ENOMEM) : "Internal error (bug)"); @@ -392,10 +241,6 @@ uncompress(lzma_stream *strm, FILE *file, const char *filename) msg = strerror(ENOMEM); break; - case LZMA_MEMLIMIT_ERROR: - msg = "Memory usage limit reached"; - break; - case LZMA_FORMAT_ERROR: msg = "File format not recognized"; break; @@ -431,10 +276,6 @@ main(int argc, char **argv) // Initialize progname which we will be used in error messages. tuklib_progname_init(argv); - // Set the default memory usage limit. This is needed before parsing - // the command line arguments. - memlimit_init(); - // Parse the command line options. parse_options(argc, argv); |