diff --git a/ChangeLog b/ChangeLog index c7795974ba23..2d36d7bb1043 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,12662 +1,13908 @@ +commit 9331ce4009ddc839f5191d234cc41b2d4797376d +Author: Lasse Collin +Date: 2024-10-01 12:21:22 +0300 + + Bump version and soname for 5.6.3 + + src/liblzma/Makefile.am | 2 +- + src/liblzma/api/lzma/version.h | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +commit f52857ffde768058db0e0e13f68a2660ca9f1330 +Author: Lasse Collin +Date: 2024-10-01 12:17:39 +0300 + + Add NEWS for 5.6.3 + + NEWS | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 125 insertions(+) + +commit b8f52990b5d47a50902bf33cd2305ce985457bac +Author: Lasse Collin +Date: 2024-10-01 12:10:23 +0300 + + Update THANKS + + (cherry picked from commit 1ebbe915d4e0d877154261b5f8103719a6722975) + + THANKS | 2 ++ + 1 file changed, 2 insertions(+) + +commit 51f6f455873911894f155e6997bc23a9be8f42ba +Author: Lasse Collin +Date: 2024-10-01 12:10:23 +0300 + + Tests/Windows: Add the application manifest to the test programs + + This ensures that the test programs get executed the same way as + the binaries that are installed. + + (cherry picked from commit 74702ee00ecfd080d8ab11118cd25dbe6c437ec0) + + CMakeLists.txt | 14 ++++++++++---- + tests/Makefile.am | 10 ++++++++++ + tests/tests.cmake | 33 ++++++++++++++++++++++++++++++++- + tests/tests_w32res.rc | 18 ++++++++++++++++++ + 4 files changed, 70 insertions(+), 5 deletions(-) + +commit bf518b9ba446327a062ddfe67e7e0a5baed2394f +Author: Lasse Collin +Date: 2024-10-01 12:10:23 +0300 + + Windows: Embed an application manifest in the EXE files + + IMPORTANT: This includes a security fix to command line tool + argument handling. + + Some toolchains embed an application manifest by default to declare + UAC-compliance. Some also declare compatibility with Vista/8/8.1/10/11 + to let the app access features newer than those of Vista. + + We want all the above but also two more things: + + - Declare that the app is long path aware to support paths longer + than 259 characters (this may also require a registry change). + + - Force the code page to UTF-8. This allows the command line tools + to access files whose names contain characters that don't exist + in the current legacy code page (except unpaired surrogates). + The UTF-8 code page also fixes security issues in command line + argument handling which can be exploited with malicious filenames. + See the new file w32_application.manifest.comments.txt. + + Thanks to Orange Tsai and splitline from DEVCORE Research Team + for discovering this issue. + + Thanks to Vijay Sarvepalli for reporting the issue to me. + + Thanks to Kelvin Lee for testing with MSVC and helping with + the required build system fixes. + + (cherry picked from commit 46ee0061629fb075d61d83839e14dd193337af59) + + CMakeLists.txt | 18 +++ + src/Makefile.am | 4 +- + src/common/common_w32res.rc | 5 + + src/common/w32_application.manifest | 28 ++++ + src/common/w32_application.manifest.comments.txt | 178 +++++++++++++++++++++++ + 5 files changed, 232 insertions(+), 1 deletion(-) + +commit 5718ce932e6ad4262d5fffc9e2a7a838f963d7e5 +Author: Lasse Collin +Date: 2024-09-29 14:46:52 +0300 + + Windows: Set DLL name accurately in StringFileInfo on Cygwin and MSYS2 + + Now the information in the "Details" tab in the file properties + dialog matches the naming convention of Cygwin and MSYS2. This + is only a cosmetic change. + + (cherry picked from commit dad153091552b52a41b95ec4981c6951f1cae487) + + src/liblzma/liblzma_w32res.rc | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +commit e77c0ca61d12ebac433b7661840cb18d7031700a +Author: Lasse Collin +Date: 2024-09-25 15:47:55 +0300 + + common_w32res.rc: White space edits + + LANGUAGE and VS_VERSION_INFO begin new statements so put an empty line + between them. + + (cherry picked from commit 8940ecb96fe9f0f2a9cfb8b66fe9ed31ffbea904) + + src/common/common_w32res.rc | 15 ++++++++------- + 1 file changed, 8 insertions(+), 7 deletions(-) + +commit e0ba0f26d9f3f53cedc92fb13303924c39d00392 +Author: Lasse Collin +Date: 2024-09-28 20:09:50 +0300 + + CMake: Add the resource files to the Cygwin and MSYS2 builds + + Autotools-based build has always done this so this is for consistency. + + However, the CMake build won't create the DEF file when building + for Cygwin or MSYS2 because in that context it should be useless. + (If Cygwin or MSYS2 is used to host building of normal Windows + binaries then the DEF file is still created.) + + (cherry picked from commit c3b9dad07d3fd9319f88386b7095019bcea45ce1) + + CMakeLists.txt | 16 ++++++++++------ + 1 file changed, 10 insertions(+), 6 deletions(-) + +commit 69637d0c323c0d7d9619cff637c7ce97dabc4f02 +Author: Lasse Collin +Date: 2024-09-28 15:19:14 +0300 + + CMake: Fix Windows resource file dependencies + + If common_w32res.rc is modified, the resource files need to be rebuilt. + In contrast, the liblzma*.map files truly are link dependencies. + + (cherry picked from commit da4f275bd1c18b897e5c2dd0043546de3accce0a) + + CMakeLists.txt | 17 +++++++++-------- + 1 file changed, 9 insertions(+), 8 deletions(-) + +commit af8533459c60d7bc5b55f2f516251af4572169e4 +Author: Lasse Collin +Date: 2024-09-29 01:20:03 +0300 + + CMake: Checking for CYGWIN covers MSYS2 too + + On MSYS2, both CYGWIN and MSYS are set. + + (cherry picked from commit 1c673c0aac7f7dee8dda2c1140351c8417a71e47) + + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit eca08e4c204db404911e513f95110dcb0fb919bd +Author: Lasse Collin +Date: 2024-09-28 09:37:30 +0300 + + Translations: Add the SPDX license identifier to pt_BR.po + + (cherry picked from commit 6aaa0173b839e28429d43a8b62d257ad2f3b4521) + + po/pt_BR.po | 2 ++ + 1 file changed, 2 insertions(+) + +commit 85801c96c32456300177fbbad1506b07f5dd0a47 +Author: Lasse Collin +Date: 2024-09-25 16:41:37 +0300 + + Windows/CMake: Use the correct resource file for lzmadec.exe + + CMakeLists.txt was using xzdec_w32res.rc for both xzdec and lzmadec. + + Fixes: 998d0b29536094a89cf385a3b894e157db1ccefe + (cherry picked from commit dc7b9f24b737e4e55bcbbdde6754883f991c2cfb) + + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit a341d19c835a8c10fcf561b00b548c53af43381e +Author: Lasse Collin +Date: 2024-09-25 21:29:59 +0300 + + Translations: Update the Brazilian Portuguese translation + + (cherry picked from commit b834ae5f80911a3819d6cdb484f61b257174c544) + + po/pt_BR.po | 144 ++++++++++++++++++++++-------------------------------------- + 1 file changed, 53 insertions(+), 91 deletions(-) + +commit e69c0b9b2e00ade984393ef9cabac57342072328 +Author: Lasse Collin +Date: 2024-09-17 01:21:15 +0300 + + Update THANKS + + (cherry picked from commit eceb023d4c129fd63ee881a2d8696eaf52ad1532) + + THANKS | 1 + + 1 file changed, 1 insertion(+) + +commit aef9a25b3200457c16846b046222fb2c7967afe0 +Author: Tobias Stoeckmann +Date: 2024-09-16 23:19:46 +0200 + + lzmainfo: Avoid integer overflow + + The MB output can overflow with huge numbers. Most likely these are + invalid .lzma files anyway, but let's avoid garbage output. + + lzmadec was adapted from LZMA Utils. The original code with this bug + was written in 2005, over 19 years ago. + + Co-authored-by: Lasse Collin + Closes: https://github.com/tukaani-project/xz/pull/144 + (cherry picked from commit 76cfd0a9bb33ae8e534b1f73f6359dc825589f2f) + + src/lzmainfo/lzmainfo.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +commit 40a7f163f56aca6b3c8b83e9382f5e5cb4f8e93b +Author: Tobias Stoeckmann +Date: 2024-09-16 22:04:40 +0200 + + xzdec: Remove unused short option -M + + "xzdec -M123" exited with exit status 1 without printing + any messages. The "M:" entry should have been removed when + the memory usage limiter support was removed from xzdec. + + Fixes: 792331bdee706aa852a78b171040ebf814c6f3ae + Closes: https://github.com/tukaani-project/xz/pull/143 + [ Lasse: Commit message edits ] + + (cherry picked from commit 78355aebb7fb654302e5e33692ba109909dacaff) + + src/xzdec/xzdec.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit c98714a57058ac381365c2ff1e1d1cd63a5742c4 +Author: Lasse Collin +Date: 2024-09-10 13:54:47 +0300 + + Update THANKS + + (cherry picked from commit e5758db7bd75587a2499e0771907521a4aa86908) + + THANKS | 1 + + 1 file changed, 1 insertion(+) + +commit 4ed449517817b3659b35d19f39703e3c460f46c2 +Author: Firas Khalil Khana +Date: 2024-09-10 12:30:32 +0300 + + Build: Fix a typo in autogen.sh + + Fixes: e9be74f5b129fe8a5388d588e68b1b7f5168a310 + Closes: https://github.com/tukaani-project/xz/pull/141 + (cherry picked from commit 80ffa38f56657257ed4d90d76f6bd2f2bcb8163c) + + autogen.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 3b83577a1547e72cb78a905ad3d308a799ded485 +Author: Lasse Collin +Date: 2024-09-02 20:08:40 +0300 + + Translations: Update Chinese (simplified) translation + + Differences to the zh_CN.po file from the Translation Project: + + - Two uses of \v were fixed. + + - Missing "OPTS" translation in --riscv[=OPTS] was copied from + previous lines. + + - "make update-po" was run to remove line numbers from comments. + + (cherry picked from commit 68c54e45d042add64a4cb44bfc87ca74d29b87e2) + + po/zh_CN.po | 102 ++++++++++++++++++++++++------------------------------------ + 1 file changed, 40 insertions(+), 62 deletions(-) + +commit 06f4c7edda0387eb6a2d6303804b59dcf4d3db1f +Author: Lasse Collin +Date: 2024-09-02 19:40:50 +0300 + + Translations: Update the Catalan translation + + Differences to the ca.po file from the Translation Project: + + - An overlong line translating --filters-help was wrapped. + + - "make update-po" was used to remove line numbers from the comments + to match the changes in fccebe2b4fd513488fc920e4dac32562ed3c7637 + and 093490b58271e9424ce38a7b1b38bcf61b9c86c6. xz.pot in the TP + is older than these commits. + + (cherry picked from commit 2230692aa1bcebb586100183831e3daf1714d60a) + + po/ca.po | 171 ++++++++++++++++++++++++++------------------------------------- + 1 file changed, 69 insertions(+), 102 deletions(-) + +commit 406cb5b669e47c0e45c98f1afb7be998084a93d0 +Author: Lasse Collin +Date: 2024-08-22 11:01:07 +0300 + + Update THANKS + + (cherry picked from commit 5e375987509fab484b7bef0b90be92f241c58c91) + + THANKS | 1 + + 1 file changed, 1 insertion(+) + +commit 3a4a05d75eb41ddc41899324df0511670ceaaf1e +Author: Yifeng Li +Date: 2024-08-22 02:18:49 +0000 + + liblzma: Fix x86-64 movzw compatibility in range_decoder.h + + Support for instruction "movzw" without suffix in "GNU as" was + added in commit [1] and stabilized in binutils 2.27, released + in August 2016. Earlier systems don't accept this instruction + without a suffix, making range_decoder.h's inline assembly + unable to build on old systems such as Ubuntu 16.04, creating + error messages like: + + lzma_decoder.c: Assembler messages: + lzma_decoder.c:371: Error: no such instruction: `movzw 2(%r11),%esi' + lzma_decoder.c:373: Error: no such instruction: `movzw 4(%r11),%edi' + lzma_decoder.c:388: Error: no such instruction: `movzw 6(%r11),%edx' + lzma_decoder.c:398: Error: no such instruction: `movzw (%r11,%r14,4),%esi' + + Change "movzw" to "movzwl" for compatibility. + + [1] https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=c07315e0c610e0e3317b4c02266f81793df253d2 + + Suggested-by: Lasse Collin + Tested-by: Yifeng Li + Signed-off-by: Yifeng Li + Fixes: 3182a330c1512cc1f5c87b5c5a272578e60a5158 + Fixes: https://github.com/tukaani-project/xz/issues/121 + Closes: https://github.com/tukaani-project/xz/pull/136 + (cherry picked from commit 6cd7c8607843c337edfe2c472aa316602a393754) + + src/liblzma/rangecoder/range_decoder.h | 24 ++++++++++++------------ + 1 file changed, 12 insertions(+), 12 deletions(-) + +commit 4669f06d1a8d31de4b8b5861b5e8afd82cacd721 +Author: Lasse Collin +Date: 2024-07-19 20:02:43 +0300 + + Build: Comment that elf_aux_info(3) will be available on OpenBSD >= 7.6 + + (cherry picked from commit bf901dee5d4c46609645e50311c0cb2dfdcf9738) + + CMakeLists.txt | 2 +- + configure.ac | 17 +++++++++++------ + 2 files changed, 12 insertions(+), 7 deletions(-) + +commit 9edddda5636d7b3504a033c31e8ea763e293fd35 +Author: Lasse Collin +Date: 2024-07-13 22:10:37 +0300 + + liblzma: Tweak a comment + + (cherry picked from commit 7c292dd0bf23cefcdf4b1509f3666322e08a7ede) + + src/liblzma/simple/arm64.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +commit 1a93ab55d1563f5eb9b2c1b8240384046fe4bb97 +Author: Lasse Collin +Date: 2024-07-11 22:17:56 +0300 + + CMake: Bump maximum policy version to 3.30 + + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit cfe4465742ad2963fb0d9795e258615d7c1cf32d +Author: Lasse Collin +Date: 2024-07-09 14:27:51 +0300 + + Update THANKS + + (cherry picked from commit 028185dd4889e3d6235ff13560160ebca6985021) + + THANKS | 1 + + 1 file changed, 1 insertion(+) + +commit 0f47db18d04434203b350bde4909a5e468f197cc +Author: Lasse Collin +Date: 2024-07-06 14:04:48 +0300 + + xz: Remove the TODO comment about --recursive + + It won't be implemented. find + xargs is more flexible, for example, + it allows compressing small files in parallel. An example for that + has been included in the xz man page since 2010. + + (cherry picked from commit baecfa142644eb5f5c6dd6f8e2f531c362fa3747) + + src/xz/args.c | 1 - + 1 file changed, 1 deletion(-) + +commit 07f52c3528e43c4a925a3fc59a933c89f5604d92 +Author: Lasse Collin +Date: 2024-07-03 20:45:48 +0300 + + CMake: Link xz against Threads::Threads if using pthreads + + The liblzma target was recently changed to link against Threads::Threads + with the PRIVATE keyword. I had forgotten that xz itself depends on + pthreads too due to pthread_sigmask(). Thus, the build broke when + building shared liblzma and pthread_sigmask() wasn't in libc. + + Thanks to Peter Seiderer for the bug report. + + Fixes: ac05f1b0d7cda1e7ae79775a8dfecc54601d7f1c + Fixes: https://github.com/tukaani-project/xz/issues/129#issuecomment-2204522994 + (cherry picked from commit b3e53122f42796aaebd767bab920cf7bedf69966) + + CMakeLists.txt | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +commit eccb4d258b01651d06a2a31b8b68be9b04b7998c +Author: Lasse Collin +Date: 2024-07-02 22:49:33 +0300 + + Update THANKS + + (cherry picked from commit 5742ec1fc7f2cf1c82cfe3477bb90594a4658374) + + THANKS | 1 + + 1 file changed, 1 insertion(+) + +commit c9bd00327f064778babb014302718a18d65cf7d3 +Author: Sam James +Date: 2024-06-28 14:18:35 +0300 + + CI: Speed up Valgrind job by using --trace-children-skip-by-arg=... + + This addresses the issue I mentioned in + 6c095a98fbec70b790253a663173ecdb669108c4 and speeds up the Valgrind + job a bit, because non-xz tools aren't run unnecessarily with + Valgrind by the script tests. + + (cherry picked from commit 7e99856f66c07852c4e0de7aa01951e9147d86b0) + + .github/workflows/ci.yml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 495de6ec9d7834c4ef4d5286844ef7b784eb951b +Author: Lasse Collin +Date: 2024-06-25 16:00:22 +0300 + + Build: Prepend, not append, PTHREAD_CFLAGS to LIBS + + It shouldn't make any difference because LIBS should be empty + at that point in configure. But prepending is the correct way + because in general the libraries being added might require other + libraries that come later on the command line. + + (cherry picked from commit 2402e8a1ae92676fa0d4cb1b761d7f62f005c098) + + configure.ac | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 55bf3f49a812e20a21e42323e39526bb31d9341a +Author: Lasse Collin +Date: 2024-06-25 14:24:29 +0300 + + Build: Use AC_LINK_IFELSE to handle implicit function declarations + + It's more robust in case the compiler allows pre-C99 implicit function + declarations. If an x86 intrinsic is missing and gets treated as + implicit function, the linking step will very probably fail. This + isn't the only way to workaround implicit function declarations but + it might be the simplest and cleanest. + + The problem hasn't been observed in the wild. + + There are a couple more AC_COMPILE_IFELSE uses in configure.ac. + Of these, Landlock check calls prctl() and in theory could have + the same problem. In practice it doesn't as the check program + looks for several other things too. However, it was changed to + AC_LINK_IFELSE still to look more correct. + + Similarly, m4/tuklib_cpucores.m4 and m4/tuklib_physmem.m4 were + updated although they haven't given any trouble either. They + have worked all these years because those check programs rely + on specific headers and types: if headers or types are missing, + compilation will fail. Using the linker makes these checks more + similar to the ones in cmake/tuklib_*.cmake which always link. + + (cherry picked from commit 7bb46f2b7b3989c1b589a247a251470f65e91cda) + + configure.ac | 8 ++++++-- + m4/tuklib_cpucores.m4 | 8 ++++---- + m4/tuklib_physmem.m4 | 17 +++++++++++------ + 3 files changed, 21 insertions(+), 12 deletions(-) + +commit b45270d88f0de1b2e8bf510f0e370a5db4067e1f +Author: Lasse Collin +Date: 2024-06-24 23:35:59 +0300 + + Build: Use AC_LINK_IFELSE instead of -Werror + + AC_COMPILE_IFELSE needed -Werror because Clang <= 14 would merely + warn about the unsupported attribute and implicit function declaration. + Changing to AC_LINK_IFELSE handles the implicit declaration because + the symbol __crc32d is unlikely to exist in libc. + + Note that the other part of the check is that #include + must work. If the header is missing, most compilers give an error + and the linking step won't be attempted. + + Avoiding -Werror makes the check more robust in case CFLAGS contains + warning flags that break -Werror anyway (but this isn't the only check + in configure.ac that has this problem). Using AC_LINK_IFELSE also makes + the check more similar to how it is done in CMakeLists.txt. + + (cherry picked from commit 35eb57355ad1c415a838d26192d5af84abb7cf39) + + configure.ac | 12 +----------- + 1 file changed, 1 insertion(+), 11 deletions(-) + +commit 2c3e4cbbdcefe214ef3033a725049034b73e9756 +Author: Lasse Collin +Date: 2024-06-24 23:34:34 +0300 + + Build: Sync the compile check changes from CMakeLists.txt + + It's nice to keep these in sync. The use of main() will later allow + AC_LINK_IFELSE usage too which may avoid the more fragile -Werror. + + (cherry picked from commit 5a728813c378cc3c4c9c95793762452418d08f1b) + + configure.ac | 15 ++++++++------- + 1 file changed, 8 insertions(+), 7 deletions(-) + +commit 809e69f1f574dad3c9b00d4f01b9ef1a492319f3 +Author: Lasse Collin +Date: 2024-06-25 16:11:13 +0300 + + CMake: Use configure_file() to copy a file + + I had missed this simpler method before. It does create a dependency + so that if .in.h changes the copying is done again. + + (cherry picked from commit de215a0517645d16343f3a5336d3df884a4f665f) + + CMakeLists.txt | 17 +++++++---------- + 1 file changed, 7 insertions(+), 10 deletions(-) + +commit 52a8c87f37f4bd133f670722d2d4b73a74e352bc +Author: Lasse Collin +Date: 2024-06-25 15:51:48 +0300 + + CMake: Always add pthread flags into CMAKE_REQUIRED_LIBRARIES + + It was weird to add CMAKE_THREAD_LIBS_INIT in CMAKE_REQUIRED_LIBRARIES + only if CLOCK_MONOTONIC is available. Alternative would be to remove + the thread libs from CMAKE_REQUIRED_LIBRARIES after the check for + pthread_condattr_setclock() but keeping the libs should be fine too. + Then it's ready in case more pthread functions were wanted some day. + + (cherry picked from commit e620f35097c0ad20cd76d8258750aa706758ced9) + + CMakeLists.txt | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +commit 1591747bf692d10c3b2fd92c9dc8ba931626fd84 +Author: Lasse Collin +Date: 2024-06-24 22:41:10 +0300 + + CMake: Fix three checks if building with -flto + + In CMake, check_c_source_compiles() always links too. With + link-time optimization, unused functions may get omitted if + main() doesn't depend on them. Consider the following which + tries to check if somefunction() is available when + has been included: + + #include + int foo(void) { return somefunction(); } + int main(void) { return 0; } + + LTO may omit foo() completely because the program as a whole doesn't + need it and then the program will link even if the symbol somefunction + isn't available in libc or other library being linked in, and then + the test may pass when it shouldn't. + + What happens if doesn't declare somefunction()? + Shouldn't the test fail in the compilation phase already? It should + but many compilers don't follow the C99 and later standards that + prohibit implicit function declarations. Instead such compilers + assume that somefunction() exists, compilation succeeds (with a + warning), and then linker with LTO omits the call to somefunction(). + + Change the tests so that they are part of main(). If compiler accepts + implicitly declared functions, LTO cannot omit them because it has to + assume that they might have side effects and thus linking will fail. + On the other hand, if the functions/intrinsics being used are supported, + they might get optimized away but in that case it's fine because they + really are supported. + + It is fine to use __attribute__((target(...))) for main(). At least + it works with GCC 4.9 to 14.1 on x86-64. + + Reported-by: Sam James + (cherry picked from commit 114cba69dbb96003e676c8c87a2e9943b12d065f) + + CMakeLists.txt | 19 ++++++++----------- + 1 file changed, 8 insertions(+), 11 deletions(-) + +commit cc386f4ff4b87ff895fbc30fd3b13ee6e6152ace +Author: Lasse Collin +Date: 2024-06-24 21:06:18 +0300 + + CMake: Improve the comment about LIBS + + (cherry picked from commit d3f20382fc1bd865eb70a65455d5022ed05caac8) + + CMakeLists.txt | 6 ++++++ + 1 file changed, 6 insertions(+) + +commit 65aaa0f87048f78a3f69c4ec0ad03723a2354fa7 +Author: Lasse Collin +Date: 2024-06-24 17:39:54 +0300 + + CI: Workaround buggy config.guess on Ubuntu 22.04LTS and 24.04LTS + + Check for the wrong triplet from config.guess and override it with + the --build option on the configure command line. Then i386 assembly + autodetection will work. + + These Ubuntu versions (and as of writing, also Debian unstable) + ship config.guess version 2022-01-09 which contains a bug that + was fixed in version 2022-05-08. It results in a wrong configure + triplet when using CC="gcc -m32" to build i386 binaries. + + Upstream fix: + https://git.savannah.gnu.org/cgit/config.git/commit/?id=f56a7140386d08a531bcfd444d632b28c61a6329 + + More information: + https://mail.gnu.org/archive/html/config-patches/2022-05/msg00003.html + + (cherry picked from commit 1bf83cded2955282fe1a868f08c83d4e5d6dca4a) + + build-aux/ci_build.bash | 9 +++++++++ + 1 file changed, 9 insertions(+) + +commit 810f1a8aee9edb3bff430559f4b832cd0ec50797 +Author: Lasse Collin +Date: 2024-06-24 15:24:52 +0300 + + CI: Use CC="gcc -m32" to get i386 compiler on x86-64 + + The old method put it in CFLAGS which is a wrong place because + config.guess doesn't read CFLAGS. + + (cherry picked from commit dbcdabf68fee9ed694b68c3a82e6adbeff20b679) + + .github/workflows/ci.yml | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +commit dde14ded9a3240fd524d9bc01c9ceeb4d7909e95 +Author: Lasse Collin +Date: 2024-06-24 14:54:17 +0300 + + CI: Let CMake use the CC environment variable + + CC from environment is used to initialize CMAKE_C_COMPILER so + setting CMAKE_C_COMPILER explicitly isn't needed. + + The syntax in ci_build.bash was broken in case one wished to put + spaces in CC. + + (cherry picked from commit 0c1e6d900bac127464fb30a854776e1810ab5f16) + + build-aux/ci_build.bash | 4 ---- + 1 file changed, 4 deletions(-) + +commit 85a55e1120bebac2f3cd9af8965f4a6335eeeb9b +Author: Lasse Collin +Date: 2024-06-20 18:12:21 +0300 + + CMake: Keep existing options in LIBS when adding -lrt + + This makes no difference yet because -lrt is currently the only option + that might be added to LIBS. + + (cherry picked from commit 75ce4797d49621710e6da95d8cb91541028c6d68) + + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit e24a762f1be6bf379df73b7fe0a115ccae139a35 +Author: Lasse Collin +Date: 2024-06-15 18:07:04 +0300 + + CMake: Fix indentation + + (cherry picked from commit c715dec8e800b65145918cfb0ee9bbc90faa8aad) + + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 99555b721b55263a6892b1093f2806f09a92e1fb +Author: Lasse Collin +Date: 2024-06-15 23:34:29 +0300 + + CMake: Link Threads::Threads as PRIVATE to liblzma + + This way pthread options aren't passed to the linker when linking + against shared liblzma but they are still passed when linking against + static liblzma. (Also, one never needs the include path of the + threading library to use liblzma since liblzma's API headers + don't #include . But tends to be in the + default include path so here this change makes no difference.) + + One cannot mix target_link_libraries() calls that use the scope + (PRIVATE, PUBLIC, or INTERFACE) keyword and calls that don't use it. + The calls without the keyword are like PUBLIC except perhaps when + they aren't, or something like that... It seems best to always + specify a scope keyword as the meanings of those three keywords + at least are clear. + + (cherry picked from commit ac05f1b0d7cda1e7ae79775a8dfecc54601d7f1c) + + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 258bae30a2040138c783b5c380cef0ca603663ed +Author: Lasse Collin +Date: 2024-06-16 19:39:32 +0300 + + CMake: Add empty lines + + (cherry picked from commit 82986d8c691a294c78b48d8391303e5c428b5437) + + CMakeLists.txt | 2 ++ + 1 file changed, 2 insertions(+) + +commit a95a9601a109f0d0d059dea7a5a44efa87ef1401 +Author: Lasse Collin +Date: 2024-06-16 19:37:36 +0300 + + CMake: Use CMAKE_THREAD_LIBS_INIT in liblzma.pc only with pthreads + + This shouldn't make much difference in practice as on Windows + no flags are needed anyway and unitialized variable (when threading + is disabled) expands to empty. But it's clearer this way. + + (cherry picked from commit 2aecffe0f0e14f3ef635e8cd7b405420f2385de2) + + CMakeLists.txt | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +commit 65a10ddd439ad435d2c0176106b1e2d6b9c1b3a1 +Author: Lasse Collin +Date: 2024-06-17 18:20:14 +0300 + + Update THANKS + + (cherry picked from commit 664918bd3635ea8e773f06022286ecb0c485166c) + + THANKS | 3 +++ + 1 file changed, 3 insertions(+) + +commit 6ad5739094ac69ac448a84493f2c7ddfc6eb0688 +Author: Lasse Collin +Date: 2024-06-16 19:25:07 +0300 + + CMake: Use native newlines in liblzma.pc + + vcpkg doesn't specify the newline type so it should be fine to + use native newlines in liblzma.pc on Windows. + + (cherry picked from commit 5ca96a93488d0f5a530c78b274cac317453807ff) + + CMakeLists.txt | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +commit 4107f2066764bb3a31d114852bc20722d582fd82 +Author: Lasse Collin +Date: 2024-06-16 19:18:56 +0300 + + CMake: Use relative paths in liblzma.pc if possible + + Now liblzma.pc can be relocatable only if using CMake >= 3.20 + but that should be OK as now we shouldn't get broken liblzma.pc + if CMAKE_INSTALL_LIBDIR or CMAKE_INSTALL_INCLUDEDIR contain an + absolute path. + + Thanks to Eli Schwartz. + + (cherry picked from commit ebd155c3a1b87411edae06d3bdaa9659ec057522) + + CMakeLists.txt | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +commit ff697eb154361417d94284e0c569aa08cacf9031 +Author: Lasse Collin +Date: 2024-06-16 13:39:37 +0300 + + liblzma: CRC CLMUL: Omit is_arch_extension_supported() when not needed + + On E2K the function compiles only due to compiler emulation but the + function is never used. It's cleaner to omit the function when it's + not needed even though it's a "static inline" function. + + Thanks to Ilya Kurdyukov. + + (cherry picked from commit 30a2d5d51006301a3ddab5ef1f5ff0a9d74dce6f) + + src/liblzma/check/crc_x86_clmul.h | 4 ++++ + 1 file changed, 4 insertions(+) + +commit 4e4a568f6a089c867891c2388a19624e312eb2f3 +Author: Lasse Collin +Date: 2024-06-12 14:26:44 +0300 + + CMake: Prefer C11 with a fallback to C99 + + There is no need to make a similar change in configure.ac. + With Autoconf 2.72, the deprecated macro AC_PROG_CC_C99 + is an alias for AC_PROG_CC which prefers a C11 compiler. + + (cherry picked from commit 2178acf8a4d40a93e970cfcf9b807d5ef6c8da92) + + CMakeLists.txt | 17 +++++++++++------ + 1 file changed, 11 insertions(+), 6 deletions(-) + +commit 849e757a8cce41bfd6acfaa7dd3b07324363de90 +Author: Lasse Collin +Date: 2024-06-12 14:20:21 +0300 + + Update THANKS + + (cherry picked from commit c97e9c12fef4d1093ee2a75236742481361f50f5) + + THANKS | 4 ++++ + 1 file changed, 4 insertions(+) + +commit 1305056a54e68895e052506bceb26274f52bbc9a +Author: Lasse Collin +Date: 2024-06-11 11:15:49 +0300 + + Tests: Improve the CRC32 test + + A similar one was already there for CRC64 but nowadays also CRC32 + has a CLMUL implementation, so it's good to test it better too. + + (cherry picked from commit 89e9f12e03324b8a186e807b268f34f92d1b2f41) + + tests/test_check.c | 17 +++++++++++++---- + 1 file changed, 13 insertions(+), 4 deletions(-) + +commit a44493ec41edc98f24ed9933668e7372f5267a40 +Author: Lasse Collin +Date: 2024-06-11 22:42:26 +0300 + + xz: Fix white space + + (cherry picked from commit c7164b1927e3fe7cdba70ee4687e1a590a81043b) + + src/xz/list.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +commit 5e74a6a8138b3c102193d731120139d5a854f2cf +Author: Lasse Collin +Date: 2024-06-11 21:59:09 +0300 + + liblzma: Fix a typo in a comment + + Thanks to Sam James for spotting it. + + Fixes: f644473a211394447824ea00518d0a214ff3f7f2 + (cherry picked from commit 0a32d2072c598de281058b26dc08920fbf0cd2a1) + + src/liblzma/check/crc_x86_clmul.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 3f7edc673cf21b3e4db3e2f11746905e0a393db7 +Author: Lasse Collin +Date: 2024-05-10 15:52:26 +0300 + + liblzma: Fix a comment indentation + + (cherry picked from commit afd9b4d282a10186808c3331dad4caf79c02d55f) + + src/liblzma/check/crc_common.h | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +commit 8a9cc7ca0867494f39990f0d4cbe0972042f6d59 +Author: Lasse Collin +Date: 2024-05-09 22:09:12 +0300 + + liblzma: Fix white space + + (cherry picked from commit 50e6bff274568c568930e15094da8217e7d47d28) + + src/liblzma/check/crc32_table.c | 10 +++++----- + src/liblzma/check/crc_x86_clmul.h | 6 +++--- + src/liblzma/check/sha256.c | 2 +- + 3 files changed, 9 insertions(+), 9 deletions(-) + +commit b29b13082fe578a3bb9384a5939c82055f796a34 +Author: RainRat +Date: 2024-06-05 15:21:49 -0700 + + Fix typos + + Closes: https://github.com/tukaani-project/xz/pull/124 + (cherry picked from commit 9e73918a4f14be754a23f74dda45ca431939a4a0) + + INSTALL | 2 +- + doc/examples/03_compress_custom.c | 2 +- + src/common/tuklib_integer.h | 2 +- + src/liblzma/api/lzma/container.h | 2 +- + src/xz/mytime.c | 2 +- + tests/test_filter_str.c | 2 +- + 6 files changed, 6 insertions(+), 6 deletions(-) + +commit 6f66155e01a6467e70db48cddbe790bdb8d87754 +Author: Lasse Collin +Date: 2024-06-07 15:47:20 +0300 + + tuklib_integer: Fix building on OpenBSD/sparc64 that uses GCC 4.2 + + GCC 4.2 doesn't have __builtin_bswap16() and friends so tuklib_integer.h + tries to use OS-specific byte swap methods instead. On OpenBSD those + macros are swap16/32/64 instead of bswap16/32/64 like on other *BSDs + and Darwin. + + An alternative to "#ifdef __OpenBSD__" could be "#ifdef swap16" as it + is a macro. But since OpenBSD seems to be a special case under this + special case of "*BSDs and Darwin", checking for __OpenBSD__ seems + the more conservative choice now. + + Thanks to Christian Weisgerber and Brad Smith who both submitted + the same patch a few hours apart. + + Co-authored-by: Christian Weisgerber + Co-authored-by: Brad Smith + Closes: https://github.com/tukaani-project/xz/pull/126 + (cherry picked from commit 04b23addf3733873667675df2439725f076c2f36) + + src/common/tuklib_integer.h | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +commit 5522759d31e0f1513fffbdf39a955f12d373f121 +Author: Lasse Collin +Date: 2024-06-05 13:55:43 +0300 + + Update THANKS + + (cherry picked from commit f5c2ae58ec68c665e62c790b842657afcb31474c) + + THANKS | 2 ++ + 1 file changed, 2 insertions(+) + +commit 45aed6f37f17e5fac215290204e03894965cf1d5 +Author: Lasse Collin +Date: 2024-06-05 13:30:28 +0300 + + CMake: Fix wrong version variable + + liblzma_VERSION has never existed in the repository. xz_VERSION from + the project() command was used for liblzma SOVERSION so use xz_VERSION + here too. + + The wrong variable did no harm in practice as PROJECT_VERSION + was used as the fallback. It has the same value as xz_VERSION. + + Fixes: 7e3493d40eac0c3fa3d5124097745a70e15c41f6 + (cherry picked from commit 1d3c61575fda0be6b2d50c9e32a343349d5cd5c0) + + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 198271a6ed0e6ac6820f8f44172a203aa44abe39 +Author: Lasse Collin +Date: 2024-06-04 23:59:29 +0300 + + CMake: Fix liblzma filename in Windows environments + + This is a mess because liblzma DLL outside Cygwin and MSYS2 + is liblzma.dll instead of lzma.dll to avoid a conflict with + lzma.dll from LZMA SDK. + + On Cygwin the name was "liblzma-5.dll" while "cyglzma-5.dll" + would have been correct (and match what Libtool produces). + MSYS2 likely was broken too as it uses the "msys-" prefix. + + This change has no effect with MinGW-w64 because with that + the "lib" prefix was correct already. + + With MSVC builds this is a small breaking change that requires developers + to adjust the library name when linking against liblzma. The liblzma.dll + name is kept as is but the import library and static library are now + lzma.lib instead of liblzma.lib. This is helpful when using pkgconf + because "pkgconf --msvc-syntax --libs liblzma" outputs "lzma.lib" + (it's converted from "-llzma" in liblzma.pc). It would be easy to + keep the liblzma.lib naming but the pkgconf compatibility seems worth + it in the long run. The lzma.lib name is compatible with MinGW-w64 + too as -llzma will find also lzma.lib. + + vcpkg had been patching CMakeLists.txt this way since 2022 but I + learned this only recently. The reasoning for the patch makes sense, + and while this is a small breaking change with MSVC, it seems like + a decent compromise as it keeps the DLL name the same. + + 2022 patch in vcpkg: https://github.com/microsoft/vcpkg/blob/0707a17ecf1466d64cf1a3c1ee18c8ff02aadb2d/ports/liblzma/win_output_name.patch + See the discussion: https://github.com/microsoft/vcpkg/pull/39024 + + Thanks to Vincent Torri for confirming the naming issue on Cygwin. + + (cherry picked from commit e0d6d05ce0d464e966c0669bbf869202a43cc2f7) + + CMakeLists.txt | 34 ++++++++++++++++++++++++++++++---- + 1 file changed, 30 insertions(+), 4 deletions(-) + +commit 92e5425979199407080fd80e67c15f2cbf85392b +Author: Lasse Collin +Date: 2024-06-03 16:55:03 +0300 + + Fix version.sh compatiblity with Solaris + + The ancient /bin/tr on Solaris doesn't support '\n'. + With /usr/xpg4/bin/tr it works but it might not be in PATH. + + Another problem was that sed was given input that didn't have a newline + at the end. Text files must end with a newline to be portable. + + Fix both problems: + + - Handle multiline input within sed itself to avoid one tr invocation. + The default sed even on Solaris does understand \n. + + - Use octals in tr -d. \012 works for ASCII "line feed", it's even + used as an example in the Solaris man page. But we must strip + also ASCII "carriage return" \015 and EBCDIC "next line" \025. + The EBCDIC case got handled with \n previously. Stripping \012 + and \015 on EBCDIC system won't matter as those control chars + won't be present in the string in the first place. + + An awk-based solution could be an alternative but it might need + special casing on Solaris to used nawk instead of awk. The changes + in this commit are smaller and should have a smaller risk for + regressions. It's also possible that version.sh will be dropped + entirely at some point. + + (cherry picked from commit e7a42cda7c827e016619e8cab15e2faf5d4181ae) + + build-aux/version.sh | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +commit 0c089a33a5b1f5b9451b332484c68e1d6f02631a +Author: Lasse Collin +Date: 2024-06-03 17:07:11 +0300 + + CI: Don't require po4a on Solaris + + (cherry picked from commit a61c9ab4751f2710dcd5459c7d74bbf20781f0f9) + + .github/workflows/solaris.yml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 83d3792711295656a3de69bbcd98dcb4b06be1c2 +Author: Lasse Collin +Date: 2024-06-03 15:08:15 +0300 + + CI: Use set -e on Solaris too + + (cherry picked from commit 5229bdf5335ce18ed54beb7e646e39927663be86) + + .github/workflows/solaris.yml | 1 + + 1 file changed, 1 insertion(+) + +commit 9c64d4fd787ea7bca3795be55367504a9f47a68c +Author: Lasse Collin +Date: 2024-06-03 17:44:50 +0300 + + CMake: Install liblzma.pc even with MSVC + + I had misunderstood that it wouldn't be useful with MSVC. + vcpkg had been installing liblzma.pc with custom rules since 2020, + years before liblzma.pc support was added to CMakeLists.txt. + + See: + https://github.com/microsoft/vcpkg/blob/eb895b95aac6fd7485373702f29f508c42a180a0/ports/liblzma/portfile.cmake + https://github.com/microsoft/vcpkg/pull/39024#issuecomment-2145064670 + (cherry picked from commit afa938e429c1ce07d26d02999352fb014b62ff3d) + + CMakeLists.txt | 8 +++----- + 1 file changed, 3 insertions(+), 5 deletions(-) + +commit 42754176bd84c4539db55a9e70bdcdd5700c709f +Author: Sam James +Date: 2024-06-03 06:16:23 +0100 + + ci: don't pin official GH actions via commit, just tag + + There's no real value in doing it via commit for official GH actions. We + can keep using pinned commits for unofficial actions. It's hassle for no + gain. + + Maybe going forward we can limit this further by only being paranoid + for the jobs with any access to tokens. + + (cherry picked from commit 35f8649f08341639a627fd06350e938124ca3622) + + .github/workflows/ci.yml | 4 ++-- + .github/workflows/freebsd.yml | 2 +- + .github/workflows/netbsd.yml | 2 +- + .github/workflows/openbsd.yml | 2 +- + .github/workflows/solaris.yml | 2 +- + .github/workflows/windows-ci.yml | 4 ++-- + 6 files changed, 8 insertions(+), 8 deletions(-) + +commit 9a5fee7022eddffdfcee32a7e43f64635581b393 +Author: Christoph Junghans +Date: 2024-04-30 07:49:26 -0600 + + ci: set -e on openbsd + + Closes: https://github.com/tukaani-project/xz/pull/116 + (cherry picked from commit e885dae37ff5b1dbc760dabc1e03e866a7302ef2) + + .github/workflows/openbsd.yml | 1 + + 1 file changed, 1 insertion(+) + +commit a2d66de54f234999a7d42305988cf2c3e0b1b8f6 +Author: Christoph Junghans +Date: 2024-04-30 07:48:58 -0600 + + ci: set -e on netbsd + + (cherry picked from commit 21b02dd128cf9e8c76325ec124f70381862dcf19) + + .github/workflows/netbsd.yml | 1 + + 1 file changed, 1 insertion(+) + +commit 1bdc70176b59b0e22c0a580c518dc5d0f2fd0723 +Author: Christoph Junghans +Date: 2024-04-25 14:56:06 -0700 + + ci: actually fail on FreeBSD + + Without "set -e" the job will always be successful. + + See vmactions/freebsd-vm#72 + + (cherry picked from commit 8641f0c24c041136670c975b23408184b45431bc) + + .github/workflows/freebsd.yml | 1 + + 1 file changed, 1 insertion(+) + +commit 4132277103acdf1c01f8b5a4c12c0992c330ade4 +Author: Andrew Murray +Date: 2024-04-25 09:24:46 +1000 + + Updated actions + + Closes: https://github.com/tukaani-project/xz/pull/115 + (cherry picked from commit ef616683ef11f11ffdfbe0624da33905e28a70f9) + + .github/workflows/ci.yml | 4 ++-- + .github/workflows/windows-ci.yml | 6 +++--- + 2 files changed, 5 insertions(+), 5 deletions(-) + +commit 1575414636104773cefc62cf075726c6ee7ae37d +Author: Sam James +Date: 2024-06-03 02:49:40 +0100 + + ci: add po4a + + (cherry picked from commit 57b440d316da9ac9cb312ee7e6890f5382556f10) + + .github/workflows/netbsd.yml | 2 +- + .github/workflows/openbsd.yml | 3 ++- + 2 files changed, 3 insertions(+), 2 deletions(-) + +commit c3e293037e1bb2bd9efedbb0e75387d1282cc03f +Author: Sam James +Date: 2024-04-13 21:02:04 +0100 + + ci: add Solaris + + Inspired by https://github.com/RsyncProject/rsync/commit/3f2a38b01184cae9a931280b534acf5a3dae2e94. + + It runs on Solaris 5.11 via a VirtualBox VM. + + (cherry picked from commit 08cdf4be9a673d78efe393b53dd73bf43c81dd95) + + .github/workflows/solaris.yml | 31 +++++++++++++++++++++++++++++++ + 1 file changed, 31 insertions(+) + +commit dc6b6011b45b0d0ddd0650f4885e24c68b37fddf +Author: Sam James +Date: 2024-04-14 08:08:00 +0100 + + xz: list: suppress -Wformat-nonliteral for Solaris + + Solaris' GCC can't understand that our use is fine, unlike modern compilers: + ``` + list.c: In function 'print_totals_basic': + list.c:1191:4: error: format not a string literal, argument types not checked [-Werror=format-nonliteral] + uint64_to_str(totals.files, 0)); + ^~~~~~~~~~~~~ + cc1: all warnings being treated as errors + ``` + + It's presumably because of older gettext missing format attributes. + + This is with `gcc (GCC) 7.3.0`. + + (cherry picked from commit b69768c8bd1a34fde311935c551d061ba52d9a3f) + + src/xz/list.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +commit 7ce2ac795a812ecf1eb2d6b62f51b55ac799c2a5 +Author: Lasse Collin +Date: 2024-05-31 21:36:26 +0300 + + Update THANKS + + (cherry picked from commit b8d134e61ede9f4a296226d97f5c20721fb4e8e2) + + THANKS | 3 +++ + 1 file changed, 3 insertions(+) + commit 3ec664d3f652133136587a51d4505b1abe1acdd7 Author: Lasse Collin Date: 2024-05-29 18:03:51 +0300 Bump version and soname for 5.6.2 src/liblzma/Makefile.am | 2 +- src/liblzma/api/lzma/version.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit 3cc0aa702e50b786c52c6f3d3f831a635c4df197 Author: Lasse Collin Date: 2024-05-29 18:03:04 +0300 Add NEWS for 5.6.2 NEWS | 130 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) commit 526d3f7f2c2d5e134157d08b37fb5fd0b125799e Author: Lasse Collin Date: 2024-05-29 18:03:04 +0300 Add NEWS for 5.4.7 NEWS | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) commit 660b09279e8f544acf120d29194d5c3051b484eb Author: Lasse Collin Date: 2024-05-29 18:03:04 +0300 Add NEWS for 5.2.13 NEWS | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) commit 7d76282dac766c0ced8ae24e0f7ce0005f3e377d Author: Lasse Collin Date: 2024-05-29 17:47:13 +0300 Translations: Run po4a/update-po Now the files are in the new formatting without source file line numbers. Future updates should keep the diffs much smaller. po4a/de.po | 1592 ++++++++++--------- po4a/fr.po | 4450 +++++++++++++++++----------------------------------- po4a/ko.po | 1592 ++++++++++--------- po4a/pt_BR.po | 4817 ++++++++++++++++++--------------------------------------- po4a/ro.po | 1592 ++++++++++--------- po4a/uk.po | 1592 ++++++++++--------- 6 files changed, 6114 insertions(+), 9521 deletions(-) commit 4470c3f7d8954bb47b280ec07ad0bd4be2223083 Author: Lasse Collin Date: 2024-05-29 17:44:53 +0300 Translations: Run "make -C po update-po" In the past this wasn't done before releases; the Git repository just contained the files from the Translation Project. But this way it is clearer when comparing release tarballs against the Git repository. In future releases this might no longer be necessary within a stable branch as the .po files won't change so easily anymore when creating a tarball. po/ca.po | 567 +++++++++++++++++++++++++--------------- po/cs.po | 821 +++++++++++++++++++++++++++++++++++++-------------------- po/da.po | 809 +++++++++++++++++++++++++++++++++++--------------------- po/de.po | 403 ++++++++++++++-------------- po/eo.po | 403 ++++++++++++++-------------- po/es.po | 403 ++++++++++++++-------------- po/fi.po | 578 +++++++++++++++++++++++++--------------- po/fr.po | 538 +++++++++++++++++++++++--------------- po/hr.po | 403 ++++++++++++++-------------- po/hu.po | 403 ++++++++++++++-------------- po/it.po | 854 +++++++++++++++++++++++++++++++++++++++--------------------- po/ko.po | 403 ++++++++++++++-------------- po/pl.po | 403 ++++++++++++++-------------- po/pt.po | 842 +++++++++++++++++++++++++++++++++++++++-------------------- po/pt_BR.po | 567 +++++++++++++++++++++++++--------------- po/ro.po | 403 ++++++++++++++-------------- po/sr.po | 838 ++++++++++++++++++++++++++++++++++++++-------------------- po/sv.po | 403 ++++++++++++++-------------- po/tr.po | 567 +++++++++++++++++++++++++--------------- po/uk.po | 403 ++++++++++++++-------------- po/vi.po | 403 ++++++++++++++-------------- po/zh_CN.po | 417 +++++++++++++++-------------- po/zh_TW.po | 558 ++++++++++++++++++++++++--------------- 23 files changed, 7257 insertions(+), 5132 deletions(-) commit 33b8a85face5392b5ac843bdbe3a72f024cad6ef Author: Lasse Collin Date: 2024-05-29 16:33:24 +0300 Build: Update po/*.po files only when needed When po/xz.pot doesn't exist, running "make" or "make dist" will create it. Then the .po files will be updated but only if they actually would change more than the POT-Creation-Date line. Then the .gmo files would be generated from the .po files. This is the case before and after this commit. However, "make dist" and thus "make mydist" did a forced update to the files, updating them even if the only change was the POT-Creation-Date line. This had pros and cons: It made it clear that the .po file really is in sync with the recent strings in the package. On the other hand, it added noise in form of changed files in the source tree and distribution tarballs. It can be ignored with something like "diff -I'^"POT-Creation-Date: '" but it's still a minor annoyance *if* there's not enough value in having the most recent timestamp. Setting DIST_DEPENDS_ON_UPDATE_PO = no means that such forced update won't happen in "make dist" anymore. However, the "mydist" target will use xz.pot-update target which is the same target that is run when xz.pot doesn't exist at all yet. Thus "mydist" will ensure that the translations are up to date, without noise from changes that would affect only the POT-Creation-Date line. Note that po4a always uses msgmerge with --update, so POT-Creation-Date in the man page translations is never the only change in .po files. In that sense this commit makes the message translations behave more similarly to the man page translations. Distribution tarballs will still have non-reproducible POT-Creation-Date in po/xz.pot and po4a/xz-man.pot but those are just two files. Even they could be made reproducible from a Git timestamp if desired. (cherry picked from commit 9284f1aea31f0eb23e2ea72f7218b271e2234762) Makefile.am | 3 ++- po/Makevars | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) commit 09daebd66b55799bbc495b84310a86c91bbfc1c8 Author: Lasse Collin Date: 2024-05-28 21:10:33 +0300 po4a/update-po: Disable wrapping in .pot and .po files The .po files from the Translation Project come with unwrapped strings so this matches it. This may reduce the noise in diffs too. When the beginning of a paragraph had changed, the rest of the lines got rewrapped in msgsid. Now it's just one very long line that changes when a paragraph has been edited. The --add-location=file option was removed as redundant. The line numbers don't exist in the .pot file due to --porefs file and thus they cannot get copied to the .po files either. (cherry picked from commit 4beba1cd62d7f8f7a6f1e899b68292d94c53b599) po4a/update-po | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) commit 51ad72dae4e516e9292f6f399bd1e4970b77f7c1 Author: Lasse Collin Date: 2024-05-28 18:36:53 +0300 Update contact info in README (cherry picked from commit b14c130a58a649f9a73392eeb122cb252327c569) README | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) commit 18463917f9b255b8f925fa54ab9388319735b14a Author: Lasse Collin Date: 2024-05-28 13:25:07 +0300 Translations: Use --package-name=xz-man with po4a This is to match reality. See the added comment. (cherry picked from commit 75f5f2e014b0ee646963f36bc6a9c840fb272353) po4a/update-po | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) commit 26bbcb13cd2bbb56fe406544a484b4edfc7e0837 Author: Lasse Collin Date: 2024-05-28 13:03:40 +0300 Translations: Omit --package-name from po/Makevars This is closer to the reality in the po/*.po files. (cherry picked from commit eb217d016cfbbba1babc19a61095b3ea25898af6) po/Makevars | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) commit c35ee804b89556d15bc8cdc16867f4316e69392f Author: Lasse Collin Date: 2024-05-28 01:17:45 +0300 Translations: Omit man page line numbers from .pot and .po files (cherry picked from commit 9114267038deaecf4832a5cacb5acbe6591ac839) po4a/update-po | 5 +++++ 1 file changed, 5 insertions(+) commit 0f4429d47f9cfe2cdfbad115a7bc2f11221cb217 Author: Lasse Collin Date: 2024-05-28 01:06:30 +0300 Translations: Use the xgettext option --add-location=file (cherry picked from commit 093490b58271e9424ce38a7b1b38bcf61b9c86c6) po/Makevars | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit a93e2c2d1d34a6f609d24a8e62072ce78df7a734 Author: Lasse Collin Date: 2024-05-28 00:43:53 +0300 Translations: Use the msgmerge option --add-location=file This way the PO file diffs are less noisy but the locations of the strings are still present at file level, just without line numbers. The option is available since gettext 0.19 (2014). configure.ac requires 0.19.6. (cherry picked from commit fccebe2b4fd513488fc920e4dac32562ed3c7637) po/Makevars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit d4389895592e9a8e0f6391fdad816ae0537bb07b Author: Lasse Collin Date: 2024-05-27 12:22:08 +0300 Build: Use $(SHELL) instead of sh to run scripts in Makefile.am (cherry picked from commit f361d9ae85707a87eb28db400eb7229cec103d58) Makefile.am | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) commit 5781414b6e3120098b0060d073aa2b0580ff6f40 Author: Lasse Collin Date: 2024-05-23 17:25:13 +0300 Translations: Change the home page URLs in man page translations Since the source strings have changed, these would get marked as fuzzy and the original string would be used instead. The original and translated strings are identical in this case so it wouldn't matter. But patching the translations helps still because then po4a will show the correct translation percentage. (cherry picked from commit a26dece34793a09aac2476f954d162d03e9cf62b) po4a/de.po | 8 ++++---- po4a/fr.po | 4 ++-- po4a/ko.po | 4 ++-- po4a/pt_BR.po | 4 ++-- po4a/ro.po | 8 ++++---- po4a/uk.po | 8 ++++---- 6 files changed, 18 insertions(+), 18 deletions(-) commit 3670e0616eb9d86e7519d2b76242fd32c6e0c1ae Author: Lasse Collin Date: 2024-05-23 15:15:18 +0300 CMake: Add manual support for 32-bit x86 assembly files One has to pass -DENABLE_X86_ASM=ON to cmake to enable the CRC assembly code. Autodetection isn't done. Looking at CMAKE_SYSTEM_PROCESSOR might not work as it comes from uname unless cross-compilation is done using a CMake toolchain file. On top of this, if the code is run on modern processors that support the CLMUL instruction, then the C code should be faster (but then one should also be using a x86-64 build if possible). (cherry picked from commit 24387c234b4eed1ef9a7eaa107391740b4095568) CMakeLists.txt | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) commit c1b001b09e902ecacabb8a2ae1fc991018a4d1f8 Author: Lasse Collin Date: 2024-05-23 14:26:45 +0300 CMake: Rename USE_DOXYGEN to ENABLE_DOXYGEN It's more consistent with the other option() uses. (cherry picked from commit 0fb3c9c3f684f5a25bd425ed079a20a79f0c969d) CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 7213fe39c717d4623c92af715484a71d9a6ff8d0 Author: Lasse Collin Date: 2024-05-22 15:21:53 +0300 Use more confident language in COPYING (cherry picked from commit 62733592a1cc6f0b41f46ef52e06d1a6fe1ff38a) COPYING | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) commit 15358be94a4e3f9c20f331b64b3980f3e5283760 Author: Lasse Collin Date: 2024-04-29 17:16:38 +0300 Add SPDX license identifiers to files under tests/ossfuzz (cherry picked from commit 9ae2ebc1e504a1814b0788de95fb5c58c0328dde) tests/ossfuzz/Makefile | 2 ++ tests/ossfuzz/config/fuzz_decode_alone.options | 2 ++ tests/ossfuzz/config/fuzz_decode_stream.options | 2 ++ tests/ossfuzz/config/fuzz_encode_stream.options | 2 ++ tests/ossfuzz/config/fuzz_lzma.dict | 2 ++ tests/ossfuzz/config/fuzz_xz.dict | 2 ++ 6 files changed, 12 insertions(+) commit 1aa92c7ffd0bf8f9738ebf3bd1263bd6f5f096a2 Author: Lasse Collin Date: 2024-04-29 17:16:06 +0300 Add SPDX license identifier to .codespellrc (cherry picked from commit 9000d70eb9815bd7f43ffddc1c3316c507aa0e05) .codespellrc | 2 ++ 1 file changed, 2 insertions(+) commit 3c7e400fdcabc0a1b78863948fc17964667a9401 Author: Lasse Collin Date: 2024-05-22 15:12:09 +0300 Move entries po4a/.gitignore to the top level .gitignore The po4a directory is in EXTRA_DIST and thus all files there are included in the package. .gitignore doesn't belong in the package so keep that file out of the po4a directory. (cherry picked from commit 903c16fcfa5bfad0cdb2a7383d941243bcb12e76) .gitignore | 4 ++++ po4a/.gitignore | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) commit 8a99272d4a9358dabdb5bc0b72f4c5240a9dc066 Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 CMake: Add comments (cherry picked from commit 9d997d6f9d4f042412e45c7b7a23a14ad2e4f9aa) tests/tests.cmake | 2 ++ 1 file changed, 2 insertions(+) commit c35259c9e2400f6f88c269d95ecafdb223ff45d2 Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 CMake: Remove the note that some tests aren't run They are now in the common build configurations. (cherry picked from commit d35368b33e54bad2f566df99fac29ffea38e34de) CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) commit 30982a215395f19b3837c3da540e1cb3f913569f Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 CMake: Add support for test_files.sh (cherry picked from commit dc232d584619b2819a9c52d6ad5d8b5d56b392ba) tests/tests.cmake | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) commit 3a8f81e0ad4cd1c102a03ff09e703cf8cb074afc Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 Tests: Make test_files.sh more flexible Add a new optional argument to specify the directory of the xz and xzdec executables. If ../config.h doesn't exist, assume that all encoders and decoders are available. (cherry picked from commit a7e9230af9d1f87f474fe38886eb977d4149dc9b) tests/test_files.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) commit 0644675c829143112c85455f8a6aa91bfc4e1bbb Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 CMake: Add support for test_compress.sh tests (cherry picked from commit b40e6efbb48d740b9b5b303e59e344801cbb5bd8) tests/tests.cmake | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) commit dcc02a6ca0e0ac4e330e820683754badbcf9815b Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 Tests: Make test_compress.sh more flexible Add a new optional second argument: directory of the xz and xzdec executables. This is need with the CMake build where the binaries end up in the top-level build directory. If ../config.h doesn't exist, assume that all encoders and decoders are available. This will make this script usable from CMake in the most common build configuration. NOTE: Since the existence of ../config.h is checked, the working directory of the test script must be a subdir in the build tree! Otherwise ../config.h would look outside the build tree. Use the default check type instead of forcing CRC32 or CRC64. Now the script doesn't need to check if CRC64 is available. (cherry picked from commit ac3222d2cb1ff3a15eb6d58f9ea9bc78e8bc3bb2) tests/test_compress.sh | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) commit c761b7051fb2ebb6da3cbecafe695fb5af7b2c9c Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 CMake: Prepare to support the test_*.sh tests This is a bit hacky since the scripts grep config.h to know which features were built but the CMake build doesn't create config.h. So instead those test scripts will be run only when all relevant features have been enabled. (cherry picked from commit 006040b29c83104403621e950ada0c8956c56b3d) tests/tests.cmake | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) commit a71bc2d75b95f85fe046f0fd1fb25d36be2b20ba Author: Lasse Collin Date: 2024-05-20 16:55:00 +0300 Tests: test_suffix.sh: Add a comment (cherry picked from commit 6167607a6ea72fb74eefb943c4566e3cab528cd2) tests/test_suffix.sh | 3 +++ 1 file changed, 3 insertions(+) commit 8fda5ce872632e464a1f9660b3ab8dac939a03c6 Author: Lasse Collin Date: 2024-05-18 00:34:07 +0300 Fix typos Thanks to xx on #tukaani. (cherry picked from commit 4e9023857d287f624562156b60dc23d2b64c0f10) src/common/mythread.h | 2 +- src/common/tuklib_integer.h | 2 +- src/liblzma/api/lzma/base.h | 2 +- src/liblzma/common/filter_buffer_decoder.c | 2 +- src/liblzma/common/filter_common.c | 2 +- src/scripts/xzgrep.in | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) commit 2729079bcb8dd1c3ab1a79426690d17f6f8e6f7d Author: Lasse Collin Date: 2024-05-18 00:23:52 +0300 liblzma: Fix white space Thanks to xx on #tukaani. (cherry picked from commit b14d08fbbc254485ace9ccfe7908674f608a62ae) src/liblzma/simple/simple_coder.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) commit a289c4dfeb3ded35e129c48b13f46605f0138704 Author: Lasse Collin Date: 2024-05-13 17:15:04 +0300 xz: Document the static function get_chains_memusage() (cherry picked from commit 142e670a413a7bce1a2647f1cf1f33f8ee2dbe88) src/xz/coder.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) commit 6f0db31713845386ce2419c55b2df89b53b80dd3 Author: Lasse Collin Date: 2024-05-13 17:07:22 +0300 xz: Rename filters_memusage_max() to get_chains_memusage() (cherry picked from commit 78e984399a64bfee5d11e7308e0bdbc1006db2ca) src/xz/coder.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) commit d7e2bf7e2dc9289a7a5dd0311d19d10de6d7ea1b Author: Lasse Collin Date: 2024-05-13 17:04:05 +0300 xz: Rename filter_memusages to chains_memusages (cherry picked from commit 54c3db0a83d3e67d89aba92a0957f2dce9b111a7) src/xz/coder.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 58f200b6d1dc4cbc1ab3315a359120ab6eb84878 Author: Lasse Collin Date: 2024-05-12 22:26:30 +0300 xz: Simplify the memory usage scaling code This is closer to what it was before the --filtersX support was added, just extended to support for scaling all filter chains. The method before this commit was an extended version of the original too but it was done in a more complex way for no clear reason. In case of an error, the complex version printed fewer informative messages (a good thing) but it's not a sigificant benefit. In the limit is too low even for single-threaded mode, the required amount of memory is now reported like in 5.4.x instead of like in 5.5.1alpha - 5.6.1 which showed the original non-scaled usage. It had been a FIXME in the old code but it's not clear what message makes the most sense. Fixes: 5f0c5a04388f8334962c70bc37a8c2ff8f605e0a (cherry picked from commit d9e1ae79ec90d6a7eafeaceaf0ece4f0c83d4417) src/xz/coder.c | 163 ++++++++++++++++++++------------------------------------- 1 file changed, 57 insertions(+), 106 deletions(-) commit 41bdc9fa5cc2fc2a70f4331329ac724773cc2f26 Author: Lasse Collin Date: 2024-05-13 12:14:00 +0300 xz: Edit comments (cherry picked from commit 0ee56983d198b776878432703de664049b1be32e) src/xz/coder.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) commit 52e40c1912dfdbf8c7aa85e3a4c3eb138fa73d5d Author: Lasse Collin Date: 2024-05-13 12:03:51 +0300 xz: Rename chain_idx to chain_num (cherry picked from commit ec82a49c3553f7206104582dbfb8b64fa433b491) src/xz/coder.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 8a019633319c694423691f58c55fa23a46e45ded Author: Lasse Collin Date: 2024-05-12 22:29:11 +0300 xz: Edit coding style (cherry picked from commit a731a6993c34bbbd55abaf9c166718682b1da24f) src/xz/coder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit e3ad7eda74caea29849e2e9ec01212f5f7d0f574 Author: Lasse Collin Date: 2024-05-12 22:16:05 +0300 xz: Edit comments Fixes: 5f0c5a04388f8334962c70bc37a8c2ff8f605e0a (cherry picked from commit 32eb176b89243fce3112347fe43a8ad14a9fd2be) src/xz/coder.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) commit 09cabae2ab47a06f6eee02419a815d4bfd0d9490 Author: Lasse Collin Date: 2024-05-12 21:57:49 +0300 xz: Fix grammar in a comment Fixes: cb3111e3ed84152912b5138d690c8d9f00c6ef02 (cherry picked from commit b90339f4daa510d2b1b8c550f855a99667f1d004) src/xz/coder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit c10b66fbf9b2442741a1f052bdb4ce7009af9cda Author: Lasse Collin Date: 2024-05-12 21:46:56 +0300 xz: Rename filter_memusages to encoder_memusages (cherry picked from commit 4c0bdaf13d651b22ba13bd93f8379724d6ccdc13) src/xz/coder.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) commit 9132ce3564b2c003bffd6de6294a3d98dccf314e Author: Lasse Collin Date: 2024-05-12 21:42:05 +0300 xz: Edit coding style (cherry picked from commit b54aa023e0ec291b06e976e5f094ab0549e7b09b) src/xz/coder.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) commit d642e13874e93b03959d1de523f1c8ebe9428838 Author: Lasse Collin Date: 2024-05-12 21:31:02 +0300 xz: Rename filters_index to chain_num The reason is the same as in bd0782c1f13e52cd0fd8415208e30e47004a4c68. (cherry picked from commit 49f67d3d3f42b640a7dfc4ca04c8934f658e10ce) src/xz/args.c | 8 ++++---- src/xz/coder.c | 8 ++++---- src/xz/coder.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) commit 47599f3b73f0a2bc18e0a8367d723f1eb0f11b63 Author: Lasse Collin Date: 2024-05-12 21:22:43 +0300 xz: Replace a few uint32_t with "unsigned" to reduce the number of casts These hold only tiny values. (cherry picked from commit ff9e8b3d069ecfa52ec43dcdb198542d1692a492) src/xz/args.c | 2 +- src/xz/coder.c | 17 ++++++++--------- src/xz/coder.h | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) commit 8f5ab75c454ea8676ed09c7f6eda8afe87b008ad Author: Lasse Collin Date: 2024-05-12 21:10:45 +0300 xz: Rename filters_used_mask to chains_used_mask The reason is the same as in bd0782c1f13e52cd0fd8415208e30e47004a4c68. (cherry picked from commit b5e6c1113b1ba02c282bd9163eccdb521c937a78) src/xz/coder.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) commit 3eb7cf9dd5b90a074f741234225d7de51ad88774 Author: Lasse Collin Date: 2024-05-12 17:14:43 +0300 xz: Move the setting of "check" in coder_set_compression_settings() It's more logical to do it in the beginning instead of in the middle of the filter chain handling. Fixes: d6af7f347077b22403133239592e478931307759 (cherry picked from commit 32500dfaadae2ea36fda2e17b49ae7d9ac1acf52) src/xz/coder.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) commit 067961ee0e1adaa66a43fbf8c3be31697554a839 Author: Lasse Collin Date: 2024-05-12 17:09:17 +0300 xz: Rename "filters" to "chains" The convention is that lzma_filter filters[LZMA_FILTERS_MAX + 1]; contains the filters of a single filter chain. It was so here as well before the commit d6af7f347077b22403133239592e478931307759. It changes "filters" to a ten-element array of filter chains. It's clearer to call this array-of-arrays "chains". This also renames "filter_idx" to "chain_idx" which is used as an index as in chains[chain_idx]. (cherry picked from commit ad146b1f42bbb678175a503a45ce525e779f9b8b) src/xz/coder.c | 68 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) commit 6822f6f891d43c97ea379a51223ce8ea69439161 Author: Lasse Collin Date: 2024-05-12 16:56:15 +0300 xz: Clean up a comment (cherry picked from commit 5a4ae4e4d0105404184e9a82ee08f94e1b7783e0) src/xz/coder.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) commit 0e5e3e7bdcfcdc4b4607665ff0f6ad794e5195af Author: Lasse Collin Date: 2024-05-12 16:52:09 +0300 xz: Add clarifying assertions (cherry picked from commit 2de80494ed9a4dc7db395a32a5efb770ce769804) src/xz/coder.c | 4 ++++ 1 file changed, 4 insertions(+) commit 77bcf6b76a26833923e62b2dec717474d5d44700 Author: Lasse Collin Date: 2024-05-10 20:23:33 +0300 xz: Add a clarifying assertion Fixes: 5f0c5a04388f8334962c70bc37a8c2ff8f605e0a (cherry picked from commit 1eaad004bf7748976324672db028e34f42802e61) src/xz/coder.c | 1 + 1 file changed, 1 insertion(+) commit df3efc058a256629ea0153b4750d3df308757038 Author: Lasse Collin Date: 2024-05-12 16:47:17 +0300 xz: Clarify a comment (cherry picked from commit 605094329b986244833c967c04963cacc41a868d) src/xz/coder.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) commit 4ebfe11cd33439675f03e1e3725abf03d6f8251b Author: Lasse Collin Date: 2024-05-12 16:28:25 +0300 xz: Use the info collected in parse_block_list() This is slightly simpler and it avoids looping through the opt_block_list array. (cherry picked from commit 8fac2577f2dbb9491afd8500f60d004c9071df3b) src/xz/coder.c | 95 ++++++++++++++++++++++++---------------------------------- 1 file changed, 39 insertions(+), 56 deletions(-) commit bfea6913618357a7034a1d79079bccb688262124 Author: Lasse Collin Date: 2024-05-12 15:48:45 +0300 xz: Remember the filter chains and the largest Block in parse_block_list() (cherry picked from commit 81d350dab864b985b740742772f3b132d4c52914) src/xz/args.c | 18 ++++++++++++++++++ src/xz/coder.c | 2 ++ src/xz/coder.h | 13 +++++++++++++ 3 files changed, 33 insertions(+) commit d4e33e73922427a0f5277b91b239af538fd41c06 Author: Lasse Collin Date: 2024-05-12 15:38:48 +0300 xz: Update a comment and initialization of filters_used_mask (cherry picked from commit 46ab56968f7dfdac187710a1223659d832fa1565) src/xz/coder.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) commit 3c130737c9bb4a5021bb14eb19e9ceae30ffef3a Author: Lasse Collin Date: 2024-05-12 15:08:10 +0300 xz: parse_block_list: Edit integer type casting (cherry picked from commit e89293a0baeb8663707c6b4a74fbb310ec698a8f) src/xz/args.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) commit 40c8513b4ee42b8c0fae9b2a229e078ac7e0f87a Author: Lasse Collin Date: 2024-05-12 14:51:37 +0300 xz: Make filter_memusages a local variable (cherry picked from commit 87011e40c168255cd2edea129ee68c901770603b) src/xz/coder.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) commit cacaf25aa71cd1110cc049d037c11e4075602c35 Author: Lasse Collin Date: 2024-05-10 20:33:08 +0300 xz: Remove unused code and simplify opt_mode == MODE_COMPRESS isn't possible when HAVE_ENCODERS isn't defined. Thus, when *encoding*, the message about *decoder* memory usage is possible to show only when both encoder and decoder have been built. Since the message is shown only at V_DEBUG, skip the memusage calculation if verbosity level isn't high enough. Fixes: 5f0c5a04388f8334962c70bc37a8c2ff8f605e0a (cherry picked from commit 347b412a9374e0456bef9da0d7d79174c0b6f1a5) src/xz/coder.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) commit 3495a6b291f49079485854bb185a52c29d06cd2f Author: Lasse Collin Date: 2024-05-10 20:22:58 +0300 xz: Fix integer type from uint64_t to uint32_t lzma_options_lzma.dict_size is uint32_t so use it here too. Fixes: 5f0c5a04388f8334962c70bc37a8c2ff8f605e0a (cherry picked from commit 31358c057c9de9d6aba96bae112b2d17942de7cb) src/xz/coder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 2861d856deb557734f067c5c471d670f0b0c6684 Author: Lasse Collin Date: 2024-05-08 21:40:07 +0300 debug/translation.bash: Remove an outdated test command Since 5.3.5beta, "xz --lzma2=mf=bt4,nice=2" works even though bt4 needs at least nice=4. It is rounded up internally by liblzma when needed. Fixes: 5cd9f0df78cc4f8a7807bf6104adea13034fbb45 (cherry picked from commit 3f71e0f3a118e1012526f94fd640a626d30cb599) debug/translation.bash | 1 - 1 file changed, 1 deletion(-) commit 54546babc3feb2786e541b80f9e7216b8f1bd543 Author: Lasse Collin Date: 2024-05-07 20:41:28 +0300 Fix the date of NEWS for 5.4.5 (cherry picked from commit b05a516830095a0e1937aeb31c937fb0400408b6) NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit a7e58d1fdb493d58854ac599347cf64da0cecca4 Author: Lasse Collin Date: 2024-05-07 16:21:15 +0300 Build: Update visibility.m4 from Gnulib This fixes the syntax of the "serial" line and renames a temporary variable. (cherry picked from commit 6d336aeb97b69c496ddc626af403f6f21c753658) m4/visibility.m4 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) commit 07a9cda037042b262ba6c8c18fae4a5b3333d508 Author: Lasse Collin Date: 2024-05-07 15:05:21 +0300 po4a/update-po: Delete the *.po.authors files These are temporary files that are needed only when running po4a. The top-level Makefile.am puts the whole po4a directory into distribution tarball (it's simpler) so deleting these temporary files is needed to prevent them from getting into tarballs. (cherry picked from commit ab51e8ee610e2a893906859848f93d5cb0d5ba83) po4a/update-po | 4 ++++ 1 file changed, 4 insertions(+) commit 1b4e7dca243d8ef297a245b5ee3ce9cd1ca20f56 Author: Lasse Collin Date: 2024-05-07 13:12:17 +0300 xz: Edit comments and coding style (cherry picked from commit e4780244a17420cc95d5498cd6e02ad10eac6e5f) src/xz/coder.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) commit 18683525a78e96ec6d7c2b4e841e94ad39be7096 Author: Lasse Collin Date: 2024-05-06 23:08:22 +0300 xz: Omit an incorrect comment It likely was a leftover from a development version of the code. Fixes: 183819bfd9efac8c184d9bf123325719b7eee30f (cherry picked from commit fe4d8b0c80eaeca3381be302eeb89aba871a7e7c) src/xz/coder.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) commit 005f0398645b0342c9c1915d422743c77ec1d435 Author: Lasse Collin Date: 2024-05-06 23:04:31 +0300 xz: Add braces to a for-statement and to an if-statement No functional changes. Fixes: 5f0c5a04388f8334962c70bc37a8c2ff8f605e0a Fixes: 479fd58d60622331fcbe48fddf756927b9f80d9a (cherry picked from commit 9bef5b8d17dd5e009d6a6b2becc2dc535da53937) src/xz/coder.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit 34be4e6aa62376314fde250ea4f142c18274272f Author: Lasse Collin Date: 2024-05-06 23:00:09 +0300 liblzma: Omit an unneeded array from the x86 filter Fixes: 6aa2a6deeba04808a0fe4461396e7fb70277f3d4 (cherry picked from commit de06b9f0c0a3f72569829ecadbc9c0a3ef099f57) src/liblzma/simple/x86.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) commit 79e329b771210c30ea317dd4d99e8968f3e6f9b2 Author: Lasse Collin Date: 2024-05-06 22:56:31 +0300 CMake: Add test_suffix.sh to the tests (cherry picked from commit 7da488cb933fdf51cfc14cb5810beb0766224380) tests/tests.cmake | 13 +++++++++++++ 1 file changed, 13 insertions(+) commit 86f33bb90c6cfe6950f1d36c9e5dd7fdc9798124 Author: Lasse Collin Date: 2024-05-06 22:55:54 +0300 Test: Add CMake support to test_suffix.sh It needs to find the xz executable from a different directory and work without config.h. (cherry picked from commit a805594ed0b4cbf7b81aa28ff46a8ab3c83c6876) tests/test_suffix.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) commit 1e243ab378e8f78ebb3af741fb38354954cf20f9 Author: Lasse Collin Date: 2024-05-06 20:45:34 +0300 Update INSTALL about MINIX 3 The latest stable is 3.3.0 and it's from 2014. Don't mention the older versions in INSTALL. 3.3.0 ships with Clang already. Testing with 3.4.0beta6 shows that tuklib_physmem works too so omit comments about that from INSTALL. Visibility warnigns weren't a problem either. Thus it's enough to mention the need for --disable-threads as configure doesn't autodetect the lack of pthreads. (cherry picked from commit 50e19489387774bab3c4a988397d0d9c7a142a46) INSTALL | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) commit 8595b5ab3ba766eb6daed890bfe91a16fe329c2c Author: Lasse Collin Date: 2024-05-02 23:00:16 +0300 Windows: Remove the "doc/api" line from README-Windows.txt Fixes: 252aa1d67bc015eeba462803ab72edeb7744d864 (cherry picked from commit 68d18aea1422a2b86b98b71d0b019233d84e01b0) windows/README-Windows.txt | 2 -- 1 file changed, 2 deletions(-) commit a3f163a4ad97189744107e964e4dea505fbcc252 Author: Lasse Collin Date: 2024-05-02 22:59:04 +0300 Build: Don't copy doc/api from source tree to distribution tarball It was copied if it existed. This was intentional when autogen.sh still built liblzma API docs with Doxygen. Fixes: d3a77ebc04bf1db8d52de2d9b0f07877bc4fd139 (cherry picked from commit 8ede961374613aa302a13571d662cfaea1cf91f7) Makefile.am | 5 ----- 1 file changed, 5 deletions(-) commit cb0e847fe07099c1ef6d8076f6a46e17bc431acb Author: Sam James Date: 2024-05-02 13:26:40 +0100 ci: add SPDX headers I've checked over each of these and they're straightforward applications of the relevant Github Actions. (cherry picked from commit 9a6761aa35ed84d30bd2fda2333a4fdf3f46ecdc) .github/workflows/freebsd.yml | 2 ++ .github/workflows/netbsd.yml | 2 ++ .github/workflows/openbsd.yml | 2 ++ 3 files changed, 6 insertions(+) commit c3c854dc759fe0c5549aa0a730be9e259243edb6 Author: Yaroslav Halchenko Date: 2024-03-29 14:37:24 -0400 codespell: Ignore the THANKS file and debbugs.gnu.org URL This way "codespell -i 0" is silent. This is the first commit from https://github.com/tukaani-project/xz/pull/93 with trivial edits by Lasse Collin. (cherry picked from commit 81efe6119f86e3274e512c9eca5ec22b2196c2b3) .codespellrc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) commit 3216301aa20fcf9d5a7485e35a295d5c451d9658 Author: Lasse Collin Date: 2024-04-30 14:37:11 +0300 Add .gitattributes to clean up git-archive output (cherry picked from commit 905bfc74fe2670fd9c39014803017ab53d325401) .gitattributes | 7 +++++++ 1 file changed, 7 insertions(+) commit f99e7c69ada9e0db0ee1ebbc38c8ce9390cd9788 Author: Lasse Collin Date: 2024-04-19 12:11:09 +0300 xzdec: Support Landlock ABI version 4 This was added to xz in 02e3505991233901575b7eabc06b2c6c62a96899 but I forgot to do the same in xzdec. The Landlock sandbox in xzdec could be stricter as now it's active only for the last file being decompressed. In xz, read-only sandbox is used for multi-file case. On the other hand, xz doesn't go to the strictest mode when processing the last file when more than one file was specified; xzdec does. (cherry picked from commit 3334c71d3d4294a4f6569df3ba9bcf2443dfa501) src/xzdec/xzdec.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) commit bfe9be7a46cfd3b3069c15f7ba1432192bca1f5b Author: Lasse Collin Date: 2024-04-30 22:22:45 +0300 liblzma: Fix incorrect function type error from sanitizer Clang 17 with -fsanitize=address,undefined: src/liblzma/common/filter_common.c:366:8: runtime error: call to function encoder_find through pointer to incorrect function type 'const lzma_filter_coder *(*)(unsigned long)' src/liblzma/common/filter_encoder.c:187: note: encoder_find defined here Use a wrapper function to get the correct type neatly. This reduces the number of casts needed too. This issue could be a problem with control flow integrity (CFI) methods that check the function type on indirect function calls. Fixes: 3b34851de1eaf358cf9268922fa0eeed8278d680 (cherry picked from commit 278563ef8f2b8d98d7f2c85e1a64ec1bc21d26d8) src/liblzma/common/filter_decoder.c | 15 ++++++++++++--- src/liblzma/common/filter_encoder.c | 17 +++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) commit 882eadc5b820b6b1495fc91ba3573ac2aa6c1df3 Author: Lasse Collin Date: 2024-04-30 21:41:11 +0300 xz: Avoid arithmetic on a null pointer It's undefined behavior. The result wasn't ever used as it occurred in the last iteration of a loop. Clang 17 with -fsanitize=address,undefined: $ src/xz/xz --block-list=123 src/xz/args.c:164:12: runtime error: applying non-zero offset 1 to null pointer Fixes: 88ccf47205d7f3aa314d358c72ef214f10f68b43 Co-authored-by: Sam James (cherry picked from commit 77c8f60547decefca8f2d0c905d9c708c38ee8ff) src/xz/args.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) commit ec5458e1c9b2beb416781e81ad4ff22b0149b99d Author: Lasse Collin Date: 2024-04-27 20:42:00 +0300 CMake: Support building liblzma API docs using Doxygen This is disabled by default to match the default in Autotools. Use -DUSE_DOXYGEN=ON to enable Doxygen usage. This uses the update-doxygen script, thus this is under if(UNIX) although Doxygen itself can run on Windows too. (cherry picked from commit 64503cc2b76a388ced4ec5f68234a07f0dcddcd5) CMakeLists.txt | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) commit 8c93ced56bcb23df723dab23b7477d580720f522 Author: Lasse Collin Date: 2024-04-20 23:36:39 +0300 CMake: List API headers in LIBLZMA_API_HEADERS variable This way the same list will be usable in more than one location. (cherry picked from commit 0a7f5a80d8532a1d8cfa0a902c9d1ad7651eca37) CMakeLists.txt | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) commit f7c9bab0372db357511e42c9c610a2cfe5fca9b1 Author: Lasse Collin Date: 2024-04-19 15:16:42 +0300 PACKAGERS: Document the optional Doxygen usage Also add a note that packagers should check the licensing of the Doxygen output. (cherry picked from commit 541406bee3f09e9813103c6406b10fc6ab2e0d30) PACKAGERS | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) commit 28e7d130cb843e96d7e6b0358f8dd58bd1b2a275 Author: Lasse Collin Date: 2024-04-27 17:47:09 +0300 Build: Add --enable-doxygen to generate and install API docs It requires Doxygen. This option is disabled by default. (cherry picked from commit e21efdf96f39378fe417479f89e97046680406f5) INSTALL | 6 ++++++ configure.ac | 10 +++++++++- src/liblzma/api/Makefile.am | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) commit cca7e6c05bc6cc51c0271c36856b7fe29f65c648 Author: Lasse Collin Date: 2024-04-19 15:15:17 +0300 Doxygen: update-doxygen: Support out-of-tree builds Also, now $0 is used to refer to the script itself. (cherry picked from commit 0ece09a575d7e542bda8825808ddd6cf7de8cc4b) doxygen/update-doxygen | 110 ++++++++++++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 42 deletions(-) commit 8090d3dc7f0eea4a3a61f4f6d46a0d0866e345fe Author: Lasse Collin Date: 2024-04-28 21:08:00 +0300 Doxygen: Simplify Doxyfile and add SPDX license identifier This omits all comments and a few non-default options that weren't needed. Now it contains no copyrighted content from Doxygen itself. (cherry picked from commit 2c519f641f266fd897edf680827d9c905f411440) doxygen/Doxyfile | 2698 +----------------------------------------------------- 1 file changed, 25 insertions(+), 2673 deletions(-) commit 0721b8bfe558502669f06c97601fe59ad0d52541 Author: Lasse Collin Date: 2024-04-19 15:14:02 +0300 Doxygen: Don't strip JavaScript anymore The stripping method worked well with Doxygen 1.8 and 1.9 but it doesn't work with Doxygen 1.10 anymore. Since we won't ship pre-generated liblzma API docs anymore, the extra bloat and extra license info of the JavaScript files won't affect the upstream source package anymore. (cherry picked from commit bdba39a57530d11b88440df8024002be3d09e4a1) doxygen/update-doxygen | 21 --------------------- 1 file changed, 21 deletions(-) commit 1ddb40f6fd286c3c6ef510735112db1ac1b60936 Author: Lasse Collin Date: 2024-04-19 17:26:41 +0300 Build: Remove old Doxygen rules from top-level Makefile.am (cherry picked from commit d3a77ebc04bf1db8d52de2d9b0f07877bc4fd139) Makefile.am | 12 ------------ 1 file changed, 12 deletions(-) commit 092af76234b1bc79380427456b3215aa0b80f339 Author: Lasse Collin Date: 2024-04-19 15:10:06 +0300 Update COPYING to match the autogen.sh and mydist changes (cherry picked from commit fd7faa4c338a42a6a40e854b837d285ae2e8c609) COPYING | 11 ----------- 1 file changed, 11 deletions(-) commit 77bce9a0a250cfb20333ee0dca036b3193dd4941 Author: Lasse Collin Date: 2024-04-19 17:23:43 +0300 Build: Don't run update-doxygen as part of "make mydist" (cherry picked from commit b2bc55d8a0a9f2f59bfd4302067300e650f6baa3) Makefile.am | 1 - 1 file changed, 1 deletion(-) commit 3a2fc62f59b2e8cc45f8d8fd9988b4305efe4bff Author: Lasse Collin Date: 2024-04-19 15:09:48 +0300 autogen.sh: Don't generated Doxygen docs anymore (cherry picked from commit e9be74f5b129fe8a5388d588e68b1b7f5168a310) autogen.sh | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) commit b04c16f9a5a8675a87783305568cadfa3f17d999 Author: Lasse Collin Date: 2024-04-19 17:41:36 +0300 windows/build.bash: Omit Doxygen docs from the package They will be omitted from the source tarball and I don't want to make Doxygen a dependency of build.bash. (cherry picked from commit 252aa1d67bc015eeba462803ab72edeb7744d864) windows/build.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit d4dd3c8f6169adf50cad8fe6872e0f5fcb82475c Author: Lasse Collin Date: 2024-04-19 14:14:47 +0300 README: Don't mention PDF man pages anymore (cherry picked from commit 634095364d87444d62d8ec54c134c0cd4705f5d7) README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit be90720d6cd7fbb1b170794445815f579b444a6f Author: Lasse Collin Date: 2024-04-19 14:10:39 +0300 Build: Omit PDF man pages from the package pdf-local rule was added to create the PDFs still with "make pdf". The install rules are missing but that likely doesn't matter at all. (cherry picked from commit dc684bf76ea23574ee9d88382057381e04e6089a) Makefile.am | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) commit f724552d0c1ae2e3aa693d80d8d0da962dfac4e8 Author: Lasse Collin Date: 2024-04-19 13:54:39 +0300 windows/build.bash: Don't copy PDF man pages to the package (cherry picked from commit e3531ab4125cbd5c01ebd3200791350960547189) windows/README-Windows.txt | 2 +- windows/build.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit 00e774819c6550a8eac219e9f6f083ab2b155505 Author: Lasse Collin Date: 2024-04-28 01:34:50 +0300 Tests: test_index: Fix failures when features are disabled Fixes: cd88423e76d54eb72aea037364f3ebb21f122503 (cherry picked from commit 710a4573ef2cbd19c66318c3b2d1388e418e26c7) tests/test_index.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) commit 51133ad71eecc19bdb3ab287a0732fd9441753f4 Author: Lasse Collin Date: 2024-04-20 17:09:11 +0300 CMake: Keep the build working if the "tests" directory is missing This moves the tests section as is from CMakeLists.txt into tests/tests.cmake. CMakeLists.txt now includes tests/tests.cmake if the latter file exists. Now it's possible to delete the whole "tests" directory and building with CMake will still work normally, just without the tests. This way the tests are readily available for those who want them, and those who won't run the tests anyway have a straightforward way to ensure that nothing from the "tests" directory can affect the build process. (cherry picked from commit aaff75c3486c4489ce88b0efb36b41cf138af7c3) CMakeLists.txt | 76 ++--------------------------------------------- tests/Makefile.am | 1 + tests/tests.cmake | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 73 deletions(-) commit 85b5595b67f0081b2a900104ed7589de4bb75e12 Author: Lasse Collin Date: 2024-04-20 13:12:50 +0300 Tests: Remove x86 and SPARC BCJ tests These are very old but the exact test file isn't easy to reproduce as it was compiled from a short C program (bcj_test.c) long ago. These tests weren't very good anyway, just a little better than nothing. (cherry picked from commit a5f2aa5618fe9183706c9c514c3067985f6c338b) tests/Makefile.am | 7 ---- tests/bcj_test.c | 64 --------------------------------- tests/compress_prepared_bcj_sparc | Bin 1240 -> 0 bytes tests/compress_prepared_bcj_x86 | Bin 1388 -> 0 bytes tests/files/README | 8 ----- tests/files/good-1-sparc-lzma2.xz | Bin 612 -> 0 bytes tests/files/good-1-x86-lzma2.xz | Bin 716 -> 0 bytes tests/test_compress_prepared_bcj_sparc | 4 --- tests/test_compress_prepared_bcj_x86 | 4 --- 9 files changed, 87 deletions(-) commit d8228d1ea08155a17acaadd76ed95805d3b0a929 Author: Lasse Collin Date: 2024-04-27 18:30:40 +0300 Tests: test_index: Edit a misleading test (cherry picked from commit d879686469c9c4bf2a7c0bb6420ebe4530fc8f07) tests/test_index.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) commit 2358ef8238f166c49e66f438e7494d4d352eb113 Author: Lasse Collin Date: 2024-04-27 16:46:01 +0300 Tests: test_index: Use minimal values to test integer overflow (cherry picked from commit 612005bbdb0dea9dc09e9e2e9cc16a15c1480acd) tests/test_index.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 54f4a4162aae8796580489013583d6148be5a473 Author: Lasse Collin Date: 2024-04-27 15:13:39 +0300 Tests: test_index: Test lzma_index_buffer_decode() more (cherry picked from commit 4ad88b2544c2aaf8de8f38af54587098cbe66c1d) tests/test_index.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) commit 85ab59a6b70db33f320a3ea7a854249cb693dea2 Author: Lasse Collin Date: 2024-04-27 15:08:29 +0300 Tests: test_index: Test that *i = NULL is done on LZMA_PROG_ERROR On LZMA_DATA_ERROR from lzma_index_buffer_decode(), *i = NULL was already done but this adds a test for that case too. (cherry picked from commit 575b11b0d291e66c5fce31ce7a72f11436d57c83) tests/test_index.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) commit fb42599e44dde417305c7d92fd782147ca923079 Author: Lasse Collin Date: 2024-04-27 15:01:25 +0300 Tests: test_index: Test lzma_index_buffer_encode() with empty output buf (cherry picked from commit 2c970debdb285823f01f75e875561d893345ac2b) tests/test_index.c | 3 +++ 1 file changed, 3 insertions(+) commit 20cac20f63a96a39391f2d613bef0f7bd6553495 Author: Lasse Collin Date: 2024-04-27 14:59:55 +0300 Tests: test_index: Replace if-statements with tuktest assertions (cherry picked from commit cd88423e76d54eb72aea037364f3ebb21f122503) tests/test_index.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) commit 91e3ea8735752db5d0373991e84607196070aeaa Author: Lasse Collin Date: 2024-04-27 14:56:16 +0300 Tests: test_index: Make it clear that my_alloc() has no integer overflows liblzma guarantees that the product of the allocation size arguments will fit in size_t. Putting the pre-increment in the if-statement was clearly wrong although in practice it didn't matter here as the function is called only a couple of times. (cherry picked from commit 7f865577a6224fbbb5f5ca52574b62ea8ac9bf51) tests/test_index.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) commit df1659a6c8367db69e82e2ea59ad5f959cf4e615 Author: Lasse Collin Date: 2024-04-27 14:51:52 +0300 Tests: test_index: Verify also iter.block.number_in_stream (cherry picked from commit 12313a3b6596cdcf012e180597f84d231f8730d3) tests/test_index.c | 2 ++ 1 file changed, 2 insertions(+) commit e083e95dbfda73900109cca4c82c8713d0a1da21 Author: Lasse Collin Date: 2024-04-27 14:51:06 +0300 Tests: test_index: Check cases that aren't a multiple of 4 bytes (cherry picked from commit ad2654010d9d641ce1601beeff00630027e6bcd4) tests/test_index.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) commit b0d3b86ecf1881d10e6614b64b0fcc6c16a3b08f Author: Lasse Collin Date: 2024-04-27 14:40:25 +0300 Tests: test_index: Edit comments and white space (cherry picked from commit 2524fcf2b68b662035437cee8edbe80067c0c240) tests/test_index.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) commit bae288ea6ffb976c36e2387c03d75ce84a8a1034 Author: Lasse Collin Date: 2024-04-27 14:33:38 +0300 liblzma: index_decoder: Fix missing initializations on LZMA_PROG_ERROR If the arguments to lzma_index_decoder() or lzma_index_buffer_decode() were such that LZMA_PROG_ERROR was returned, the lzma_index **i argument wasn't touched even though the API docs say that *i = NULL is done if an error occurs. This obviously won't be done even now if i == NULL but otherwise it is best to do it due to the wording in the API docs. In practice this matters very little: The problem can occur only if the functions are called with invalid arguments, that is, the calling application must already have a bug. (cherry picked from commit 71eed2520e2eecae89bade9dceea16e56cfa2ea0) src/liblzma/common/index_decoder.c | 11 +++++++++++ 1 file changed, 11 insertions(+) commit f10cb93f335900a29e50f990b751996ef026b3a3 Author: Lasse Collin Date: 2024-04-26 18:25:18 +0300 CMake: Bump maximum policy version to 3.29 (cherry picked from commit 0478473953f50716a2bc37b619b1c7dc2682b1ad) CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 59055d70cdd3df091264ae9da793821bfd65314d Author: Sam James Date: 2024-04-13 22:30:44 +0100 ci: add NetBSD (cherry picked from commit a607e2b40d23f7d998dbaba76692aa30b4c3d9d3) .github/workflows/netbsd.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) commit 812c1f95f37751aaa1e020fc2360949a674842fd Author: Sam James Date: 2024-04-13 23:49:26 +0100 ci: add FreeBSD (cherry picked from commit 72c210336de26fb87a928160d025fa10a638d23b) .github/workflows/freebsd.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) commit d2a4f963c28b864aa179464f7827cc10c6e1365d Author: Sam James Date: 2024-04-13 23:16:08 +0100 ci: add OpenBSD (cherry picked from commit b526ec2dbfb5889845ea60548c4f5b1f97d84ab2) .github/workflows/openbsd.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) commit 493bc57c33385bda5ad32d01ab73dcfe8f5e7ced Author: Sam James Date: 2024-04-15 05:53:01 +0100 liblzma: outqueue: add header guard Reported by github's codeql. (cherry picked from commit c7ef767c49351743d8d011574abb9e200bf6b24f) src/liblzma/common/outqueue.h | 5 +++++ 1 file changed, 5 insertions(+) commit cede418d4f8e1fb4c8a30839fa5d3b14743e83d4 Author: Sam James Date: 2024-04-15 05:53:56 +0100 liblzma: easy_preset: add header guard Reported by github's codeql. (cherry picked from commit 55dcae3056d95cb2ddb8b560c12ba7596bc79f2c) src/liblzma/common/easy_preset.h | 5 +++++ 1 file changed, 5 insertions(+) commit 6e76a25df28b47407a201bf0381fa6d3c80cb0bb Author: Lasse Collin Date: 2024-04-25 14:00:57 +0300 tuklib_integer: Rename bswapXX to byteswapXX The __builtin_bswapXX from GCC and Clang are preferred when they are available. This can allow compilers to emit the x86 MOVBE instruction instead of doing a load + byteswap as two instructions (which would happen if the byteswapping is done in inline asm). bswap16, bswap32, and bswap64 exist in system headers on *BSDs and Darwin. #defining bswap16 on NetBSD results in a warning about macro redefinition. It's safest to avoid this namespace conflict completely. No OS supported by tuklib_integer.h uses byteswapXX names and a web search doesn't immediately find any obvious danger of namespace conflicts. So let's try these still-pretty-short names for the macros. Thanks to Sam James for pointing out the compiler warning on NetBSD 10.0. (cherry picked from commit 4ffc60f32397371769b7d6b5e3ed8626292d58df) src/common/tuklib_integer.h | 47 ++++++++++++++++++++------------------ src/liblzma/check/crc32_fast.c | 4 ++-- src/liblzma/check/crc32_tablegen.c | 2 +- src/liblzma/check/crc64_fast.c | 4 ++-- src/liblzma/check/crc64_tablegen.c | 2 +- 5 files changed, 31 insertions(+), 28 deletions(-) commit 0ca14871f306b97ce81bfe44c4a39b6b2af31bb3 Author: Lasse Collin Date: 2024-04-24 01:20:26 +0300 liblzma: API doc cleanups (cherry picked from commit 08ab0966a75b501aa7c717622223f0c13a113c75) src/liblzma/api/lzma/container.h | 2 +- src/liblzma/api/lzma/index.h | 6 +++--- src/liblzma/api/lzma/vli.h | 5 ++--- 3 files changed, 6 insertions(+), 7 deletions(-) commit 94a462850bc8718f5dd5b30116bce2165b2403c2 Author: Lasse Collin Date: 2024-04-23 16:35:33 +0300 Tests: test_filter_str: Add a few assertions (cherry picked from commit 3ac8a9bb4cccbee88350696dc9c645c48d77c989) tests/test_filter_str.c | 4 ++++ 1 file changed, 4 insertions(+) commit 72058ca22a7f3c9c67ed58be624f8302c6337cd7 Author: Lasse Collin Date: 2024-04-23 16:35:08 +0300 Tests: test_filter_str: Move one assertion and add a comment (cherry picked from commit 26c69be80523b05c84dea86c47c4ddd9a10945d7) tests/test_filter_str.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit c59ebbe1c6dd18b78a046aae3133702dd52c352e Author: Lasse Collin Date: 2024-04-23 16:26:06 +0300 Tests: test_filter_str: Tweak comments and white space (cherry picked from commit 4f6af853bc99904efb8b6c28a0af7b81a8476c1b) tests/test_filter_str.c | 3 +++ 1 file changed, 3 insertions(+) commit ceda860934b0272689d0722ceeb490cf9c559956 Author: Lasse Collin Date: 2024-04-23 16:25:22 +0300 Tests: test_filter_str: Add missing RISC-V case Fixes: 89ea1a22f4ed3685b053b7260bc5acf6c75d1664 (cherry picked from commit c92663aa1bd576e0615498a4189acf0df12e84b9) tests/test_filter_str.c | 3 +++ 1 file changed, 3 insertions(+) commit 2234b7cc472e62f3401216a71261579342fa2959 Author: Lasse Collin Date: 2024-04-22 22:23:32 +0300 Tests: test_filter_str: Test *error_pos more thoroughly (cherry picked from commit b0366df1d7ed26268101f9303a001c91c0806dfc) tests/test_filter_str.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) commit 3ba3ef57f929670adb1f9c5e5207a81a29374237 Author: Lasse Collin Date: 2024-04-22 21:54:39 +0300 liblzma: lzma_str_to_filters: Set *error_pos on all errors The API docs clearly say that if error_pos isn't NULL then *error is always set on any error. However, it wasn't touched if str == NULL or filters == NULL or unsupported flags were specified. Fixes: cedeeca2ea6ada5b0411b2ae10d7a859e837f203 (cherry picked from commit 70d12dd069bb9bb0d6bb1c8fafc4e6f77780263d) src/liblzma/common/string_conversion.c | 6 ++++++ 1 file changed, 6 insertions(+) commit 57ad820e15381344a812c78ce9b67a77a60b9cf3 Author: Lasse Collin Date: 2024-04-22 20:31:25 +0300 liblzma: Clean up white space (cherry picked from commit ed8e552395701fbf046027cebc8be4a6755b263f) src/liblzma/lz/lz_encoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit ba0b5bfe7cb3cdbd9a4e3c268e10c304cb834e8a Author: Lasse Collin Date: 2024-04-22 18:35:19 +0300 Tests: test_filter_flags: Edit comments and style (cherry picked from commit 2f06920f20b1ad63b7953dc09569e1d424998849) tests/test_filter_flags.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) commit d2ed6759596185ac6a9c69ea713c27cd4bd1d9ba Author: Lasse Collin Date: 2024-04-22 16:39:44 +0300 Tests: Fix C99/C11 compatibility when features are disabled The array could become empty and then the initializer would be simply {} which is allowed only in GNU-C and C23. (cherry picked from commit b101e1d1dbc81577c0c9aa0cb89cf2e46a15eb82) tests/test_filter_flags.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) commit 9a70e93fef3fd5943484e56f1881a7c6e3296027 Author: Lasse Collin Date: 2024-04-21 20:32:16 +0300 DOS: Omit useless defines from config.h (cherry picked from commit f8f3a220ac8afcb8cb2812917d3b77e00c2eab0d) dos/config.h | 12 ------------ 1 file changed, 12 deletions(-) commit dc4740f720e08bdd496aa2736db3b7aea6dd3d1e Author: Lasse Collin Date: 2024-04-21 20:27:50 +0300 Build: Omit useless checks for fcntl.h, limits.h, and sys/time.h (cherry picked from commit fc1921b04b8840caaa777c2bd5340d41b259da20) configure.ac | 6 ------ 1 file changed, 6 deletions(-) commit 6e210d5766b25d36729152a13c5889bb0605a1e3 Author: Lasse Collin Date: 2024-04-19 22:04:21 +0300 liblzma: Silence a warning from Coverity static analysis It is logical why it cannot know for sure that the value has to be at most 4 if it is less than 16. The x86 filter is based on a very old LZMA SDK version. Newer ones have quite a different implementation for the same filter. Thanks to Sam James. (cherry picked from commit 6aa2a6deeba04808a0fe4461396e7fb70277f3d4) src/liblzma/simple/x86.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) commit 4019b012f29008ea6545aba6fe6c141a2d920ae2 Author: Lasse Collin Date: 2024-04-19 23:18:19 +0300 Update .gitignore (cherry picked from commit e89d3e83b4496d0b5410870634970c0aa9721d59) .gitignore | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) commit 09a0311a1e8cdefbcfab9e490cdd41c97a459d24 Author: Lasse Collin Date: 2024-04-19 20:53:24 +0300 Tests: test_lzip_decoder: Tweak coding style and comments (cherry picked from commit 86fc4ee859709da0ff9617a1490f13ddac0a109b) tests/test_lzip_decoder.c | 58 +++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 30 deletions(-) commit 3117336a0291309ddd2a54d2966a589f9f806850 Author: Lasse Collin Date: 2024-04-19 20:51:36 +0300 Tests: test_lzip_decoder: Remove redundant initializations (cherry picked from commit 38be573a279bd7b608ee7d8509ec10884e6fb0d5) tests/test_lzip_decoder.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) commit f78081eb12c804ec4f5a3dc569b859646b16e9e5 Author: Lasse Collin Date: 2024-04-19 20:47:24 +0300 Tests: test_lzip_decoder: Remove unneeded tuktest_malloc() calls (cherry picked from commit d7e4bc53eacfab9f3de95d8252bdfdc9419079c9) tests/test_lzip_decoder.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) commit 7413383e4280065b79ca70abe4d8ebc78055b35a Author: Lasse Collin Date: 2024-04-15 20:35:07 +0300 xz: Fix white space error. Thanks to xx on #tukaani. (cherry picked from commit eeca8f7c5baf1ad69606bb734d5001763466d58f) src/xz/args.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit eed2f26c0edb6e31a50d48bab4ff619778690a1e Author: Sam James Date: 2024-04-11 23:01:44 +0100 xz: add missing noreturn for message_filters_help Fixes: a165d7df1964121eb9df715e6f836a31c865beef (cherry picked from commit 462ca9409940a19f743daee6b3bcc611277d0007) src/xz/message.h | 1 + 1 file changed, 1 insertion(+) commit 2633d8df616405bd54fd748d7bf887ebc4505b88 Author: Sam James Date: 2024-04-11 19:34:04 +0100 xz: signals: suppress -Wsign-conversion on macOS On macOS, we get: ``` signals.c: In function 'signals_init': signals.c:76:17: error: conversion to 'sigset_t' {aka 'unsigned int'} from 'int' may change the sign of the result [-Werror=sign-conversion] 76 | sigaddset(&hooked_signals, sigs[i]); | ^~~~~~~~~ signals.c:81:17: error: conversion to 'sigset_t' {aka 'unsigned int'} from 'int' may change the sign of the result [-Werror=sign-conversion] 81 | sigaddset(&hooked_signals, message_progress_sigs[i]); | ^~~~~~~~~ signals.c:86:9: error: conversion to 'sigset_t' {aka 'unsigned int'} from 'int' may change the sign of the result [-Werror=sign-conversion] 86 | sigaddset(&hooked_signals, SIGTSTP); | ^~~~~~~~~ ``` We use `int` for `hooked_signals` but we can't just cast to whatever `sigset_t` is because `sigset_t` is an opaque type. It's an unsigned int on macOS. On macOS, `sigaddset` is implemented as a macro. Just suppress -Wsign-conversion for `signals_init` for macOS given there's no real nice way of fixing this. (cherry picked from commit 863f13d2828b99b0539ce73f9cf85bde32358034) src/xz/signals.c | 7 +++++++ 1 file changed, 7 insertions(+) commit 50fb269c7a9cf62a9f3fe08859e2aa4348b600a7 Author: Lasse Collin Date: 2024-04-13 22:19:40 +0300 Tests: test_microlzma: Add a "FIXME?" about LZMA_FINISH handling (cherry picked from commit fcbd0d199933a69713cb293cbd7409a757d854cd) tests/test_microlzma.c | 8 ++++++++ 1 file changed, 8 insertions(+) commit 3e2ff2d38c54c8fc7ce15aaf91185dc105d9c92c Author: Lasse Collin Date: 2024-04-13 18:05:31 +0300 Tests: test_microlzma: Tweak comments, coding style, and minor details A few lines were reordered, a few ARRAY_SIZE were changed to sizeof, and a few uint32_t were changed to size_t. No real functional changes were intended. (cherry picked from commit 0fe2dfa68355d2b165544b2bc8babf77dcc2039e) tests/test_microlzma.c | 149 +++++++++++++++++++++++++++---------------------- 1 file changed, 83 insertions(+), 66 deletions(-) commit ebc8b8de19d641c37ab7959a224bcd0ff4c0833f Author: Ryan Carsten Schmidt Date: 2024-04-12 19:31:13 -0500 CI: Use only the active CPUs on macOS hw.ncpu counts all CPUs including inactive ones. hw.activecpu counts only the active CPUs. (cherry picked from commit 97f0ee0f1f903f4e7c4ea23e9b89d687025d2992) build-aux/ci_build.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 1e63f7d53648beb6dd5acb5771850d7c4bc30477 Author: Sam James Date: 2024-04-10 18:33:55 +0100 ci: rename ci_build.sh -> ci_build.bash We discussed the name and it's less cognitive load to just call it '.bash' so you don't have an immediate question about if bashisms are OK. (cherry picked from commit 73f629e321b74f68c9954728fa4f19261afccf46) .github/workflows/ci.yml | 52 ++++++++++++++++---------------- .github/workflows/windows-ci.yml | 20 ++++++------ build-aux/{ci_build.sh => ci_build.bash} | 0 3 files changed, 36 insertions(+), 36 deletions(-) commit aea54a4724414466a20afd7493156d40d0a2741c Author: Sam James Date: 2024-04-10 17:42:23 +0100 ci: build in parallel by default (cherry picked from commit 8709407a9ef8e7e8aec117879400e4dd3e227ada) build-aux/ci_build.sh | 2 ++ 1 file changed, 2 insertions(+) commit 4381fcf00b2fabb6dcc9fd5cf35d520feb9e775a Author: Sam James Date: 2024-04-10 15:41:08 +0100 ci: default to -O2 We need this for when we're passing sanitizer flags or -gdwarf-4 for Clang with Valgrind. Just always start with -O2 if CFLAGS isn't set in the environment and append what was passed on the command line. (cherry picked from commit 65bf7e0a1ca6386f17608e8afb84ac470c18d23f) build-aux/ci_build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 752ba5ed99ec754bafbdc4d87a2876cb2566ecc4 Author: Sam James Date: 2024-04-10 15:17:47 +0100 ci: make automake's test runner verbose on failures This is a lot easier to work with than the save-logs thing the action tries to do... (cherry picked from commit bc899f9e0700ad153bd65f4804c4de7515c8a847) build-aux/ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit cc21af171599ffe0419fc32a30edd3ef7d479865 Author: Sam James Date: 2024-04-10 12:38:51 +0100 ci: make UBSAN abort on errors Unfortunately, UBSAN doesn't do this by default. See also the change I made in Meson for this in October [0]. [0] https://github.com/mesonbuild/meson/commit/7b7d2e060b447de9c2642848847370a58711ac1c (cherry picked from commit b5e3470442531717b2457b40ab412740296af1bc) .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) commit 2d2d5f14b392cd1aeddab7ce34fd50ba5422e5b5 Author: Sam James Date: 2024-04-10 11:43:10 +0100 ci: test Valgrind Using `--trace-children=yes` has a trade-off here, as it makes `test_scripts.sh` pretty slow when calling various non-xz utilities. But I also feel like it's not useless to have Valgrind used there and it's not easy to exclude Valgrind just for that one test... I did consider using AX_VALGRIND_CHECK [0][1] but I couldn't get it working immediately with some conditionally-built tests and I wondered if it was worth spending time on at least while we're debating xz's future build system situation. [0] https://www.gnu.org/software/autoconf-archive/ax_valgrind_check.html [1] https://tecnocode.co.uk/2014/12/23/automatically-valgrinding-code-with-ax_valgrind_check/ (cherry picked from commit 6c095a98fbec70b790253a663173ecdb669108c4) .github/workflows/ci.yml | 11 ++++++++++- build-aux/ci_build.sh | 8 +++++--- 2 files changed, 15 insertions(+), 4 deletions(-) commit 5d20a612051fac3ca6d99abe3cd7e0e3370e5b67 Author: Lasse Collin Date: 2024-04-10 23:20:02 +0300 liblzma: CRC: Simplify table omission macros A macro is useful to prevent a single #if directive from getting too ugly but only one macro is needed for all archs. (cherry picked from commit 6286c1900c2d2ca33d9b1b397122c7bcdb9a4d59) src/liblzma/check/crc32_table.c | 10 ++++------ src/liblzma/check/crc64_table.c | 4 ++-- src/liblzma/check/crc_common.h | 5 +++-- 3 files changed, 9 insertions(+), 10 deletions(-) commit 2a80827e23169c624560ac89714bf5084cbead43 Author: Lasse Collin Date: 2024-04-10 23:09:40 +0300 liblzma: ARM64 CRC: Fix omission of CRC32 table The macro name had an odd typo so the table wasn't omitted when it should have. Fixes: 1940f0ec28f08c0ac72c1413d9706fb82eabe6ad (cherry picked from commit 45da936c879acf4f053a3055665bf1b10ded4462) src/liblzma/check/crc32_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit a54117377151356c1e2494ba1febc245cb71b51c Author: Lasse Collin Date: 2024-04-10 22:21:51 +0300 Build: If ARM64 feature detection func is found, stop looking for others This can speed up configure a tiny bit. Fixes: c5f6d79cc9515a7f22d7ea4860c6cc394b295732 (cherry picked from commit 308a9af85400b0e2019f0f012c8354e831d06d65) configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 9223ad6e78a666cc9f9aba135d1755fec184a24a Author: Lasse Collin Date: 2024-04-10 22:04:27 +0300 liblzma: ARM64 CRC32: Change style of the macOS code to match FreeBSD I didn't test this but it shouldn't change any functionality. Fixes: 761f5b69a4c778c8bcb09279b845b07c28790575 (cherry picked from commit fc43cecd32bf9d5f8caa599206b15c9569af1eb6) src/liblzma/check/crc32_arm64.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) commit 32ceb2c36a0e450037bbe906c2a1ea42607b9d21 Author: Lasse Collin Date: 2024-04-10 21:59:27 +0300 liblzma: ARM64 CRC32: Add error checking to FreeBSD-specific code Also add parenthesis to the return statement. I didn't test this. Fixes: 761f5b69a4c778c8bcb09279b845b07c28790575 (cherry picked from commit 1024cd4cd966b998fedec51e385e9ee9a49b3c57) src/liblzma/check/crc32_arm64.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit 42915101e914dba353c236925bc1d5e4826d3f7a Author: Lasse Collin Date: 2024-04-10 21:56:33 +0300 liblzma: ARM64 CRC32: Use negation instead of subtracting from 8 Subtracting from 0 is negation, this just keeps warnings away. Fixes: 761f5b69a4c778c8bcb09279b845b07c28790575 (cherry picked from commit 2337f7021c860b026e3e849e60a9ae8d09ec0ea0) src/liblzma/check/crc32_arm64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 42a9482b48f0171852fbaddbdc729a56f2daa547 Author: Lasse Collin Date: 2024-04-10 21:55:10 +0300 liblzma: ARM64 CRC32: Tweak coding style and comments (cherry picked from commit d8fffd01aa1a3c18e437a222abd34699e23ff5e7) src/liblzma/check/crc32_arm64.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) commit 38a3ec5a7e2ddeee3686be64b037aa1377f31fd1 Author: Lasse Collin Date: 2024-04-09 17:47:01 +0300 CI: Remove ifunc support. (cherry picked from commit 986865ea2f9d1f8dbef4a130926df106b0f6d41a) .github/workflows/ci.yml | 13 +++---------- build-aux/ci_build.sh | 5 +---- 2 files changed, 4 insertions(+), 14 deletions(-) commit 34d1252f093944ff350a88a6196539f95902ad41 Author: Lasse Collin Date: 2024-04-09 17:43:16 +0300 liblzma: Remove ifunc support. This is *NOT* done for security reasons even though the backdoor relied on the ifunc code. Instead, the reason is that in this project ifunc provides little benefits but it's quite a bit of extra code to support it. The only case where ifunc *might* matter for performance is if the CRC functions are used directly by an application. In normal compression use it's completely irrelevant. (cherry picked from commit 689ae2427342a2ea1206eb5ca08301baf410e7e0) CMakeLists.txt | 79 --------------------------------------- INSTALL | 8 ---- configure.ac | 79 --------------------------------------- src/liblzma/check/crc32_fast.c | 48 +++--------------------- src/liblzma/check/crc64_fast.c | 21 ----------- src/liblzma/check/crc_common.h | 9 +---- src/liblzma/check/crc_x86_clmul.h | 11 +----- 7 files changed, 8 insertions(+), 247 deletions(-) commit a594b39685051cd1ec866360bc4dd6c22f301bb4 Author: Lasse Collin Date: 2024-04-08 22:04:41 +0300 tests/files/README: Update the main heading. (cherry picked from commit 6b4c859059a7eb9b0547590c081668e14ecf8af6) tests/files/README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit fa76e3ef597ee2e9d150461a42d270a386204042 Author: Lasse Collin Date: 2024-04-08 22:02:45 +0300 tests/files/README: Explain how to recreate the ARM64 test files. (cherry picked from commit 2a851e06b891ce894f918faff32a6cca6fdecee6) tests/files/README | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) commit 112fa0aba6be30968811c9131f1b995cf9e92e75 Author: Lasse Collin Date: 2024-04-08 21:51:55 +0300 debug: Add generator for the ARM64 test file data. (cherry picked from commit 3d09b721b94e18fe1f853a04799697f5de10b291) debug/Makefile.am | 3 +- debug/testfilegen-arm64.c | 116 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) commit 1a1f3d0323d5991a3238566e8f517d5116358b5c Author: Lasse Collin Date: 2024-04-08 21:19:38 +0300 xz man page: Use .ft CR instead of CW to silence warnings from groff. (cherry picked from commit 31ef676567c9d6fcc4ec9fc833c312f7a7c21c48) src/xz/xz.1 | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) commit 9f9203f574f895c40a86a83c45c6bb79c25bb5d2 Author: Lasse Collin Date: 2024-04-08 19:28:35 +0300 Fix NEWS for 5.6.0 and 5.6.1. (cherry picked from commit 780cbf29d5a88db2b546e9b7b019c4c33ca72685) NEWS | 6 ++++++ 1 file changed, 6 insertions(+) commit 12876b33c79e36d7e51e8ba6ab7162bd2129cb5b Author: Lasse Collin Date: 2024-04-08 19:22:26 +0300 Remove the XZ logo. (cherry picked from commit bfd0c7c478e93a1911b845459549ff94587b6ea2) COPYING | 5 - COPYING.CC-BY-SA-4.0 | 427 --------------------------------------------------- Makefile.am | 2 - README | 2 - doc/xz-logo.png | Bin 6771 -> 0 bytes doxygen/Doxyfile | 6 +- doxygen/footer.html | 13 -- 7 files changed, 3 insertions(+), 452 deletions(-) commit 879295d91f06c241fd8a8fc1ca95776dbeb45f93 Author: Lasse Collin Date: 2024-04-08 18:27:39 +0300 Update maintainer and author info. The other maintainer suddenly disappeared. (cherry picked from commit 77a294d98a9d2d48f7e4ac273711518bf689f5c4) AUTHORS | 9 +++++++-- README | 10 +++------- THANKS | 1 - src/liblzma/api/lzma.h | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) commit 859617d30d81317236e004b323fed0883f932dcf Author: Lasse Collin Date: 2024-04-08 18:05:32 +0300 Docs: Update .xz file format specification to 1.2.1. This only reverts the XZ URL changes. (cherry picked from commit 8dd03d4484ccf80022722a16d0ed9b37f2b58072) doc/xz-file-format.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) commit eeb74fba1f6ea334a519015938b4a26c6ba5d4eb Author: Lasse Collin Date: 2024-04-08 17:33:56 +0300 Update website URLs back to tukaani.org. The XZ projects were moved back to their original URLs. (cherry picked from commit 17aa2e1a796d3f758802df29afc89dcf335db567) CMakeLists.txt | 2 +- COPYING | 3 +-- README | 4 ++-- configure.ac | 2 +- doc/faq.txt | 2 +- doc/lzma-file-format.txt | 12 ++++++------ dos/config.h | 2 +- src/liblzma/api/lzma.h | 2 +- src/xz/xz.1 | 6 +++--- src/xzdec/xzdec.1 | 4 ++-- windows/README-Windows.txt | 2 +- 11 files changed, 20 insertions(+), 21 deletions(-) commit a7b9cd70004bfc1abadc7e865dfce765f7b8b59d Author: Lasse Collin Date: 2024-04-08 17:07:08 +0300 xzdec: Tweak coding style and comments. (cherry picked from commit 2739db981023373a2ddabc7b456c7e658bb4f582) src/xzdec/xzdec.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) commit ebe9d6d8cb27168706078009b3f64da8fde63833 Author: Lasse Collin Date: 2024-04-08 15:53:46 +0300 tests/ossfuzz: Tiny fix to a comment. (cherry picked from commit 408b6adb2a07d07c6535f859571cca38837caaf3) tests/ossfuzz/fuzz_decode_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 78ab47d65d916207233abbcdb0ccfd6efb946c05 Author: Lasse Collin Date: 2024-03-30 14:36:28 +0200 CMake: Fix sabotaged Landlock sandbox check. It never enabled it. (cherry picked from commit f9cf4c05edd14dedfe63833f8ccbe41b55823b00) CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 5f178c364c3b5c6fe87099b7624d5c76995ff8e6 Author: Lasse Collin Date: 2024-05-22 14:08:33 +0300 Delete SECURITY.md from v5.6 It's too easily out of date in the stable branches. It's not included in the release packages anyway. .github/SECURITY.md | 29 ----------------------------- 1 file changed, 29 deletions(-) commit b3a756188004a16de5956c368e3b0efd1a9bccb0 Author: Lasse Collin Date: 2024-03-22 17:46:30 +0200 liblzma: memcmplen.h: Add a comment why subtraction is used. (cherry picked from commit 0b99783d63f27606936bb79a16c52d0d70c0b56f) src/liblzma/common/memcmplen.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) commit 94939a145f362ff8b09fb37fc72901743f7f5cb2 Author: Lasse Collin Date: 2024-03-15 17:43:39 +0200 INSTALL: Document arguments of --enable-symbol-versions. (cherry picked from commit 8a25ba024d55610c448c6e4f1400a00bae51b493) INSTALL | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) commit fa14c8aaf0d0266b7e0c3b7c766159299c1a0f18 Author: Lasse Collin Date: 2024-03-15 17:15:50 +0200 Build: Use only the generic symbol versioning with NVIDIA HPC Compiler. This does the previous commit with CMake. AC_EGREP_CPP uses AC_REQUIRE so the outermost if-commands must be changed to AS_IF to ensure that things wont break some day. See 5a5bd7f871818029d5ccbe189f087f591258c294. (cherry picked from commit 49324b711f9d42b3543bf2f3ae598eaa03360bd5) configure.ac | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) commit 73baa8d99b51c7623ed95afe6411302d9ff56864 Author: Lasse Collin Date: 2024-03-15 16:36:35 +0200 CMake: Use only the generic symbol versioning with NVIDIA HPC Compiler. It doesn't support the __symver__ attribute or __asm__(".symver ..."). The generic symbol versioning can still be used since it only needs linker support. (cherry picked from commit c273123ed0ebaebf49994057a7fe98aae7f42c40) CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) commit 886633f42376f4648d931917733c8a59fb2e1f6c Author: Lasse Collin Date: 2024-03-13 21:38:24 +0200 Update THANKS. (cherry picked from commit df7f487648d18a3992386a59b8a061edca862d17) THANKS | 1 + 1 file changed, 1 insertion(+) commit 760f622f0d73632df2347aaca7ac7ff5761e98b6 Author: Lasse Collin Date: 2024-03-13 21:30:18 +0200 liblzma: Minor comment edits. (cherry picked from commit 3217b82b3ec023bf8338249134a076bea0ea30ec) src/liblzma/common/string_conversion.c | 4 ++-- src/liblzma/delta/delta_decoder.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) commit 403b4c78b81f67bc3787542f55f555407253316c Author: Sergey Kosukhin Date: 2024-03-13 13:07:13 +0100 liblzma: Fix building with NVHPC (NVIDIA HPC SDK). NVHPC compiler has several issues that make it impossible to build liblzma: - the compiler cannot handle unions that contain pointers that are not the first members; - the compiler cannot handle the assembler code in range_decoder.h (LZMA_RANGE_DECODER_CONFIG has to be set to zero); - the compiler fails to produce valid code for delta_decode if the vectorization is enabled, which results in failed tests. This introduces NVHPC-specific workarounds that address the issues. (cherry picked from commit 096bc0e3f8fb4bfc4d2f3f64a7f219401ffb4c31) src/liblzma/common/string_conversion.c | 6 ++++-- src/liblzma/delta/delta_decoder.c | 3 +++ src/liblzma/rangecoder/range_decoder.h | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) commit 1888fb49f629340758e98e69d5aa328f6f73c5e1 Author: Lasse Collin Date: 2024-03-13 21:17:10 +0200 CMake: Disable symbol versioning on non-glibc Linux. This better matches what configure.ac does. For example, musl has only basic symbol versioning support: https://wiki.musl-libc.org/functional-differences-from-glibc.html#Symbol_versioning configure.ac tries to enable symbol versioning only with glibc so now CMake does the same. (cherry picked from commit 2ad7fad67080e88fa7fc191f9d613d8b7add9c62) CMakeLists.txt | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) commit 4b3c84e8eebbcf712fc2396dbb8117cce2d72464 Author: Lasse Collin Date: 2024-03-13 20:32:46 +0200 CMake: Make symbol versioning configurable. (cherry picked from commit 82f0c0d39eb2c026b1d96ee706f70ace868d4ed4) CMakeLists.txt | 62 +++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 20 deletions(-) commit 69d1e20208eb9bd1f4f1c8ee4e49cc82d681a877 Author: Lasse Collin Date: 2024-03-13 19:47:36 +0200 Build: Style tweaks to configure.ac. The AC_MSG_ERROR line is overlong anyway as are a few other AC_MSG_ERROR lines already. (cherry picked from commit 45d33bfc45e4295b8ad743bc2ae61cc724f98076) configure.ac | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) commit 051d6b5c85a874c78249693865fd751088f403a2 Author: Sergey Kosukhin Date: 2024-03-12 20:03:49 +0100 Build: Let the users override the symbol versioning variant. There are cases when the users want to decide themselves whether they want to have the generic (even on GNU/Linux) or the linux (even if we do not recommend that) symbol versioning variant. The former might be needed to circumvent compiler issues (i.e. the compiler does not support all features that are required for the linux versioning), the latter might help in overriding the assumptions made in the configure script. (cherry picked from commit f56ed6fac6619b56b005878d3b5210e2f0d721c0) configure.ac | 91 +++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 50 insertions(+), 41 deletions(-) commit 95dcea4b5df0b180af461e4584d2bcf7725e3aef Author: Lasse Collin Date: 2024-04-09 18:22:16 +0300 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 1107712e372f7593ad729764c0c2644d0e4aa675 Author: Lasse Collin Date: 2024-04-08 15:32:58 +0300 Remove the backdoor found in 5.6.0 and 5.6.1 (CVE-2024-3094). While the backdoor was inactive (and thus harmless) without inserting a small trigger code into the build system when the source package was created, it's good to remove this anyway: - The executable payloads were embedded as binary blobs in the test files. This was a blatant violation of the Debian Free Software Guidelines. - On machines that see lots bots poking at the SSH port, the backdoor noticeably increased CPU load, resulting in degraded user experience and thus overwhelmingly negative user feedback. - The maintainer who added the backdoor has disappeared. - Backdoors are bad for security. This reverts the following without making any other changes: 6e636819 Tests: Update two test files. a3a29bbd Tests: Test --single-stream can decompress bad-3-corrupt_lzma2.xz. 0b4ccc91 Tests: Update RISC-V test files. 8c9b8b20 liblzma: Fix typos in crc32_fast.c and crc64_fast.c. 82ecc538 liblzma: Fix false Valgrind error report with GCC. cf44e4b7 Tests: Add a few test files. 3060e107 Tests: Use smaller dictionary size in RISC-V test files. e2870db5 Tests: Add two RISC-V Filter test files. The RISC-V test files also have real content that tests the filter but the real content would fit into much smaller files. A generator program would need to be available as well. Thanks to Andres Freund for finding and reporting it and making it public quickly so others could act without a delay. See: https://www.openwall.com/lists/oss-security/2024/03/29/4 src/liblzma/check/crc32_fast.c | 7 +++++-- src/liblzma/check/crc64_fast.c | 4 +++- src/liblzma/check/crc_common.h | 25 ------------------------- tests/files/README | 27 --------------------------- tests/files/bad-3-corrupt_lzma2.xz | Bin 512 -> 0 bytes tests/files/bad-dict_size.lzma | Bin 41 -> 0 bytes tests/files/good-1-riscv-lzma2-1.xz | Bin 7424 -> 0 bytes tests/files/good-1-riscv-lzma2-2.xz | Bin 7432 -> 0 bytes tests/files/good-2cat.xz | Bin 136 -> 0 bytes tests/files/good-large_compressed.lzma | Bin 35421 -> 0 bytes tests/files/good-small_compressed.lzma | Bin 258 -> 0 bytes tests/test_files.sh | 11 ----------- 12 files changed, 8 insertions(+), 66 deletions(-) commit fd1b975b7851e081ed6e5cf63df946cd5cbdbb94 Author: Jia Tan Date: 2024-03-09 11:42:50 +0800 Bump version and soname for 5.6.1. src/liblzma/Makefile.am | 2 +- src/liblzma/api/lzma/version.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit a2cda572498e96163fe4e2bde096d5dd7b814668 Author: Jia Tan Date: 2024-03-09 11:27:27 +0800 Add NEWS for 5.6.1 NEWS | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) commit 8583c6021124e388bce044a09f00ebabfd6165a7 Author: Jia Tan Date: 2024-03-09 10:43:20 +0800 Translations: Add missing --riscv option to man page translations. po4a/de.po | 702 +++++++++++++++++++++++++++++----------------------------- po4a/fr.po | 549 ++++++++++++++++++++++----------------------- po4a/ko.po | 702 +++++++++++++++++++++++++++++----------------------------- po4a/pt_BR.po | 641 +++++++++++++++++++++++++++-------------------------- po4a/ro.po | 702 +++++++++++++++++++++++++++++----------------------------- po4a/uk.po | 702 +++++++++++++++++++++++++++++----------------------------- 6 files changed, 2024 insertions(+), 1974 deletions(-) commit 74b138d2a6529f2c07729d7c77b1725a8e8b16f1 Author: Jia Tan Date: 2024-03-09 10:18:29 +0800 Tests: Update two test files. The original files were generated with random local to my machine. To better reproduce these files in the future, a constant seed was used to recreate these files. tests/files/bad-3-corrupt_lzma2.xz | Bin 484 -> 512 bytes tests/files/good-large_compressed.lzma | Bin 35430 -> 35421 bytes 2 files changed, 0 insertions(+), 0 deletions(-) commit 3ec6dfd656bdd40ede2a5f11e6be338988e38be4 Author: Jia Tan Date: 2024-03-09 10:08:32 +0800 Tests: Test --single-stream can decompress bad-3-corrupt_lzma2.xz. The first stream in this file is valid, so this tests that xz properly stops after decompressing it. tests/test_files.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) commit a67dcce6109c2f932a0a86abb0d7a95d3c31fb3e Author: Jia Tan Date: 2024-03-09 10:05:32 +0800 Tests: Update RISC-V test files. This increases code coverage and tests for possible shifting bugs. tests/files/good-1-riscv-lzma2-1.xz | Bin 7512 -> 7424 bytes tests/files/good-1-riscv-lzma2-2.xz | Bin 7512 -> 7432 bytes 2 files changed, 0 insertions(+), 0 deletions(-) commit 058337b0f1da9f166049ecc972fa5c499c1af08c Author: Jia Tan Date: 2024-03-09 09:52:32 +0800 liblzma: Fix typos in crc32_fast.c and crc64_fast.c. src/liblzma/check/crc32_fast.c | 4 ++-- src/liblzma/check/crc64_fast.c | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) commit cd5de9c1bbab3dd41b34b37a89c193fb6ff51ca5 Author: Jia Tan Date: 2024-03-09 09:49:55 +0800 Tests: Replace HAVE_MICROLZMA usage in CMake and Autotools builds. This reverts commit adaacafde6661496ca2814b1e94a3ba5186428cb. CMakeLists.txt | 15 ++++++++++----- configure.ac | 9 ++------- tests/Makefile.am | 9 ++++++--- tests/test_microlzma.c | 12 ++++-------- 4 files changed, 22 insertions(+), 23 deletions(-) commit 651a1545c8b6150051a0b44857136efd419afc6f Author: Jia Tan Date: 2024-03-09 09:20:57 +0800 liblzma: Fix false Valgrind error report with GCC. With GCC and a certain combination of flags, Valgrind will falsely trigger an invalid write. This appears to be due to the omission of instructions to properly save, set up, and restore the frame pointer. The IFUNC resolver is a leaf function since it only calls a function that is inlined. So sometimes GCC omits the frame pointer instructions in the resolver unless this optimization is explictly disabled. This fixes https://bugzilla.redhat.com/show_bug.cgi?id=2267598. src/liblzma/check/crc32_fast.c | 9 +++------ src/liblzma/check/crc64_fast.c | 7 +++---- src/liblzma/check/crc_common.h | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) commit 6e97b299f1b22e366ec42ba5dc5b9d0746e87b84 Author: Lasse Collin Date: 2024-03-05 23:21:26 +0200 liblzma: Fix a typo in a comment in the RISC-V filter. src/liblzma/simple/riscv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 4e1c97052b5f14f4d6dda99d12cbbd01e66e3712 Author: Jia Tan Date: 2024-03-05 00:34:46 +0800 liblzma: Use attribute no_profile_instrument_function with ifunc. Thanks to Sam James for determining this was the attribute needed to workaround the GCC bug and for his version of the patch in Gentoo. src/liblzma/check/crc32_fast.c | 5 +++++ src/liblzma/check/crc64_fast.c | 3 +++ 2 files changed, 8 insertions(+) commit ed957d39426695e948b06de0ed952a2fbbe84bd1 Author: Jia Tan Date: 2024-03-05 00:27:31 +0800 Build: Require attribute no_profile_instrument_function for ifunc usage. Using __attribute__((__no_profile_instrument_function__)) on the ifunc resolver works around a bug in GCC -fprofile-generate: it adds profiling code even to ifunc resolvers which can make the ifunc resolver crash at program startup. This attribute was not introduced until GCC 7 and Clang 13, so ifunc won't be used with prior versions of these compilers. This bug was brought to our attention by: https://bugs.gentoo.org/925415 And was reported to upstream GCC by: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=11411 CMakeLists.txt | 7 +++++++ configure.ac | 7 +++++++ 2 files changed, 14 insertions(+) commit e98ddaf85a1a8fb3cc863637f83356cc9db31e13 Author: Lasse Collin Date: 2024-03-04 19:23:18 +0200 liblzma: Fix a comment in the RISC-V filter. src/liblzma/simple/riscv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 319cec142f67fe294e0486402f1569f223d9a83d Author: Lasse Collin Date: 2024-02-29 16:35:52 +0200 CMake: Warn if translated man pages are missing. CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) commit 46c3e113d8eeb1a731a60829fa7f5d1b519f7f26 Author: Lasse Collin Date: 2024-02-29 16:35:52 +0200 CMake: Warn if gettext tools and pre-created .gmo files are missing. It's only done with CMake >= 3.20 and if library support for translation was already found. Sort of fixes: https://github.com/tukaani-project/xz/issues/82 CMakeLists.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) commit 86bec8334bb1dcb6d9293a11cdccd895b17f364b Author: Lasse Collin Date: 2024-02-28 18:26:25 +0200 xz: Add comments. src/xz/coder.c | 10 ++++++++++ 1 file changed, 10 insertions(+) commit 5c91b454c24e043ca8f2cc7d2b09bd091dafe655 Author: Jia Tan Date: 2024-02-27 23:42:41 +0800 xz: Change logging level for thread reduction to highest verbosity only. Now that multi threaded encoding is the default, users do not need to see a warning message everytime the number of threads is reduced. On some machines, this could happen very often. It is not unreasonable for users to need to set double verbose mode to see this kind of information. To see these warning messages -vv or --verbose --verbose must be passed to set xz into the highest possible verbosity mode. These warnings had caused automated testing frameworks to fail when they expected no output to stderr. Thanks to Sebastian Andrzej Siewior for reporting this and for the initial version of the patch. src/xz/coder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit d0e57b2f159f8fd03a9a89f2f593a768d0487898 Author: Lasse Collin Date: 2024-02-26 23:06:13 +0200 Fix sorting in THANKS. THANKS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit d416be55ac02af1144fed455fb18b710147bb490 Author: Jia Tan Date: 2024-02-26 23:39:29 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit f06b33edd2aeabdb11836a2bf0b681768dad29d3 Author: Chien Wong Date: 2024-02-25 21:38:13 +0800 xz: Add missing RISC-V on the filter list in the man page Signed-off-by: Chien Wong src/xz/xz.1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) commit a100f9111c8cc7f5b5f0e4a5e8af3de7161c7975 Author: Jia Tan Date: 2024-02-26 23:02:06 +0800 Build: Fix Linux Landlock feature test in Autotools and CMake builds. The previous Linux Landlock feature test assumed that having the linux/landlock.h header file was enough. The new feature tests also requires that prctl() and the required Landlock system calls are supported. CMakeLists.txt | 25 ++++++++++++++++++++++--- configure.ac | 27 ++++++++++++++++++++++++++- src/xz/sandbox.c | 2 +- src/xz/sandbox.h | 2 +- src/xzdec/xzdec.c | 8 ++++---- 5 files changed, 54 insertions(+), 10 deletions(-) commit d85efdc8911e6e8964ec920af44c8a6fe0a4c3c2 Author: Jia Tan Date: 2024-02-26 20:06:10 +0800 Tests: Add test_microlzma to .gitignore and CMakeLists.txt. .gitignore | 1 + CMakeLists.txt | 1 + 2 files changed, 2 insertions(+) commit 42ee4256739779005a7f921946c8a8e483d1f2ed Author: Jia Tan Date: 2024-02-26 19:56:25 +0800 Tests: Correct license header in test_microlzma.c. tests/test_microlzma.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) commit c83349dfd9cf9c495005b6d30e2fd34a9cafc18a Author: Jia Tan Date: 2024-02-25 21:41:55 +0800 Fix typos in NEWS and CMakeLists. CMakeLists.txt | 2 +- NEWS | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit 2d7d862e3ffa8cec4fd3fdffcd84e984a17aa429 Author: Jia Tan Date: 2024-02-24 15:55:08 +0800 Bump version and soname for 5.6.0. src/liblzma/Makefile.am | 2 +- src/liblzma/api/lzma/version.h | 6 +++--- src/liblzma/liblzma_generic.map | 2 +- src/liblzma/liblzma_linux.map | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) commit a18fb1edef0d0aac12a09eed05e9c448c777af7b Author: Jia Tan Date: 2024-02-24 15:50:36 +0800 Add NEWS for 5.6.0. NEWS | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) commit 24355c5280bc95e3d594432d60bb8432aa6af173 Author: Jia Tan Date: 2024-02-22 22:27:01 +0800 Translations: Remove obsolete and fuzzy matches from some translations. The French and Brazilian Portuguese man page translations have not been updated since the switch from public domain to 0BSD. The old GPLv2 strings have now been removed from these files. po4a/fr.po | 4702 +++++++++++++++++++++++++++++++++++++---------------- po4a/pt_BR.po | 4987 ++++++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 6832 insertions(+), 2857 deletions(-) commit 02ca4a7d7b703e2ec63e00b70feec825e919dbc1 Author: Jia Tan Date: 2024-02-21 00:31:54 +0800 Translations: Patch man pages to avoid fuzzy matches. This will be fixed in the next round of translations, but this avoids having a fuzzy match or not fixing the English version. po4a/de.po | 2 +- po4a/ko.po | 2 +- po4a/ro.po | 2 +- po4a/uk.po | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) commit 898aad9fc711e03452d24d9e2c5b7f77a6f9ce64 Author: Jia Tan Date: 2024-02-21 00:30:43 +0800 xzmore: Fix typo in xzmore.1. Thanks to Yuri Chornoivan. src/scripts/xzmore.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 5631aa206c8d16b4eeab85a46b8b698f4fc4cdba Author: Jia Tan Date: 2024-02-24 12:12:16 +0800 Translations: Update the Vietnamese translation. po/vi.po | 505 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 309 insertions(+), 196 deletions(-) commit a65fd7ce9d6228e87faf61dc56a35984d0088248 Author: Jia Tan Date: 2024-02-24 12:06:40 +0800 Translations: Update the Esperanto translation. po/eo.po | 502 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 306 insertions(+), 196 deletions(-) commit cf44e4b7f5dfdbf8c78aef377c10f71e274f63c0 Author: Jia Tan Date: 2024-02-23 23:09:59 +0800 Tests: Add a few test files. tests/files/README | 19 +++++++++++++++++++ tests/files/bad-3-corrupt_lzma2.xz | Bin 0 -> 484 bytes tests/files/bad-dict_size.lzma | Bin 0 -> 41 bytes tests/files/good-2cat.xz | Bin 0 -> 136 bytes tests/files/good-large_compressed.lzma | Bin 0 -> 35430 bytes tests/files/good-small_compressed.lzma | Bin 0 -> 258 bytes 6 files changed, 19 insertions(+) commit 39f4a1a86ad80b2d064b812cee42668e6c8b8c73 Author: Jia Tan Date: 2024-02-23 20:58:36 +0800 Tests: Add MicroLZMA test. tests/Makefile.am | 4 +- tests/test_microlzma.c | 548 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 551 insertions(+), 1 deletion(-) commit adaacafde6661496ca2814b1e94a3ba5186428cb Author: Jia Tan Date: 2024-02-23 20:57:59 +0800 Build: Define HAVE_MICROLZMA when it is configured. CMakeLists.txt | 4 ++++ configure.ac | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) commit eea78216d27182ca917bf00e02feaab058a4d21e Author: Jia Tan Date: 2024-02-23 20:27:15 +0800 xz: Fix Capsicum sandbox compile error. user_abort_pipe[] was still being used instead of the parameters. src/xz/sandbox.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 32b0a3ce19224f9074d01a4ffbc1655b05fcb82d Author: Jia Tan Date: 2024-02-23 16:12:32 +0800 Build: Fix ARM64 CRC32 instruction feature test. Old versions of Clang reported the unsupported function attribute and __crc32d() function as warnings instead of errors, so the feature test passed when it shouldn't have, causing a compile error at build time. -Werror was added to this feature test to fix this. The change is not needed for CMake because check_c_source_compiles() also performs linking and the error is caught then. Thanks to Sebastian Andrzej Siewior for reporting this. configure.ac | 10 ++++++++++ 1 file changed, 10 insertions(+) commit 4c81c9611f8b2e1ad65eb7fa166afc570c58607e Author: Lasse Collin Date: 2024-02-22 19:16:35 +0200 CMake: Add LOCALEDIR to the windres workaround. LOCALEDIR may contain spaces like in "C:\Program Files". CMakeLists.txt | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) commit de4337fd89ca7db5feb97b5c40143404f6e22986 Author: Lasse Collin Date: 2024-02-22 15:18:25 +0200 xz: Landlock: Fix error message if input file is a directory. If xz is given a directory, it should look like this: $ xz /usr/bin xz: /usr/bin: Is a directory, skipping The Landlock rules didn't allow opening directories for reading: $ xz /usr/bin xz: /usr/bin: Permission denied The simplest fix was to allow opening directories for reading. While it's a bit silly to allow it solely for the error message, it shouldn't make the sandbox significantly weaker. The single-file use case (like when called from GNU tar) is still as strict as possible: all Landlock restrictions are enabled before (de)compression starts. src/xz/sandbox.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) commit 120da10ae139ea52ca4275452adf8eda02d07cc8 Author: Lasse Collin Date: 2024-02-22 14:41:29 +0200 liblzma: Disable branchless C version in range decoder. Thanks to Sebastian Andrzej Siewior and Sam James for benchmarking on various systems. src/liblzma/rangecoder/range_decoder.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) commit 00440f52be9ac2c7438c7b0cb1082f12399632c6 Author: Lasse Collin Date: 2024-02-21 17:41:32 +0200 INSTALL: Clarify that --disable-assembler affects only 32-bit x86. INSTALL | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) commit 11405be84ea294497e12d03d7219f607063f4a00 Author: Lasse Collin Date: 2024-02-19 18:41:37 +0200 Windows: build.bash: Include COPYING.0BSD in the package. windows/build.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit c27cf64e3e27f4968431d65be7098a12a3a80d30 Author: Lasse Collin Date: 2024-02-18 17:59:46 +0200 Windows: build.bash: include liblzma-crt-mixing.txt in the package. windows/build.bash | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit 8d38941baed05de4ff7cc775de14833108f62184 Author: Lasse Collin Date: 2024-02-18 17:17:43 +0200 Windows: Major update to Windows build instructions. INSTALL | 68 ++++----- windows/INSTALL-MSVC.txt | 23 +-- windows/INSTALL-MinGW-w64_with_Autotools.txt | 49 +++++++ windows/INSTALL-MinGW-w64_with_CMake.txt | 203 +++++++++++++++++++++++++++ windows/INSTALL-MinGW.txt | 138 ------------------ windows/README-Windows.txt | 2 + windows/build-with-cmake.bat | 35 +++++ windows/liblzma-crt-mixing.txt | 70 +++++++++ 8 files changed, 404 insertions(+), 184 deletions(-) commit 4b5b0d352348ff510ffb50a3b5b71788857d37a1 Author: Lasse Collin Date: 2024-02-18 15:15:04 +0200 Windows: Update windows/README-Windows.txt. It's for binary packages built with windows/build.bash. windows/README-Windows.txt | 104 ++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 63 deletions(-) commit 1ee716f74085223c8fbcae1d5a384e6bf53c0f6a Author: Lasse Collin Date: 2024-02-18 15:15:04 +0200 Windows: Update windows/build.bash. Support for the old MinGW was dropped. Only MinGW-w64 with GCC is supported now. The script now supports also cross-compilation from GNU/Linux (tests are not run). MSYS2 and also the old MSYS 1.0.11 work for building on Windows. The i686 and x86_64 toolchains must be in PATH to build both 32-bit and 64-bit versions. Parallel builds are done if "nproc" from GNU coreutils is available. MinGW-w64 runtime copyright information file was renamed from COPYING-Windows.txt to COPYING.MinGW-w64-runtime.txt which is the filename used by MinGW-w64 itself. Its existence is now mandatory, it's checked at the beginning of the script. The file TODO is no longer copied to the package. windows/build.bash | 191 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 112 insertions(+), 79 deletions(-) commit 60462e42609a1d961868a1d1ebecc713c6d27e2e Author: Jia Tan Date: 2024-02-20 23:32:22 +0800 Translations: Update the Romanian man page translations. po4a/ro.po | 1715 +++++++++++++++++++++++++++++++----------------------------- 1 file changed, 875 insertions(+), 840 deletions(-) commit 10d733e5b8929c642e00891cfa9ead9c2cdd2e05 Author: Jia Tan Date: 2024-02-20 23:30:25 +0800 Translations: Update the Korean man page translations. po4a/ko.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 797a34b72ac6baff237d7a546fa941d8f78f2f62 Author: Jia Tan Date: 2024-02-20 21:03:53 +0800 Translations: Update the Spanish translation. po/es.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 5c3751d019f023e091df9a653e2bb1f6ea8b0d49 Author: Jia Tan Date: 2024-02-20 20:18:07 +0800 Translations: Update the Romanian translation. po/ro.po | 470 ++++++++++++++++++++++++++++++--------------------------------- 1 file changed, 227 insertions(+), 243 deletions(-) commit e2d31154ecc750935436e8b62c6b073b2cfa84e3 Author: Jia Tan Date: 2024-02-20 20:15:50 +0800 Translations: Update the Croatian translation. po/hr.po | 648 ++++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 355 insertions(+), 293 deletions(-) commit 704500f994d5ac271bfcfd592275c5a7da4dc8d2 Author: Jia Tan Date: 2024-02-20 20:05:44 +0800 Translations: Update the German man page translations. po4a/de.po | 1696 +++++++++++++++++++++++++++++++----------------------------- 1 file changed, 873 insertions(+), 823 deletions(-) commit 1cfd3dca3fef321b06db73c3c9e13f347c2e2f5f Author: Jia Tan Date: 2024-02-20 19:58:25 +0800 Translations: Update the German translation. po/de.po | 427 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 225 insertions(+), 202 deletions(-) commit 28b9b3f16cc7c6e5b42e691994569c17f4561c9a Author: Jia Tan Date: 2024-02-20 19:56:52 +0800 Translations: Update the Hungarian translation. po/hu.po | 556 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 338 insertions(+), 218 deletions(-) commit 00b06cd0af6ad2ee93d3006bf80417db060c2b04 Author: Lasse Collin Date: 2024-02-19 16:48:05 +0200 CMake: Fix building of lzmainfo when translations are enabled. CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) commit b0d1422b6037bfea6f6723683bd82a8e6d77026c Author: Lasse Collin Date: 2024-02-19 13:38:42 +0200 CMake: Don't assume that -fvisibility=hidden is supported outside Windows. The original code was good enough for supporting GNU/Linux and a few others but it wasn't very portable. CMake doesn't support Solaris Studio's -xldscope=hidden. If it ever does, things should still work with this commit as Solaris Studio supports not only its own __global but also the GNU C __attribute__((visibility("default"))). Support for the attribute was added in 2007 to Sun Studio 12 compiler version 5.9. CMakeLists.txt | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) commit 2ced9d34bef4dce52ecbbf84d0903ab0aae1442c Author: Lasse Collin Date: 2024-02-19 12:20:59 +0200 CMake: Revise the component splitting. CMakeLists.txt | 57 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 26 deletions(-) commit 426bdc709c169d39b31dec410016779de117ef69 Author: Lasse Collin Date: 2024-02-17 21:45:07 +0200 CMake: Update the main comment and document CMAKE_BUILD_TYPE=Release. CMakeLists.txt | 79 ++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 16 deletions(-) commit 4430e075f7ccfc47972d6ca0aa1c3779fc265e10 Author: Lasse Collin Date: 2024-02-17 21:27:48 +0200 CMake: Use -O2 instead of -O3 in CMAKE_BUILD_TYPE=Release. -O3 doesn't seem useful for speed but it makes the code bigger. CMake makes is difficult for users to simply override the optimization level: CFLAGS / CMAKE_C_FLAGS aren't helpful because they go before CMAKE_C_FLAGS_RELEASE. Of course, users can override CMAKE_C_FLAGS_RELEASE directly but then they have to remember to add also -DNDEBUG to disable assertions. This commit changes -O3 to -O2 in CMAKE_C_FLAGS_RELEASE if and only if CMAKE_C_FLAGS_RELEASE cache variable doesn't already exist. So if a custom value is passed on the command line (or reconfiguring an already-configured build), the cache variable won't be modified. CMakeLists.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) commit 025eb6d7879e4c4e8cb29716b371e0f4c1aea660 Author: Lasse Collin Date: 2024-02-18 14:59:52 +0200 CMake: Handle symbol versioning on MicroBlaze specially. This is to match configure.ac. CMakeLists.txt | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) commit 2edd1a35b2507d1ce68b52dbaebe23c4850a74ce Author: Lasse Collin Date: 2024-02-17 22:18:12 +0200 CMake: Keep build working even if lib/*.[ch] are removed. CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) commit d753e2ce4715552884afadc4ed6fbf8ccca6efac Author: Lasse Collin Date: 2024-02-17 18:10:40 +0200 CMake: Install documentation. CMakeLists.txt | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) commit 7a0405bea9cb0df9318b70f779f82b2c473e98ac Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 CMake: Bump maximum policy version to 3.28. CMP0154 doesn't affect us since we don't use FILE_SET. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit c2264ffbe3892d28930b89b0123efc369cabc143 Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 CMake: Build lzmainfo. CMakeLists.txt | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) commit 998d0b29536094a89cf385a3b894e157db1ccefe Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 CMake: Build lzmadec. CMakeLists.txt | 76 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 34 deletions(-) commit 74e8bc7417a0f37ca7ed5ee0127d33c69b3100b9 Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 CMake: Add test_scripts.sh to the tests. In contrast to Automake, skipping of this test when decoders are disabled is handled at CMake side instead of test_scripts.sh because CMake-build doesn't create config.h. CMakeLists.txt | 14 ++++++++++++++ tests/test_scripts.sh | 13 ++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) commit 4808f238a731befcd46c2117c62a1caaf4403989 Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 CMake: Install scripts. Compared to the Autotools-based build, this has simpler handling for the shell (@POSIX_SHELL@) and extra PATH entry for the scripts (configure has --enable-path-for-scripts=PREFIX). The simpler metho should be enough for non-ancient systems and Solaris. CMakeLists.txt | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) commit 3462362ebd94d835c664e94ad8f414cfe7590ca7 Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 Scripts: Use @PACKAGE_VERSION@ instead of @VERSION@. PACKAGE_VERSION was already used in liblzma.pc.in. This way only one version @foo@ is used. src/scripts/xzdiff.in | 2 +- src/scripts/xzgrep.in | 2 +- src/scripts/xzless.in | 2 +- src/scripts/xzmore.in | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) commit 67610c245ba6c68cf65991693bab9312b7dc987b Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 CMake: Simplify symlink creation and install translated man pages. It helps that cmake_install.cmake doesn't parallelize installation so symlinks can be created so that the target is always known to exist (a requirement on Windows in some cases). This bumps the minimum CMake version from 3.13 to 3.14 to use file(CREATE_LINK ...). It could be made to work on 3.13 by calling "cmake -E create_symlink" but it's uglier code and slower in "make install". 3.14 should be a reasonable version to require nowadays, especially since the Autotools build is still the primary build system for most OSes. CMakeLists.txt | 195 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 98 insertions(+), 97 deletions(-) commit 50cc1d8a5a8154428bf240c7e4972e32b17d99bf Author: Lasse Collin Date: 2024-02-17 15:35:35 +0200 CMake: Add support for building and installing xz with translations. If gettext tools are available, the .po files listed in po/LINGUAS are converted using msgfmt. This allows building with translations directly from xz.git without Autotools. If gettext tools aren't available, the Autotools-created .gmo files in the "po" directory will be used. This allows CMake-based build to use translations from Autotools-generated tarball. If translation support is found (Intl_FOUND) but both the gettext tools and the pre-generated .gmo files are missing, then "make" will fail. CMakeLists.txt | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) commit 746c471643009947f94a3494a1148f74c7381b56 Author: Lasse Collin Date: 2024-02-19 11:58:33 +0200 liblzma: Remove commented-out code. src/liblzma/rangecoder/range_decoder.h | 3 --- 1 file changed, 3 deletions(-) commit 4ce300ce0884c6e552de2af9ae8050b47b01f0e7 Author: Lasse Collin Date: 2024-02-17 23:07:35 +0200 xz: Delete old commented-out code. src/xz/message.c | 19 ------------------- 1 file changed, 19 deletions(-) commit cae9a5e0bf422e6c5e64180805904f7ed02dc3aa Author: Lasse Collin Date: 2024-02-17 23:07:35 +0200 xz: Use stricter pledge(2) and Landlock sandbox. This makes these sandboxing methods stricter when no files are created or deleted. That is, it's a middle ground between the initial sandbox and the strictest single-file-to-stdout sandbox: this allows opening files for reading but output has to go to stdout. src/xz/main.c | 46 +++++++++++++++++++++++++++++++++------------- src/xz/sandbox.c | 32 ++++++++++++++++++++++++++++++++ src/xz/sandbox.h | 4 ++++ 3 files changed, 69 insertions(+), 13 deletions(-) commit 02e3505991233901575b7eabc06b2c6c62a96899 Author: Lasse Collin Date: 2024-02-17 23:07:35 +0200 xz: Support Landlock ABI version 4. Linux 6.7 added support for ABI version 4 which restricts TCP connections which xz won't need and thus those can be forbidden now. Since the ABI version is handled at runtime, supporting version 4 won't cause any compatibility issues. Note that new enough kernel headers are required to get version 4 support enabled at build time. src/xz/sandbox.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) commit 374868d81d473ab56556a1cfd6b1b36a1fab348b Author: Lasse Collin Date: 2024-02-17 23:07:35 +0200 xz: Move sandboxing code to sandbox.c and improve Landlock sandbox. Landlock is now always used just like pledge(2) is: first in more permissive mode and later (under certain common conditions) in a strict mode that doesn't allow opening more files. I put pledge(2) first in sandbox.c because it's the simplest API to use and still somewhat fine-grained for basic applications. So it's the simplest thing to understand for anyone reading sandbox.c. CMakeLists.txt | 2 + src/xz/Makefile.am | 2 + src/xz/file_io.c | 170 +----------------------------- src/xz/file_io.h | 6 -- src/xz/main.c | 50 +++------ src/xz/private.h | 6 +- src/xz/sandbox.c | 295 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/xz/sandbox.h | 39 +++++++ 8 files changed, 357 insertions(+), 213 deletions(-) commit 7312dfbb02197c7f990c7a3cefd027a9387d1473 Author: Lasse Collin Date: 2024-02-17 23:07:35 +0200 xz: Tweak comments. src/xz/main.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) commit c701a5909ad9882469fbab4fab5d2d5556d3ba78 Author: Lasse Collin Date: 2024-02-17 23:07:35 +0200 xz: Fix message_init() description. Also explicitly initialize progress_automatic to make it clear that it can be read before message_init() sets it. Static variable was initialized to false by default already so this is only for clarity. src/xz/main.c | 3 ++- src/xz/message.c | 2 +- src/xz/message.h | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) commit 9466306719f3b76e92fac4e55fbfd89ec92295fa Author: Lasse Collin Date: 2024-02-17 19:35:47 +0200 Build: Makefile.am: Sort EXTRA_DIST. Dirs first, then files in case-sensitive ASCII order. Makefile.am | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) commit f3440e78c9517db75bfa52e1a378fad60b073bbe Author: Lasse Collin Date: 2024-02-17 19:25:05 +0200 Build: Don't install TODO. Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit a7a3b62e2ab03c82b2bd5c78da1d1fb8b8490381 Author: Jia Tan Date: 2024-02-18 01:09:11 +0800 Translations: Update the Korean man page translations. po4a/ko.po | 1707 +++++++++++++++++++++++++++++++----------------------------- 1 file changed, 871 insertions(+), 836 deletions(-) commit 9b315db2d5e74700f3dc0755eb86c27947c0b393 Author: Jia Tan Date: 2024-02-18 01:08:32 +0800 Translations: Update the Korean translation. po/ko.po | 423 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 223 insertions(+), 200 deletions(-) commit 56246607dff177b0410d140fcca4a42c865723dc Author: Lasse Collin Date: 2024-02-17 16:23:14 +0200 Build: Install translated lzmainfo man pages. All other translated man pages were being installed but lzmainfo had been forgotten. src/lzmainfo/Makefile.am | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) commit f1d6b88aefcced538403c5c2606ba57065b16e70 Author: Lasse Collin Date: 2024-02-17 16:01:32 +0200 liblzma: Avoid implementation-defined behavior in the RISC-V filter. GCC docs promise that it works and a few other compilers do too. Clang/LLVM is documented source code only but unsurprisingly it behaves the same as others on x86-64 at least. But the certainly-portable way is good enough here so use that. src/liblzma/simple/riscv.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) commit 843ddc5f617b91ae132d6bab0f2f2d9c9fcd214a Author: Lasse Collin Date: 2024-02-17 15:48:28 +0200 liblzma: Wrap a line exceeding 80 chars. src/liblzma/rangecoder/range_decoder.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit e9053c907250c70d98b319d95fa54cb94fc76869 Author: Sebastian Andrzej Siewior Date: 2024-02-16 21:50:15 +0100 liblzma/rangecoder: Exclude x32 from the x86-64 optimisation. The x32 port has a x86-64 ABI in term of all registers but uses only 32bit pointer like x86-32. The assembly optimisation fails to compile on x32. Given the state of x32 I suggest to exclude it from the optimisation rather than trying to fix it. Signed-off-by: Sebastian Andrzej Siewior src/liblzma/rangecoder/range_decoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 3d198fb13b87f8803442e5799d465f7434a70555 Author: Jia Tan Date: 2024-02-17 21:05:07 +0800 Translations: Update the Spanish translation. po/es.po | 427 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 226 insertions(+), 201 deletions(-) commit cf278bfe60a25b54b3786f06503bc61272970820 Author: Jia Tan Date: 2024-02-17 20:43:29 +0800 Translations: Update the Swedish translation. po/sv.po | 434 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 230 insertions(+), 204 deletions(-) commit b0f1a41be50560cc6cb528e8e96b02b2067c52c2 Author: Jia Tan Date: 2024-02-17 20:41:38 +0800 Translations: Update the Polish translation. po/pl.po | 424 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 224 insertions(+), 200 deletions(-) commit d74ed48b30c631b6a4c7e7858b06828293bf8520 Author: Jia Tan Date: 2024-02-17 20:41:02 +0800 Translations: Update the Ukrainian translation. po/uk.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 711e22d5c5f3bac39ac904efb3ede874a66e2045 Author: Lasse Collin Date: 2024-02-16 17:53:34 +0200 Translations: Use the same sentence in xz.pot-header that the TP uses. po/xz.pot-header | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit fb5f6aaf18584672d0fee5dbe41fd30fc6bf5422 Author: Jia Tan Date: 2024-02-16 22:53:46 +0800 Fix typos discovered by codespell. AUTHORS | 2 +- NEWS | 2 +- src/liblzma/rangecoder/range_decoder.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) commit c64723bbb094e29b4edd98f6fcce866e1b569b42 Author: Jia Tan Date: 2024-02-16 22:52:41 +0800 Translations: Update the Ukrainian man page translations. po4a/uk.po | 1710 +++++++++++++++++++++++++++++++----------------------------- 1 file changed, 873 insertions(+), 837 deletions(-) commit 2895195ed0f68b245c7bd568c126ba6e685fa1d6 Author: Jia Tan Date: 2024-02-16 22:51:04 +0800 Translations: Update the Ukrainian translation. po/uk.po | 466 ++++++++++++++++++++++++++++++--------------------------------- 1 file changed, 225 insertions(+), 241 deletions(-) commit 4c20781f4c8f04879b64d631a4f44b4909147bde Author: Lasse Collin Date: 2024-02-15 22:32:52 +0200 Translations: Omit the generic copyright line from man page headers. po4a/update-po | 1 + 1 file changed, 1 insertion(+) commit 4323bc3e0c1e1d2037d5e670a3bf6633e8a3031e Author: Jia Tan Date: 2024-02-15 22:26:43 +0800 Update m4/.gitignore. m4/.gitignore | 1 + 1 file changed, 1 insertion(+) commit 5394a1665b7a108a54cb8b4ef3ebe59d3dbcca3a Author: Lasse Collin Date: 2024-02-14 21:11:49 +0200 Tests: tuktest.h: Treat Clang separately from GCC. Don't assume that Clang defines __GNUC__ as the extensions are available in clang-cl as well (and possibly in some other Clang variants?). tests/tuktest.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit cce7330b9f23485a0879422e0c3395a7065439ac Author: Lasse Collin Date: 2024-02-14 21:11:03 +0200 Tests: tuktest.h: Add a missing word to a comment. tests/tuktest.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 5dd8fc9452a3373cedc27379067ce638f992c741 Author: Lasse Collin Date: 2024-02-14 21:10:10 +0200 Tests: tuktest.h: Fix the comment about STest. tests/tuktest.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 6f1790254a03c5edf0f2976f773220f070450acd Author: Jia Tan Date: 2024-02-15 01:53:40 +0800 Bump version for 5.5.2beta. src/liblzma/api/lzma/version.h | 4 ++-- src/liblzma/liblzma_generic.map | 2 +- src/liblzma/liblzma_linux.map | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) commit 924fdeedf48113fb1e0646d86bd89a356d21a055 Author: Lasse Collin Date: 2024-02-14 19:46:11 +0200 liblzma: Fix validate_map.sh. Adding the SPDX license identifier changed the line numbers. src/liblzma/validate_map.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 22140a2df6161b0110e6b4afa5ea0a07c5b60b01 Author: Lasse Collin Date: 2024-02-14 19:38:34 +0200 Build: Start the generated ChangeLog from around 5.4.0 instead of 5.2.0. Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 0b8cefa136c21d403a01b78517f4decb50172bdb Author: Lasse Collin Date: 2024-02-14 19:27:46 +0200 Fixed NEWS for 5.5.2beta. NEWS | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) commit a4557bad96361d93ea171ed859ac5a696fca824f Author: Lasse Collin Date: 2024-02-14 19:21:45 +0200 liblzma: Silence warnings in --enable-small build. src/liblzma/lzma/lzma_decoder.c | 2 ++ src/liblzma/rangecoder/range_decoder.h | 1 + 2 files changed, 3 insertions(+) commit 38edf473236d00b3e100dc4c4f0bf43a4993fed2 Author: Lasse Collin Date: 2024-02-14 19:15:58 +0200 Build: Install COPYING.0BSD as part of docs. Makefile.am | 1 + 1 file changed, 1 insertion(+) commit b74e10bd839bcdc239afb5300ffaee195f34c217 Author: Lasse Collin Date: 2024-02-14 19:14:05 +0200 Docs: List COPYING.0BSD in README. README | 1 + 1 file changed, 1 insertion(+) commit dfdb60ffe933a1f1497d300dbb4513ed17ec6f0e Author: Lasse Collin Date: 2024-02-14 19:11:48 +0200 Docs: Include doc/examples/11_file_info.c in tarballs. It was added in 2017 in c2e29f06a7d1e3ba242ac2fafc69f5d6e92f62cd but it never got into any release tarballs because it was forgotten to be added to Makefile.am. Makefile.am | 1 + 1 file changed, 1 insertion(+) commit 160b6862646d95dfdbd73ab7f1031ede0f54992d Author: Lasse Collin Date: 2024-02-14 19:05:58 +0200 liblzma: Silence a warning. src/liblzma/rangecoder/range_decoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit eeedd4d0925ea417add04ceb42a6c0829244b50c Author: Lasse Collin Date: 2024-02-14 18:32:27 +0200 Add NEWS for 5.5.2beta. NEWS | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) commit 8af7db854f903068d72a9a0d21103cb0c5027fa8 Author: Lasse Collin Date: 2024-02-13 14:32:47 +0200 xz: Mention lzmainfo if trying to use 'lzma --list'. This kind of fixes the problem reported here: https://bugs.launchpad.net/ubuntu/+source/xz-utils/+bug/1291020 src/xz/list.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) commit 0668907ff736e4cd16738c10d39a2bc9e851aefb Author: Lasse Collin Date: 2024-02-14 14:58:36 +0200 liblzma: Add comments. src/liblzma/lzma/lzma_decoder.c | 9 +++++++++ src/liblzma/rangecoder/range_decoder.h | 11 +++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) commit 109f1913d4824c8214d5bbd38ebebf62c37572da Author: Lasse Collin Date: 2024-02-13 17:00:17 +0200 Scripts: Add lz4 support to xzgrep and xzdiff. src/scripts/xzdiff.1 | 8 +++++--- src/scripts/xzdiff.in | 14 +++++++++----- src/scripts/xzgrep.1 | 6 ++++-- src/scripts/xzgrep.in | 1 + 4 files changed, 19 insertions(+), 10 deletions(-) commit de55485cb23af56c5adbe3239b935c957ff8ac4f Author: Lasse Collin Date: 2024-02-13 14:05:13 +0200 liblzma: Choose the range decoder variants using a bitmask macro. src/liblzma/rangecoder/range_decoder.h | 64 ++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 11 deletions(-) commit 0709c2b2d7c1d8f437b003f691880fd7810e5be5 Author: Lasse Collin Date: 2024-02-13 11:38:10 +0200 xz: Fix outdated threading related info on the man page. src/xz/xz.1 | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) commit 3182a330c1512cc1f5c87b5c5a272578e60a5158 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: Range decoder: Add x86-64 inline assembly. It's compatible with GCC and Clang. src/liblzma/rangecoder/range_decoder.h | 491 +++++++++++++++++++++++++++++++++ 1 file changed, 491 insertions(+) commit cba2edc991dffba7cd4891dbc1bd26cb950cf053 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: Range decoder: Add branchless C code. It's used only for basic bittrees and fixed-size reverse bittree because those showed a clear benefit on x86-64 with GCC and Clang. The other methods were more mixed and thus are commented out but they should be tested on other archs. src/liblzma/rangecoder/range_decoder.h | 76 ++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) commit e290a72d6dee71faf3a90c9678b2f730083666a7 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: Clarify a comment. src/liblzma/lzma/lzma_decoder.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) commit 5e04706b91ca90d6befd4da24a588a55e631d4a9 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: LZMA decoder: Optimize loop comparison. But now it needs one more local variable. src/liblzma/lzma/lzma_decoder.c | 5 ++--- src/liblzma/rangecoder/range_decoder.h | 10 +++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) commit 88276f9f2cb4871c7eb86952d93d07c1cf6caa66 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: Optimize literal_subcoder() macro slightly. src/liblzma/lzma/lzma_common.h | 22 ++++++++++++---------- src/liblzma/lzma/lzma_decoder.c | 12 ++++++------ src/liblzma/lzma/lzma_encoder.c | 6 +++--- src/liblzma/lzma/lzma_encoder_optimum_normal.c | 2 +- src/liblzma/lzma/lzma_encoder_private.h | 4 ++-- 5 files changed, 24 insertions(+), 22 deletions(-) commit 5938f6de4d8ec9656776cd69e78ddfd6c3ad84e5 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: LZ decoder: Add unlikely(). src/liblzma/lz/lz_decoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 9c252e3ed086c6b72590b2531586c42596d4a9d9 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: LZ decoder: Remove a useless unlikely(). src/liblzma/lz/lz_decoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit f3872a59475456c5d365cad9f1c5be514cfa54b5 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: Optimize LZ decoder slightly. Now extra buffer space is reserved so that repeating bytes for any single match will never need to copy from two places (both the beginning and the end of the buffer). This simplifies dict_repeat() and helps a little with speed. This seems to reduce .lzma decompression time about 2 %, so with .xz and CRC it could be slightly less. The small things add up still. src/liblzma/lz/lz_decoder.c | 43 ++++++++++++----- src/liblzma/lz/lz_decoder.h | 101 +++++++++++++++++++++------------------- src/liblzma/lzma/lzma_decoder.c | 4 +- 3 files changed, 88 insertions(+), 60 deletions(-) commit eb518446e578acf079abae5f1ce28db7b6e59bc1 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: LZMA decoder: Get rid of next_state[]. It's not completely obvious if this is better in the decoder. It should be good if compiler can avoid creating a branch (like using CMOV on x86). This also makes lzma_encoder.c use the new macros. src/liblzma/lzma/lzma_common.h | 14 ++++++++++++++ src/liblzma/lzma/lzma_decoder.c | 30 ++++++++---------------------- src/liblzma/lzma/lzma_encoder.c | 4 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) commit e0c0ee475c0800c08291ae45e0d66aa00d5ce604 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: LZMA decoder improvements. This adds macros for bittree decoding which prepares the code for alternative C versions and inline assembly. src/liblzma/lzma/lzma_decoder.c | 264 ++++++++++----------------------- src/liblzma/rangecoder/range_common.h | 4 + src/liblzma/rangecoder/range_decoder.h | 142 ++++++++++++++++-- 3 files changed, 210 insertions(+), 200 deletions(-) commit de5c5e417645ad8906ef914bc059d08c1462fc29 Author: Jia Tan Date: 2024-02-12 17:09:10 +0200 liblzma: Creates Non-resumable and Resumable modes for lzma_decoder. The new decoder resumes the first decoder loop in the Resumable mode. Then, the code executes in Non-resumable mode until it detects that it cannot guarantee to have enough input/output to decode another symbol. The Resumable mode is how the decoder has always worked. Before decoding every input bit, it checks if there is enough space and will save its location to be resumed later. When the decoder has more input/output, it jumps back to the correct sequence in the Resumable mode code. When the input/output buffers are large, the Resumable mode is much slower than the Non-resumable because it has more branches and is harder for the compiler to optimize since it is in a large switch block. Early benchmarking shows significant time improvement (8-10% on gcc and clang x86) by using the Non-resumable code as much as possible. src/liblzma/lz/lz_decoder.h | 14 +- src/liblzma/lzma/lzma_decoder.c | 720 ++++++++++++++++++++++++++++------------ 2 files changed, 521 insertions(+), 213 deletions(-) commit e446ab7a18abfde18f8d1cf02a914df72b1370e3 Author: Jia Tan Date: 2024-02-12 17:09:10 +0200 liblzma: Creates separate "safe" range decoder mode. The new "safe" range decoder mode is the same as old range decoder, but now the default behavior of the range decoder will not check if there is enough input or output to complete the operation. When the buffers are close to fully consumed, the "safe" operations must be used instead. This will improve speed because it will reduce the number of branches needed for most of the range decoder operations. src/liblzma/lzma/lzma_decoder.c | 108 ++++++++------------------------- src/liblzma/rangecoder/range_decoder.h | 77 +++++++++++++++++------ 2 files changed, 82 insertions(+), 103 deletions(-) commit 7f6d9ca329ff3e01d4b0be7366eb4f5c93da41b9 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 doxygen/footer.html: Add missing closing tags and don't open a new tab. The footer template from Doxygen has the closing as Doxygen doesn't add them otherwise. target="_blank" was omitted as it's not useful here but it can be slightly annoying as one cannot just go back in the browser history. Since the footer links to the license file in the same directory and not to CC website, the rel attributes can be omitted. doxygen/footer.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit 26d1527d34d52b0f5d632d4fb636fb33d0867e92 Author: Lasse Collin Date: 2024-02-13 13:19:10 +0200 Tweak the expressions in AUTHORS. AUTHORS | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) commit d231d56580175fa040fdd3c6207a58243ce6217b Author: Lasse Collin Date: 2024-02-13 13:07:33 +0200 Translations: Add the man page translators into man page header comment. It looked odd to only have the original English authors listed in the header comments of the translated files. po4a/.gitignore | 1 + po4a/po4a.conf | 14 +++++++------- po4a/update-po | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) commit 6d35fcb936474fca1acaebfd9502c097b6fde88e Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Translations: Translate also messages of lzmainfo. lzmainfo has had translation support since 2009 at least but it was never added to po/POTFILES.in so the messages weren't translated. It's a very rarely needed tool so it's not too bad. This also adds src/xz/mytime.c to po/POTFILES.in although there are no translatable strings. It's simpler this way so that it won't be forgotten if strings were ever added to that file. po/POTFILES.in | 2 ++ 1 file changed, 2 insertions(+) commit a9f369dd54b05f9ac4e00ead9d765d04fc259868 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Translations: Add custom .pot header with SPDX license identifier. The same is used for both po/xz.pot and po4a/xz-man.pot. Makefile.am | 1 + po/xz.pot-header | 7 +++++++ po4a/update-po | 8 ++++++++ 3 files changed, 16 insertions(+) commit 469cd6653bb96e83c5cf1031c204d34566b15f44 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Translations: po4a/update-po: Add copyright notice to xz-man.pot. All man pages are under 0BSD now so this is simple now. po4a/update-po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 28ce45e38fbed4b5f54f2013e38dab47d22bf699 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Update COPYING about the man pages of the scripts. COPYING | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit e48287bf51afd5184ea74de1dcade9e153f873f7 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 xzdiff, xzgrep, and xzmore: Rewrite the man pages. The main reason is a kind of silly one: xz-man.pot contains strings from all man pages in XZ Utils. The man pages of xzdiff, xzgrep, and xzmore were under GPLv2 and the rest under 0BSD. Thus xz-man.pot contained strings under two licences. po4a creates the translated man pages from the combined 0BSD+GPLv2 xz-man.pot. I haven't liked this mixing in xz-man.pot but the Translation Project requires that all man pages must be in the same .pot file. So a separate xz-man-gpl.pot wasn't an option. Since these man pages are short, rewriting them was quick enough. Now xz-man.pot is entirely under 0BSD and marking the per-file licenses is simpler. As a bonus, some wording hopefully is now slightly better although it's perhaps a matter of taste. NOTE: In xzgrep.1, the EXIT STATUS section was written by me in the commit d796b6d7fdb8b7238b277056cf9146cce25db604 so that's why that section could be taken as is from the old xzgrep.1. src/scripts/xzdiff.1 | 94 ++++++++++++++++++++++++----------------- src/scripts/xzgrep.1 | 116 ++++++++++++++++++++++++++++++++------------------- src/scripts/xzmore.1 | 79 ++++++++++++++++++++--------------- 3 files changed, 173 insertions(+), 116 deletions(-) commit 3e551b111b8ae8150f1a1040364dbafc034f22be Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 xzless: Update man page slightly. The xz tool can decompress three file formats and xzless has always supported uncompressed files too. src/scripts/xzless.1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) commit 40f36da2262d13d6e1ba8449caa855512ae626d7 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Translations: Change po/Makevars to add a copyright notice to po/xz.pot. po/Makevars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 24192854e2ea5c06997431a98bda3c36c5da1497 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Translations: Update po/Makevars to use the template from gettext 0.22.4. Also add SPDX license identifier now that there is a known license. po/Makevars | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) commit b94154957370116480b43bcabca25fc52deb9853 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: Include the SPDX license identifier 0BSD to generated files. Perhaps the generated files aren't even copyrightable but using the same license for them as for the rest of the liblzma keeps things more consistent for tools that look for license info. src/liblzma/check/crc32_table_be.h | 4 +++- src/liblzma/check/crc32_table_le.h | 4 +++- src/liblzma/check/crc32_tablegen.c | 16 ++++++++++------ src/liblzma/check/crc64_table_be.h | 4 +++- src/liblzma/check/crc64_table_le.h | 4 +++- src/liblzma/check/crc64_tablegen.c | 8 +++++--- src/liblzma/lz/lz_encoder_hash_table.h | 4 +++- src/liblzma/lzma/fastpos_table.c | 4 +++- src/liblzma/lzma/fastpos_tablegen.c | 12 +++++++----- src/liblzma/rangecoder/price_table.c | 4 +++- src/liblzma/rangecoder/price_tablegen.c | 12 +++++++----- 11 files changed, 50 insertions(+), 26 deletions(-) commit 8e4ec794836bc1701d8c9bd5e347b8ce8cc5bbb4 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 liblzma: Fix compilation of price_tablegen.c. It is built and run only manually so this didn't matter unless one wanted to regenerate the price_table.c. src/liblzma/rangecoder/price_tablegen.c | 5 +++++ src/liblzma/rangecoder/range_common.h | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) commit e99bff3ffbcdf2634fd5bd13887627ec7dbfecaf Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Add SPDX license identifiers to GPL, LGPL, and FSFULLR files. extra/scanlzma/scanlzma.c | 2 ++ lib/Makefile.am | 2 ++ lib/getopt-cdefs.h | 2 ++ lib/getopt-core.h | 2 ++ lib/getopt-ext.h | 2 ++ lib/getopt-pfx-core.h | 2 ++ lib/getopt-pfx-ext.h | 2 ++ lib/getopt.c | 2 ++ lib/getopt.in.h | 2 ++ lib/getopt1.c | 2 ++ lib/getopt_int.h | 2 ++ m4/ax_pthread.m4 | 2 ++ m4/getopt.m4 | 2 ++ m4/posix-shell.m4 | 2 ++ m4/visibility.m4 | 2 ++ src/scripts/xzdiff.1 | 3 +-- src/scripts/xzdiff.in | 1 + src/scripts/xzgrep.1 | 3 +-- src/scripts/xzgrep.in | 1 + src/scripts/xzless.in | 1 + src/scripts/xzmore.1 | 3 +-- src/scripts/xzmore.in | 1 + 22 files changed, 37 insertions(+), 6 deletions(-) commit 22af94128b89a131f5e58ae69bee5e50227c15da Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Add SPDX license identifier into 0BSD source code files. .github/workflows/ci.yml | 2 ++ .github/workflows/windows-ci.yml | 2 ++ CMakeLists.txt | 2 ++ Makefile.am | 3 +-- autogen.sh | 1 + build-aux/ci_build.sh | 3 ++- build-aux/manconv.sh | 3 ++- build-aux/version.sh | 3 ++- cmake/remove-ordinals.cmake | 2 ++ cmake/tuklib_common.cmake | 4 ++++ cmake/tuklib_cpucores.cmake | 4 ++++ cmake/tuklib_integer.cmake | 4 ++++ cmake/tuklib_large_file_support.cmake | 4 ++++ cmake/tuklib_mbstr.cmake | 4 ++++ cmake/tuklib_physmem.cmake | 4 ++++ cmake/tuklib_progname.cmake | 4 ++++ configure.ac | 4 +++- debug/Makefile.am | 3 +-- debug/crc32.c | 2 ++ debug/full_flush.c | 2 ++ debug/hex2bin.c | 2 ++ debug/known_sizes.c | 2 ++ debug/memusage.c | 2 ++ debug/repeat.c | 2 ++ debug/sync_flush.c | 2 ++ debug/translation.bash | 1 + doc/examples/01_compress_easy.c | 2 ++ doc/examples/02_decompress.c | 2 ++ doc/examples/03_compress_custom.c | 2 ++ doc/examples/04_compress_easy_mt.c | 2 ++ doc/examples/11_file_info.c | 2 ++ doc/examples/Makefile | 3 +-- dos/Makefile | 2 ++ dos/config.h | 2 ++ doxygen/update-doxygen | 3 ++- extra/7z2lzma/7z2lzma.bash | 3 ++- m4/tuklib_common.m4 | 8 ++++++-- m4/tuklib_cpucores.m4 | 8 ++++++-- m4/tuklib_integer.m4 | 8 ++++++-- m4/tuklib_mbstr.m4 | 8 ++++++-- m4/tuklib_physmem.m4 | 8 ++++++-- m4/tuklib_progname.m4 | 8 ++++++-- po/POTFILES.in | 2 ++ po4a/po4a.conf | 2 ++ po4a/update-po | 3 ++- src/Makefile.am | 3 +-- src/common/common_w32res.rc | 2 ++ src/common/mythread.h | 2 ++ src/common/sysdefs.h | 2 ++ src/common/tuklib_common.h | 2 ++ src/common/tuklib_config.h | 2 ++ src/common/tuklib_cpucores.c | 2 ++ src/common/tuklib_cpucores.h | 2 ++ src/common/tuklib_exit.c | 2 ++ src/common/tuklib_exit.h | 2 ++ src/common/tuklib_gettext.h | 2 ++ src/common/tuklib_integer.h | 2 ++ src/common/tuklib_mbstr.h | 2 ++ src/common/tuklib_mbstr_fw.c | 2 ++ src/common/tuklib_mbstr_width.c | 2 ++ src/common/tuklib_open_stdxxx.c | 2 ++ src/common/tuklib_open_stdxxx.h | 2 ++ src/common/tuklib_physmem.c | 2 ++ src/common/tuklib_physmem.h | 2 ++ src/common/tuklib_progname.c | 2 ++ src/common/tuklib_progname.h | 2 ++ src/liblzma/Makefile.am | 3 +-- src/liblzma/api/Makefile.am | 3 +-- src/liblzma/api/lzma.h | 2 ++ src/liblzma/api/lzma/base.h | 2 ++ src/liblzma/api/lzma/bcj.h | 2 ++ src/liblzma/api/lzma/block.h | 2 ++ src/liblzma/api/lzma/check.h | 2 ++ src/liblzma/api/lzma/container.h | 2 ++ src/liblzma/api/lzma/delta.h | 2 ++ src/liblzma/api/lzma/filter.h | 2 ++ src/liblzma/api/lzma/hardware.h | 2 ++ src/liblzma/api/lzma/index.h | 2 ++ src/liblzma/api/lzma/index_hash.h | 2 ++ src/liblzma/api/lzma/lzma12.h | 2 ++ src/liblzma/api/lzma/stream_flags.h | 2 ++ src/liblzma/api/lzma/version.h | 2 ++ src/liblzma/api/lzma/vli.h | 2 ++ src/liblzma/check/Makefile.inc | 4 ++-- src/liblzma/check/check.c | 2 ++ src/liblzma/check/check.h | 2 ++ src/liblzma/check/crc32_arm64.h | 2 ++ src/liblzma/check/crc32_fast.c | 2 ++ src/liblzma/check/crc32_small.c | 2 ++ src/liblzma/check/crc32_table.c | 2 ++ src/liblzma/check/crc32_tablegen.c | 2 ++ src/liblzma/check/crc32_x86.S | 2 ++ src/liblzma/check/crc64_fast.c | 2 ++ src/liblzma/check/crc64_small.c | 2 ++ src/liblzma/check/crc64_table.c | 2 ++ src/liblzma/check/crc64_tablegen.c | 2 ++ src/liblzma/check/crc64_x86.S | 2 ++ src/liblzma/check/crc_common.h | 2 ++ src/liblzma/check/crc_x86_clmul.h | 2 ++ src/liblzma/check/sha256.c | 2 ++ src/liblzma/common/Makefile.inc | 3 +-- src/liblzma/common/alone_decoder.c | 2 ++ src/liblzma/common/alone_decoder.h | 2 ++ src/liblzma/common/alone_encoder.c | 2 ++ src/liblzma/common/auto_decoder.c | 2 ++ src/liblzma/common/block_buffer_decoder.c | 2 ++ src/liblzma/common/block_buffer_encoder.c | 2 ++ src/liblzma/common/block_buffer_encoder.h | 2 ++ src/liblzma/common/block_decoder.c | 2 ++ src/liblzma/common/block_decoder.h | 2 ++ src/liblzma/common/block_encoder.c | 2 ++ src/liblzma/common/block_encoder.h | 2 ++ src/liblzma/common/block_header_decoder.c | 2 ++ src/liblzma/common/block_header_encoder.c | 2 ++ src/liblzma/common/block_util.c | 2 ++ src/liblzma/common/common.c | 2 ++ src/liblzma/common/common.h | 2 ++ src/liblzma/common/easy_buffer_encoder.c | 2 ++ src/liblzma/common/easy_decoder_memusage.c | 2 ++ src/liblzma/common/easy_encoder.c | 2 ++ src/liblzma/common/easy_encoder_memusage.c | 2 ++ src/liblzma/common/easy_preset.c | 2 ++ src/liblzma/common/easy_preset.h | 2 ++ src/liblzma/common/file_info.c | 2 ++ src/liblzma/common/filter_buffer_decoder.c | 2 ++ src/liblzma/common/filter_buffer_encoder.c | 2 ++ src/liblzma/common/filter_common.c | 2 ++ src/liblzma/common/filter_common.h | 2 ++ src/liblzma/common/filter_decoder.c | 2 ++ src/liblzma/common/filter_decoder.h | 2 ++ src/liblzma/common/filter_encoder.c | 2 ++ src/liblzma/common/filter_encoder.h | 2 ++ src/liblzma/common/filter_flags_decoder.c | 2 ++ src/liblzma/common/filter_flags_encoder.c | 2 ++ src/liblzma/common/hardware_cputhreads.c | 2 ++ src/liblzma/common/hardware_physmem.c | 2 ++ src/liblzma/common/index.c | 2 ++ src/liblzma/common/index.h | 2 ++ src/liblzma/common/index_decoder.c | 2 ++ src/liblzma/common/index_decoder.h | 2 ++ src/liblzma/common/index_encoder.c | 2 ++ src/liblzma/common/index_encoder.h | 2 ++ src/liblzma/common/index_hash.c | 2 ++ src/liblzma/common/lzip_decoder.c | 2 ++ src/liblzma/common/lzip_decoder.h | 2 ++ src/liblzma/common/memcmplen.h | 2 ++ src/liblzma/common/microlzma_decoder.c | 2 ++ src/liblzma/common/microlzma_encoder.c | 2 ++ src/liblzma/common/outqueue.c | 2 ++ src/liblzma/common/outqueue.h | 2 ++ src/liblzma/common/stream_buffer_decoder.c | 2 ++ src/liblzma/common/stream_buffer_encoder.c | 2 ++ src/liblzma/common/stream_decoder.c | 2 ++ src/liblzma/common/stream_decoder.h | 2 ++ src/liblzma/common/stream_decoder_mt.c | 2 ++ src/liblzma/common/stream_encoder.c | 2 ++ src/liblzma/common/stream_encoder_mt.c | 2 ++ src/liblzma/common/stream_flags_common.c | 2 ++ src/liblzma/common/stream_flags_common.h | 2 ++ src/liblzma/common/stream_flags_decoder.c | 2 ++ src/liblzma/common/stream_flags_encoder.c | 2 ++ src/liblzma/common/string_conversion.c | 2 ++ src/liblzma/common/vli_decoder.c | 2 ++ src/liblzma/common/vli_encoder.c | 2 ++ src/liblzma/common/vli_size.c | 2 ++ src/liblzma/delta/Makefile.inc | 3 +-- src/liblzma/delta/delta_common.c | 2 ++ src/liblzma/delta/delta_common.h | 2 ++ src/liblzma/delta/delta_decoder.c | 2 ++ src/liblzma/delta/delta_decoder.h | 2 ++ src/liblzma/delta/delta_encoder.c | 2 ++ src/liblzma/delta/delta_encoder.h | 2 ++ src/liblzma/delta/delta_private.h | 2 ++ src/liblzma/liblzma.pc.in | 3 +-- src/liblzma/liblzma_generic.map | 2 ++ src/liblzma/liblzma_linux.map | 2 ++ src/liblzma/liblzma_w32res.rc | 2 ++ src/liblzma/lz/Makefile.inc | 3 +-- src/liblzma/lz/lz_decoder.c | 2 ++ src/liblzma/lz/lz_decoder.h | 2 ++ src/liblzma/lz/lz_encoder.c | 2 ++ src/liblzma/lz/lz_encoder.h | 2 ++ src/liblzma/lz/lz_encoder_hash.h | 2 ++ src/liblzma/lz/lz_encoder_mf.c | 2 ++ src/liblzma/lzma/Makefile.inc | 3 +-- src/liblzma/lzma/fastpos.h | 2 ++ src/liblzma/lzma/fastpos_tablegen.c | 2 ++ src/liblzma/lzma/lzma2_decoder.c | 2 ++ src/liblzma/lzma/lzma2_decoder.h | 2 ++ src/liblzma/lzma/lzma2_encoder.c | 2 ++ src/liblzma/lzma/lzma2_encoder.h | 2 ++ src/liblzma/lzma/lzma_common.h | 2 ++ src/liblzma/lzma/lzma_decoder.c | 2 ++ src/liblzma/lzma/lzma_decoder.h | 2 ++ src/liblzma/lzma/lzma_encoder.c | 2 ++ src/liblzma/lzma/lzma_encoder.h | 2 ++ src/liblzma/lzma/lzma_encoder_optimum_fast.c | 2 ++ src/liblzma/lzma/lzma_encoder_optimum_normal.c | 2 ++ src/liblzma/lzma/lzma_encoder_presets.c | 2 ++ src/liblzma/lzma/lzma_encoder_private.h | 2 ++ src/liblzma/rangecoder/Makefile.inc | 3 +-- src/liblzma/rangecoder/price.h | 2 ++ src/liblzma/rangecoder/price_tablegen.c | 2 ++ src/liblzma/rangecoder/range_common.h | 2 ++ src/liblzma/rangecoder/range_decoder.h | 2 ++ src/liblzma/rangecoder/range_encoder.h | 2 ++ src/liblzma/simple/Makefile.inc | 3 +-- src/liblzma/simple/arm.c | 2 ++ src/liblzma/simple/arm64.c | 2 ++ src/liblzma/simple/armthumb.c | 2 ++ src/liblzma/simple/ia64.c | 2 ++ src/liblzma/simple/powerpc.c | 2 ++ src/liblzma/simple/riscv.c | 2 ++ src/liblzma/simple/simple_coder.c | 2 ++ src/liblzma/simple/simple_coder.h | 2 ++ src/liblzma/simple/simple_decoder.c | 2 ++ src/liblzma/simple/simple_decoder.h | 2 ++ src/liblzma/simple/simple_encoder.c | 2 ++ src/liblzma/simple/simple_encoder.h | 2 ++ src/liblzma/simple/simple_private.h | 2 ++ src/liblzma/simple/sparc.c | 2 ++ src/liblzma/simple/x86.c | 2 ++ src/liblzma/validate_map.sh | 1 + src/lzmainfo/Makefile.am | 3 +-- src/lzmainfo/lzmainfo.c | 2 ++ src/lzmainfo/lzmainfo_w32res.rc | 2 ++ src/scripts/Makefile.am | 3 +-- src/xz/Makefile.am | 3 +-- src/xz/args.c | 2 ++ src/xz/args.h | 2 ++ src/xz/coder.c | 2 ++ src/xz/coder.h | 2 ++ src/xz/file_io.c | 2 ++ src/xz/file_io.h | 2 ++ src/xz/hardware.c | 2 ++ src/xz/hardware.h | 2 ++ src/xz/list.c | 2 ++ src/xz/list.h | 2 ++ src/xz/main.c | 2 ++ src/xz/main.h | 2 ++ src/xz/message.c | 2 ++ src/xz/message.h | 2 ++ src/xz/mytime.c | 2 ++ src/xz/mytime.h | 2 ++ src/xz/options.c | 2 ++ src/xz/options.h | 2 ++ src/xz/private.h | 2 ++ src/xz/signals.c | 2 ++ src/xz/signals.h | 2 ++ src/xz/suffix.c | 2 ++ src/xz/suffix.h | 2 ++ src/xz/util.c | 2 ++ src/xz/util.h | 2 ++ src/xz/xz_w32res.rc | 2 ++ src/xzdec/Makefile.am | 3 +-- src/xzdec/lzmadec_w32res.rc | 2 ++ src/xzdec/xzdec.c | 2 ++ src/xzdec/xzdec_w32res.rc | 2 ++ tests/Makefile.am | 3 +-- tests/bcj_test.c | 2 ++ tests/code_coverage.sh | 1 + tests/create_compress_files.c | 2 ++ tests/ossfuzz/fuzz_common.h | 2 ++ tests/ossfuzz/fuzz_decode_alone.c | 2 ++ tests/ossfuzz/fuzz_decode_stream.c | 2 ++ tests/ossfuzz/fuzz_encode_stream.c | 2 ++ tests/test_bcj_exact_size.c | 2 ++ tests/test_block_header.c | 2 ++ tests/test_check.c | 2 ++ tests/test_compress.sh | 1 + tests/test_compress_generated_abc | 1 + tests/test_compress_generated_random | 1 + tests/test_compress_generated_text | 1 + tests/test_compress_prepared_bcj_sparc | 1 + tests/test_compress_prepared_bcj_x86 | 1 + tests/test_files.sh | 1 + tests/test_filter_flags.c | 2 ++ tests/test_filter_str.c | 2 ++ tests/test_hardware.c | 2 ++ tests/test_index.c | 2 ++ tests/test_index_hash.c | 2 ++ tests/test_lzip_decoder.c | 2 ++ tests/test_memlimit.c | 2 ++ tests/test_scripts.sh | 1 + tests/test_stream_flags.c | 2 ++ tests/test_suffix.sh | 1 + tests/test_vli.c | 2 ++ tests/tests.h | 2 ++ tests/tuktest.h | 2 ++ windows/build.bash | 3 ++- 290 files changed, 588 insertions(+), 58 deletions(-) commit 23de53421ea258cde6a3c33a038b1e9d08f771d1 Author: Lasse Collin Date: 2024-02-12 23:25:54 +0200 liblzma: Sync the AUTHORS fix about SHA-256 to lzma.h. src/liblzma/api/lzma.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) commit 689e0228baeb95232430e90d628379db89583d71 Author: Lasse Collin Date: 2024-02-12 17:09:10 +0200 Change most public domain parts to 0BSD. Translations and doc/xz-file-format.txt and doc/lzma-file-format.txt were not touched. COPYING.0BSD was added. .github/workflows/ci.yml | 3 - .github/workflows/windows-ci.yml | 3 - CMakeLists.txt | 3 - COPYING | 112 ++++++++++++++----------- COPYING.0BSD | 11 +++ Makefile.am | 3 - PACKAGERS | 11 +-- autogen.sh | 3 - build-aux/ci_build.sh | 3 - build-aux/manconv.sh | 3 - build-aux/version.sh | 3 - cmake/remove-ordinals.cmake | 3 - cmake/tuklib_common.cmake | 3 - cmake/tuklib_cpucores.cmake | 3 - cmake/tuklib_integer.cmake | 3 - cmake/tuklib_large_file_support.cmake | 3 - cmake/tuklib_mbstr.cmake | 3 - cmake/tuklib_physmem.cmake | 3 - cmake/tuklib_progname.cmake | 3 - configure.ac | 3 - debug/Makefile.am | 3 - debug/crc32.c | 3 - debug/full_flush.c | 3 - debug/hex2bin.c | 3 - debug/known_sizes.c | 3 - debug/memusage.c | 3 - debug/repeat.c | 3 - debug/sync_flush.c | 3 - debug/translation.bash | 3 - doc/examples/01_compress_easy.c | 3 - doc/examples/02_decompress.c | 3 - doc/examples/03_compress_custom.c | 3 - doc/examples/04_compress_easy_mt.c | 3 - doc/examples/11_file_info.c | 3 - doc/examples/Makefile | 3 - dos/Makefile | 3 - doxygen/update-doxygen | 3 - extra/7z2lzma/7z2lzma.bash | 3 - m4/tuklib_common.m4 | 3 - m4/tuklib_cpucores.m4 | 3 - m4/tuklib_integer.m4 | 3 - m4/tuklib_mbstr.m4 | 3 - m4/tuklib_physmem.m4 | 3 - m4/tuklib_progname.m4 | 3 - po4a/update-po | 3 - src/Makefile.am | 3 - src/common/common_w32res.rc | 3 - src/common/mythread.h | 3 - src/common/sysdefs.h | 3 - src/common/tuklib_common.h | 3 - src/common/tuklib_cpucores.c | 3 - src/common/tuklib_cpucores.h | 3 - src/common/tuklib_exit.c | 3 - src/common/tuklib_exit.h | 3 - src/common/tuklib_gettext.h | 3 - src/common/tuklib_integer.h | 3 - src/common/tuklib_mbstr.h | 3 - src/common/tuklib_mbstr_fw.c | 3 - src/common/tuklib_mbstr_width.c | 3 - src/common/tuklib_open_stdxxx.c | 3 - src/common/tuklib_open_stdxxx.h | 3 - src/common/tuklib_physmem.c | 3 - src/common/tuklib_physmem.h | 3 - src/common/tuklib_progname.c | 3 - src/common/tuklib_progname.h | 3 - src/liblzma/Makefile.am | 3 - src/liblzma/api/Makefile.am | 3 - src/liblzma/api/lzma.h | 13 ++- src/liblzma/api/lzma/base.h | 3 - src/liblzma/api/lzma/bcj.h | 3 - src/liblzma/api/lzma/block.h | 3 - src/liblzma/api/lzma/check.h | 3 - src/liblzma/api/lzma/container.h | 3 - src/liblzma/api/lzma/delta.h | 3 - src/liblzma/api/lzma/filter.h | 3 - src/liblzma/api/lzma/hardware.h | 3 - src/liblzma/api/lzma/index.h | 3 - src/liblzma/api/lzma/index_hash.h | 3 - src/liblzma/api/lzma/lzma12.h | 3 - src/liblzma/api/lzma/stream_flags.h | 3 - src/liblzma/api/lzma/version.h | 3 - src/liblzma/api/lzma/vli.h | 3 - src/liblzma/check/Makefile.inc | 3 - src/liblzma/check/check.c | 3 - src/liblzma/check/check.h | 3 - src/liblzma/check/crc32_arm64.h | 3 - src/liblzma/check/crc32_fast.c | 3 - src/liblzma/check/crc32_small.c | 3 - src/liblzma/check/crc32_table.c | 3 - src/liblzma/check/crc32_tablegen.c | 3 - src/liblzma/check/crc32_x86.S | 3 - src/liblzma/check/crc64_fast.c | 3 - src/liblzma/check/crc64_small.c | 3 - src/liblzma/check/crc64_table.c | 3 - src/liblzma/check/crc64_tablegen.c | 3 - src/liblzma/check/crc64_x86.S | 3 - src/liblzma/check/crc_common.h | 3 - src/liblzma/check/crc_x86_clmul.h | 3 - src/liblzma/check/sha256.c | 3 - src/liblzma/common/Makefile.inc | 3 - src/liblzma/common/alone_decoder.c | 3 - src/liblzma/common/alone_decoder.h | 3 - src/liblzma/common/alone_encoder.c | 3 - src/liblzma/common/auto_decoder.c | 3 - src/liblzma/common/block_buffer_decoder.c | 3 - src/liblzma/common/block_buffer_encoder.c | 3 - src/liblzma/common/block_buffer_encoder.h | 3 - src/liblzma/common/block_decoder.c | 3 - src/liblzma/common/block_decoder.h | 3 - src/liblzma/common/block_encoder.c | 3 - src/liblzma/common/block_encoder.h | 3 - src/liblzma/common/block_header_decoder.c | 3 - src/liblzma/common/block_header_encoder.c | 3 - src/liblzma/common/block_util.c | 3 - src/liblzma/common/common.c | 3 - src/liblzma/common/common.h | 3 - src/liblzma/common/easy_buffer_encoder.c | 3 - src/liblzma/common/easy_decoder_memusage.c | 3 - src/liblzma/common/easy_encoder.c | 3 - src/liblzma/common/easy_encoder_memusage.c | 3 - src/liblzma/common/easy_preset.c | 3 - src/liblzma/common/easy_preset.h | 3 - src/liblzma/common/file_info.c | 3 - src/liblzma/common/filter_buffer_decoder.c | 3 - src/liblzma/common/filter_buffer_encoder.c | 3 - src/liblzma/common/filter_common.c | 3 - src/liblzma/common/filter_common.h | 3 - src/liblzma/common/filter_decoder.c | 3 - src/liblzma/common/filter_decoder.h | 3 - src/liblzma/common/filter_encoder.c | 3 - src/liblzma/common/filter_encoder.h | 3 - src/liblzma/common/filter_flags_decoder.c | 3 - src/liblzma/common/filter_flags_encoder.c | 3 - src/liblzma/common/hardware_cputhreads.c | 3 - src/liblzma/common/hardware_physmem.c | 3 - src/liblzma/common/index.c | 3 - src/liblzma/common/index.h | 3 - src/liblzma/common/index_decoder.c | 3 - src/liblzma/common/index_decoder.h | 3 - src/liblzma/common/index_encoder.c | 3 - src/liblzma/common/index_encoder.h | 3 - src/liblzma/common/index_hash.c | 3 - src/liblzma/common/lzip_decoder.c | 3 - src/liblzma/common/lzip_decoder.h | 3 - src/liblzma/common/memcmplen.h | 3 - src/liblzma/common/microlzma_decoder.c | 3 - src/liblzma/common/microlzma_encoder.c | 3 - src/liblzma/common/outqueue.c | 3 - src/liblzma/common/outqueue.h | 3 - src/liblzma/common/stream_buffer_decoder.c | 3 - src/liblzma/common/stream_buffer_encoder.c | 3 - src/liblzma/common/stream_decoder.c | 3 - src/liblzma/common/stream_decoder.h | 3 - src/liblzma/common/stream_decoder_mt.c | 3 - src/liblzma/common/stream_encoder.c | 3 - src/liblzma/common/stream_encoder_mt.c | 3 - src/liblzma/common/stream_flags_common.c | 3 - src/liblzma/common/stream_flags_common.h | 3 - src/liblzma/common/stream_flags_decoder.c | 3 - src/liblzma/common/stream_flags_encoder.c | 3 - src/liblzma/common/string_conversion.c | 3 - src/liblzma/common/vli_decoder.c | 3 - src/liblzma/common/vli_encoder.c | 3 - src/liblzma/common/vli_size.c | 3 - src/liblzma/delta/Makefile.inc | 3 - src/liblzma/delta/delta_common.c | 3 - src/liblzma/delta/delta_common.h | 3 - src/liblzma/delta/delta_decoder.c | 3 - src/liblzma/delta/delta_decoder.h | 3 - src/liblzma/delta/delta_encoder.c | 3 - src/liblzma/delta/delta_encoder.h | 3 - src/liblzma/delta/delta_private.h | 3 - src/liblzma/liblzma.pc.in | 3 - src/liblzma/liblzma_w32res.rc | 3 - src/liblzma/lz/Makefile.inc | 3 - src/liblzma/lz/lz_decoder.c | 3 - src/liblzma/lz/lz_decoder.h | 3 - src/liblzma/lz/lz_encoder.c | 3 - src/liblzma/lz/lz_encoder.h | 3 - src/liblzma/lz/lz_encoder_hash.h | 3 - src/liblzma/lz/lz_encoder_mf.c | 3 - src/liblzma/lzma/Makefile.inc | 3 - src/liblzma/lzma/fastpos.h | 3 - src/liblzma/lzma/fastpos_tablegen.c | 3 - src/liblzma/lzma/lzma2_decoder.c | 3 - src/liblzma/lzma/lzma2_decoder.h | 3 - src/liblzma/lzma/lzma2_encoder.c | 3 - src/liblzma/lzma/lzma2_encoder.h | 3 - src/liblzma/lzma/lzma_common.h | 3 - src/liblzma/lzma/lzma_decoder.c | 3 - src/liblzma/lzma/lzma_decoder.h | 3 - src/liblzma/lzma/lzma_encoder.c | 3 - src/liblzma/lzma/lzma_encoder.h | 3 - src/liblzma/lzma/lzma_encoder_optimum_fast.c | 3 - src/liblzma/lzma/lzma_encoder_optimum_normal.c | 3 - src/liblzma/lzma/lzma_encoder_presets.c | 3 - src/liblzma/lzma/lzma_encoder_private.h | 3 - src/liblzma/rangecoder/Makefile.inc | 3 - src/liblzma/rangecoder/price.h | 3 - src/liblzma/rangecoder/price_tablegen.c | 3 - src/liblzma/rangecoder/range_common.h | 3 - src/liblzma/rangecoder/range_decoder.h | 3 - src/liblzma/rangecoder/range_encoder.h | 3 - src/liblzma/simple/Makefile.inc | 3 - src/liblzma/simple/arm.c | 3 - src/liblzma/simple/arm64.c | 3 - src/liblzma/simple/armthumb.c | 3 - src/liblzma/simple/ia64.c | 3 - src/liblzma/simple/powerpc.c | 3 - src/liblzma/simple/riscv.c | 3 - src/liblzma/simple/simple_coder.c | 3 - src/liblzma/simple/simple_coder.h | 3 - src/liblzma/simple/simple_decoder.c | 3 - src/liblzma/simple/simple_decoder.h | 3 - src/liblzma/simple/simple_encoder.c | 3 - src/liblzma/simple/simple_encoder.h | 3 - src/liblzma/simple/simple_private.h | 3 - src/liblzma/simple/sparc.c | 3 - src/liblzma/simple/x86.c | 3 - src/liblzma/validate_map.sh | 3 - src/lzmainfo/Makefile.am | 3 - src/lzmainfo/lzmainfo.1 | 4 +- src/lzmainfo/lzmainfo.c | 3 - src/lzmainfo/lzmainfo_w32res.rc | 3 - src/scripts/Makefile.am | 3 - src/scripts/xzless.1 | 4 +- src/xz/Makefile.am | 3 - src/xz/args.c | 3 - src/xz/args.h | 3 - src/xz/coder.c | 3 - src/xz/coder.h | 3 - src/xz/file_io.c | 3 - src/xz/file_io.h | 3 - src/xz/hardware.c | 3 - src/xz/hardware.h | 3 - src/xz/list.c | 3 - src/xz/list.h | 3 - src/xz/main.c | 3 - src/xz/main.h | 3 - src/xz/message.c | 3 - src/xz/message.h | 3 - src/xz/mytime.c | 3 - src/xz/mytime.h | 3 - src/xz/options.c | 3 - src/xz/options.h | 3 - src/xz/private.h | 3 - src/xz/signals.c | 3 - src/xz/signals.h | 3 - src/xz/suffix.c | 3 - src/xz/suffix.h | 3 - src/xz/util.c | 3 - src/xz/util.h | 3 - src/xz/xz.1 | 4 +- src/xz/xz_w32res.rc | 3 - src/xzdec/Makefile.am | 3 - src/xzdec/lzmadec_w32res.rc | 3 - src/xzdec/xzdec.1 | 4 +- src/xzdec/xzdec.c | 3 - src/xzdec/xzdec_w32res.rc | 3 - tests/Makefile.am | 3 - tests/bcj_test.c | 3 - tests/code_coverage.sh | 3 - tests/create_compress_files.c | 3 - tests/files/README | 3 +- tests/ossfuzz/fuzz_common.h | 3 - tests/ossfuzz/fuzz_decode_alone.c | 3 - tests/ossfuzz/fuzz_decode_stream.c | 3 - tests/ossfuzz/fuzz_encode_stream.c | 3 - tests/test_bcj_exact_size.c | 3 - tests/test_block_header.c | 3 - tests/test_check.c | 3 - tests/test_compress.sh | 3 - tests/test_files.sh | 3 - tests/test_filter_flags.c | 3 - tests/test_filter_str.c | 3 - tests/test_hardware.c | 3 - tests/test_index.c | 3 - tests/test_index_hash.c | 3 - tests/test_lzip_decoder.c | 3 - tests/test_memlimit.c | 3 - tests/test_scripts.sh | 3 - tests/test_stream_flags.c | 3 - tests/test_suffix.sh | 3 - tests/test_vli.c | 3 - tests/tests.h | 3 - tests/tuktest.h | 3 - windows/README-Windows.txt | 11 +-- windows/build.bash | 3 - 288 files changed, 100 insertions(+), 911 deletions(-) commit 76946dc4336c831fe2cc26696a035d807dd3cf13 Author: Lasse Collin Date: 2024-02-09 17:20:31 +0200 Fix SHA-256 authors. The initial commit 5d018dc03549c1ee4958364712fb0c94e1bf2741 in 2007 had a comment in sha256.c that the code is based on Crypto++ Library 5.5.1. In 2009 the Authors list in sha256.c and the AUTHORS file was updated with information that the code had come from Crypto++ but via 7-Zip. I know I had viewed 7-Zip's SHA-256 code but back then the C code has been identical enough with Crypto++, so I don't why I thought the author info would need that extra step via 7-Zip for this single file. Another error is that I had mixed sha.* and shacal2.* files when checking for author info in Crypto++. The shacal2.* files aren't related to liblzma's sha256.c and thus Kevin Springle's code in Crypto++ isn't either. AUTHORS | 6 ++---- src/liblzma/check/sha256.c | 14 ++++---------- 2 files changed, 6 insertions(+), 14 deletions(-) commit 21d9cbae9eecca28ce373d3d9464defd2cf5d851 Author: Lasse Collin Date: 2024-02-09 17:20:31 +0200 Remove macosx/build.sh. It was last updated in 2013. Makefile.am | 1 - macosx/build.sh | 113 -------------------------------------------------------- 2 files changed, 114 deletions(-) commit eac2c3c67f9113a225fb6667df862edd30366931 Author: Lasse Collin Date: 2024-02-09 17:20:31 +0200 Doc: Remove doc/examples_old. It was good to keep these around in parallel with the newer examples but I think it's OK to remove the old ones at this point. Makefile.am | 5 -- doc/examples_old/xz_pipe_comp.c | 127 -------------------------------------- doc/examples_old/xz_pipe_decomp.c | 123 ------------------------------------ 3 files changed, 255 deletions(-) commit 89ea1a22f4ed3685b053b7260bc5acf6c75d1664 Author: Jia Tan Date: 2024-02-13 22:38:58 +0800 Tests: Add RISC-V filter support in a few places. tests/test_filter_flags.c | 6 ++++++ tests/test_filter_str.c | 6 ++++++ 2 files changed, 12 insertions(+) commit 45663443eb2b377e6171529380fee312f1adcdf4 Author: Jia Tan Date: 2024-02-13 22:37:07 +0800 liblzma: Fix build error if only RISC-V BCJ filter is enabled. If any other BCJ filter was enabled for encoding or decoding, then this was not a problem. src/liblzma/common/string_conversion.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) commit 2f15597d677bc35743c777d4cf3bfa698b478681 Author: Jia Tan Date: 2024-02-13 22:56:24 +0800 Translations: Update the Korean translation. po/ko.po | 526 ++++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 284 insertions(+), 242 deletions(-) commit df873143ad1615c6d6aaa1bf8808b1676091dfe3 Author: Jia Tan Date: 2024-02-13 01:55:53 +0800 Translations: Update the Korean man page translations. po4a/ko.po | 1375 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 770 insertions(+), 605 deletions(-) commit b3f415eddb150341865a1af47959c3baba076b33 Author: Jia Tan Date: 2024-02-13 01:53:33 +0800 Translations: Update the Chinese (simplified) translation. po/zh_CN.po | 424 ++++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 268 insertions(+), 156 deletions(-) commit 9860d418d296eb3c721e5384fb367c0499b579c8 Author: Lasse Collin Date: 2024-02-09 23:21:01 +0200 xzless: Use ||- in LESSOPEN with with "less" 451 and newer. src/scripts/xzless.in | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) commit fd0692b0525e6c26b496492be9e2c865cab734f8 Author: Lasse Collin Date: 2024-02-09 23:00:05 +0200 xzless: Use --show-preproc-errors with "less" 632 and newer. This makes "less" show a warning if a decompression error occurred. src/scripts/xzless.in | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) commit adb073da76a920b5a81e6b32254f4ddb054dc57a Author: Jia Tan Date: 2024-02-09 23:59:54 +0800 liblzma: Fix typo discovered by codespell. src/liblzma/check/crc32_arm64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 55d9fc883d221cbace951a370f1fb144698f8c2e Author: Jia Tan Date: 2024-02-09 20:01:06 +0800 Translations: Update the Swedish translation. po/sv.po | 420 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 254 insertions(+), 166 deletions(-) commit 55ba4a1ea321499c805eedfa811ffde690bae311 Author: Jia Tan Date: 2024-02-08 20:09:04 +0800 Translations: Update the Spanish translation. po/es.po | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) commit 7f2293cd804a89d3c3b2d3ed573560ca9e1520ae Author: Jia Tan Date: 2024-02-07 21:34:35 +0800 Translations: Update the Spanish translation. po/es.po | 419 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 253 insertions(+), 166 deletions(-) commit f4af2036bc625739d6d33d9e1fede583a25c3828 Author: Jia Tan Date: 2024-02-07 21:28:32 +0800 Translations: Update the Polish translation. po/pl.po | 411 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 249 insertions(+), 162 deletions(-) commit e5e93bb816043c559cddf03a3b7ba13bec353ee4 Author: Jia Tan Date: 2024-02-07 19:40:12 +0800 Translations: Update the German translation. po/de.po | 396 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 242 insertions(+), 154 deletions(-) commit 28f18ff8e26902762fb007c13be235b4ac1ac071 Author: Jia Tan Date: 2024-02-07 19:27:25 +0800 Translations: Update the German man page translations. po4a/de.po | 1353 +++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 752 insertions(+), 601 deletions(-) commit cabfbc7947da05aa5dfe39bec9759e076f940e3c Author: Jia Tan Date: 2024-02-06 23:44:06 +0800 Translations: Update the Romanian translation. po/ro.po | 416 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 252 insertions(+), 164 deletions(-) commit bf20c94f5d748cea2147779f4fa7e2fd2eb8555e Author: Jia Tan Date: 2024-02-06 23:45:02 +0800 Translations: Update the Romanian man page translations. po4a/ro.po | 1759 +++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 966 insertions(+), 793 deletions(-) commit 7c25ec9feb0241e4affb7432681cc4f5696f3a96 Author: Jia Tan Date: 2024-02-07 20:56:57 +0800 Translations: Update the Ukrainian translation. po/uk.po | 397 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 242 insertions(+), 155 deletions(-) commit b3523250e9eef10b017473754c1e1c9e31f10374 Author: Jia Tan Date: 2024-02-06 23:30:03 +0800 Translations: Update the Ukrainian man page translations. po4a/uk.po | 1363 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 764 insertions(+), 599 deletions(-) commit a5c177f514f4c90e0d2f6045636fca6c2e80a20d Author: Jia Tan Date: 2024-02-02 01:39:28 +0800 Update AUTHORS. AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 7f68a68c19d0ae57bd0e802be0ea8f974e41299f Author: Jia Tan Date: 2024-02-02 01:38:51 +0800 liblzma: Update Authors list in crc32_arm64.h. src/liblzma/check/crc32_arm64.h | 1 + 1 file changed, 1 insertion(+) commit 97f9ba50b84e67b3dcb5b17dd5d3e1d14f9ad1d0 Author: Jia Tan Date: 2024-02-01 16:07:03 +0800 liblzma: Check HAVE_USABLE_CLMUL before omitting CRC32 table. This was split from the prior commit so it could be easily applied to the 5.4 branch. Closes: https://github.com/tukaani-project/xz/pull/77 src/liblzma/check/crc32_table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit ca9015f4dead2c97b48f5a6933631b0a448b65b9 Author: Jia Tan Date: 2024-02-01 16:06:29 +0800 liblzma: Check HAVE_USABLE_CLMUL before omitting CRC64 table. If liblzma is configured with --disable-clmul-crc CFLAGS="-msse4.1 -mpclmul", then it will fail to compile because the generic version must be used but the CRC tables were not included. src/liblzma/check/crc64_table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 2f1552a91c825e87013925e1a67a0930e7aef592 Author: Jia Tan Date: 2024-01-23 18:02:13 +0800 liblzma: Only use ifunc in crcXX_fast.c if its needed. The code was using HAVE_FUNC_ATTRIBUTE_IFUNC instead of CRC_USE_IFUNC. With ARM64, ifunc is incompatible because it requires non-inline function calls for runtime detection. src/liblzma/check/crc32_fast.c | 6 +++--- src/liblzma/check/crc64_fast.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) commit 30a25f3742287697bc57a1bef86c19ecf5129322 Author: Jia Tan Date: 2024-01-22 22:08:45 +0800 Docs: Add --disable-arm64-crc32 description to INSTALL. INSTALL | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) commit 1940f0ec28f08c0ac72c1413d9706fb82eabe6ad Author: Jia Tan Date: 2024-01-22 21:36:09 +0800 liblzma: Omit CRC tables when not needed with ARM64 optimizations. This is similar to the existing x86-64 CLMUL conditions to omit the tables. They were slightly refactored to improve readability. src/liblzma/check/crc32_table.c | 18 +++++++++++++++--- src/liblzma/check/crc64_table.c | 7 ++++++- src/liblzma/check/crc_common.h | 5 ++++- 3 files changed, 25 insertions(+), 5 deletions(-) commit 761f5b69a4c778c8bcb09279b845b07c28790575 Author: Jia Tan Date: 2024-01-22 20:54:56 +0800 liblzma: Rename crc32_aarch64.h to crc32_arm64.h. Even though the proper name for the architecture is aarch64, this project uses ARM64 throughout. So the rename is for consistency. Additionally, crc32_arm64.h was slightly refactored for the following changes: * Added MSVC, FreeBSD, and macOS support in is_arch_extension_supported(). * crc32_arch_optimized() now checks the size when aligning the buffer. * crc32_arch_optimized() loop conditions were slightly modified to avoid both decrementing the size and incrementing the buffer pointer. * Use the intrinsic wrappers defined in because GCC and Clang name them differently. * Minor spacing and comment changes. CMakeLists.txt | 2 +- src/liblzma/check/Makefile.inc | 2 +- src/liblzma/check/crc32_aarch64.h | 109 ---------------------------------- src/liblzma/check/crc32_arm64.h | 119 ++++++++++++++++++++++++++++++++++++++ src/liblzma/check/crc32_fast.c | 3 +- src/liblzma/check/crc64_fast.c | 3 - 6 files changed, 122 insertions(+), 116 deletions(-) commit 455a08609caa3223066a717fb01bfa42c5dba47d Author: Jia Tan Date: 2024-01-22 20:49:30 +0800 liblzma: Refactor crc_common.h. The CRC_GENERIC is now split into CRC32_GENERIC and CRC64_GENERIC, since the ARM64 optimizations will be different between CRC32 and CRC64. For the same reason, CRC_ARCH_OPTIMIZED is split into CRC32_ARCH_OPTIMIZED and CRC64_ARCH_OPTIMIZED. ifunc will only be used with x86-64 CLMUL because the runtime detection methods needed with ARM64 are not compatible with ifunc. src/liblzma/check/crc32_fast.c | 8 +-- src/liblzma/check/crc64_fast.c | 8 +-- src/liblzma/check/crc_common.h | 108 ++++++++++++++++++++++++++++------------- 3 files changed, 82 insertions(+), 42 deletions(-) commit 61908e816049af7a9f43ea804a57ee8570e2e644 Author: Jia Tan Date: 2024-01-22 00:42:28 +0800 CMake: Add support for ARM64 CRC32 instruction detection. CMakeLists.txt | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) commit c5f6d79cc9515a7f22d7ea4860c6cc394b295732 Author: Jia Tan Date: 2024-01-22 00:36:47 +0800 Build: Add support for ARM64 CRC32 instruction detection. This adds --enable-arm64-crc32/--disable-arm64-crc32 (enabled by default) for using the ARM64 CRC32 instruction. This can be disabled if one knows the binary will never need to run on an ARM64 machine with this instruction extension. configure.ac | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) commit 849d0f282a6a890c5cf5a0e0f02980b12d9ebb0f Author: Chenxi Mao Date: 2024-01-09 17:23:11 +0800 Speed up CRC32 calculation on ARM64 The CRC32 instructions in ARM64 can calculate the CRC32 result for 8 bytes in a single operation, making the use of ARM64 instructions much faster compared to the general CRC32 algorithm. Optimized CRC32 will be enabled if ARM64 has CRC extension running on Linux. Signed-off-by: Chenxi Mao CMakeLists.txt | 1 + src/liblzma/check/Makefile.inc | 3 +- src/liblzma/check/crc32_aarch64.h | 109 ++++++++++++++++++++++++++++++++++++++ src/liblzma/check/crc32_fast.c | 5 +- src/liblzma/check/crc64_fast.c | 5 +- src/liblzma/check/crc_common.h | 16 +++--- 6 files changed, 130 insertions(+), 9 deletions(-) commit b43c3e48bf6097095eef36d44cdbec811074940a Author: Jia Tan Date: 2024-01-26 19:05:51 +0800 Bump version number for 5.5.1alpha. src/liblzma/api/lzma/version.h | 2 +- src/liblzma/liblzma_generic.map | 2 +- src/liblzma/liblzma_linux.map | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) commit c7a7ae1500ea90bd3c2d54533e4f433933eb598f Author: Jia Tan Date: 2024-01-26 19:00:52 +0800 Add NEWS for 5.5.1alpha NEWS | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) commit 0ef8192e8d5af4e6200d5d4aee22d1f177f7a2df Author: Jia Tan Date: 2024-01-26 18:54:24 +0800 Add NEWS for 5.4.6. NEWS | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) commit 93de7e751d17731315a899264f2a7239d7d2d316 Author: Lasse Collin Date: 2024-01-24 20:00:57 +0200 Move doc/logo/xz-logo.png to "doc" and Doxygen footer to "doxygen". The footer isn't a complete HTML file so having it in the doxygen directory is a tiny bit clearer. Makefile.am | 2 +- doc/{logo => }/xz-logo.png | Bin doxygen/Doxyfile | 4 ++-- doc/logo/copyright.html => doxygen/footer.html | 0 4 files changed, 3 insertions(+), 3 deletions(-) commit 00fa01698df51c58ae2acf8c7fa4e1fb159f75a9 Author: Jia Tan Date: 2024-01-09 17:05:01 +0800 README: Add COPYING.CC-BY-SA-4.0 entry to section 1.1. The Overall documentation section (1.1) table spacing had to be adjusted since the filename was very long. README | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) commit e280470040b27c5e58d78b25b9e2bb71fc6c3882 Author: Jia Tan Date: 2024-01-09 16:56:16 +0800 Build: Add the logo and license to the release. Makefile.am | 2 ++ 1 file changed, 2 insertions(+) commit b1ee6cf259bb49ce91abe9f622294524e37edf4c Author: Jia Tan Date: 2024-01-09 16:44:42 +0800 COPYING: Add the license for the XZ logo. COPYING | 5 + COPYING.CC-BY-SA-4.0 | 427 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 432 insertions(+) commit 31293ae7074802cc7286089a89c7b552d930c97f Author: Jia Tan Date: 2024-01-09 16:40:56 +0800 Doxygen: Added the XZ logo and copyright information. The PROJECT_LOGO field is now used to include the XZ logo. The footer of each page now lists the copyright information instead of the default footer. The license is also copied to statisfy the copyright and so the link in the documentation can be local. doc/logo/copyright.html | 11 +++++++++++ doc/logo/xz-logo.png | Bin 0 -> 6771 bytes doxygen/Doxyfile | 6 +++--- 3 files changed, 14 insertions(+), 3 deletions(-) commit 6daa4d0ea46a8441f21f609149f3633158bf4704 Author: Lasse Collin Date: 2024-01-23 18:29:28 +0200 xz: Use threaded mode by defaut (as if --threads=0 was used). This hopefully does more good than bad: + It's faster by default. + Only the threaded compressor creates files that can be decompressed in threaded mode. - Compression ratio is worse, usually not too much though. When it matters, -T1 must be used. - Memory usage increases. - Scripts that assume single-threaded mode but don't use -T1 will possibly use too much resources, for example, if they run multiple xz processes in parallel to compress multiple files. - Output from single-threaded and multi-threaded compressors differ but such changes could happen for other reasons too (they just haven't happened since 5.0.0). src/xz/hardware.c | 6 +++++- src/xz/message.c | 4 ++-- src/xz/xz.1 | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) commit a2dd2dc8e5307a7280bb99868bc478560facba2c Author: Jia Tan Date: 2024-01-23 23:52:49 +0800 CI: Use RISC-V filter when building with BCJ support. build-aux/ci_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 3060e1070b2421b26c0e17794c1307ec5622f11d Author: Jia Tan Date: 2024-01-23 23:52:14 +0800 Tests: Use smaller dictionary size in RISC-V test files. tests/files/good-1-riscv-lzma2-1.xz | Bin 7512 -> 7512 bytes tests/files/good-1-riscv-lzma2-2.xz | Bin 7516 -> 7512 bytes 2 files changed, 0 insertions(+), 0 deletions(-) commit 44ff2fa5c94dc345c4dd69195a19fc5238df60b3 Author: Jia Tan Date: 2024-01-23 23:50:57 +0800 Tests: Skip RISC-V test files if decoder was not built. tests/test_files.sh | 5 +++++ 1 file changed, 5 insertions(+) commit 6133a3f30049d3beaf7d22535b1e5d38e109be4e Author: Lasse Collin Date: 2024-01-23 16:11:54 +0200 xz: Man page: Add more examples of LZMA2 options with BCJ filters. src/xz/xz.1 | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) commit 50255feeaabcc7e7db22b858a6bd64a9b5b4f16d Author: Lasse Collin Date: 2024-01-23 00:09:48 +0200 liblzma: RISC-V filter: Use byte-by-byte access. Not all RISC-V processors support fast unaligned access so it's better to read only one byte in the main loop. This can be faster even on x86-64 when compared to reading 32 bits at a time as half the time the address is only 16-bit aligned. The downside is larger code size on archs that do support fast unaligned access. src/liblzma/simple/riscv.c | 114 +++++++++++++++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 30 deletions(-) commit db5eb5f563e8baa8d912ecf576f53391ff861596 Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 xz: Update xz -lvv for RISC-V filter. Version 5.6.0 will be shown, even though upcoming alphas and betas will be able to support this filter. 5.6.0 looks nicer in the output and people shouldn't be encouraged to use an unstable version in production in any way. src/xz/list.c | 10 ++++++++++ 1 file changed, 10 insertions(+) commit e2870db5be1503e6a489fc3d47daf950d6f62723 Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 Tests: Add two RISC-V Filter test files. These test files achieve 100% code coverage in src/liblzma/simple/riscv.c. They contain all of the instructions that should be filtered and a few cases that should not. tests/files/README | 8 ++++++++ tests/files/good-1-riscv-lzma2-1.xz | Bin 0 -> 7512 bytes tests/files/good-1-riscv-lzma2-2.xz | Bin 0 -> 7516 bytes 3 files changed, 8 insertions(+) commit b26a89869315ece2f6d9d10d32d45f672550f245 Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 xz: Update message in --long-help for RISC-V Filter. src/xz/message.c | 1 + 1 file changed, 1 insertion(+) commit 283f778908873eca61388029fc418fa800c9d7d7 Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 xz: Update the man page for the RISC-V Filter. A special note was added to suggest using four-byte alignment when the compressed instruction extension is not present in a RISC-V binary. src/xz/xz.1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit ac3691ccca051d67f60b4a3b05b88e511d0b1b28 Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 Tests: Add RISC-V Filter test in test_compress.sh. tests/test_compress.sh | 1 + 1 file changed, 1 insertion(+) commit 2959dbc7358efcf421ce51bc9cd7eae8fdd8fec4 Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 liblzma: Update string_conversion.c to support RISC-V Filter. src/liblzma/common/string_conversion.c | 5 +++++ 1 file changed, 5 insertions(+) commit 34372a5adbe5a7f6bf29498410ba3a463a720966 Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 CMake: Support RISC-V BCJ Filter for encoding and decoding. CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) commit 440a2eccb082dc13400c09e22308a58fef85146c Author: Jia Tan Date: 2024-01-22 23:33:39 +0800 liblzma: Add RISC-V BCJ filter. The new Filter ID is 0x0B. Thanks to Chien Wong for the initial version of the Filter, the xz CLI updates, and the Autotools build system modifications. Thanks to Igor Pavlov for his many contributions to the design of the filter. configure.ac | 4 +- src/liblzma/api/lzma/bcj.h | 5 + src/liblzma/common/filter_common.c | 9 + src/liblzma/common/filter_decoder.c | 8 + src/liblzma/common/filter_encoder.c | 10 + src/liblzma/simple/Makefile.inc | 4 + src/liblzma/simple/riscv.c | 688 ++++++++++++++++++++++++++++++++++++ src/liblzma/simple/simple_coder.h | 9 + src/xz/args.c | 7 + 9 files changed, 742 insertions(+), 2 deletions(-) commit 5540f4329bbdb4deb4850d4af48b18ad074bba19 Author: Jia Tan Date: 2024-01-19 23:08:14 +0800 Docs: Update .xz file format specification to 1.2.0. The new RISC-V filter was added to the specification, in addition to updating the specification URL. doc/xz-file-format.txt | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) commit 22d86192f8cf00902a1f90ee2a83ca600794459b Author: Jia Tan Date: 2024-01-19 23:08:14 +0800 xz: Update website URLs in the man pages. src/xz/xz.1 | 6 +++--- src/xzdec/xzdec.1 | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) commit 6b63c4c6139fa1bb21b570521d3d2b4a608bc34d Author: Jia Tan Date: 2024-01-19 23:08:14 +0800 liblzma: Update website URL. dos/config.h | 2 +- src/liblzma/api/lzma.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) commit fce4758018f3a3589236f3fe7999fd9dd08c77e9 Author: Jia Tan Date: 2024-01-19 23:08:14 +0800 Docs: Update website URLs. .github/SECURITY.md | 2 +- COPYING | 3 ++- README | 4 ++-- doc/faq.txt | 2 +- doc/lzma-file-format.txt | 18 +++++++++--------- windows/README-Windows.txt | 3 ++- 6 files changed, 17 insertions(+), 15 deletions(-) commit c26812c5b2c8a2a47f43214afe6b0b840c73e4f5 Author: Jia Tan Date: 2024-01-19 23:08:14 +0800 Build: Update website URL. CMakeLists.txt | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit fbb3ce541ef79cad1710e88a27a5babb5f6f8e5b Author: Lasse Collin Date: 2024-01-11 15:01:50 +0200 liblzma: CRC: Add a comment to crc_x86_clmul.h about BUILDING_ macros. src/liblzma/check/crc_x86_clmul.h | 6 ++++++ 1 file changed, 6 insertions(+) commit 4f518c1b6b7b7ce5dcefea81acd44d7a086a8882 Author: Lasse Collin Date: 2024-01-11 15:22:36 +0200 liblzma: CRC: Remove crc_always_inline, use lzma_always_inline instead. Now crc_simd_body() in crc_x86_clmul.h is only called once in a translation unit, we no longer need to be so cautious about ensuring the always-inline behavior. src/liblzma/check/crc_common.h | 20 -------------------- src/liblzma/check/crc_x86_clmul.h | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) commit 35c03ec6bf66f1b159964c9721a2dce0e2859b20 Author: Lasse Collin Date: 2024-01-11 14:39:46 +0200 liblzma: CRC: Update CLMUL comments to more generic wording. src/liblzma/check/crc32_fast.c | 16 ++++++++-------- src/liblzma/check/crc64_fast.c | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) commit 66f080e8016129576536482ac377e2ecac7a2b90 Author: Lasse Collin Date: 2024-01-10 18:23:31 +0200 liblzma: Rename arch-specific CRC functions and macros. CRC_CLMUL was split to CRC_ARCH_OPTIMIZED and CRC_X86_CLMUL. CRC_ARCH_OPTIMIZED is defined when an arch-optimized version is used. Currently the x86 CLMUL implementations are the only arch-optimized versions, and these also use the CRC_x86_CLMUL macro to tell when crc_x86_clmul.h needs to be included. is_clmul_supported() was renamed to is_arch_extension_supported(). crc32_clmul() and crc64_clmul() were renamed to crc32_arch_optimized() and crc64_arch_optimized(). This way the names make sense with arch-specific non-CLMUL implementations as well. src/liblzma/check/crc32_fast.c | 13 +++++++------ src/liblzma/check/crc64_fast.c | 13 +++++++------ src/liblzma/check/crc_common.h | 9 ++++++--- src/liblzma/check/crc_x86_clmul.h | 21 +++++++++++---------- 4 files changed, 31 insertions(+), 25 deletions(-) commit 3dbed75b0b9c7087c76fe687acb5cf582cd57b99 Author: Lasse Collin Date: 2024-01-10 18:19:21 +0200 liblzma: Fix a comment in crc_common.h. src/liblzma/check/crc_common.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 419f55f9dfc2df8792902b8953d50690121afeea Author: Lasse Collin Date: 2023-10-20 23:35:10 +0300 liblzma: Avoid extern lzma_crc32_clmul() and lzma_crc64_clmul(). A CLMUL-only build will have the crcxx_clmul() inlined into lzma_crcxx(). Previously a jump to the extern lzma_crcxx_clmul() was needed. Notes about shared liblzma on ELF platforms: - On platforms that support ifunc and -fvisibility=hidden, this was silly because CLMUL-only build would have that single extra jump instruction of extra overhead. - On platforms that support neither -fvisibility=hidden nor linker version script (liblzma*.map), jumping to lzma_crcxx_clmul() would go via PLT so a few more instructions of overhead (still not a big issue but silly nevertheless). There was a downside with static liblzma too: if an application only needs lzma_crc64(), static linking would make the linker include the CLMUL code for both CRC32 and CRC64 from crc_x86_clmul.o even though the CRC32 code wouldn't be needed, thus increasing code size of the executable (assuming that -ffunction-sections isn't used). Also, now compilers are likely to inline crc_simd_body() even if they don't support the always_inline attribute (or MSVC's __forceinline). Quite possibly all compilers that build the code do support such an attribute. But now it likely isn't a problem even if the attribute wasn't supported. Now all x86-specific stuff is in crc_x86_clmul.h. If other archs The other archs can then have their own headers with their own is_clmul_supported() and crcxx_clmul(). Another bonus is that the build system doesn't need to care if crc_clmul.c is needed. is_clmul_supported() stays as inline function as it's not needed when doing a CLMUL-only build (avoids a warning about unused function). CMakeLists.txt | 7 +- configure.ac | 1 - src/liblzma/check/Makefile.inc | 6 +- src/liblzma/check/crc32_fast.c | 9 ++- src/liblzma/check/crc64_fast.c | 9 ++- src/liblzma/check/crc_common.h | 64 ---------------- src/liblzma/check/{crc_clmul.c => crc_x86_clmul.h} | 86 ++++++++++++++++++---- 7 files changed, 91 insertions(+), 91 deletions(-) commit e3833e297dfb5021a197bda34ba2a795e30aaf8a Author: Lasse Collin Date: 2023-10-21 00:06:52 +0300 liblzma: crc_clmul.c: Add crc_attr_target macro. This reduces the number of the complex #if directives. src/liblzma/check/crc_clmul.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) commit d164ac0e62904126f7920c25f9a2875c8cd28b97 Author: Lasse Collin Date: 2023-10-20 22:49:48 +0300 liblzma: Simplify existing cases with lzma_attr_no_sanitize_address. src/liblzma/check/crc_clmul.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) commit 9523c1300d22fa715765c181cf991d14d6112fb1 Author: Lasse Collin Date: 2023-10-20 21:53:35 +0300 liblzma: #define crc_attr_no_sanitize_address in crc_common.h. src/liblzma/check/crc_common.h | 10 ++++++++++ 1 file changed, 10 insertions(+) commit 93d144f0930821590524247bd174afd38003d7f0 Author: Lasse Collin Date: 2023-10-20 23:25:14 +0300 liblzma: CRC: Add empty lines. And remove one too. src/liblzma/check/crc32_fast.c | 2 ++ src/liblzma/check/crc64_fast.c | 3 +++ src/liblzma/check/crc_clmul.c | 1 - 3 files changed, 5 insertions(+), 1 deletion(-) commit 0c7e854ffd27f1cec2e9b0e61601d6f90bfa10ae Author: Lasse Collin Date: 2023-10-20 23:19:33 +0300 liblzma: crc_clmul.c: Tidy up the location of MSVC pragma. It makes no difference in practice. src/liblzma/check/crc_clmul.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 15cf3f04f270d707a5c91cc0208b23b6db42b774 Author: Lasse Collin Date: 2023-12-20 21:16:24 +0200 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit cd64dd70d5665b6048829c45772d08606f44672e Author: Lasse Collin Date: 2023-12-20 21:15:16 +0200 liblzma: Use 8-byte method in memcmplen.h on ARM64. It requires fast unaligned access to 64-bit integers and a fast instruction to count leading zeros in a 64-bit integer (__builtin_ctzll()). This perhaps should be enabled on some other archs too. Thanks to Chenxi Mao for the original patch: https://github.com/tukaani-project/xz/pull/75 (the first commit) According to the numbers there, this may improve encoding speed by about 3-5 %. This enables the 8-byte method on MSVC ARM64 too which should work but wasn't tested. src/liblzma/common/memcmplen.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) commit 12c90c00f05e19da3c0c91d8cd8e0d0d45965606 Author: Lasse Collin Date: 2023-12-20 21:01:06 +0200 liblzma: Check also for __clang__ in memcmplen.h. This change hopefully makes no practical difference as Clang likely was detected via __GNUC__ or _MSC_VER already. src/liblzma/common/memcmplen.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 133c5851eb917c6d99d0b623c1689c8518e65f38 Author: Jia Tan Date: 2023-12-21 21:39:08 +0800 Translations: Update the French translation. po/fr.po | 632 +++++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 370 insertions(+), 262 deletions(-) commit 710cbc186cad0ac601c38bd6bf31167648a5581e Author: Jia Tan Date: 2023-12-21 16:39:53 +0800 xz: Add a comment to Capsicum sandbox setup. This comment is repeated in xzdec.c to help remind us why all the capabilities are removed from stdin in certain situations. src/xz/file_io.c | 1 + 1 file changed, 1 insertion(+) commit 4e1c695676bafbaecc9fb307f6ee94138ae72c12 Author: Jia Tan Date: 2023-12-20 22:19:19 +0800 Docs: Update --enable-sandbox option in INSTALL. xzdec now also uses the sandbox when its configured. INSTALL | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) commit ebddf20214143a8e002ab897e95e880bb4c5ac44 Author: Jia Tan Date: 2023-12-20 22:39:13 +0800 CMake: Move sandbox detection outside of xz section. The sandbox is now enabled for xzdec as well, so it no longer belongs in just the xz section. xz and xzdec are always built, except for older MSVC versions, so there isn't a need to conditionally show the sandbox configuration. CMake will do a little unecessary work on older MSVC versions that can't build xz or xzdec, but this is a very small downside. CMakeLists.txt | 178 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 98 insertions(+), 80 deletions(-) commit 5feb09266fd2928ec0a4dcb98c1dc7f053111316 Author: Jia Tan Date: 2023-12-20 22:43:44 +0800 Build: Allow sandbox to be configured for just xzdec. If xz is disabled, then xzdec can still use the sandbox. configure.ac | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) commit d74fb5f060b76db709b50f5fd37490394e52f975 Author: Jia Tan Date: 2023-12-19 21:18:28 +0800 xzdec: Add sandbox support for Pledge, Capsicum, and Landlock. A very strict sandbox is used when the last file is decompressed. The likely most common use case of xzdec is to decompress a single file. The Pledge sandbox is applied to the entire process with slightly more relaxed promises, until the last file is processed. Thanks to Christian Weisgerber for the initial patch adding Pledge sandboxing. src/xzdec/xzdec.c | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 139 insertions(+), 7 deletions(-) commit b34b6a9912d6165e34ba0db151b7f9941d2e06d5 Author: Jia Tan Date: 2023-12-20 21:31:34 +0800 liblzma: Initialize lzma_lz_encoder pointers with NULL. This fixes the recent change to lzma_lz_encoder that used memzero instead of the NULL constant. On some compilers the NULL constant (always 0) may not equal the NULL pointer (this only needs to guarentee to not point to valid memory address). Later code compares the pointers to the NULL pointer so we must initialize them with the NULL pointer instead of 0 to guarentee code correctness. src/liblzma/lz/lz_encoder.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) commit 183a62f0b540ff4d23cc19b2b6bc2525f0bd64df Author: Jia Tan Date: 2023-12-16 20:51:38 +0800 liblzma: Set all values in lzma_lz_encoder to NULL after allocation. The first member of lzma_lz_encoder doesn't necessarily need to be set to NULL since it will always be set before anything tries to use it. However the function pointer members must be set to NULL since other functions rely on this NULL value to determine if this behavior is supported or not. This fixes a somewhat serious bug, where the options_update() and set_out_limit() function pointers are not set to NULL. This seems to have been forgotten since these function pointers were added many years after the original two (code() and end()). The problem is that by not setting this to NULL we are relying on the memory allocation to zero things out if lzma_filters_update() is called on a LZMA1 encoder. The function pointer for set_out_limit() is less serious because there is not an API function that could call this in an incorrect way. set_out_limit() is only called by the MicroLZMA encoder, which must use LZMA1 where set_out_limit() is always set. Its currently not possible to call set_out_limit() on an LZMA2 encoder at this time. So calling lzma_filters_update() on an LZMA1 encoder had undefined behavior since its possible that memory could be manipulated so the options_update member pointed to a different instruction sequence. This is unlikely to be a bug in an existing application since it relies on calling lzma_filters_update() on an LZMA1 encoder in the first place. For instance, it does not affect xz because lzma_filters_update() can only be used when encoding to the .xz format. This is fixed by using memzero() to set all members of lzma_lz_encoder to NULL after it is allocated. This ensures this mistake will not occur here in the future if any additional function pointers are added. src/liblzma/lz/lz_encoder.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) commit 1a1bb381db7a20cf86cb45a350e5cca35224d017 Author: Jia Tan Date: 2023-12-16 20:30:55 +0800 liblzma: Tweak a comment. src/liblzma/lz/lz_encoder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 55810780e04f759747b02683fb8020b8cd022a85 Author: Jia Tan Date: 2023-12-16 20:28:21 +0800 liblzma: Make parameter names in function definition match declaration. lzma_raw_encoder() and lzma_raw_encoder_init() used "options" as the parameter name instead of "filters" (used by the declaration). "filters" is more clear since the parameter represents the list of filters passed to the raw encoder, each of which contains filter options. src/liblzma/common/filter_encoder.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) commit 5dad6f628af742bab826819760deb677597445f7 Author: Jia Tan Date: 2023-12-16 20:18:47 +0800 liblzma: Improve lzma encoder init function consistency. lzma_encoder_init() did not check for NULL options, but lzma2_encoder_init() did. This is more of a code style improvement than anything else to help make lzma_encoder_init() and lzma2_encoder_init() more similar. src/liblzma/lzma/lzma_encoder.c | 3 +++ 1 file changed, 3 insertions(+) commit e1b1a9d6370b788bd6078952c6c201e12bc27cbf Author: Jia Tan Date: 2023-12-16 11:20:20 +0800 Docs: Update repository URL in Changelog. ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit f9b82bc64a9405e486575c65c1729229eb0a8198 Author: Jia Tan Date: 2023-12-15 16:56:31 +0800 CI: Update Upload Artifact Action. .github/workflows/ci.yml | 2 +- .github/workflows/windows-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit d0b24efe6cdc47db5b0fdf6306f70a2e0e63e49e Author: Jia Tan Date: 2023-12-07 21:48:07 +0800 Tests: Silence -Wsign-conversion warning on GCC version < 10. Since GCC version 10, GCC no longer complains about simple implicit integer conversions with Arithmetic operators. For instance: uint8_t a = 5; uint32_t b = a + 5; Give a warning on GCC 9 and earlier but this: uint8_t a = 5; uint32_t b = (a + 5) * 2; Gives a warning with GCC 10+. tests/test_block_header.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 4a972a8ee3ed88ac14067c1d2f15b78988e5dae8 Author: Jia Tan Date: 2023-12-06 18:39:03 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit ee2f48350099201694a7586e41d7aa2f09fc74da Author: Jia Tan Date: 2023-12-06 18:30:25 +0800 Tests: Minor cleanups to OSS-Fuzz files. Most of these fixes are small typos and tweaks. A few were caused by bad advice from me. Here is the summary of what is changed: - Author line edits - Small comment changes/additions - Using the return value in the error messages in the fuzz targets' coder initialization code - Removed fuzz_encode_stream.options. This set a max length, which may prevent some worthwhile code paths from being properly exercised. - Removed the max_len option from fuzz_decode_stream.options for the same reason as fuzz_encode_stream. The alone decoder fuzz target still has this restriction. - Altered the dictionary contents for fuzz_lzma.dict. Instead of keeping the properties static and varying the dictionary size, the properties are varied and the dictionary size is kept small. The dictionary size doesn't have much impact on the code paths but the properties do. Closes: https://github.com/tukaani-project/xz/pull/73 tests/ossfuzz/Makefile | 3 ++ tests/ossfuzz/config/fuzz_decode_stream.options | 1 - tests/ossfuzz/config/fuzz_lzma.dict | 34 +++++++++++----------- tests/ossfuzz/fuzz_common.h | 16 +++++------ tests/ossfuzz/fuzz_decode_alone.c | 15 +++++----- tests/ossfuzz/fuzz_decode_stream.c | 15 +++++----- tests/ossfuzz/fuzz_encode_stream.c | 38 +++++++++++++++---------- 7 files changed, 66 insertions(+), 56 deletions(-) commit 483bb90eec7c83e1c2bcd06287714afd62d8c17d Author: Maksym Vatsyk Date: 2023-12-05 16:31:09 +0100 Tests: Add fuzz_encode_stream ossfuzz target. This fuzz target handles .xz stream encoding. The first byte of input is used to dynamically set the preset level in order to increase the fuzz coverage of complex critical code paths. tests/ossfuzz/config/fuzz_encode_stream.options | 2 + tests/ossfuzz/fuzz_encode_stream.c | 79 +++++++++++++++++++++++++ 2 files changed, 81 insertions(+) commit 7ca8c9869df82756c3128c4fcf1058da4d18aa48 Author: Maksym Vatsyk Date: 2023-12-04 17:23:24 +0100 Tests: Add fuzz_decode_alone OSS-Fuzz target This fuzz target that handles LZMA alone decoding. A new fuzz dictionary .dict was also created with common LZMA header values to help speed up the discovery of valid headers. tests/ossfuzz/config/fuzz_decode_alone.options | 3 ++ tests/ossfuzz/config/fuzz_lzma.dict | 22 ++++++++++++++ tests/ossfuzz/fuzz_decode_alone.c | 41 ++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) commit 37581a77ad5a49615325b1d1925fdc402b1e1d5a Author: Maksym Vatsyk Date: 2023-12-04 17:21:29 +0100 Tests: Update OSS-Fuzz Makefile. All .c files can be built as separate fuzz targets. This simplifies the Makefile by allowing us to use wildcards instead of having a Makefile target for each fuzz target. tests/ossfuzz/Makefile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) commit 28ce6a1c2a74866c51f7996a6869679c236d3c94 Author: Maksym Vatsyk Date: 2023-12-04 17:20:08 +0100 Tests: Move common OSS-Fuzz target code to .h file. tests/ossfuzz/fuzz_common.h | 56 ++++++++++++++++++++++++++++++++++++ tests/ossfuzz/fuzz_decode_stream.c | 59 ++++++++++---------------------------- 2 files changed, 71 insertions(+), 44 deletions(-) commit bf0521ea1591c25b9d510c1b8be86073e9d847c6 Author: Maksym Vatsyk Date: 2023-12-04 17:18:20 +0100 Tests: Rename OSS-Fuzz files. tests/ossfuzz/config/fuzz.options | 2 -- tests/ossfuzz/config/fuzz_decode_stream.options | 3 +++ tests/ossfuzz/config/{fuzz.dict => fuzz_xz.dict} | 0 tests/ossfuzz/{fuzz.c => fuzz_decode_stream.c} | 0 4 files changed, 3 insertions(+), 2 deletions(-) commit 685094b8e1c1aa1bf934de0366ca42ef599d25f7 Author: Jia Tan Date: 2023-11-30 23:10:43 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 3b3023e00b0071e10f589bbc3674e0ec432b8add Author: Kian-Meng Ang Date: 2023-11-30 23:01:19 +0800 Tests: Fix typos tests/test_index.c | 2 +- tests/test_lzip_decoder.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) commit 424d46ead8cbc0da57f406b76926ec4ed47437f5 Author: Kian-Meng Ang Date: 2023-11-30 22:59:47 +0800 xz: Fix typo src/xz/file_io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 35558adf9c45e5597f2c8dbd969885dd484038d2 Author: Jia Tan Date: 2023-11-30 20:41:00 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit fd170e8557727bed6bec0518c16415064d972e4e Author: Jia Tan Date: 2023-11-22 21:20:12 +0800 CI: Test musl libc builds on Ubuntu runner. .github/workflows/ci.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) commit db2b4aa068a492c0013279a4ed43803e8ff9bb3e Author: Jia Tan Date: 2023-11-22 21:12:15 +0800 CI: Allow ci_build.sh to set a different C compiler. build-aux/ci_build.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) commit ff7badef53c2cd698d4b72b945f34dfd0835e13c Author: Jia Tan Date: 2023-11-24 21:19:12 +0800 CMake: Use consistent indentation with check_c_source_compiles(). CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit d4af167570f2c14b002ee18a39d5b1e7e5a892b1 Author: Jia Tan Date: 2023-11-22 20:33:36 +0800 CMake: Change __attribute__((__ifunc__())) detection. This renames ALLOW_ATTR_IFUNC to USE_ATTR_IFUNC and applies the ifunc detection changes that were made to the Autotools build. Fixes: https://github.com/tukaani-project/xz/issues/70 CMakeLists.txt | 53 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 8 deletions(-) commit 20ecee40a0053fd16371ef0628046bf45e548d72 Author: Jia Tan Date: 2023-11-24 20:19:11 +0800 Docs: Update INSTALL for --enable_ifunc change. INSTALL | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) commit ffb456593d695d70052a2f71c7a2e6269217d194 Author: Jia Tan Date: 2023-11-21 20:56:55 +0800 Build: Change --enable-ifunc handling. Some compilers support __attribute__((__ifunc__())) even though the dynamic linker does not. The compiler is able to create the binary but it will fail on startup. So it is not enough to just test if the attribute is supported. The default value for enable_ifunc is now auto, which will attempt to compile a program using __attribute__((__ifunc__())). There are additional checks in this program if glibc is being used or if it is running on FreeBSD. Setting --enable-ifunc will skip this test and always enable __attribute__((__ifunc__())), even if is not supported. configure.ac | 61 +++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 17 deletions(-) commit 12b89bcc9915090eb42ae638e565af44b6832a23 Author: Lasse Collin Date: 2023-11-23 17:39:10 +0200 xz: Tweak a comment. src/xz/util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 2ab2e4b5a542eab93902985ce4e642719a8b7a4e Author: Jia Tan Date: 2023-11-23 22:13:39 +0800 xz: Use is_tty() in message.c. src/xz/message.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) commit 584e3a258f32d579b1d07f99b4dc6e856c10ac7e Author: Jia Tan Date: 2023-11-23 22:04:35 +0800 xz: Create separate is_tty() function. The new is_tty() will report if a file descriptor is a terminal or not. On POSIX systems, it is a wrapper around isatty(). However, the native Windows implementation of isatty() will return true for all character devices, not just terminals. So is_tty() has a special case for Windows so it can use alternative Windows API functions to determine if a file descriptor is a terminal. This fixes a bug with MSVC and MinGW-w64 builds that refused to read from or write to non-terminal character devices because xz thought it was a terminal. For instance: xz foo -c > /dev/null would fail because /dev/null was assumed to be a terminal. src/xz/util.c | 30 +++++++++++++++++++++++------- src/xz/util.h | 14 ++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) commit 6b05f827f50e686537e9a23c49c5aa4c0aa6b23d Author: Jia Tan Date: 2023-11-22 20:39:41 +0800 tuklib_integer: Fix typo discovered by codespell. Based on internet dictionary searches, 'choise' is an outdated spelling of 'choice'. src/common/tuklib_integer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 659aca0d695807c0762d4101765189e4e33d1e2c Author: Lasse Collin Date: 2023-11-17 19:35:19 +0200 xz: Move the check for --suffix with --format=raw a few lines earlier. Now it reads from argv[] instead of args->arg_names. src/xz/args.c | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) commit ca278eb2b7f5a4940f5ab18955297b398d423824 Author: Jia Tan Date: 2023-11-17 20:35:11 +0800 Tests: Create test_suffix.sh. This tests some complicated interactions with the --suffix= option. The suffix option must be used with --format=raw, but can optionally be used to override the default .xz suffix. This test also verifies some recent bugs have been correctly solved and to hopefully avoid further regressions in the future. tests/Makefile.am | 2 + tests/test_suffix.sh | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) commit 2a732aba22da1b0d4a1241cb32280ed010ba03ce Author: Jia Tan Date: 2023-11-17 20:19:26 +0800 xz: Fix a bug with --files and --files0 in raw mode without a suffix. The following command caused a segmentation fault: xz -Fraw --lzma1 --files=foo when foo was a valid file. The usage of --files or --files0 was not being checked when compressing or decompressing in raw mode without a suffix. The suffix checking code was meant to validate that all files to be processed are "-" (if not writing to standard out), meaning the data is only coming from standard in. In this case, there were no file names to check since --files and --files0 store their file name in a different place. Later code assumed the suffix was set and caused a segmentation fault. Now, the above command results in an error. src/xz/args.c | 5 +++++ 1 file changed, 5 insertions(+) commit 299920bab9ae258a247366339264e8aefca9e3ce Author: Jia Tan Date: 2023-11-17 20:04:58 +0800 Tests: Fix typo in a comment. tests/test_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit f481523baac946fa3bc13d79186ffaf0c0b818a7 Author: Jia Tan Date: 2023-11-15 23:40:13 +0800 xz: Refactor suffix test with raw format. The previous version set opt_stdout, but this caused an issue with copying an input file to standard out when decompressing an unknown file type. The following needs to result in an error: echo foo | xz -df since -c, --stdout is not used. This fixes the previous error by not setting opt_stdout. src/xz/args.c | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) commit 837ea40b1c9d4998cac4500b55171bf33e0c31a6 Author: Jia Tan Date: 2023-11-14 20:27:46 +0800 xz: Move suffix check after stdout mode is detected. This fixes a bug introduced in cc5aa9ab138beeecaee5a1e81197591893ee9ca0 when the suffix check was initially moved. This caused a situation that previously worked: echo foo | xz -Fraw --lzma1 | wc -c to fail because the old code knew that this would write to standard out so a suffix was not needed. src/xz/args.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) commit d4f4a4d040ef47a5e82dffd0f067e92716606ddf Author: Jia Tan Date: 2023-11-14 20:27:04 +0800 xz: Detect when all data will be written to standard out earlier. If the -c, --stdout argument is not used, then we can still detect when the data will be written to standard out if all of the provided filenames are "-" (denoting standard in) or if no filenames are provided. src/xz/args.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) commit 2ade7246e7ba729a91460d2fab0f4c7b89d3998b Author: Jia Tan Date: 2023-11-09 01:21:53 +0800 liblzma: Add missing comments to lz_encoder.h. src/liblzma/lz/lz_encoder.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) commit 5fe1450603dc625340b8b7866fb4a83ff748ad06 Author: Jia Tan Date: 2023-11-01 20:18:30 +0800 Add NEWS for 5.4.5. NEWS | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) commit 46007049cd42e606543dbe650feb17bdf4469c29 Author: Lasse Collin Date: 2023-10-31 21:41:09 +0200 liblzma: Fix compilation of fastpos_tablegen.c. The macro lzma_attr_visibility_hidden has to be defined to make fastpos.h usable. The visibility attribute is irrelevant to fastpos_tablegen.c so simply #define the macro to an empty value. fastpos_tablegen.c is never built by the included build systems and so the problem wasn't noticed earlier. It's just a standalone program for generating fastpos_table.c. Fixes: https://github.com/tukaani-project/xz/pull/69 Thanks to GitHub user Jamaika1. src/liblzma/lzma/fastpos_tablegen.c | 2 ++ 1 file changed, 2 insertions(+) commit 148e20607e95781558bdfc823ecba07b7af4b590 Author: Jia Tan Date: 2023-10-31 21:51:40 +0800 Build: Fix text wrapping in an output message. configure.ac | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) commit 8c36ab79cbf23104ce7a3d533d5ac98cd492e57c Author: Lasse Collin Date: 2023-10-30 18:09:53 +0200 liblzma: Add a note why crc_always_inline exists for now. Solaris Studio is a possible example (not tested) which supports the always_inline attribute but might not get detected by the common.h #ifdefs. src/liblzma/check/crc_common.h | 5 +++++ 1 file changed, 5 insertions(+) commit e7a86b94cd247435ac96bc79ba528b690b9ca388 Author: Lasse Collin Date: 2023-10-22 17:59:11 +0300 liblzma: Use lzma_always_inline in memcmplen.h. src/liblzma/common/memcmplen.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) commit dcfe5632992fb7f06f921da13fcdd84f83d0d285 Author: Lasse Collin Date: 2023-10-30 17:43:03 +0200 liblzma: #define lzma_always_inline in common.h. src/liblzma/common/common.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) commit 41113fe30a47f6fd3e30cb4494dd538e86212edf Author: Lasse Collin Date: 2023-10-22 17:15:32 +0300 liblzma: Use lzma_attr_visibility_hidden on private extern declarations. These variables are internal to liblzma and not exposed in the API. src/liblzma/check/check.h | 7 +++++++ src/liblzma/common/stream_flags_common.h | 3 +++ src/liblzma/lz/lz_encoder_hash.h | 1 + src/liblzma/lzma/fastpos.h | 1 + src/liblzma/rangecoder/price.h | 1 + 5 files changed, 13 insertions(+) commit a2f5ca706acc6f7715b8d260a8c6ed50d7717478 Author: Lasse Collin Date: 2023-10-22 17:08:39 +0300 liblzma: #define lzma_attr_visibility_hidden in common.h. In ELF shared libs: -fvisibility=hidden affects definitions of symbols but not declarations.[*] This doesn't affect direct calls to functions inside liblzma as a linker can replace a call to lzma_foo@plt with a call directly to lzma_foo when -fvisibility=hidden is used. [*] It has to be like this because otherwise every installed header file would need to explictly set the symbol visibility to default. When accessing extern variables that aren't defined in the same translation unit, compiler assumes that the variable has the default visibility and thus indirection is needed. Unlike function calls, linker cannot optimize this. Using __attribute__((__visibility__("hidden"))) with the extern variable declarations tells the compiler that indirection isn't needed because the definition is in the same shared library. About 15+ years ago, someone told me that it would be good if the CRC tables would be defined in the same translation unit as the C code of the CRC functions. While I understood that it could help a tiny amount, I didn't want to change the code because a separate translation unit for the CRC tables was needed for the x86 assembly code anyway. But when visibility attributes are supported, simply marking the extern declaration with the hidden attribute will get identical result. When there are only a few affected variables, this is trivial to do. I wish I had understood this back then already. src/liblzma/common/common.h | 11 +++++++++++ 1 file changed, 11 insertions(+) commit 2c7ee92e44e1e66f0a427555233eb22c78f6c4f8 Author: Lasse Collin Date: 2023-09-30 22:54:28 +0300 liblzma: Refer to MinGW-w64 instead of MinGW in the API headers. MinGW (formely a MinGW.org Project, later the MinGW.OSDN Project at ) has GCC 9.2.0 as the most recent GCC package (released 2021-02-02). The project might still be alive but majority of people have switched to MinGW-w64. Thus it seems clearer to refer to MinGW-w64 in our API headers too. Building with MinGW is likely to still work but I haven't tested it in the recent years. src/liblzma/api/lzma.h | 4 ++-- src/liblzma/api/lzma/version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) commit 597f49b61475438a43a417236989b2acc968a686 Author: Lasse Collin Date: 2023-09-27 00:58:17 +0300 CMake: Use -D_FILE_OFFSET_BITS=64 if (and only if) needed. A CMake option LARGE_FILE_SUPPORT is created if and only if -D_FILE_OFFSET_BITS=64 affects sizeof(off_t). This is needed on many 32-bit platforms and even with 64-bit builds with MinGW-w64 to get support for files larger than 2 GiB. CMakeLists.txt | 7 ++++- cmake/tuklib_large_file_support.cmake | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) commit 1bc548b8210366e44ba35b0b11577a8e328c1228 Author: Lasse Collin Date: 2023-09-30 02:14:25 +0300 CMake: Generate and install liblzma.pc if not using MSVC. Autotools based build uses -pthread and thus adds it to Libs.private in liblzma.pc. CMake doesn't use -pthread at all if pthread functions are available in libc so Libs.private doesn't get -pthread either. CMakeLists.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) commit 2add71966f891d315105d6245f724ed4f43a4eff Author: Lasse Collin Date: 2023-09-30 01:13:13 +0300 CMake: Rearrange the PACKAGE_ variables. The windres workaround now replaces spaces with \x20 so the package name isn't repeated. These changes will help with creation of liblzma.pc. CMakeLists.txt | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) commit a7d1b2825c49dc83f1910eeb8ba0f1dfbd886d91 Author: Lasse Collin Date: 2023-09-29 20:46:11 +0300 liblzma: Add Cflags.private to liblzma.pc.in for MSYS2. It properly adds -DLZMA_API_STATIC when compiling code that will be linked against static liblzma. Having it there on systems other than Windows does no harm. See: https://www.msys2.org/docs/pkgconfig/ src/liblzma/liblzma.pc.in | 1 + 1 file changed, 1 insertion(+) commit 80e0750e3996c1c659e972ce9cf789ca2e99f702 Author: Lasse Collin Date: 2023-09-27 22:46:20 +0300 CMake: Create liblzma.def when building liblzma.dll with MinGW-w64. CMakeLists.txt | 20 ++++++++++++++++++++ cmake/remove-ordinals.cmake | 26 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) commit 08d12595f486890cf601b87f36ee0ddbce57728e Author: Lasse Collin Date: 2023-10-26 21:44:42 +0300 CMake: Change one CMAKE_CURRENT_SOURCE_DIR to CMAKE_CURRENT_LIST_DIR. In this case they have identical values. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit e67aaf698de75c73443a5ec786781cbf2034461d Author: Lasse Collin Date: 2023-10-01 19:10:57 +0300 CMake/Windows: Fix the import library filename. Both PREFIX and IMPORT_PERFIX have to be set to "" to get liblzma.dll and liblzma.dll.a. CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) commit 88588b1246d8c26ffbc138b3e5c413c5f14c3179 Author: Lasse Collin Date: 2023-10-25 19:13:25 +0300 Build: Detect -fsanitize= in CFLAGS and incompatible build options. Now configure will fail if -fsanitize= is found in CFLAGS and sanitizer-incompatible ifunc or Landlock sandboxing would be used. These are incompatible with one or more sanitizers. It's simpler to reject all -fsanitize= uses instead of trying to pass those that might not cause problems. CMake-based build was updated similarly. It lets the configuration finish (SEND_ERROR instead of FATAL_ERROR) so that both error messages can be seen at once. CMakeLists.txt | 29 +++++++++++++++++++++++++++++ configure.ac | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 4 deletions(-) commit 5e3d890f8862a7d4fbef5e38e11b6c9fbd98f468 Author: Jia Tan Date: 2023-10-24 00:50:08 +0800 CI: Disable sandboxing in fsanitize=address,undefined job. The sandboxing on Linux now supports Landlock, which restricts all supported filesystem actions after xz opens the files it needs. The sandbox is only enabled when one file is input and we are writing to standard out. With fsanitize=address,undefined, the instrumentation needs to read additional files after the sandbox is in place. This forces all xz based test to fail, so the sandbox must instead be disabled. .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) commit b1408987ea832e2760e478ae960a636df17a1363 Author: Jia Tan Date: 2023-10-24 00:15:39 +0800 CI: Allow disabling the sandbox in ci_build.sh. build-aux/ci_build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) commit 91c435cf1c7a1e893706d4d716dfd361621ed824 Author: Lasse Collin Date: 2023-10-11 19:47:44 +0300 CMake: Don't shadow the cache entry ENABLE_THREADS with a normal variable. Using set(ENABLE_THREADS "posix") is confusing because it sets a new normal variable and leaves the cache entry with the same name unchanged. The intent wasn't to change the cache entry so this switches to a different variable name. CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) commit fa1609eb9393ecd30decfed4891c907829f06710 Author: Lasse Collin Date: 2023-10-09 22:28:49 +0300 Docs: Update INSTALL about sandboxing support. INSTALL | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) commit 8276c7f41c671eee4aa3239490658b23dcfd3021 Author: Lasse Collin Date: 2023-10-09 22:07:52 +0300 xz: Support basic sandboxing with Linux Landlock (ABI versions 1-3). It is enabled only when decompressing one file to stdout, similar to how Capsicum is used. Landlock was added in Linux 5.13. CMakeLists.txt | 12 +++++++++++- configure.ac | 11 ++++++++--- src/xz/file_io.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/xz/main.c | 19 +++++++++++++++++++ src/xz/private.h | 3 ++- 5 files changed, 98 insertions(+), 5 deletions(-) commit 3a1e9fd031b9320d769d63b503ef4e82e1b6ea8c Author: Lasse Collin Date: 2023-10-09 21:12:31 +0300 CMake: Edit threading related messages. It's mostly to change from "thread method" to "threading method". CMakeLists.txt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) commit bf011352528ae3539ea7b780b45b96736ee57a99 Author: Lasse Collin Date: 2023-10-09 20:59:24 +0300 CMake: Use FATAL_ERROR if user-supplied options aren't understood. This way typos are caught quickly and compounding error messages are avoided (a single typo could cause more than one error). This keeps using SEND_ERROR when the system is lacking a feature (like threading library or sandboxing method). This way the whole configuration log will be generated in case someone wishes to report a problem upstream. CMakeLists.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) commit 3f53870c249945d657ca3d75e0993e6267d71f75 Author: Lasse Collin Date: 2023-10-09 18:37:32 +0300 CMake: Add sandboxing support. CMakeLists.txt | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) commit 2e2cd11535ad77364cf021297e0b3f162fa3a3d0 Author: Lasse Collin Date: 2023-10-09 18:13:08 +0300 Simplify detection of Capsicum support. This removes support for FreeBSD 10.0 and 10.1 which used instead of . Support for FreeBSD 10.1 ended on 2016-12-31. So now FreeBSD >= 10.2 is required to enable Capsicum support. This also removes support for Capsicum on Linux (libcaprights) which seems to have been unmaintained since 2017 and Linux 4.11: https://github.com/google/capsicum-linux configure.ac | 4 +-- m4/ax_check_capsicum.m4 | 85 ------------------------------------------------- src/xz/Makefile.am | 2 +- src/xz/file_io.c | 14 +++----- src/xz/private.h | 2 +- 5 files changed, 9 insertions(+), 98 deletions(-) commit c57858b60e186d020b2dbaf7aabd9b32c71da824 Author: Lasse Collin Date: 2023-09-25 01:46:36 +0300 xz/Windows: Allow clock_gettime with POSIX threads. If winpthreads are used for threading, it's OK to use clock_gettime() from winpthreads too. src/xz/mytime.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) commit dd32f628bb5541ef4e8ce66966ef456a1934084c Author: Lasse Collin Date: 2023-09-25 01:39:26 +0300 mythread.h: Make MYTHREAD_POSIX compatible with MinGW-w64's winpthreads. This might be almost useless but it doesn't need much extra code either. src/common/mythread.h | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) commit 680e52cdd086e92691d8a0bca2c98815565f60ca Author: Lasse Collin Date: 2023-09-23 03:06:36 +0300 CMake: Check for clock_gettime() even on Windows. This mirrors configure.ac although currently MinGW-w64 builds don't use clock_gettime() even if it is found. CMakeLists.txt | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) commit 1c1a8c3ee4dad0064dbe63b8dbc4ac4bc679f419 Author: Lasse Collin Date: 2023-09-23 03:23:32 +0300 Build: Check for clock_gettime() even if not using POSIX threads. See the new comment in the code. This also makes the check for clock_gettime() run with MinGW-w64 with which we don't want to use clock_gettime(). The previous commit already took care of this situation. configure.ac | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) commit 46fd991cd2808ef62554853864c946232e7547f0 Author: Lasse Collin Date: 2023-09-24 22:58:53 +0300 xz/Windows: Ensure that clock_gettime() isn't used with MinGW-w64. This commit alone doesn't change anything in the real-world: - configure.ac currently checks for clock_gettime() only when using pthreads. - CMakeLists.txt doesn't check for clock_gettime() on Windows. So clock_gettime() wasn't used with MinGW-w64 before either. clock_gettime() provides monotonic time and it's better than gettimeofday() in this sense. But clock_gettime() is defined in winpthreads, and liblzma or xz needs nothing else from winpthreads. By avoiding clock_gettime(), we avoid the dependency on libwinpthread-1.dll or the need to link against the static version. As a bonus, GetTickCount64() and MinGW-w64's gettimeofday() can be faster than clock_gettime(CLOCK_MONOTONIC, &tv). The resolution is more than good enough for the progress indicator in xz. src/xz/mytime.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) commit cdb4d91f2464b50c985ef7b9517314ea237ddda7 Author: Lasse Collin Date: 2023-09-24 00:21:22 +0300 xz/Windows: Use GetTickCount64() with MinGW-w64 if using Vista threads. src/xz/mytime.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) commit 988e09f27b9b04a43d45d10f92782e0092ee27a9 Author: Jia Tan Date: 2023-10-20 19:17:46 +0800 liblzma: Move is_clmul_supported() back to crc_common.h. This partially reverts creating crc_clmul.c (8c0f9376f58c0696d5d6719705164d35542dd891) where is_clmul_supported() was moved, extern'ed, and renamed to lzma_is_clmul_supported(). This caused a problem when the function call to lzma_is_clmul_supported() results in a call through the PLT. ifunc resolvers run very early in the dynamic loading sequence, so the PLT may not be setup properly at this point. Whether the PLT is used or not for lzma_is_clmul_supported() depened upon the compiler-toolchain used and flags. In liblzma compiled with GCC, for instance, GCC will go through the PLT for function calls internal to liblzma if the version scripts and symbol visibility hiding are not used. If lazy-binding is disabled, then it would have made any program linked with liblzma fail during dynamic loading in the ifunc resolver. src/liblzma/check/crc32_fast.c | 2 +- src/liblzma/check/crc64_fast.c | 2 +- src/liblzma/check/crc_clmul.c | 45 ------------------------------------ src/liblzma/check/crc_common.h | 52 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 51 insertions(+), 50 deletions(-) commit 105c7ca90d4152942e0798580a37f736d02faa22 Author: Jia Tan Date: 2023-10-19 16:23:32 +0800 Build: Remove check for COND_CHECK_CRC32 in check/Makefile.inc. Currently crc32 is always enabled, so COND_CHECK_CRC32 must always be set. Because of this, it makes the recent change to conditionally compile check/crc_clmul.c appear wrong since that file has CLMUL implementations for both CRC32 and CRC64. src/liblzma/check/Makefile.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 139757170468f0f1fafdf0a8ffa74363d1ea1d0c Author: Jia Tan Date: 2023-10-19 16:09:01 +0800 CMake: Add ALLOW_CLMUL_CRC option to enable/disable CLMUL. The option is enabled by default, but will only be visible to a user listing cache variables or using a CMake GUI application if the immintrin.h header file is found. This mirrors our Autotools build --disable-clmul-crc functionality. CMakeLists.txt | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) commit c60b25569d414bb73b705977a4dd342f8f9f1965 Author: Jia Tan Date: 2023-10-19 00:22:50 +0800 liblzma: Fix -fsanitize=address failure with crc_clmul functions. After forcing crc_simd_body() to always be inlined it caused -fsanitize=address to fail for lzma_crc32_clmul() and lzma_crc64_clmul(). The __no_sanitize_address__ attribute was added to lzma_crc32_clmul() and lzma_crc64_clmul(), but not removed from crc_simd_body(). ASAN and inline functions behavior has changed over the years for GCC specifically, so while strictly required we will keep __attribute__((__no_sanitize_address__)) on crc_simd_body() in case this becomes a requirement in the future. Older GCC versions refuse to inline a function with ASAN if the caller and callee do not agree on sanitization flags (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89124#c3). If the function was forced to be inlined, it will not compile if the callee function has __no_sanitize_address__ but the caller doesn't. src/liblzma/check/crc_clmul.c | 6 ++++++ 1 file changed, 6 insertions(+) commit 9a78971261bc67622cbd7dae02f6966968ac1393 Author: Lasse Collin Date: 2023-10-14 20:16:13 +0300 tuklib_integer: Update the CMake test for fast unaligned access. cmake/tuklib_integer.cmake | 69 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 15 deletions(-) commit 2f81ac852bc5aafc91c8e2adc66b5114761703c4 Author: Lasse Collin Date: 2023-09-23 23:28:48 +0300 Build: Enabled unaligned access by default on PowerPC64LE and some RISC-V. PowerPC64LE wasn't tested but it seems like a safe change. POWER8 supports unaligned access in little endian mode. Testing on godbolt.org shows that GCC uses unaligned access by default. The RISC-V macro __riscv_misaligned_fast is very new and not in any stable compiler release yet. Documentation in INSTALL was updated to match. Documentation about an autodetection bug when using ARM64 GCC with -mstrict-align was added to INSTALL. CMake files weren't updated yet. INSTALL | 39 +++++++++++++++++++++++++++++++++++++-- m4/tuklib_integer.m4 | 34 +++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 9 deletions(-) commit c8f715f1bca4c30db814fcf1fd2fe88b8992ede2 Author: Lasse Collin Date: 2023-10-14 17:56:59 +0300 tuklib_integer: Revise unaligned reads and writes on strict-align archs. In XZ Utils context this doesn't matter much because unaligned reads and writes aren't used in hot code when TUKLIB_FAST_UNALIGNED_ACCESS isn't #defined. src/common/tuklib_integer.h | 256 ++++++++++++++++++++++++++++++++------------ 1 file changed, 189 insertions(+), 67 deletions(-) commit 6828242735cbf61b93d140383336e1e51a006f2d Author: Lasse Collin Date: 2023-09-23 02:21:49 +0300 tuklib_integer: Add missing write64be and write64le fallback functions. src/common/tuklib_integer.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) commit 1c8884f0af28b3a4690bb573cdf3240a8ec73416 Author: Jia Tan Date: 2023-10-18 19:57:10 +0800 liblzma: Set the MSVC optimization fix to only cover lzma_crc64_clmul(). After testing a 32-bit Release build on MSVC, only lzma_crc64_clmul() has the bug. crc_simd_body() and lzma_crc32_clmul() do not need the optimizations disabled. src/liblzma/check/crc_clmul.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) commit 5ce0f7a48bdf5c3b45430850a4487307afac6143 Author: Lasse Collin Date: 2023-10-18 14:30:00 +0300 liblzma: CRC_USE_GENERIC_FOR_SMALL_INPUTS cannot be used with ifunc. src/liblzma/check/crc_common.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) commit 27735380491bb5ce0d0f41d5244d89c1d0825f6b Author: Lasse Collin Date: 2023-10-17 21:53:11 +0300 liblzma: Include common.h in crc_common.h. crc_common.h depends on common.h. The headers include common.h except when there is a reason to not do so. src/liblzma/check/crc_clmul.c | 1 - src/liblzma/check/crc_common.h | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) commit e13b7947b92355c334edd594295d3a2c99c4bca1 Author: Jia Tan Date: 2023-10-18 01:23:26 +0800 liblzma: Add include guards to crc_common.h. src/liblzma/check/crc_common.h | 5 +++++ 1 file changed, 5 insertions(+) commit 40abd88afcc61a8157fcd12d78d491caeb8e12be Author: Jia Tan Date: 2023-10-18 22:50:25 +0800 liblzma: Add the crc_always_inline macro to crc_simd_body(). Forcing this to be inline has a significant speed improvement at the cost of a few repeated instructions. The compilers tested on did not inline this function since it is large and is used twice in the same translation unit. src/liblzma/check/crc_clmul.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit a5966c276bd6fa975f0389f8a8dc61393de750b0 Author: Jia Tan Date: 2023-10-18 22:48:19 +0800 liblzma: Create crc_always_inline macro. This macro must be used instead of the inline keyword. On MSVC, it is a replacement for __forceinline which is an MSVC specific keyword that should not be used with inline (it will issue a warning if it is). It does not use a build system check to determine if __attribute__((__always_inline__)) since all compilers that can use CLMUL extensions (except the special case for MSVC) should support this attribute. If this assumption is incorrect then it will result in a bug report instead of silently producing slow code. src/liblzma/check/crc_common.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) commit 96b663f67c0e738a99ba8f35d9f4ced9add74544 Author: Jia Tan Date: 2023-10-14 13:23:23 +0800 liblzma: Refactor CRC comments. A detailed description of the three dispatch methods was added. Also, duplicated comments now only appear in crc32_fast.c or were removed from both crc32_fast.c and crc64_fast.c if they appeared in crc_clmul.c. src/liblzma/check/crc32_fast.c | 64 +++++++++++++++++++++++++++++------------- src/liblzma/check/crc64_fast.c | 61 ++++++---------------------------------- 2 files changed, 53 insertions(+), 72 deletions(-) commit 8c0f9376f58c0696d5d6719705164d35542dd891 Author: Jia Tan Date: 2023-10-14 12:17:57 +0800 liblzma: Create crc_clmul.c. Both crc32_clmul() and crc64_clmul() are now exported from crc32_clmul.c as lzma_crc32_clmul() and lzma_crc64_clmul(). This ensures that is_clmul_supported() (now lzma_is_clmul_supported()) is not duplicated between crc32_fast.c and crc64_fast.c. Also, it encapsulates the complexity of the CLMUL implementations into a single file and reduces the complexity of crc32_fast.c and crc64_fast.c. Before, CLMUL code was present in crc32_fast.c, crc64_fast.c, and crc_common.h. During the conversion, various cleanups were applied to code (thanks to Lasse Collin) including: - Require using semicolons with MASK_/L/H/LH macros. - Variable typing and const handling improvements. - Improvements to comments. - Fixes to the pragmas used. - Removed unneeded variables. - Whitespace improvements. - Fixed CRC_USE_GENERIC_FOR_SMALL_INPUTS handling. - Silenced warnings and removed the need for some #pragmas CMakeLists.txt | 6 +- configure.ac | 6 +- src/liblzma/check/Makefile.inc | 3 + src/liblzma/check/crc32_fast.c | 120 +----------- src/liblzma/check/crc64_fast.c | 128 +------------ src/liblzma/check/crc_clmul.c | 414 +++++++++++++++++++++++++++++++++++++++++ src/liblzma/check/crc_common.h | 190 +------------------ 7 files changed, 444 insertions(+), 423 deletions(-) commit a3ebc2c516b09616638060806c841bd4bcf7bce3 Author: Jia Tan Date: 2023-10-14 10:23:03 +0800 liblzma: Define CRC_USE_IFUNC in crc_common.h. When ifunc is supported, we can define a simpler macro instead of repeating the more complex check in both crc32_fast.c and crc64_fast.c. src/liblzma/check/crc32_fast.c | 3 +-- src/liblzma/check/crc64_fast.c | 3 +-- src/liblzma/check/crc_common.h | 5 +++++ 3 files changed, 7 insertions(+), 4 deletions(-) commit f1cd9d7194f005cd66ec03c6635ceae75f90ef17 Author: Hans Jansen Date: 2023-10-12 19:37:01 +0200 liblzma: Added crc32_clmul to crc32_fast.c. src/liblzma/check/crc32_fast.c | 247 ++++++++++++++++++++++++++++++++++++++-- src/liblzma/check/crc32_table.c | 19 +++- 2 files changed, 255 insertions(+), 11 deletions(-) commit 93e6fb08b22c7c13be2dd1e7274fe78413436254 Author: Hans Jansen Date: 2023-10-12 19:23:40 +0200 liblzma: Moved CLMUL CRC logic to crc_common.h. crc64_fast.c was updated to use the code from crc_common.h instead. src/liblzma/check/crc64_fast.c | 257 ++--------------------------------------- src/liblzma/check/crc_common.h | 230 +++++++++++++++++++++++++++++++++++- 2 files changed, 240 insertions(+), 247 deletions(-) commit 233885a437f8b55a5c8442984ebc0aaa579e92de Author: Hans Jansen Date: 2023-10-12 19:07:50 +0200 liblzma: Rename crc_macros.h to crc_common.h. CMakeLists.txt | 2 +- src/liblzma/check/Makefile.inc | 2 +- src/liblzma/check/crc32_fast.c | 2 +- src/liblzma/check/crc64_fast.c | 2 +- src/liblzma/check/{crc_macros.h => crc_common.h} | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) commit 37947d4a7565b87e4cec8b89229d35b0a3f8d2cd Author: Gabriela Gutierrez Date: 2023-09-26 15:55:13 +0000 CI: Bump and ref actions by commit SHA in windows-ci.yml Referencing actions by commit SHA in GitHub workflows guarantees you are using an immutable version. Actions referenced by tags and branches are more vulnerable to attacks, such as the tag being moved to a malicious commit or a malicious commit being pushed to the branch. It's important to make sure the SHA's are from the original repositories and not forks. For reference: https://github.com/msys2/setup-msys2/releases/tag/v2.20.1 https://github.com/msys2/setup-msys2/commit/27b3aa77f672cb6b3054121cfd80c3d22ceebb1d https://github.com/actions/checkout/releases/tag/v4.1.0 https://github.com/actions/checkout/commit/8ade135a41bc03ea155e62e844d188df1ea18608 https://github.com/actions/upload-artifact/releases/tag/v3.1.3 https://github.com/actions/upload-artifact/commit/a8a3f3ad30e3422c9c7b888a15615d19a852ae32 Signed-off-by: Gabriela Gutierrez .github/workflows/windows-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit f28cc9bd481ce493da11f98c18526d324211599a Author: Gabriela Gutierrez Date: 2023-09-26 14:35:08 +0000 CI: Bump and ref actions by commit SHA in ci.yml Referencing actions by commit SHA in GitHub workflows guarantees you are using an immutable version. Actions referenced by tags and branches are more vulnerable to attacks, such as the tag being moved to a malicious commit or a malicious commit being pushed to the branch. It's important to make sure the SHA's are from the original repositories and not forks. For reference: https://github.com/actions/checkout/releases/tag/v4.1.0 https://github.com/actions/checkout/commit/8ade135a41bc03ea155e62e844d188df1ea18608 https://github.com/actions/upload-artifact/releases/tag/v3.1.3 https://github.com/actions/upload-artifact/commit/a8a3f3ad30e3422c9c7b888a15615d19a852ae32 Signed-off-by: Gabriela Gutierrez .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit f74f1740067b75042497edbfa6ea457ff75484b9 Author: Jia Tan Date: 2023-10-12 20:12:18 +0800 Build: Update visibility.m4 from Gnulib. Updating from version 6 -> 8 from upstream. Declarations for variables and function bodies were added to avoid unnecessary failures with -Werror. m4/visibility.m4 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) commit 5c4bca521e6fb435898a0012b3276eee70a6dadf Author: Lasse Collin Date: 2023-10-06 19:36:35 +0300 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit d91cb6e884c73d0b05d7e7d68ad4e6eb29f4b44b Author: Lasse Collin Date: 2023-10-06 18:55:57 +0300 CMake/Windows: Fix when the windres workaround is applied. CMake doesn't set WIN32 on CYGWIN but the workaround is probably needed on Cygwin too. Same for MSYS and MSYS2. The workaround must not be used with Clang that is acting in MSVC mode. This fixes it by checking for the known environments that need the workaround instead of using "NOT MSVC". Thanks to Martin Storsjö. https://github.com/tukaani-project/xz/commit/0570308ddd9c0e39e85597ebc0e31d4fc81d436f#commitcomment-129098431 CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 01e34aa1171b04f8b28960b1cc6135a903e0c13d Author: Jia Tan Date: 2023-09-29 22:11:54 +0800 CI: Disable CLANG64 MSYS2 environment until bug is resolved. lld 17.0.1 searches for libraries to link first in the toolchain directories before the local directory when building. The is a problem for us because liblzma.a is installed in MSYS2 CLANG64 by default and xz.exe will thus use the installed library instead of the one being built. This causes tests to fail when they are expecting features to be disabled. More importantly, it will compile xz.exe with an incorrect liblzma and could cause unexpected behavior by being unable to update liblzma code in static builds. The CLANG64 environment can be tested again once this is fixed. Link to bug: https://github.com/llvm/llvm-project/issues/67779. .github/workflows/windows-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) commit 30d0c35327f3639cb11224872aa58fdbf0b1526e Author: Jia Tan Date: 2023-09-29 20:14:39 +0800 CMake: Rename xz and man page symlink custom targets. The Ninja Generator for CMake cannot have a custom target and its BYPRODUCTS have the same name. This has prevented Ninja builds on Unix-like systems since the xz symlinks were introduced in 80a1a8bb838842a2be343bd88ad1462c21c5e2c9. CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 506d03127a8565442b028ec991e1578124fd3025 Author: Jia Tan Date: 2023-09-29 19:58:44 +0800 CMake: Specify LINKER_LANGUAGE for libgnu target to fix Ninja Generator. CMake is unable to guess the linker language for just a header file so it must be explicitly set. CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) commit 0570308ddd9c0e39e85597ebc0e31d4fc81d436f Author: Lasse Collin Date: 2023-09-27 19:54:35 +0300 CMake: Fix Windows build with Clang/LLVM 17. llvm-windres 17.0.0 has more accurate emulation of GNU windres, so the hack for GNU windres must now be used with llvm-windres too. LLVM 16.0.6 has the old behavior and there likely won't be more 16.x releases. So we can simply check for >= 17.0.0. See also: https://github.com/llvm/llvm-project/commit/2bcc0fdc58a220cb9921b47ec8a32c85f2511a47 CMakeLists.txt | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) commit 5a9af95f85a7e5d4f9c10cb8cf737651a921f1d1 Author: Lasse Collin Date: 2023-09-26 21:47:13 +0300 liblzma: Update a comment. The C standards don't allow an empty translation unit which can be avoided by declaring something, without exporting any symbols. When I committed f644473a211394447824ea00518d0a214ff3f7f2 I had a feeling that some specific toolchain somewhere didn't like empty object files (assembler or maybe "ar" complained) but I cannot find anything to confirm this now. Quite likely I remembered nonsense. I leave this here as a note to my future self. :-) src/liblzma/check/crc64_table.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) commit 8ebaf3f665ddc7e4f19c613005050dde5ccbe499 Author: Jia Tan Date: 2023-09-27 00:02:11 +0800 liblzma: Avoid compiler warning without creating extra symbol. When the generic fast crc64 method is used, then we omit lzma_crc64_table[][]. Similar to d9166b52cf3458a4da3eb92224837ca8fc208d79, we can avoid compiler warnings with -Wempty-translation-unit (Clang) or -pedantic (GCC) by creating a never used typedef instead of an extra symbol. src/liblzma/check/crc64_table.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) commit 092d21db2e5eea19fe079264ce48c178989c7606 Author: Lasse Collin Date: 2023-09-26 17:24:15 +0300 Build: Update the comment about -Werror usage in checks. configure.ac | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) commit a37a2763383e6c204fe878e1416dd35e7711d3a9 Author: Lasse Collin Date: 2023-09-26 15:00:43 +0300 Build: Fix __attribute__((ifunc(...))) detection with clang -Wall. Now if user-supplied CFLAGS contains -Wall -Wextra -Wpedantic the two checks that need -Werror will still work. At CMake side there is add_compile_options(-Wall -Wextra) but it didn't affect the -Werror tests. So with both Autotools and CMake only user-supplied CFLAGS could make the checks fail when they shouldn't. This is not a full fix as things like -Wunused-macros in user-supplied CFLAGS will still cause problems with both GCC and Clang. CMakeLists.txt | 8 ++++++++ configure.ac | 8 ++++++++ 2 files changed, 16 insertions(+) commit 9c42f936939b813f25d0ff4e99c3eb9c2d17a0d2 Author: Lasse Collin Date: 2023-09-26 13:51:31 +0300 Build: Fix underquoted AC_LANG_SOURCE. It made no practical difference in this case. configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 9f1444a8a5c0e724b2c7ef83424f642f07a95982 Author: Lasse Collin Date: 2023-09-26 13:14:37 +0300 Build: Silence two Autoconf warnings. There were two uses of AC_COMPILE_IFELSE that didn't use AC_LANG_SOURCE and Autoconf warned about these. The omission had been intentional but it turned out that this didn't do what I thought it would. Autoconf 2.71 manual gives an impression that AC_LANG_SOURCE inserts all #defines that have been made with AC_DEFINE so far (confdefs.h). The idea was that omitting AC_LANG_SOURCE would mean that only the exact code included in the AC_COMPILE_IFELSE call would be compiled. With C programs this is not true: the #defines get added without AC_LANG_SOURCE too. There seems to be no neat way to avoid this. Thus, with the C language at least, adding AC_LANG_SOURCE makes no other difference than silencing a warning from Autoconf. The generated "configure" remains identical. (Docs of AC_LANG_CONFTEST say that the #defines have been inserted since Autoconf 2.63b and that AC_COMPILE_IFELSE uses AC_LANG_CONFTEST. So the behavior is documented if one also reads the docs of macros that one isn't calling directly.) Any extra code, including #defines, can cause problems for these two tests because these tests must use -Werror. CC=clang CFLAGS=-Weverything is the most extreme example. It enables -Wreserved-macro-identifier which warns about #define __EXTENSIONS__ 1 because it begins with two underscores. It's possible to write a test file that passes -Weverything but it becomes impossible when Autoconf inserts confdefs.h. So this commit adds AC_LANG_SOURCE to silence Autoconf warnings. A different solution is needed for -Werror tests. configure.ac | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) commit 519e47c2818acde571fadc79551294527fe6cc22 Author: Jia Tan Date: 2023-09-26 01:17:11 +0800 CMake: Remove accidental extra newline. CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) commit bbb42412da6a02705ba3e668e90840c2683e4e67 Author: Jia Tan Date: 2023-09-26 00:47:26 +0800 Build: Remove Gnulib dependency from tests. The tests do not use any Gnulib replacements so they do not need to link libgnu.a or have /lib in the include path. tests/Makefile.am | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) commit d265f6b75691c6c8fa876eb5320c3ff5aed17dfa Author: Jia Tan Date: 2023-09-26 00:43:43 +0800 CMake: Remove /lib from tests include path. The tests never included anything from /lib, so this was not needed. CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) commit 9fb5de41f2fb654ca952d4bda15cf3777c2b720f Author: Jia Tan Date: 2023-09-24 22:10:41 +0800 Scripts: Change quoting style from `...' to '...'. src/scripts/xzdiff.in | 2 +- src/scripts/xzgrep.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit eaebdef4d4de3c088b0905f42626b74e0d23abf3 Author: Jia Tan Date: 2023-09-24 22:10:18 +0800 xz: Change quoting style from `...' to '...'. src/xz/args.c | 6 +++--- src/xz/file_io.c | 2 +- src/xz/main.c | 4 ++-- src/xz/message.c | 14 +++++++------- src/xz/options.c | 2 +- src/xz/suffix.c | 2 +- src/xz/util.c | 6 +++--- 7 files changed, 18 insertions(+), 18 deletions(-) commit f6667702bf075a05fbe336dbf3576ad1a82ec645 Author: Jia Tan Date: 2023-09-24 22:09:47 +0800 liblzma: Change quoting style from `...' to '...'. This was done for both internal and API headers. src/liblzma/api/lzma/base.h | 18 +++++++++--------- src/liblzma/api/lzma/container.h | 10 +++++----- src/liblzma/api/lzma/filter.h | 6 +++--- src/liblzma/api/lzma/index.h | 8 ++++---- src/liblzma/api/lzma/lzma12.h | 2 +- src/liblzma/lz/lz_encoder.h | 2 +- src/liblzma/rangecoder/range_decoder.h | 2 +- 7 files changed, 24 insertions(+), 24 deletions(-) commit be012b8097a4eaee335b51357d6befa745f753ce Author: Jia Tan Date: 2023-09-24 22:09:16 +0800 Build: Change quoting style from `...' to '...'. configure.ac | 18 +++++++++--------- dos/config.h | 6 +++--- m4/getopt.m4 | 2 +- m4/tuklib_progname.m4 | 2 +- windows/build.bash | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) commit ce162db07f03495bd333696e66883c8f36abdc1e Author: Jia Tan Date: 2023-09-24 22:05:02 +0800 Docs: Change quoting style from `...' to '...'. These days the ` and ' do not look symmetric. This quoting style has been changed in various apps over the years including the GNU tools. INSTALL | 6 +++--- doc/examples/01_compress_easy.c | 2 +- doc/examples/11_file_info.c | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) commit db17656721e43939bfa4ec13506e7c76f4b86da6 Author: Jia Tan Date: 2023-09-24 21:25:01 +0800 lib: Silence -Wsign-conversion in getopt.c. lib/getopt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit a6234f677d66888f435010bc0b67de6a32fefcf6 Author: Jia Tan Date: 2023-09-24 20:48:52 +0800 Build: Update getopt.m4 from Gnulib. This file was modified from upstream since we do not need to replace getopt() and can avoid complexity and feature tests. m4/getopt.m4 | 79 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 40 deletions(-) commit 84808b68f1075e8603a8ef95d361a61fdc6a5b10 Author: Jia Tan Date: 2023-09-26 00:09:53 +0800 CMake: Add /lib to include path. CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) commit 01804a0b4b64e0f33568e947e0579263808c59d3 Author: Jia Tan Date: 2023-09-24 20:36:34 +0800 CMake: Update libgnu target with new header files. CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) commit d34558388fe1d8929f6478d61dc322eb4f2900af Author: Jia Tan Date: 2023-09-23 00:47:52 +0800 lib: Update Makefile.am for new header files. lib/Makefile.am | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) commit 52bf644bdf536e20fcc743b712cede135e05eec5 Author: Jia Tan Date: 2023-09-24 20:34:03 +0800 lib: Update getopt1.c from Gnulib. The only difference was maintaining the conditional inclusion for config.h. lib/getopt1.c | 56 ++++++++++++++++++++++---------------------------------- 1 file changed, 22 insertions(+), 34 deletions(-) commit 7e884c00d0093c38339f17fb1d280eec493f42ca Author: Jia Tan Date: 2023-09-23 03:27:00 +0800 lib: Update getopt.in.h from Gnulib with modifications. We can still avoid modifying the contents of this file during configuration to simplify the build systems. Gnulib added replacements for inclusions guards for Cygwin. Cygwin should not need getopt_long replacement so this feature can be omitted. is conditionally included to avoid MSVC since it is not available. The definition for _GL_ARG_NONNULL was also copied into this file from Gnulib since this stage is usually done during gnulib-tool. lib/getopt.in.h | 228 +++++++------------------------------------------------- 1 file changed, 29 insertions(+), 199 deletions(-) commit cff05f82066ca3ce9425dafdb086325a8eef8de3 Author: Jia Tan Date: 2023-09-23 00:31:55 +0800 lib: Update getopt_int.h from Gnulib. lib/getopt_int.h | 109 ++++++++++++++++++++++++------------------------------- 1 file changed, 48 insertions(+), 61 deletions(-) commit 04bd86a4b010d43c6a016a3857ecb38dc1d5b024 Author: Jia Tan Date: 2023-09-23 00:27:23 +0800 lib: Update getopt.c from Gnulib with modifications. The code maintains the prior modifications of conditionally including config.h and disabling NLS support. _GL_UNUSED is repalced with the simple cast to void trick. _GL_UNUSED is only used for these two parameters so its simpler than having to define it. lib/getopt.c | 1134 +++++++++++++++++++--------------------------------------- 1 file changed, 377 insertions(+), 757 deletions(-) commit 56b42be7287844db20b3a3bc1372c6ae8c040d63 Author: Jia Tan Date: 2023-09-23 00:18:56 +0800 lib: Add getopt-cdefs.h for getopt_long update. This was modified slightly from Gnulib. In Gnulib, it expects the @HAVE_SYS_CDEFS_H@ to be replaced. Instead, we can set HAVE_SYS_CDEFS_H on systems that have it and avoid copying another file into the build directory. Since we are not using gnulib-tool, copying extra files requires extra build system updates (and special handling with CMake) so we should avoid when possible. lib/getopt-cdefs.h | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) commit 9834e591a4cf9dc2f49e42e26bf28d1d247bc196 Author: Jia Tan Date: 2023-09-23 00:15:25 +0800 lib: Copy new header files from Gnulib without modification. The getopt related files have changed from Gnulib by splitting up getopt.in.h into more modular header files. We could have kept everything in just getopt.in.h, but this will help us continue to update in the future. lib/getopt-core.h | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/getopt-ext.h | 77 +++++++++++++++++++++++++++++++++++++++++ lib/getopt-pfx-core.h | 66 +++++++++++++++++++++++++++++++++++ lib/getopt-pfx-ext.h | 70 +++++++++++++++++++++++++++++++++++++ 4 files changed, 309 insertions(+) commit 5b7a6f06e93d99d6635a740fd2e12fab66096c93 Author: Lasse Collin Date: 2023-09-22 21:16:52 +0300 Windows: Update the version requirement comments from Win95 to W2k. windows/README-Windows.txt | 10 ++++------ windows/build.bash | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) commit e582f8e0fee46e7cd967f42f465d6bb608b73bc1 Author: Lasse Collin Date: 2023-09-22 21:12:54 +0300 tuklib_physmem: Comment out support for Windows versions older than 2000. src/common/tuklib_physmem.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) commit 7d73d1f0e08f96c4ab7aac91b958e37a3dadf07a Author: Lasse Collin Date: 2023-09-24 16:32:32 +0300 sysdefs.h: Update the comment about __USE_MINGW_ANSI_STDIO. src/common/sysdefs.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) commit 2a9929af0ab7e6c0ab725565034afe3293e51d71 Author: Lasse Collin Date: 2023-09-22 02:33:29 +0300 xz: Windows: Don't (de)compress to special files like "con" or "nul". Before this commit, the following writes "foo" to the console and deletes the input file: echo foo | xz > con_xz xz --suffix=_xz --decompress con_xz It cannot happen without --suffix because names like con.xz are also special and so attempting to decompress con.xz (or compress con to con.xz) will already fail when opening the input file. Similar thing is possible when compressing. The following writes to "nul" and the input file "n" is deleted. echo foo | xz > n xz --suffix=ul n Now xz checks if the destination is a special file before continuing. DOS/DJGPP version had a check for this but Windows (and OS/2) didn't. src/xz/file_io.c | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) commit 01311b81f03cce1c0ce847a3d556f84dbd439343 Author: Lasse Collin Date: 2023-09-21 20:42:52 +0300 CMake: Wrap two overlong lines that are possible to wrap. CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit 152d0771ddd0cffcac9042ad1a66f110d228eee2 Author: Lasse Collin Date: 2023-09-21 20:36:31 +0300 CMake: Add a comment about threads on Cygwin. CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) commit 6df988cceffaa3100b428ed816fad334935b27bf Author: Lasse Collin Date: 2023-09-12 23:53:25 +0300 MSVC: Remove Visual Studio project files and update INSTALL-MSVC.txt. CMake is now the preferred build file generator when building with MSVC. windows/INSTALL-MSVC.txt | 37 ++-- windows/vs2013/config.h | 157 --------------- windows/vs2013/liblzma.vcxproj | 363 --------------------------------- windows/vs2013/liblzma_dll.vcxproj | 398 ------------------------------------ windows/vs2013/xz_win.sln | 48 ----- windows/vs2017/config.h | 157 --------------- windows/vs2017/liblzma.vcxproj | 363 --------------------------------- windows/vs2017/liblzma_dll.vcxproj | 398 ------------------------------------ windows/vs2017/xz_win.sln | 48 ----- windows/vs2019/config.h | 157 --------------- windows/vs2019/liblzma.vcxproj | 364 --------------------------------- windows/vs2019/liblzma_dll.vcxproj | 399 ------------------------------------- windows/vs2019/xz_win.sln | 51 ----- 13 files changed, 12 insertions(+), 2928 deletions(-) commit edd563daf0da1d00018684614803c77ab62efcd6 Author: Lasse Collin Date: 2023-09-21 19:17:40 +0300 CMake: Require VS2015 or later for building xzdec. xzdec might build with VS2013 but it hasn't been tested. It was never supported before and VS2013 is old anyway so for simplicity only liblzma is supported with VS2013. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit daea64d158a7151ca6c255a0e4554c6d521cd589 Author: Lasse Collin Date: 2023-09-12 23:43:49 +0300 CMake: Allow building xz with Visual Studio 2015 and later. Building the command line tools xz and xzdec with the combination of CMake + Visual Studio 2015/2017/2019/2022 works now. VS2013 update 2 should still be able to build liblzma. VS2013 cannot build the xz command line tool because xz needs snprintf() that roughly conforms to C99. VS2013 is old and no extra code will be added to support it. Thanks to Kelvin Lee and Jia Tan for testing. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 8c2d197c940d246849b2ec48109bb22e54036927 Author: Lasse Collin Date: 2023-09-12 23:34:31 +0300 MSVC: #define inline and restrict only when needed. This also drops the check for _WIN32 as that shouldn't be needed. src/common/sysdefs.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) commit af66cd585902045e5689a0418103ec81f19f1d0a Author: Lasse Collin Date: 2023-09-12 22:16:56 +0300 CMake: Add support for replacement getopt_long (lib/getopt*). Thanks to Jia Tan for the initial work. I added the libgnu target and made a few related minor edits. CMakeLists.txt | 54 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 7 deletions(-) commit e3288fdb45c580cb849f6799cf419c4922004ae5 Author: Lasse Collin Date: 2023-09-12 21:12:34 +0300 CMake: Bump maximum policy version to 3.27. There are several new policies. CMP0149 may affect the Windows SDK version that CMake will choose by default. The new behavior is more predictable, always choosing the latest SDK version by default. The other new policies shouldn't affect this package. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit aff1b479c7b168652bd20305ceed4317d5db6661 Author: Lasse Collin Date: 2023-09-12 20:55:10 +0300 lib/getopt*.c: Include only HAVE_CONFIG_H is defined. The CMake-based build doesn't use config.h. Up-to-date getopt_long in Gnulib is LGPLv2 so at some point it could be included in XZ Utils too but for now this commit is enough to make CMake-based build possible. lib/getopt.c | 4 +++- lib/getopt1.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) commit aa0cd585d2ed1455d35732798e0d90e3520e8ba5 Author: Lasse Collin Date: 2023-09-08 19:08:57 +0300 Doxygen: Add more C macro names to PREDEFINED. doxygen/Doxyfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) commit ee7709bae53637e1765ce142ef102914f1423cb5 Author: Lasse Collin Date: 2023-09-11 18:47:26 +0300 liblzma: Move a few __attribute__ uses in function declarations. The API headers have many attributes but these were left as is for now. src/liblzma/common/common.c | 6 ++++-- src/liblzma/common/common.h | 8 ++++---- src/liblzma/common/memcmplen.h | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) commit 217958d88713b5dc73d366d24dd64b2b311b86fe Author: Lasse Collin Date: 2023-09-11 19:03:35 +0300 xz, xzdec, lzmainfo: Use tuklib_attr_noreturn. For compatibility with C23's [[noreturn]], tuklib_attr_noreturn must be at the beginning of declaration (before "extern" or "static", and even before any GNU C's __attribute__). This commit also moves all other function attributes to the beginning of function declarations. "extern" is kept at the beginning of a line so the attributes are listed on separate lines before "extern" or "static". src/lzmainfo/lzmainfo.c | 6 ++++-- src/xz/coder.c | 3 ++- src/xz/hardware.h | 3 ++- src/xz/message.h | 30 +++++++++++++++++------------- src/xz/options.c | 3 ++- src/xz/util.h | 8 ++++---- src/xzdec/xzdec.c | 9 ++++++--- 7 files changed, 37 insertions(+), 25 deletions(-) commit 18a66fbac031c98f9c2077fc88846e4d07849197 Author: Lasse Collin Date: 2023-09-11 18:53:31 +0300 Remove incorrect uses of __attribute__((__malloc__)). xrealloc() is obviously incorrect, modern GCC docs even mention realloc() as an example where this attribute cannot be used. liblzma's lzma_alloc() and lzma_alloc_zero() would be correct uses most of the time but custom allocators may use a memory pool or otherwise hold the pointer so aliasing issues could happen in theory. The xstrdup() case likely was correct but I removed it anyway. Now there are no __malloc__ attributes left in the code. The allocations aren't in hot paths so this should make no practical difference. src/liblzma/common/common.c | 4 ++-- src/liblzma/common/common.h | 4 ++-- src/xz/util.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) commit 74b0e900c92d5b222b36f474f1efa431f8e262f7 Author: Lasse Collin Date: 2023-09-08 18:41:25 +0300 Build: Omit -Wc99-c11-compat since it warns about _Noreturn. configure.ac | 1 - 1 file changed, 1 deletion(-) commit 90c94dddfd57b7d744bfad64c54e10d15778144b Author: Lasse Collin Date: 2023-09-08 18:19:26 +0300 tuklib: Update tuklib_attr_noreturn for C11/C17 and C23. This makes no difference for GCC or Clang as they support GNU C's __attribute__((__noreturn__)) but this helps with MSVC: - VS 2019 version 16.7 and later support _Noreturn if the options /std:c11 or /std:c17 are used. This gets handled with the check for __STDC_VERSION__ >= 201112. - When MSVC isn't in C11/C17 mode, __declspec(noreturn) is used. C23 will deprecate _Noreturn (and ) for [[noreturn]]. This commit anticipates that but the final __STDC_VERSION__ value isn't known yet. src/common/tuklib_common.h | 22 +++++++++++++++++++++- src/common/tuklib_exit.h | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) commit 189f72581329ab281ad6af37f60135910cb1b146 Author: Lasse Collin Date: 2023-09-11 17:22:44 +0300 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 79334e7f20f2bf9e0de095835b48868f1238f584 Author: Lasse Collin Date: 2023-09-05 22:42:10 +0300 MSVC: xz: Make file_io.c and file_io.h compatible with MSVC. Thanks to Kelvin Lee for the original patches and testing the modifications I made. src/xz/file_io.c | 26 ++++++++++++++++++++++++++ src/xz/file_io.h | 10 ++++++++++ 2 files changed, 36 insertions(+) commit c660b8d78b7bda43b12b285550d8c70e8ccec698 Author: Lasse Collin Date: 2023-09-05 21:33:35 +0300 MSVC: xz: Use GetTickCount64() to implement mytime_now(). It's available since Windows Vista. src/xz/mytime.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) commit 5c6f892d411670e3060f4bc309402617a209e57c Author: Kelvin Lee Date: 2023-09-05 15:05:09 +0300 MSVC: xz: Use _stricmp() instead of strcasecmp() in suffix.c. src/xz/suffix.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) commit e241051f50044259d174e8b4633dd9a1c4478408 Author: Kelvin Lee Date: 2023-09-05 15:01:10 +0300 MSVC: xz: Use _isatty() from to implement isatty(). src/xz/message.c | 5 +++++ src/xz/util.c | 5 +++++ 2 files changed, 10 insertions(+) commit d14bba8fc2be02a9fed8c9bcaaf61103451755f8 Author: Kelvin Lee Date: 2023-09-05 15:10:31 +0300 MSVC: xz: Use _fileno() instead of fileno(). src/xz/private.h | 4 ++++ 1 file changed, 4 insertions(+) commit c4edd367678e6a38c42b149856159bf417da7fe1 Author: Kelvin Lee Date: 2023-09-05 15:00:07 +0300 MSVC: xzdec: Use _fileno and _setmode. src/xzdec/xzdec.c | 4 ++++ 1 file changed, 4 insertions(+) commit cfd1054b9b539ee92524901e95d7bb5a1fe670a0 Author: Kelvin Lee Date: 2023-09-05 14:37:50 +0300 MSVC: Don't #include . lib/getopt.c | 4 +++- lib/getopt.in.h | 4 +++- src/xz/private.h | 5 ++++- src/xzdec/xzdec.c | 5 ++++- 4 files changed, 14 insertions(+), 4 deletions(-) commit adef92f23563a2cc088b31ddee9040ecc96bc996 Author: Lasse Collin Date: 2023-09-19 14:03:45 +0300 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 953e775941a25bfcfa353f802b13e66acb1edf2c Author: Jia Tan Date: 2023-09-14 21:13:23 +0800 CI: Enable CLMUL in address sanitization test. The crc64_clmul() function should be ignored by the address sanitizer now so these builds should still pass. .github/workflows/ci.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) commit f167e79bc98f3f56af2e767b83aa81c2d2b9ed77 Author: Lasse Collin Date: 2023-09-14 16:35:46 +0300 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 4f44ef86758a41a8ec814096f4cb6ee6de04c82e Author: Lasse Collin Date: 2023-09-14 16:34:07 +0300 liblzma: Mark crc64_clmul() with __attribute__((__no_sanitize_address__)). Thanks to Agostino Sarubbo. Fixes: https://github.com/tukaani-project/xz/issues/62 src/liblzma/check/crc64_fast.c | 8 ++++++++ 1 file changed, 8 insertions(+) commit 7379bb3eed428c0ae734d0cc4a1fd04359d53f08 Author: Jia Tan Date: 2023-09-12 22:36:12 +0800 CMake: Fix time.h checks not running on second CMake run. If CMake was configured more than once, HAVE_CLOCK_GETTIME and HAVE_CLOCK_MONOTONIC would not be set as compile definitions. The check for librt being needed to provide HAVE_CLOCK_GETTIME was also simplified. CMakeLists.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) commit 5d691fe58286b92d704c0dc5cd0c4df22881c6c6 Author: Jia Tan Date: 2023-09-12 22:34:06 +0800 CMake: Fix unconditionally defining HAVE_CLOCK_MONOTONIC. If HAVE_CLOCK_GETTIME was defined, then HAVE_CLOCK_MONOTONIC was always added as a compile definition even if the check for it failed. CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) commit eccf12866527b8d24c7d7f92f755142be8ef9b11 Author: Lasse Collin Date: 2023-08-31 19:50:05 +0300 xz: Refactor thousand separator detection and disable it on MSVC. Now the two variations of the format strings are created with a macro, and the whole detection code can be easily disabled on platforms where thousand separator formatting is known to not work (MSVC has no support, and on DJGPP 2.05 it can have problems in some cases). src/xz/util.c | 89 ++++++++++++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 44 deletions(-) commit f7093cd9d130477c234b40aeda613964171f8f21 Author: Lasse Collin Date: 2023-08-31 18:14:43 +0300 xz: Fix a too relaxed assertion and remove uses of SSIZE_MAX. SSIZE_MAX isn't readily available on MSVC. Removing it means that there is one thing less to worry when porting to MSVC. src/xz/file_io.c | 5 ++--- src/xz/file_io.h | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) commit 74c3449d8b816a724b12ebce7417e00fb597309a Author: Jia Tan Date: 2023-08-28 23:14:45 +0800 Tests: Improve invalid unpadded size check in test_lzma_index_append(). This check was extended to test the code added to fix a failing assert in ae5c07b22a6b3766b84f409f1b6b5c100469068a. tests/test_index.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) commit 2544274a8b8a27f4ea6c457d2c4c32eb1e4cd336 Author: Jia Tan Date: 2023-08-28 21:54:41 +0800 Tests: Improve comments in test_index.c. tests/test_index.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 49be29d6380b94e6fb26e511dd2cdbd9afce0f8b Author: Jia Tan Date: 2023-08-28 21:52:54 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 721e3d9f7a82f59f32795d5fb97e0210d1aa839a Author: Jia Tan Date: 2023-08-28 21:50:16 +0800 liblzma: Update assert in vli_ceil4(). The argument to vli_ceil4() should always guarantee the return value is also a valid lzma_vli. Thus the highest three valid lzma_vli values are invalid arguments. All uses of the function ensure this so the assert is updated to match this. src/liblzma/common/index.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit ae5c07b22a6b3766b84f409f1b6b5c100469068a Author: Jia Tan Date: 2023-08-28 21:31:25 +0800 liblzma: Add overflow check for Unpadded size in lzma_index_append(). This was not a security bug since there was no path to overflow UINT64_MAX in lzma_index_append() or when it calls index_file_size(). The bug was discovered by a failing assert() in vli_ceil4() when called from index_file_size() when unpadded_sum (the sum of the compressed size of current Stream and the unpadded_size parameter) exceeds LZMA_VLI_MAX. Previously, the unpadded_size parameter was checked to be not greater than UNPADDED_SIZE_MAX, but no check was done once compressed_base was added. This could not have caused an integer overflow in index_file_size() when called by lzma_index_append(). The calculation for file_size breaks down into the sum of: - Compressed base from all previous Streams - 2 * LZMA_STREAM_HEADER_SIZE (size of the current Streams header and footer) - stream_padding (can be set by lzma_index_stream_padding()) - Compressed base from the current Stream - Unpadded size (parameter to lzma_index_append()) The sum of everything except for Unpadded size must be less than LZMA_VLI_MAX. This is guarenteed by overflow checks in the functions that can set these values including lzma_index_stream_padding(), lzma_index_append(), and lzma_index_cat(). The maximum value for Unpadded size is enforced by lzma_index_append() to be less than or equal UNPADDED_SIZE_MAX. Thus, the sum cannot exceed UINT64_MAX since LZMA_VLI_MAX is half of UINT64_MAX. Thanks to Joona Kannisto for reporting this. src/liblzma/common/index.c | 6 ++++++ 1 file changed, 6 insertions(+) commit 1057765aaabfe0f1397b8094531846655376ae38 Author: Jia Tan Date: 2023-08-28 22:18:29 +0800 Translations: Update the Esperanto translation. po/eo.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit f2e94d064f305bb8ad77ca70f91d93e55f5cf856 Author: Jia Tan Date: 2023-08-26 20:10:23 +0800 Translations: Update the Esperanto translation. po/eo.po | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) commit 2b871f4dbffe3801d0da3f89806b5935f758d5f3 Author: Jia Tan Date: 2023-08-09 20:55:36 +0800 Docs: Update INSTALL for --enable-threads method win95. The Autotools build allows win95 threads and --enable-small together now if the compiler supports __attribute__((__constructor__)). INSTALL | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit 356ad5b26b4196f085ce3afa1869154ca81faad8 Author: Jia Tan Date: 2023-08-09 20:54:15 +0800 CMake: Conditionally allow win95 threads and --enable-small. CMakeLists.txt | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) commit de574404c4c2f87aca049f232c38526e3ce092aa Author: Jia Tan Date: 2023-08-09 20:35:16 +0800 Build: Conditionally allow win95 threads and --enable-small. When the compiler supports __attribute__((__constructor__)) mythread_once() is never used, even with --enable-small. A configuration with win95 threads and --enable-small will compile and be thread safe so it can be allowed. This isn't a very common configuration since MSVC does not support __attribute__((__constructor__)), but MINGW32 and CLANG32 environments for MSYS2 can use win95 threads and have __attribute__((__constructor__)) support. configure.ac | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) commit 6bf33b704cd31dccf25e68480464aa22d3fcad5a Author: Jamaika1 Date: 2023-08-08 14:07:59 +0200 mythread.h: Fix typo error in Vista threads mythread_once(). The "once_" variable was accidentally referred to as just "once". This prevented building with Vista threads when HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR was not defined. src/common/mythread.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 80cb961e5380a3878246d41341ff91378ca59e05 Author: Jia Tan Date: 2023-08-04 22:17:11 +0800 codespell: Add .codespellrc to set default options. The .codespellrc allows setting default options to avoid false positive matches, set additional dictionaries, etc. For now, codespell can be used locally before committing doc and comment changes. It should help prevent silly errors and fix up commits in the future. .codespellrc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) commit cd678a6077358935249b64a4a16fe8d17434f9c9 Author: Jia Tan Date: 2023-08-03 20:10:21 +0800 Tests: Style fixes to test_lzip_decoder.c. tests/test_lzip_decoder.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) commit 1cac5ed4fa45c9861d745b02d80575cb2ff01d81 Author: Jia Tan Date: 2023-08-03 15:56:20 +0800 Translations: Update the Chinese (simplified) translation. po/zh_CN.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 16068f6c30b888cdb873f6285af941d00f95741d Author: Lasse Collin Date: 2023-08-02 17:15:12 +0300 xz: Omit an empty paragraph on the man page. src/xz/xz.1 | 1 - 1 file changed, 1 deletion(-) commit 9ae4371b5106189486e850ce777e40f7b6021c0b Author: Jia Tan Date: 2023-08-02 20:30:07 +0800 Add NEWS for 5.4.4. NEWS | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) commit e8c2203b2c76466d8d3387c5212b46151de8e605 Author: Lasse Collin Date: 2023-08-02 15:19:43 +0300 build-aux/manconv.sh: Fix US-ASCII and UTF-8 output. groff defaults to SGR escapes. Using -P-c passes -c to grotty which restores the old behavior. Perhaps there is a better way to get pure plain text output but this works for now. build-aux/manconv.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 9a706167b0d903d92fd134895acb4bc6a5e3e688 Author: Lasse Collin Date: 2023-08-01 19:10:43 +0300 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 33e25a0f5650754c38bed640deedefe3b4fec5ef Author: Lasse Collin Date: 2023-08-01 18:22:24 +0300 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 81db3b889830132334d1f2129bdc93177ac2ca7d Author: ChanTsune <41658782+ChanTsune@users.noreply.github.com> Date: 2023-08-01 18:17:17 +0300 mythread.h: Disable signal functions in builds targeting Wasm + WASI. signal.h in WASI SDK doesn't currently provide sigprocmask() or sigset_t. liblzma doesn't need them so this change makes liblzma and xzdec build against WASI SDK. xz doesn't build yet and the tests don't either as tuktest needs setjmp() which isn't (yet?) implemented in WASI SDK. Closes: https://github.com/tukaani-project/xz/pull/57 See also: https://github.com/tukaani-project/xz/pull/56 (The original commit was edited a little by Lasse Collin.) src/common/mythread.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 71c638c611324e606d324c8189fef8fe79db6991 Author: Jia Tan Date: 2023-08-01 21:58:51 +0800 Add newline to end of .gitignore. Newline was accidentally removed in commit 01cbb7f023ee7fda8ddde04bd17cf7d3c2418706. .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 42df7c7aa1cca385e509eb33c65136e61890f0bf Author: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: 2023-07-31 14:02:21 +0200 Docs: Fix typos found by codespell CMakeLists.txt | 4 ++-- NEWS | 2 +- configure.ac | 2 +- src/liblzma/api/lzma/container.h | 4 ++-- src/liblzma/api/lzma/filter.h | 2 +- src/liblzma/api/lzma/lzma12.h | 4 ++-- src/liblzma/common/block_buffer_encoder.c | 2 +- src/liblzma/common/common.h | 2 +- src/liblzma/common/file_info.c | 2 +- src/liblzma/common/lzip_decoder.c | 2 +- src/liblzma/common/stream_decoder_mt.c | 8 ++++---- src/liblzma/common/string_conversion.c | 6 +++--- src/liblzma/lz/lz_encoder.h | 2 +- src/liblzma/lzma/lzma_encoder.c | 4 ++-- src/xz/hardware.c | 4 ++-- tests/test_filter_flags.c | 4 ++-- tests/test_index.c | 2 +- tests/test_vli.c | 2 +- 18 files changed, 29 insertions(+), 29 deletions(-) commit 01cbb7f023ee7fda8ddde04bd17cf7d3c2418706 Author: Jia Tan Date: 2023-07-26 20:26:23 +0800 Update .gitignore. .gitignore | 4 ++++ 1 file changed, 4 insertions(+) commit f97a1afd564c48ad9cb94682e10972a72e11fa08 Author: Jia Tan Date: 2023-07-28 22:03:08 +0800 CMake: Conditionally allow the creation of broken symlinks. The CMake build will try to create broken symlinks on Unix and Unix-like platforms. Cygwin and MSYS2 are Unix-like, but may not be able to create broken symlinks. The value of the CYGWIN or MSYS environment variables determine if broken symlinks are valid. The default for MSYS2 does not allow for broken symlinks, so the CMake build has been broken for MSYS2 since commit 80a1a8bb838842a2be343bd88ad1462c21c5e2c9. CMakeLists.txt | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 7 deletions(-) commit 7190f4cc7c9ade5b9b3675d0cbfa3b6d6ec9cb4f Author: Jia Tan Date: 2023-07-28 21:56:48 +0800 CI: Fix windows-ci dependency installation. All of the MSYS2 environments need make, and it does not come with the toolchain package. The toolchain package will install the needed compiler toolchains since without this package CMake cannot properly generate the Makefiles. .github/workflows/windows-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit a048f472cd9a2245265cb292853cbbcdd4f02001 Author: Jia Tan Date: 2023-07-28 21:54:22 +0800 CI: Update ci_build.sh CMake to always make Unix Makefiles. The default for many of the MSYS2 environments is for CMake to create Ninja build files. This would complicate the build script since we would need a different command to run the tests. Its simpler to always use Unix Makefiles so that "make test" is always a usable target for testing. build-aux/ci_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 7870396a0ca945473aa0d1d790f4cbef456610bd Author: Jia Tan Date: 2023-07-25 20:17:23 +0800 CI: Test CMake builds and test framework with MSYS2. .github/workflows/windows-ci.yml | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) commit 6497d1f8875cb7e3007f714336cc09c06fed235b Author: Jia Tan Date: 2023-07-25 20:14:53 +0800 CI: Windows CI rename system matrix variable -> msys2_env. Calling the MSYS2 environment "system" was a bit vague and should be more specific. .github/workflows/windows-ci.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) commit 785e4121d9b2921ad36bd3af1cf61fa20a9265bd Author: Jia Tan Date: 2023-07-24 23:11:45 +0800 CI: Add Clang64 MSYS2 environment to Windows CI. .github/workflows/windows-ci.yml | 1 + 1 file changed, 1 insertion(+) commit d9166b52cf3458a4da3eb92224837ca8fc208d79 Author: Jia Tan Date: 2023-07-24 21:43:44 +0800 liblzma: Prevent an empty translation unit in Windows builds. To workaround Automake lacking Windows resource compiler support, an empty source file is compiled to overwrite the resource files for static library builds. Translation units without an external declaration are not allowed by the C standard and result in a warning when used with -Wempty-translation-unit (Clang) or -pedantic (GCC). src/liblzma/Makefile.am | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) commit db5019d691f980d622fb56fdcf383af2c3519c98 Author: Jia Tan Date: 2023-07-22 18:37:56 +0800 Translations: Update the Vietnamese translation. po/vi.po | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) commit f3a055f762ba5b71b746fc2d44a6ababde2c61b5 Author: Jia Tan Date: 2023-07-22 14:55:42 +0800 CI: Add Windows runner for Autotools builds with MSYS2. Only a subset of the tests run by the Linux and MacOS Autotools builds are run. The most interesting tests are the ones that disable threads, encoders, and decoders. The Windows runner will only be run manually since these tests will likely take much longer than the Linux and MacOS runners. This runner should be used before merging any large features and before releases. Currently the clang64 environment fails to due to a warning and -Werror is enabled for the CI tests. This is still an early version since the CMake build can be done for MSVC and optionally each of the MSYS2 environments. GitHub does not allow manually running the CI tests unless the workflow is checked on the default branch so checking in a minimum version is a good idea. Thanks to Arthur S for the original proposing the original patch. Closes: https://github.com/tukaani-project/xz/pull/34 .github/workflows/windows-ci.yml | 119 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) commit 556536a3525df9e5ed78b8c7057991cfa9edfac8 Author: Jia Tan Date: 2023-07-21 22:11:01 +0800 CI: Add argument to ci_build.sh to pass flags to autogen.sh. build-aux/ci_build.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) commit 39a32d36fc465c4e70f13192eea380e518ba6e8a Author: Jia Tan Date: 2023-07-21 18:05:44 +0800 Tests: Skip .lz files in test_files.sh if not configured. Previously if the lzip decoder was not configured then test_files.sh would pass the lzip tests instead of skipping them. tests/test_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 194d12724b30fe42789d12a0184f9d412c449347 Author: Jia Tan Date: 2023-07-20 22:11:13 +0800 Tests: Add ARM64 filter test to test_compress.sh. tests/test_compress.sh | 1 + 1 file changed, 1 insertion(+) commit d850365c444368102c69beaddf849ed463c33467 Author: Jia Tan Date: 2023-07-20 20:30:05 +0800 Translations: Update the Croatian translation. po/hr.po | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) commit 24049eb7acf6d42a60f00efe4e7289fe8e1797fe Author: Jia Tan Date: 2023-07-20 20:28:32 +0800 Translations: Update the Korean man page translations. po4a/ko.po | 1255 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 629 insertions(+), 626 deletions(-) commit 4d4a4fa07de6cb9d913fb2f97712fddda2527b49 Author: Jia Tan Date: 2023-07-20 20:25:24 +0800 Translations: Update the Korean translation. po/ko.po | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) commit 237f06d9c55cf438a7538a598354bcf103f23711 Author: Jia Tan Date: 2023-07-20 20:24:05 +0800 Translations: Update the Polish translation. po/pl.po | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) commit 80c2c832136656d5ac7a1bca8bc42d95e13d281a Author: Jia Tan Date: 2023-07-20 20:22:23 +0800 Translations: Update the German man page translations. po4a/de.po | 1255 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 629 insertions(+), 626 deletions(-) commit fdbde14503ca03069d3649aa51926f5f796b89d8 Author: Jia Tan Date: 2023-07-20 20:18:44 +0800 Translations: Update the German translation. po/de.po | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) commit 9f3bf5ff5b2b5cf0b252a2bf381238ca49dc4101 Author: Jia Tan Date: 2023-07-20 20:17:10 +0800 Translations: Update the Chinese (simplified) translation. po/zh_CN.po | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) commit 376938c588011567c74f1d5a160c0ccce6336d46 Author: Jia Tan Date: 2023-07-20 20:15:47 +0800 Translations: Update the Swedish translation. po/sv.po | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) commit 26b0bc6eb82c84559936a7c7080de5c71c8276f8 Author: Jia Tan Date: 2023-07-20 20:14:00 +0800 Translations: Update the Ukrainian man page translations. po4a/uk.po | 1253 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 628 insertions(+), 625 deletions(-) commit 2d02c8b7640b54f3c5aa1c8b5990ba56f322393b Author: Jia Tan Date: 2023-07-20 20:09:15 +0800 Translations: Update the Ukrainian translation. po/uk.po | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) commit f881018b503fd334331c24a09075429558abbce1 Author: Jia Tan Date: 2023-07-20 20:06:57 +0800 Translations: Update the Spanish translation. po/es.po | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) commit 791fe6d3ffd6877fa5f852be69d9251397dfaa31 Author: Jia Tan Date: 2023-07-20 20:05:19 +0800 Translations: Update the Romanian translation. po/ro.po | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) commit 8827e90704f699fe08bb5bed56b1717a2bc0eb77 Author: Jia Tan Date: 2023-07-20 20:02:56 +0800 Translations: Update the Romanian man page translations. po4a/ro.po | 1254 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 629 insertions(+), 625 deletions(-) commit 0184d344fa4f215cd345bb131db9068e077c69b8 Author: Jia Tan Date: 2023-07-19 23:36:00 +0800 liblzma: Suppress -Wunused-function warning. Clang 16.0.0 and earlier have a bug that the ifunc resolver function triggers the -Wunused-function warning. The resolver function is static and only "used" by the __attribute__((__ifunc()__)). At this time, the bug is still unresolved, but has been reported: https://github.com/llvm/llvm-project/issues/63957 This is not a problem in GCC. src/liblzma/check/crc64_fast.c | 10 ++++++++++ 1 file changed, 10 insertions(+) commit 43845fa70fc751736c44c18f4cee42d49bfd1392 Author: Jia Tan Date: 2023-07-18 22:52:25 +0800 liblzma: Reword lzma_str_list_filters() documentation. This further improves the documentation from commit f36ca7982f6bd5e9827219ed4f3c5a1fbf5d7bdf. The previous wording of "supported options" was slightly misleading since the options that are printed are the ones that are relevant for encoding/decoding. It is not about which options can or must be specified. src/liblzma/api/lzma/filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 818701ba1c9dff780b7fbf28f9ab8eb11a25dd67 Author: Jia Tan Date: 2023-07-18 22:49:57 +0800 liblzma: Improve comment in string_conversion.c. The comment used "flag" when referring to decoder options. Just referring to them as options is more clear and consistent. src/liblzma/common/string_conversion.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit b6b7d065853cd4c3f5b8d9be8aea0b6dcb0fe090 Author: Lasse Collin Date: 2023-07-18 17:37:33 +0300 xz: Translate the second "%s: " in message.c since French needs "%s : ". This string is used to print a filename when using "xz -v" and stderr isn't a terminal. src/xz/message.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit be644042c3066d8e7a2834f989671ba74d27f749 Author: Lasse Collin Date: 2023-07-18 14:35:33 +0300 xz: Make "%s: %s" translatable because French needs "%s : %s". src/xz/args.c | 5 ++++- src/xz/coder.c | 8 ++++---- src/xz/file_io.c | 8 ++++---- src/xz/list.c | 11 ++++++----- 4 files changed, 18 insertions(+), 14 deletions(-) commit 97fd5cb669ee0afc48d2087675ab166aff89eaa2 Author: Lasse Collin Date: 2023-07-18 13:57:54 +0300 liblzma: Tweak #if condition in memcmplen.h. Maybe ICC always #defines _MSC_VER on Windows but now it's very clear which code will get used. src/liblzma/common/memcmplen.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 40392c19f71985852d75997f109dea97177d6f3f Author: Lasse Collin Date: 2023-07-18 13:49:43 +0300 liblzma: Omit unnecessary parenthesis in a preprocessor directive. src/liblzma/common/memcmplen.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit abc1d5601b7e419ebc28a1ab4b268613b52e6f98 Author: Jia Tan Date: 2023-07-18 00:51:48 +0800 xz: Update Authors list in a few files. src/xz/args.c | 3 ++- src/xz/args.h | 3 ++- src/xz/coder.c | 3 ++- src/xz/coder.h | 3 ++- src/xz/message.c | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) commit 289034a168878baa9df6ff6e159110aade69cba5 Author: Jia Tan Date: 2023-07-14 23:20:33 +0800 Docs: Add a new section to INSTALL for Tests. The new Tests section describes basic information about the tests, how to run them, and important details when cross compiling. We have had a few questions about how to compile the tests without running them, so hopefully this information will help others with the same question in the future. Fixes: https://github.com/tukaani-project/xz/issues/54 INSTALL | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 17 deletions(-) commit 1119e5f5a519b0ab71c81fc4dc84c0cc72abe513 Author: Jia Tan Date: 2023-07-14 21:10:27 +0800 Docs: Update README. This adds an entry to "Other implementations of the .xz format" for XZ for Java. README | 4 ++++ 1 file changed, 4 insertions(+) commit f99e2e4e53b7ea89e4eef32ddd4882e0416357c9 Author: Jia Tan Date: 2023-07-13 23:32:10 +0800 xz: Fix typo in man page. The Memory limit information section described three output columns when it actually has six. This was reworded to "multiple" to make it more future proof. src/xz/xz.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit f907705eb1f6c5edaafc9668a34c51a989932f1d Author: Jia Tan Date: 2023-07-13 21:46:12 +0800 xz: Minor clean up for coder.c * Moved max_block_list_size from a global to local variable. * Reworded error message in validate_block_list_filter(). * Removed helper function filter_chain_error(). * Changed 1 << X to 1U << X in many places src/xz/coder.c | 53 +++++++++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) commit 9adc9e56157ecbf2948e5036df8567809b9ae177 Author: Jia Tan Date: 2023-07-13 21:26:47 +0800 xz: Update man page Authors and date. src/xz/xz.1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) commit c12e429f2635da8d8f5749e5f733f451baca6945 Author: Jia Tan Date: 2023-06-20 20:32:59 +0800 xz: Add a section to man page for robot mode --filters-help. src/xz/xz.1 | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) commit e10f2db5d10300c16fa482a136ed31c1aa6e8e8d Author: Jia Tan Date: 2023-06-19 23:11:41 +0800 xz: Slight reword in xz man page for consistency. Changed will print => prints in xz --robot --version description to match --robot --info-memory description. src/xz/xz.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit f5dc172a402fa946f3c45a16929d7fe14c9f5e81 Author: Jia Tan Date: 2023-06-19 23:07:10 +0800 xz: Reorder robot mode subsections in the man page. The order is now consistent with the order the command line arguments are documented earlier in the man page. The new order is: 1. --list 2. --info-memory 3. --version Instead of the previous order: 1. --version 2. --info-memory 3. --list src/xz/xz.1 | 192 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 96 insertions(+), 96 deletions(-) commit 9628be23aef2784249fd9f3199799d785d2ec5cc Author: Jia Tan Date: 2023-05-13 00:46:50 +0800 xz: Update man page for new --filters-help option. src/xz/xz.1 | 10 ++++++++++ 1 file changed, 10 insertions(+) commit a165d7df1964121eb9df715e6f836a31c865beef Author: Jia Tan Date: 2023-05-13 00:44:41 +0800 xz: Add a new --filters-help option. The --filters-help can be used to help create filter chains with the --filters and --filtersX options. The message in --long-help is too short to fully explain the syntax to construct complex filter chains. In --robot mode, xz will only print the output from liblzma function lzma_str_list_filters. src/xz/args.c | 8 ++++++++ src/xz/message.c | 30 ++++++++++++++++++++++++++++++ src/xz/message.h | 5 +++++ 3 files changed, 43 insertions(+) commit 95f1a414b156ee35d3e71862a14915fdd138f913 Author: Jia Tan Date: 2023-04-21 20:28:11 +0800 xz: Update the man page for --block-list and --filtersX The --block-list option description needed updating since the new --filtersX option changes how it can be used. The new entry for --filters1=FILTERS ... --filter9=FILTERS was created right after the --filters option. src/xz/xz.1 | 106 +++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 26 deletions(-) commit 47a63cad2aa778280e0c1926b7159427ea028cb1 Author: Jia Tan Date: 2023-04-21 19:50:14 +0800 xz: Update --long-help for the new --filtersX option. src/xz/message.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) commit 8b9913a13daca2550d02dfdcdc9be15f55ca4d13 Author: Jia Tan Date: 2023-06-17 20:46:21 +0800 xz: Ignore filter chains that are set but never used in --block-list. If a filter chain is set but not used in --block-list, it introduced unexpected behavior such as requiring an unneeded amount of memory to compress, reducing the number of threads in multi-threaded encoding, and printing an incorrect amount of memory needed to decompress. This also renames filters_init_mask => filters_used_mask. A filter is assumed to be used if it is specified in --filtersX until coder_set_compression_settings() determines which filters are referenced in --block-list. src/xz/coder.c | 66 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 18 deletions(-) commit 183819bfd9efac8c184d9bf123325719b7eee30f Author: Jia Tan Date: 2023-05-13 20:11:13 +0800 xz: Set the Block size for mt encoding correctly. When opt_block_size is not used, the Block size for mt encoder is derived from the minimum of the largest Block specified by --block-list and the recommended Block size on all filter chains calculated by lzma_mt_block_size(). This avoids using unnecessary memory and ensures that all Blocks are large enough for the most memory needy filter chain. src/xz/coder.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) commit afb2dbec3d857b026486b75e42a4728e12d234cb Author: Jia Tan Date: 2023-05-11 00:09:41 +0800 xz: Validate --flush-timeout for all specified filter chains. src/xz/coder.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) commit 5f0c5a04388f8334962c70bc37a8c2ff8f605e0a Author: Jia Tan Date: 2023-05-13 19:54:33 +0800 xz: Allows --block-list filters to scale down memory usage. Previously, only the default filter chain could have its memory usage adjusted. The filter chains specified with --filtersX were not checked for memory usage. Now, all used filter chains will be adjusted if necessary. src/xz/coder.c | 269 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 214 insertions(+), 55 deletions(-) commit 479fd58d60622331fcbe48fddf756927b9f80d9a Author: Jia Tan Date: 2023-05-10 21:50:33 +0800 xz: Do not include block splitting if encoders are disabled. The block splitting logic and split_block() function are not needed if encoders are disabled. This will help slightly reduce the binary size when built without encoders and allow split_block() to use functions that require encoders being enabled. src/xz/coder.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) commit f86ede22500f7ae024ec3ec3f3489ab5a857a3b3 Author: Jia Tan Date: 2023-05-10 22:38:59 +0800 xz: Free filters[] in debug mode. This will only free filter chains created with --filters1-9 since the default filter chain may be set from a static function variable. The complexity to free the default filter chain is not worth the burden on code maintenance. src/xz/coder.c | 10 ++++++++++ 1 file changed, 10 insertions(+) commit f281cd0d692ac0c70fc7669b80dddb863ea947e1 Author: Jia Tan Date: 2023-05-13 19:28:23 +0800 xz: Add a message if --block-list is used outside of xz compresssion. --block-list is only supported with compression in xz format. This avoids silently ignoring when --block-list is unused. src/xz/args.c | 11 +++++++++++ 1 file changed, 11 insertions(+) commit d6af7f347077b22403133239592e478931307759 Author: Jia Tan Date: 2023-04-18 20:29:09 +0800 xz: Create command line options for filters[1-9]. The new command line options are meant to be combined with --block-list. They work as an optional extension to --block-list to specify a custom filter chain for each block listed. The new options allow the creation of up to 9 reusable filter chains. For instance: xz --block-list=1:10MiB,3:5MiB,,2:5MiB,1:0 --filters1=delta--lzma2 \ --filters2=x86--lzma2 --filters3=arm64--lzma2 Will create the following blocks: 1. A block of size 10 MiB with filter chain delta, lzma2. 2. A block of size 5 MiB with filter chain arm64, lzma2. 3. A block of size 5 MiB with filter chain arm64, lzma2. 4. A block of size 5 MiB with filter chain x86, lzma2. 5. A block containing the rest of the file contents with filter chain delta, lzma2. src/xz/args.c | 82 ++++++++++++++++++++++--- src/xz/coder.c | 188 ++++++++++++++++++++++++++++++++++++++++++--------------- src/xz/coder.h | 20 +++++- 3 files changed, 230 insertions(+), 60 deletions(-) commit 072d29250113268536719ad0e040ab8a66fb6435 Author: Jia Tan Date: 2023-05-13 19:36:09 +0800 xz: Use lzma_filters_free() in forget_filter_chain(). This is a little cleaner than the previous implementation of forget_filter_chain(). It is also more consistent since lzma_str_to_filters() will always terminate the filter chain so there is no need to terminate it later in coder_set_compression_settings(). src/xz/coder.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) commit 3d21da5cff4b511633cb6e0d8a1090485c0c1059 Author: Jia Tan Date: 2023-04-17 22:22:45 +0800 xz: Separate string to filter conversion into a helper function. Converting from string to filter will also need to be done for block specific filter chains. src/xz/coder.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) commit a6583726e5f950278f96abcf79c04f1056810be6 Author: Jia Tan Date: 2023-01-06 00:03:35 +0800 Tests: Use new --filters option in test_compress.sh tests/test_compress.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) commit 5f3b898d07cc9b7160c7c88b3120b7edabb8a5b0 Author: Jia Tan Date: 2023-01-06 00:03:06 +0800 xz: Update --long-help and man page for new --filters option. src/xz/message.c | 6 ++++++ src/xz/xz.1 | 41 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) commit 9ded880a0221f4d1256845fc4ab957ffd377c760 Author: Jia Tan Date: 2023-01-06 00:02:29 +0800 xz: Add --filters option to CLI. The --filters option uses the new lzma_str_to_filters() function to convert a string into a full filter chain. Using this option will reset all previous filters set by --preset, --[filter], or --filters. src/xz/args.c | 9 +++++++-- src/xz/coder.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/xz/coder.h | 3 +++ 3 files changed, 58 insertions(+), 4 deletions(-) commit 2c189bb00af73dc7ba1a67a9d274d5be03ee3a88 Author: Jia Tan Date: 2023-07-14 21:30:25 +0800 Tests: Improve feature testing for skipping. Fixed a bug where test_compress_* would all fail if arm64 or armthumb filters were enabled for compression but arm was disabled. Since the grep tests only checked for "define HAVE_ENCODER_ARM", this would match on HAVE_ENCODER_ARM64 or HAVE_ENCODER_ARMTHUMB. Now the config.h feature test requires " 1" at the end to prevent the prefix problem. have_feature() was also updated for this even though there were known current bugs affecting it. This is just in case future features have a similar prefix problem. tests/test_compress.sh | 4 ++-- tests/test_files.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) commit 80a6b9bcad016c99c9ba3f3eeb4a619fcadfd357 Author: Jia Tan Date: 2023-07-10 20:56:28 +0800 Translations: Update the Chinese (traditional) translation. po/zh_TW.po | 659 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 377 insertions(+), 282 deletions(-) commit 17f8844e6fc355abf997d77637a7447c4f7bbcbd Author: Jia Tan Date: 2023-07-08 21:24:19 +0800 liblzma: Remove non-portable empty initializer. Commit 78704f36e74205857c898a351c757719a6c8b666 added an empty initializer {} to prevent a warning. The empty initializer is a GNU extension and results in a build failure on MSVC. The -wpedantic flag warns about empty initializers. src/liblzma/common/stream_encoder_mt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 3aca4f629cd577f0c54f594d5d88722edf0b0413 Author: Jia Tan Date: 2023-07-08 20:03:59 +0800 Translations: Update the Vietnamese translation. po/vi.po | 620 +++++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 349 insertions(+), 271 deletions(-) commit 66bdcfa85fef2911cc80f5f30fed3f9610faccb4 Author: Jia Tan Date: 2023-06-28 20:46:31 +0800 Tests: Fix memory leaks in test_index. Several tests were missing calls to lzma_index_end() to clean up the lzma_index structs. The memory leaks were discovered by using -fsanitize=address with GCC. tests/test_index.c | 11 +++++++++++ 1 file changed, 11 insertions(+) commit fe3bd438fb119f9bad3f08dc29d331e4956196e1 Author: Jia Tan Date: 2023-06-28 20:43:29 +0800 Tests: Fix memory leaks in test_block_header. test_block_header was not properly freeing the filter options between calls to lzma_block_header_decode(). The memory leaks were discovered by using -fsanitize=address with GCC. tests/test_block_header.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) commit 78704f36e74205857c898a351c757719a6c8b666 Author: Jia Tan Date: 2023-06-28 20:31:11 +0800 liblzma: Prevent uninitialzed warning in mt stream encoder. This change only impacts the compiler warning since it was impossible for the wait_abs struct in stream_encode_mt() to be used before it was initialized since mythread_condtime_set() will always be called before mythread_cond_timedwait(). Since the mythread.h code is different between the POSIX and Windows versions, this warning was only present on Windows builds. Thanks to Arthur S for reporting the warning and providing an initial patch. src/liblzma/common/stream_encoder_mt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit e3356a204c5ae02db3ec4552b6c1be354e9b6142 Author: Jia Tan Date: 2023-06-28 20:22:38 +0800 liblzma: Prevent warning for MSYS2 Windows build. In lzma_memcmplen(), the header file is only included if _MSC_VER and _M_X64 are both defined but _BitScanForward64() was previously used if _M_X64 was defined. GCC for MSYS2 defines _M_X64 but not _MSC_VER so _BitScanForward64() was used without including . Now, lzma_memcmplen() will use __builtin_ctzll() for MSYS2 GCC builds as expected. src/liblzma/common/memcmplen.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) commit 45e250a9e9f3c3e8e8af2983366b170bf54f890e Author: Jia Tan Date: 2023-06-28 21:01:22 +0800 CI: Add test with -fsanitize=address,undefined. ci_build.sh was updated to accept disabling of __attribute__ ifunc and CLMUL. This will allow -fsanitize=address to pass because ifunc is incompatible with -fsanitize=address. The CLMUL implementation has optimizations that potentially read past the buffer and mask out the unwanted bytes. This test will only run on Autotools Linux. .github/workflows/ci.yml | 23 +++++++++++++++++++---- build-aux/ci_build.sh | 8 +++++++- 2 files changed, 26 insertions(+), 5 deletions(-) commit 596ee722cd7ddf0afae584fc06365adc0e735977 Author: Jia Tan Date: 2023-06-28 20:16:04 +0800 CI: Upgrade checkout action from v2 to v3. .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 86118ea320f867e09e98a8682cc08cbbdfd640e2 Author: Jia Tan Date: 2023-06-27 23:38:32 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 3d1fdddf92321b516d55651888b9c669e254634e Author: Jia Tan Date: 2023-06-27 17:27:09 +0300 Docs: Document the configure option --disable-ifunc in INSTALL. INSTALL | 8 ++++++++ 1 file changed, 8 insertions(+) commit b4cf7a2822e8d30eb2b12a1a07fd04383b10ade3 Author: Lasse Collin Date: 2023-06-27 17:24:49 +0300 Minor tweaks to style and comments. CMakeLists.txt | 8 ++++---- configure.ac | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) commit 23fb9e3a329117c2968c1e7388b6ef07c782dba1 Author: Lasse Collin Date: 2023-06-27 17:19:49 +0300 CMake: Rename CHECK_ATTR_IFUNC to ALLOW_ATTR_IFUNC. It's so that there's a clear difference in wording compared to liblzma's integrity check types. CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit ee44863ae88e377a5df10db007ba9bfadde3d314 Author: Lasse Collin Date: 2023-06-27 17:05:23 +0300 liblzma: Add ifunc implementation to crc64_fast.c. The ifunc method avoids indirection via the function pointer crc64_func. This works on GNU/Linux and probably on FreeBSD too. The previous __attribute((__constructor__)) method is kept for compatibility with ELF platforms which do support ifunc. The ifunc method has some limitations, for example, building liblzma with -fsanitize=address will result in segfaults. The configure option --disable-ifunc must be used for such builds. Thanks to Hans Jansen for the original patch. Closes: https://github.com/tukaani-project/xz/pull/53 src/liblzma/check/crc64_fast.c | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) commit b72d21202402a603db6d512fb9271cfa83249639 Author: Hans Jansen Date: 2023-06-22 19:49:30 +0200 Add ifunc check to CMakeLists.txt CMake build system will now verify if __attribute__((__ifunc__())) can be used in the build system. If so, HAVE_FUNC_ATTRIBUTE_IFUNC will be defined to 1. CMakeLists.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) commit 23b5c36fb71904bfbe16bb20f976da38dadf6c3b Author: Hans Jansen Date: 2023-06-22 19:46:55 +0200 Add ifunc check to configure.ac configure.ac will now verify if __attribute__((__ifunc__())) can be used in the build system. If so, HAVE_FUNC_ATTRIBUTE_IFUNC will be defined to 1. configure.ac | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) commit dbb3a536ed9873ffa0870321f6873e564c6a9da8 Author: Jia Tan Date: 2023-06-07 00:18:30 +0800 CI: Add apt update command before installing dependencies. Without the extra command, all of the CI tests were automatically failing because the Ubuntu servers could not be reached properly. .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) commit 6bcd516812331de42b347922913230895bebad34 Author: Jia Tan Date: 2023-06-07 00:10:38 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 0d94ba69220d894d2a86081821d2d7a89df5a10b Author: Benjamin Buch Date: 2023-06-06 15:32:45 +0200 CMake: Protects against double find_package Boost iostream uses `find_package` in quiet mode and then again uses `find_package` with required. This second call triggers a `add_library cannot create imported target "ZLIB::ZLIB" because another target with the same name already exists.` This can simply be fixed by skipping the alias part on secondary `find_package` runs. CMakeLists.txt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) commit 045d7aae286ecd2ce163be9e0d9041343a03f89a Author: Jia Tan Date: 2023-05-31 20:26:42 +0800 Translations: Update the Esperanto translation. po/eo.po | 185 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 92 insertions(+), 93 deletions(-) commit b0cc7c2dcefe4cbc4e1e697598c14fb687ed0b78 Author: Jia Tan Date: 2023-05-31 20:25:00 +0800 Translations: Update the Croatian translation. po/hr.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit af045ef6f848f02cd14c9ad195a5f87bb0c02dce Author: Jia Tan Date: 2023-05-31 20:15:53 +0800 Translations: Update the Chinese (simplified) translation. po/zh_CN.po | 317 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 157 insertions(+), 160 deletions(-) commit e6b92d5817fe91ad27a0f7f57bd0f2144311e383 Author: Jia Tan Date: 2023-05-17 23:12:13 +0800 Translations: Update German translation of man pages. po4a/de.po | 52 ++++++++++++---------------------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) commit 592961ccdbba39c7d60fe37e36764232feb57c60 Author: Jia Tan Date: 2023-05-17 23:09:18 +0800 Translations: Update the German translation. po/de.po | 189 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 94 insertions(+), 95 deletions(-) commit 13572cb2c391f5b7503e333c6e05b20bd5bbb524 Author: Jia Tan Date: 2023-05-17 20:30:01 +0800 Translations: Update the Croatian translation. po/hr.po | 187 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 93 insertions(+), 94 deletions(-) commit 4e6e425ea8f097c6fb43e69cc9540294dca3680d Author: Jia Tan Date: 2023-05-17 20:26:54 +0800 Translations: Update Korean translation of man pages. po4a/ko.po | 3015 ++++++++++++------------------------------------------------ 1 file changed, 568 insertions(+), 2447 deletions(-) commit d5ef1f6faf7c270f60093629257150085ecf19ca Author: Jia Tan Date: 2023-05-17 20:13:01 +0800 Translations: Update the Korean translation. po/ko.po | 319 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 158 insertions(+), 161 deletions(-) commit e22d0b0f2e301e7906d0106689d967ed84362028 Author: Jia Tan Date: 2023-05-16 23:49:09 +0800 Translations: Update the Spanish translation. po/es.po | 319 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 158 insertions(+), 161 deletions(-) commit f50da74d52d01f6cfd826a921249e289cf671678 Author: Jia Tan Date: 2023-05-16 23:47:23 +0800 Translations: Update the Romanian translation. po/ro.po | 195 ++++++++++++++++++++++++++++++++------------------------------- 1 file changed, 98 insertions(+), 97 deletions(-) commit 4b9ad60a7305e9841b7cb4ea611bdf5fa7271696 Author: Jia Tan Date: 2023-05-16 23:45:43 +0800 Translations: Update Romanian translation of man pages. po4a/ro.po | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) commit cb6fd57f889c5d9fab36ae8c9e10083a5fe32dea Author: Jia Tan Date: 2023-05-16 23:43:51 +0800 Translations: Update Ukrainian translation of man pages. po4a/uk.po | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) commit c3e8fcbc2db4861f92ad15606c995bd255803c52 Author: Jia Tan Date: 2023-05-16 23:37:54 +0800 Translations: Update the Ukrainian translation. po/uk.po | 321 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 159 insertions(+), 162 deletions(-) commit 27b81b84fcedbc55aa6e6b21004c44070b15b038 Author: Jia Tan Date: 2023-05-16 23:07:35 +0800 Translations: Update the Polish translation. po/pl.po | 316 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 155 insertions(+), 161 deletions(-) commit 8024ad636a65ed6ea95c94d57255be4c6724d6ed Author: Jia Tan Date: 2023-05-16 22:52:14 +0800 Translations: Update the Swedish translation. po/sv.po | 319 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 158 insertions(+), 161 deletions(-) commit 6699a29673f227c4664826db485ed9f7596320d2 Author: Jia Tan Date: 2023-05-16 21:21:38 +0800 Translations: Update the Esperanto translation. po/eo.po | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) commit f36ca7982f6bd5e9827219ed4f3c5a1fbf5d7bdf Author: Jia Tan Date: 2023-05-13 21:21:54 +0800 liblzma: Slightly rewords lzma_str_list_filters() documentation. Reword "options required" to "supported options". The previous may have suggested that the options listed were all required anytime a filter is used for encoding or decoding. The reword makes this more clear that adjusting the options is optional. src/liblzma/api/lzma/filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 3374a5359e52f1671d8f831d65827d5020fe2595 Author: Jia Tan Date: 2023-05-11 23:49:23 +0800 liblzma: Adds lzma_nothrow to MicroLZMA API functions. None of the liblzma functions may throw an exception, so this attribute should be applied to all liblzma API functions. src/liblzma/api/lzma/container.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) commit 8f236574986e7c414c0ea059f441982d1387e6a4 Author: Jia Tan Date: 2023-05-09 20:20:06 +0800 liblzma: Exports lzma_mt_block_size() as an API function. The lzma_mt_block_size() was previously just an internal function for the multithreaded .xz encoder. It is used to provide a recommended Block size for a given filter chain. This function is helpful to determine the maximum Block size for the multithreaded .xz encoder when one wants to change the filters between blocks. Then, this determined Block size can be provided to lzma_stream_encoder_mt() in the lzma_mt options parameter when intializing the coder. This requires one to know all the filter chains they are using before starting to encode (or at least the filter chain that will need the largest Block size), but that isn't a bad limitation. src/liblzma/api/lzma/container.h | 28 ++++++++++++++++++++++++++++ src/liblzma/common/filter_encoder.c | 16 ++++++++++------ src/liblzma/common/filter_encoder.h | 6 +----- src/liblzma/common/stream_encoder_mt.c | 20 +++++++++----------- src/liblzma/liblzma_generic.map | 5 +++++ src/liblzma/liblzma_linux.map | 5 +++++ src/liblzma/lzma/lzma2_encoder.c | 3 +++ 7 files changed, 61 insertions(+), 22 deletions(-) commit d0f33d672a4da7985ebb5ba8d829f885de49c171 Author: Jia Tan Date: 2023-05-08 22:58:09 +0800 liblzma: Creates IS_ENC_DICT_SIZE_VALID() macro. This creates an internal liblzma macro to test if the dictionary size is valid for encoding. src/liblzma/lz/lz_encoder.c | 4 +--- src/liblzma/lz/lz_encoder.h | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) commit c247d06e1f6cada9a76f4f6225cbd97ea760f52f Author: Jia Tan Date: 2023-05-02 20:39:56 +0800 Add NEWS for 5.4.3. NEWS | 10 ++++++++++ 1 file changed, 10 insertions(+) commit 77050b78364ffb6b0f129e742b7c31602d725c08 Author: Jia Tan Date: 2023-05-02 20:39:37 +0800 Add NEWS for 5.2.12. NEWS | 14 ++++++++++++++ 1 file changed, 14 insertions(+) commit 713e15e43eb6279a7ab4bbad3d1325ebfdcf09a0 Author: Jia Tan Date: 2023-05-04 20:38:52 +0800 Translations: Update the Croatian translation. po/hr.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 9ad64bdf309844b6ca6c3e8a4dfb6dbaedda0ca9 Author: Jia Tan Date: 2023-05-04 20:30:25 +0800 tuklib_integer.h: Reverts previous commit. Previous commit 6be460dde07113fe3f08f814b61ddc3264125a96 would cause an error if the integer size was 32 bit. src/common/tuklib_integer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 6be460dde07113fe3f08f814b61ddc3264125a96 Author: Jia Tan Date: 2023-05-04 19:25:20 +0800 tuklib_integer.h: Changes two other UINT_MAX == UINT32_MAX to >=. src/common/tuklib_integer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 44c0c5eae990a22ef04e9b88c1a15838a0d00878 Author: Lasse Collin Date: 2023-05-03 22:46:42 +0300 tuklib_integer.h: Fix a recent copypaste error in Clang detection. Wrong line was changed in 7062348bf35c1e4cbfee00ad9fffb4a21aa6eff7. Also, this has >= instead of == since ints larger than 32 bits would work too even if not relevant in practice. src/common/tuklib_integer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 2cf5ae5b5b279b0b2e69ca4724e7bd705865fe68 Author: Jia Tan Date: 2023-04-25 20:06:15 +0800 CI: Adds a build and test for small configuration. .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) commit 16b81a057a87c2f18e6ed6447f003af0cbdcfe43 Author: Jia Tan Date: 2023-04-25 20:05:26 +0800 CI: ci_build.sh allows configuring small build. build-aux/ci_build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) commit 78ccd93951f9e988d447bcdd70b24f6df5448d1d Author: Jia Tan Date: 2023-04-20 20:15:00 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit f41df2ac2fed347d3f107f3533e76e000d29c6cb Author: Jia Tan Date: 2023-04-19 22:22:16 +0800 Windows: Include when needed. Legacy Windows did not need to #include to use the MSVC intrinsics. Newer versions likely just issue a warning, but the MSVC documentation says to include the header file for the intrinsics we use. GCC and Clang can "pretend" to be MSVC on Windows, so extra checks are needed in tuklib_integer.h to only include when it will is actually needed. src/common/tuklib_integer.h | 6 ++++++ src/liblzma/common/memcmplen.h | 10 ++++++++++ 2 files changed, 16 insertions(+) commit 7062348bf35c1e4cbfee00ad9fffb4a21aa6eff7 Author: Jia Tan Date: 2023-04-19 21:59:03 +0800 tuklib_integer: Use __builtin_clz() with Clang. Clang has support for __builtin_clz(), but previously Clang would fallback to either the MSVC intrinsic or the regular C code. This was discovered due to a bug where a new version of Clang required the header file in order to use the MSVC intrinsics. Thanks to Anton Kochkov for notifying us about the bug. src/common/tuklib_integer.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 3938718ce3773c90755785c0df8777f133b7ae29 Author: Lasse Collin Date: 2023-04-14 18:42:33 +0300 liblzma: Update project maintainers in lzma.h. AUTHORS was updated earlier, lzma.h was simply forgotten. src/liblzma/api/lzma.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 2a89670ab295e377f8b44f5bda6d198deb8ea285 Author: Jia Tan Date: 2023-04-13 20:45:19 +0800 liblzma: Cleans up old commented out code. src/liblzma/common/alone_encoder.c | 11 ----------- 1 file changed, 11 deletions(-) commit 0fbb2b87a7b5a1dd9d0f4a5e84ac7919557dbe81 Author: Jia Tan Date: 2023-04-07 20:46:41 +0800 Docs: Add missing word to SECURITY.md. .github/SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit fb9c50f38a17bf37581de4034b36c8df8ec90a87 Author: Jia Tan Date: 2023-04-07 20:43:22 +0800 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 537c6cd8a9db0dd6b13683e64ddac2943190d715 Author: Jia Tan Date: 2023-04-07 20:42:12 +0800 Docs: Minor edits to SECURITY.md. .github/SECURITY.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) commit 6549df8dd53f358345957e232648fdb699930074 Author: Gabriela Gutierrez Date: 2023-04-07 12:08:30 +0000 Docs: Create SECURITY.md Signed-off-by: Gabriela Gutierrez .github/SECURITY.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) commit d0faa85df5a5d253a4625d45313cf5e9277e6cd2 Author: Jia Tan Date: 2023-03-28 22:48:24 +0800 CI: Tests for disabling threading on CMake builds. .github/workflows/ci.yml | 3 --- build-aux/ci_build.sh | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) commit 8be5cc3b1359d88b4b30a39067466c0ae0bfbc4d Author: Jia Tan Date: 2023-03-28 22:45:42 +0800 CI: Removes CMakeCache.txt between builds. If the cache file is not removed, CMake will not reset configurations back to their default values. In order to make the tests independent, it is simplest to purge the cache. Unfortunatly, this will slow down the tests a little and repeat some checks. build-aux/ci_build.sh | 2 ++ 1 file changed, 2 insertions(+) commit 2cb6028fc31de082b7f927632363bb1426b61aaa Author: Jia Tan Date: 2023-03-28 22:32:40 +0800 CMake: Update liblzma-config.cmake generation. Now that the threading is configurable, the liblzma CMake package only needs the threading library when using POSIX threads. CMakeLists.txt | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) commit 4d7fac0b07cc722825ba8d7838c558827e635611 Author: Jia Tan Date: 2023-03-28 22:25:33 +0800 CMake: Allows setting thread method. The thread method is now configurable for the CMake build. It matches the Autotools build by allowing ON (pick the best threading method), OFF (no threading), posix, win95, and vista. If both Windows and posix threading are both available, then ON will choose Windows threading. Windows threading will also not use: target_link_libraries(liblzma Threads::Threads) since on systems like MinGW-w64 it would link the posix threads without purpose. CMakeLists.txt | 144 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 104 insertions(+), 40 deletions(-) commit 20cd905d898c1494dee42b78530769bb9c9f8076 Author: Jia Tan Date: 2023-03-24 23:05:48 +0800 CI: Runs CMake feature tests. Now, CMake will run similar feature disable tests that the Autotools version did before. In order to do this without repeating lines in ci.yml, it now makes sense to use the GitHub Workflow matrix to create a loop. .github/workflows/ci.yml | 169 +++++++++++++++-------------------------------- 1 file changed, 55 insertions(+), 114 deletions(-) commit 4fabdb269f1fc5624b3b94a170c4efb329d1d229 Author: Jia Tan Date: 2023-03-24 20:35:11 +0800 CI: ci_build.sh allows CMake features to be configured. Also included various clean ups for style and helper functions for repeated work. build-aux/ci_build.sh | 233 +++++++++++++++++++++++++++++++------------------- 1 file changed, 143 insertions(+), 90 deletions(-) commit cf3d1f130e50cf63da4bb1031771605f6f443b6a Author: Jia Tan Date: 2023-03-24 20:06:33 +0800 CI: Change ci_build.sh to use bash instead of sh. This script is only meant to be run as part of the CI build/test process on machines that are known to have bash (Ubuntu and MacOS). If this assumption changes in the future, then the bash specific commands will need to be replaced with a more portable option. For now, it is convenient to use bash commands. build-aux/ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit ddfe164368e779c40d061aa4ccc376129e92f8e1 Author: Jia Tan Date: 2023-03-24 20:05:59 +0800 CMake: Only build xzdec if decoders are enabled. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 116e81f002c503d3c3cd12726db8f9116e58ef25 Author: Jia Tan Date: 2023-03-22 15:42:04 +0800 Build: Removes redundant check for LZMA1 filter support. src/liblzma/lzma/Makefile.inc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) commit 0ba234f692772595329d225462d391fe2c199d0a Author: Lasse Collin Date: 2023-03-23 15:14:29 +0200 CMake: Bump maximum policy version to 3.26. It adds only one new policy related to FOLDERS which we don't use. This makes it clear that the code is compatible with the policies up to 3.26. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit b0891684b4436aed31510fddcbb218d513bd5489 Author: Jia Tan Date: 2023-03-21 23:36:00 +0800 CMake: Conditionally build xz list.* files if decoders are enabled. CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) commit 2c1a830efb61d9d65906a09c9ee3ce27c2c49227 Author: Jia Tan Date: 2023-02-25 11:46:50 +0800 CMake: Allow configuring features as cache variables. This allows users to change the features they build either in CMakeCache.txt or by using a CMake GUI. The sources built for liblzma are affected by this too, so only the necessary files will be compiled. CMakeLists.txt | 528 ++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 391 insertions(+), 137 deletions(-) commit 8be136f667aaeb8f9e16fbd57a83cb282f0c27ff Author: Lasse Collin Date: 2023-03-21 14:07:51 +0200 Build: Add a comment that AC_PROG_CC_C99 is needed for Autoconf 2.69. It's obsolete in Autoconf >= 2.70 and just an alias for AC_PROG_CC but Autoconf 2.69 requires AC_PROG_CC_C99 to get a C99 compiler. configure.ac | 3 +++ 1 file changed, 3 insertions(+) commit 53cc475f2652d9e390ca002018dfd0af0626ef80 Author: Lasse Collin Date: 2023-03-21 14:04:37 +0200 Build: configure.ac: Use AS_IF and AS_CASE where required. This makes no functional difference in the generated configure (at least with the Autotools versions I have installed) but this change might prevent future bugs like the one that was just fixed in the commit 5a5bd7f871818029d5ccbe189f087f591258c294. configure.ac | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) commit 3b8890a40233b6c783bb101ec14405e786871775 Author: Lasse Collin Date: 2023-03-21 13:12:03 +0200 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 5a5bd7f871818029d5ccbe189f087f591258c294 Author: Lasse Collin Date: 2023-03-21 13:11:49 +0200 Build: Fix --disable-threads breaking the building of shared libs. This is broken in the releases 5.2.6 to 5.4.2. A workaround for these releases is to pass EGREP='grep -E' as an argument to configure in addition to --disable-threads. The problem appeared when m4/ax_pthread.m4 was updated in the commit 6629ed929cc7d45a11e385f357ab58ec15e7e4ad which introduced the use of AC_EGREP_CPP. AC_EGREP_CPP calls AC_REQUIRE([AC_PROG_EGREP]) to set the shell variable EGREP but this was only executed if POSIX threads were enabled. Libtool code also has AC_REQUIRE([AC_PROG_EGREP]) but Autoconf omits it as AC_PROG_EGREP has already been required earlier. Thus, if not using POSIX threads, the shell variable EGREP would be undefined in the Libtool code in configure. ax_pthread.m4 is fine. The bug was in configure.ac which called AX_PTHREAD conditionally in an incorrect way. Using AS_CASE ensures that all AC_REQUIREs get always run. Thanks to Frank Busse for reporting the bug. Fixes: https://github.com/tukaani-project/xz/issues/45 configure.ac | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) commit dfe1710784c0a3c3a90c17b80c9e1fe19b5fce06 Author: Lasse Collin Date: 2023-03-19 22:45:59 +0200 liblzma: Silence -Wsign-conversion in SSE2 code in memcmplen.h. Thanks to Christian Hesse for reporting the issue. Fixes: https://github.com/tukaani-project/xz/issues/44 src/liblzma/common/memcmplen.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit f0c580c5fc38bf49a184b48d76c1d8c057d499ce Author: Jia Tan Date: 2023-03-18 22:10:57 +0800 Add NEWS for 5.4.2. NEWS | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) commit af4925e6043113ec9b5f9c0cf13abf2a18ccb1f6 Author: Jia Tan Date: 2023-03-18 22:10:12 +0800 Add NEWS for 5.2.11. NEWS | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) commit 5a7b930efa7f9849d8da8397e8e5d8638f92be40 Author: Lasse Collin Date: 2023-03-18 16:00:54 +0200 Update the copy of GNU GPLv3 from gnu.org to COPYING.GPLv3. COPYING.GPLv3 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) commit b473a92891f7e991398a3b5eff305f6f2b6d7293 Author: Lasse Collin Date: 2023-03-18 15:51:57 +0200 Change a few HTTP URLs to HTTPS. The xz man page timestamp was intentionally left unchanged. INSTALL | 2 +- README | 8 ++++---- configure.ac | 2 +- dos/INSTALL.txt | 4 ++-- src/liblzma/api/lzma.h | 8 ++++---- src/liblzma/check/sha256.c | 2 +- src/xz/xz.1 | 2 +- windows/INSTALL-MinGW.txt | 10 +++++----- 8 files changed, 19 insertions(+), 19 deletions(-) commit 8b2f6001b4f412c259a7883427f2f2c8cea98ea8 Author: Jia Tan Date: 2023-03-18 00:40:28 +0800 CMake: Fix typo in a comment. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 76e2315e14c399c15cc90e7930fd4d3d086b0227 Author: Lasse Collin Date: 2023-03-17 18:36:22 +0200 Windows: build.bash: Copy liblzma API docs to the output package. windows/build.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 133cf55edc5ce92952d2709abd992e48ef1f45ee Author: Lasse Collin Date: 2023-03-17 08:53:38 +0200 Windows: Add microlzma_*.c to the VS project files. These should have been included in 5.3.2alpha already. windows/vs2013/liblzma.vcxproj | 2 ++ windows/vs2013/liblzma_dll.vcxproj | 2 ++ windows/vs2017/liblzma.vcxproj | 2 ++ windows/vs2017/liblzma_dll.vcxproj | 2 ++ windows/vs2019/liblzma.vcxproj | 2 ++ windows/vs2019/liblzma_dll.vcxproj | 2 ++ 6 files changed, 12 insertions(+) commit 75c9ca450fab6982fda9286b168081c9d54126cd Author: Lasse Collin Date: 2023-03-17 08:43:51 +0200 CMake: Add microlzma_*.c to the build. These should have been included in 5.3.2alpha already. CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) commit 0cc3313bd4e569c51e686e5aab8c40c35241d34b Author: Lasse Collin Date: 2023-03-17 08:41:36 +0200 Build: Update comments about unaligned access to mention 64-bit. cmake/tuklib_integer.cmake | 7 +++---- m4/tuklib_integer.m4 | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) commit 5e57e3301319f20c35f8111dea73fa58403b96b1 Author: Lasse Collin Date: 2023-03-17 00:02:30 +0200 Tests: Update .gitignore. .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 0007394d54e21bf30abb9a5e09cbc1e8d44a73ac Author: Lasse Collin Date: 2023-03-14 20:04:03 +0200 po4a/update-po: Display the script name consistently in error messages. po4a/update-po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 509157c80c500426ec853bd992d684ebafc8500c Author: Jia Tan Date: 2023-03-17 01:30:36 +0800 Doc: Rename Doxygen HTML doc directory name liblzma => api. When the docs are installed, calling the directory "liblzma" is confusing since multiple other files in the doc directory are for liblzma. This should also make it more natural for distros when they package the documentation. .gitignore | 2 +- Makefile.am | 18 +++++++++--------- PACKAGERS | 4 ++-- doxygen/Doxyfile | 2 +- doxygen/update-doxygen | 18 +++++++++--------- 5 files changed, 22 insertions(+), 22 deletions(-) commit fd90e2f4c29180b44e33c7ef726f94e4eae54ed3 Author: Jia Tan Date: 2023-03-16 22:07:15 +0800 liblzma: Remove note from lzma_options_bcj about the ARM64 exception. This was left in by mistake since an early version of the ARM64 filter used a different struct for its options. src/liblzma/api/lzma/bcj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 4f50763b981f9056c5f1763dfb26cfa4a26a181d Author: Jia Tan Date: 2023-03-16 21:44:02 +0800 CI: Add doxygen as a dependency. Autogen now requires --no-doxygen or having doxygen installed to run without errors. .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) commit f68f4b27f62f53fdac570885a1f4f23367ce6599 Author: Lasse Collin Date: 2023-03-15 19:19:13 +0200 COPYING: Add a note about the included Doxygen-generated HTML. COPYING | 11 +++++++++++ 1 file changed, 11 insertions(+) commit 8979308528c1f45cb9ee52d511f05232b4ad90a1 Author: Jia Tan Date: 2023-03-16 21:41:09 +0800 Doc: Update PACKAGERS with details about liblzma API docs install. PACKAGERS | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) commit 55ba6e93004842ae0a0792214a23504267ad8f43 Author: Jia Tan Date: 2023-03-16 21:38:32 +0800 liblzma: Add set lzma.h as the main page for Doxygen documentation. The \mainpage command is used in the first block of comments in lzma.h. This changes the previously nearly empty index.html to use the first comment block in lzma.h for its contents. lzma.h is no longer documented separately, but this is for the better since lzma.h only defined a few macros that users do not need to use. The individual API header files all have a disclaimer that they should not be #included directly, so there should be no confusion on the fact that lzma.h should be the only header used by applications. Additionally, the note "See ../lzma.h for information about liblzma as a whole." was removed since lzma.h is now the main page of the generated HTML and does not have its own page anymore. So it would be confusing in the HTML version and was only a "nice to have" when browsing the source files. src/liblzma/api/lzma.h | 1 + src/liblzma/api/lzma/base.h | 2 -- src/liblzma/api/lzma/bcj.h | 2 -- src/liblzma/api/lzma/block.h | 2 -- src/liblzma/api/lzma/check.h | 2 -- src/liblzma/api/lzma/container.h | 2 -- src/liblzma/api/lzma/delta.h | 2 -- src/liblzma/api/lzma/filter.h | 2 -- src/liblzma/api/lzma/hardware.h | 2 -- src/liblzma/api/lzma/index.h | 2 -- src/liblzma/api/lzma/index_hash.h | 4 +--- src/liblzma/api/lzma/lzma12.h | 2 -- src/liblzma/api/lzma/stream_flags.h | 2 -- src/liblzma/api/lzma/version.h | 2 -- src/liblzma/api/lzma/vli.h | 2 -- 15 files changed, 2 insertions(+), 29 deletions(-) commit 16f21255597f6a57e5692780f962cdc090f62b8c Author: Jia Tan Date: 2023-03-16 21:37:32 +0800 Build: Generate doxygen documentation in autogen.sh. Another command line option (--no-doxygen) was added to disable creating the doxygen documenation in cases where it not wanted or if the doxygen tool is not installed. autogen.sh | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) commit 1321852a3be7196bd7fcfd146221a5669e46407c Author: Jia Tan Date: 2023-03-16 21:35:55 +0800 Build: Create doxygen/update-doxygen script. This is a helper script to generate the Doxygen documentation. It can be run in 'liblzma' or 'internal' mode by setting the first argument. It will default to 'liblzma' mode and only generate documentation for the liblzma API header files. The helper script will be run during the custom mydist hook when we create releases. This hook already alters the source directory, so its fine to do it here too. This way, we can include the Doxygen generated files in the distrubtion and when installing. In 'liblzma' mode, the JavaScript is stripped from the .html files and the .js files are removed. This avoids license hassle from jQuery and other libraries that Doxygen 1.9.6 puts into jquery.js in minified form. Makefile.am | 1 + doxygen/update-doxygen | 111 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) commit b1216a7772952d2fe7fe9c6acfcbd98d30abbc7b Author: Jia Tan Date: 2023-03-16 21:34:36 +0800 Build: Install Doxygen docs and include in distribution if generated. Added a install-data-local target to install the Doxygen documentation only when it has been generated. In order to correctly remove the docs, a corresponding uninstall-local target was added. If the doxygen docs exist in the source tree, they will also be included in the distribution now too. Makefile.am | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) commit c97d12f300b2a94c9f54a44c8931c8bc08cf0a73 Author: Lasse Collin Date: 2023-03-16 21:23:48 +0800 Doxygen: Refactor Doxyfile.in to doxygen/Doxyfile. Instead of having Doxyfile.in configured by Autoconf, the Doxyfile can have the tags that need to be configured piped into the doxygen command through stdin with the overrides after Doxyfile's contents. Going forward, the documentation should be generated in two different modes: liblzma or internal. liblzma is useful for most users. It is the documentation for just the liblzma API header files. This is the default. internal is for people who want to understand how xz and liblzma work. It might be useful for people who want to contribute to the project. .gitignore | 3 +- Makefile.am | 1 - configure.ac | 40 --- Doxyfile.in => doxygen/Doxyfile | 721 +++++++++++++++++++++++++--------------- 4 files changed, 456 insertions(+), 309 deletions(-) commit 1b7661faa4bbf4a54c6b75900b5059835c382a0f Author: Jia Tan Date: 2023-02-28 23:22:36 +0800 Tests: Remove unused macros and functions. tests/tests.h | 75 ----------------------------------------------------------- 1 file changed, 75 deletions(-) commit af55191102f01e76de658c881299f0909ca0feda Author: Jia Tan Date: 2022-12-29 21:52:15 +0800 liblzma: Defines masks for return values from lzma_index_checks(). src/liblzma/api/lzma/index.h | 23 +++++++++++++++++++++++ tests/test_index.c | 22 +++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) commit 8f38cdd9ab71e2a9d5a9787550222b7578243b73 Author: Jia Tan Date: 2023-01-12 22:29:07 +0800 Tests: Refactors existing lzma_index tests. Converts the existing lzma_index tests into tuktests and covers every API function from index.h except for lzma_file_info_decoder, which can be tested in the future. tests/test_index.c | 2036 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 1492 insertions(+), 544 deletions(-) commit 717aa3651ce582807f379d8654c2516e1594df77 Author: Lasse Collin Date: 2023-03-11 18:42:08 +0200 xz: Simplify the error-label in Capsicum sandbox code. Also remove unneeded "sandbox_allowed = false;" as this code will never be run more than once (making it work with multiple input files isn't trivial). src/xz/file_io.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) commit a0eecc235d3ba8ad3453da98b46c7bc3e644de75 Author: Lasse Collin Date: 2023-03-07 19:59:23 +0200 xz: Make Capsicum sandbox more strict with stdin and stdout. src/xz/file_io.c | 8 ++++++++ 1 file changed, 8 insertions(+) commit 916448d624aaf55cef0fc3e53754affb8c4f309d Author: Jia Tan Date: 2023-03-08 23:08:46 +0800 Revert: "Add warning if Capsicum sandbox system calls are unsupported." The warning causes the exit status to be 2, so this will cause problems for many scripted use cases for xz. The sandbox usage is already very limited already, so silently disabling this allows it to be more usable. src/xz/file_io.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) commit 01587dda2a8f13fef7e12fd624e6d05da5f9624f Author: Jia Tan Date: 2023-03-07 20:02:22 +0800 xz: Fix -Wunused-label in io_sandbox_enter(). Thanks to Xin Li for recommending the fix. src/xz/file_io.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 5fb936786601a1cd013a5d436adde65982b1e13c Author: Jia Tan Date: 2023-03-06 21:37:45 +0800 xz: Add warning if Capsicum sandbox system calls are unsupported. The warning is only used when errno == ENOSYS. Otherwise, xz still issues a fatal error. src/xz/file_io.c | 2 ++ 1 file changed, 2 insertions(+) commit 61ee82cb1232a402c82282bbae42821f2b952b0d Author: Jia Tan Date: 2023-03-06 21:27:53 +0800 xz: Skip Capsicum sandbox system calls when they are unsupported. If a system has the Capsicum header files but does not actually implement the system calls, then this would render xz unusable. Instead, we can check if errno == ENOSYS and not issue a fatal error. src/xz/file_io.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) commit f070722b57ba975a0dff36492d766f03026b1d21 Author: Jia Tan Date: 2023-03-06 21:08:26 +0800 xz: Reorder cap_enter() to beginning of capsicum sandbox code. cap_enter() puts the process into the sandbox. If later calls to cap_rights_limit() fail, then the process can still have some extra protections. src/xz/file_io.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit f1ab1f6b339d16a53ac53efeb97779ecd2bae70f Author: Jia Tan Date: 2023-02-24 23:46:23 +0800 liblzma: Clarify lzma_lzma_preset() documentation in lzma12.h. lzma_lzma_preset() does not guarentee that the lzma_options_lzma are usable in an encoder even if it returns false (success). If liblzma is built with default configurations, then the options will always be usable. However if the match finders hc3, hc4, or bt4 are disabled, then the options may not be usable depending on the preset level requested. The documentation was updated to reflect this complexity, since this behavior was unclear before. src/liblzma/api/lzma/lzma12.h | 5 +++++ 1 file changed, 5 insertions(+) commit 4b7fb3bf41a0ca4c97fad3799949a2aa61b13b99 Author: Lasse Collin Date: 2023-02-27 18:38:35 +0200 CMake: Require that the C compiler supports C99 or a newer standard. Thanks to autoantwort for reporting the issue and suggesting a different patch: https://github.com/tukaani-project/xz/pull/42 CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) commit 9aa7fdeb04c486d2700967090956af88fdccab7e Author: Jia Tan Date: 2023-02-24 18:10:37 +0800 Tests: Small tweak to test-vli.c. The static global variables can be disabled if encoders and decoders are not built. If they are not disabled and -Werror is used, it will cause an usused warning as an error. tests/test_vli.c | 2 ++ 1 file changed, 2 insertions(+) commit 3cf72c4bcba5370f07477c9b9b62ae33069ef9a9 Author: Jia Tan Date: 2023-02-06 21:46:43 +0800 liblzma: Replace '\n' -> newline in filter.h documentation. The '\n' renders as a newline when the comments are converted to html by Doxygen. src/liblzma/api/lzma/filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 002006be62d77c706565fa6ec828bea64be302da Author: Jia Tan Date: 2023-02-06 21:45:37 +0800 liblzma: Shorten return description for two functions in filter.h. Shorten the description for lzma_raw_encoder_memusage() and lzma_raw_decoder_memusage(). src/liblzma/api/lzma/filter.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) commit 463d9359b8595f01d44ada1739d75aeb87f36524 Author: Jia Tan Date: 2023-02-06 21:44:45 +0800 liblzma: Reword a few lines in filter.h src/liblzma/api/lzma/filter.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) commit 01441df92c0fd6a6c02fe5ac27982a54ce887cc0 Author: Jia Tan Date: 2023-02-06 21:35:06 +0800 liblzma: Improve documentation in filter.h. All functions now explicitly specify parameter and return values. The notes and code annotations were moved before the parameter and return value descriptions for consistency. Also, the description above lzma_filter_encoder_is_supported() about not being able to list available filters was removed since lzma_str_list_filters() will do this. src/liblzma/api/lzma/filter.h | 226 ++++++++++++++++++++++++++---------------- 1 file changed, 143 insertions(+), 83 deletions(-) commit 805b45cd60bfd5da3d3d89077de3789df179b324 Author: Lasse Collin Date: 2023-02-23 20:46:16 +0200 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 30e95bb44c36ae26b2ab12a94343b215fec285e7 Author: Lasse Collin Date: 2023-02-21 22:57:10 +0200 liblzma: Avoid null pointer + 0 (undefined behavior in C). In the C99 and C17 standards, section 6.5.6 paragraph 8 means that adding 0 to a null pointer is undefined behavior. As of writing, "clang -fsanitize=undefined" (Clang 15) diagnoses this. However, I'm not aware of any compiler that would take advantage of this when optimizing (Clang 15 included). It's good to avoid this anyway since compilers might some day infer that pointer arithmetic implies that the pointer is not NULL. That is, the following foo() would then unconditionally return 0, even for foo(NULL, 0): void bar(char *a, char *b); int foo(char *a, size_t n) { bar(a, a + n); return a == NULL; } In contrast to C, C++ explicitly allows null pointer + 0. So if the above is compiled as C++ then there is no undefined behavior in the foo(NULL, 0) call. To me it seems that changing the C standard would be the sane thing to do (just add one sentence) as it would ensure that a huge amount of old code won't break in the future. Based on web searches it seems that a large number of codebases (where null pointer + 0 occurs) are being fixed instead to be future-proof in case compilers will some day optimize based on it (like making the above foo(NULL, 0) return 0) which in the worst case will cause security bugs. Some projects don't plan to change it. For example, gnulib and thus many GNU tools currently require that null pointer + 0 is defined: https://lists.gnu.org/archive/html/bug-gnulib/2021-11/msg00000.html https://www.gnu.org/software/gnulib/manual/html_node/Other-portability-assumptions.html In XZ Utils null pointer + 0 issue should be fixed after this commit. This adds a few if-statements and thus branches to avoid null pointer + 0. These check for size > 0 instead of ptr != NULL because this way bugs where size > 0 && ptr == NULL will likely get caught quickly. None of them are in hot spots so it shouldn't matter for performance. A little less readable version would be replacing ptr + offset with offset != 0 ? ptr + offset : ptr or creating a macro for it: #define my_ptr_add(ptr, offset) \ ((offset) != 0 ? ((ptr) + (offset)) : (ptr)) Checking for offset != 0 instead of ptr != NULL allows GCC >= 8.1, Clang >= 7, and Clang-based ICX to optimize it to the very same code as ptr + offset. That is, it won't create a branch. So for hot code this could be a good solution to avoid null pointer + 0. Unfortunately other compilers like ICC 2021 or MSVC 19.33 (VS2022) will create a branch from my_ptr_add(). Thanks to Marcin Kowalczyk for reporting the problem: https://github.com/tukaani-project/xz/issues/36 src/liblzma/common/block_decoder.c | 5 ++++- src/liblzma/common/block_encoder.c | 7 +++++-- src/liblzma/common/common.c | 20 ++++++++++++++------ src/liblzma/common/index_decoder.c | 13 ++++++++++--- src/liblzma/common/index_encoder.c | 11 +++++++++-- src/liblzma/common/index_hash.c | 13 ++++++++++--- src/liblzma/common/lzip_decoder.c | 6 +++++- src/liblzma/delta/delta_decoder.c | 7 ++++++- src/liblzma/delta/delta_encoder.c | 12 ++++++++++-- src/liblzma/simple/simple_coder.c | 6 ++++-- 10 files changed, 77 insertions(+), 23 deletions(-) commit fa9065fac54194fe0407fc7f0cc9633fdce13c21 Author: Jia Tan Date: 2023-02-07 00:00:44 +0800 liblzma: Adjust container.h for consistency with filter.h. src/liblzma/api/lzma/container.h | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) commit 00a721b63d82dfb658dca8d8cb599d8a245c663f Author: Jia Tan Date: 2023-02-07 00:00:09 +0800 liblzma: Fix small typos and reword a few things in filter.h. src/liblzma/api/lzma/container.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) commit 5b1c171d4ffe89ef18fa31509bb0185d6fd11d39 Author: Jia Tan Date: 2023-02-06 23:42:08 +0800 liblzma: Convert list of flags in lzma_mt to bulleted list. src/liblzma/api/lzma/container.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) commit dbd47622eb99fefb3538a22baec3def002aa56f5 Author: Jia Tan Date: 2023-01-26 23:17:41 +0800 liblzma: Fix typo in documentation in container.h lzma_microlzma_decoder -> lzma_microlzma_encoder src/liblzma/api/lzma/container.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 14cd30806d69e55906073745bcce3ee50e0ec942 Author: Jia Tan Date: 2023-01-26 23:16:34 +0800 liblzma: Improve documentation for container.h Standardizing each function to always specify parameters and return values. Also moved the parameters and return values to the end of each function description. src/liblzma/api/lzma/container.h | 146 +++++++++++++++++++++++++-------------- 1 file changed, 93 insertions(+), 53 deletions(-) commit c9c8bfae3502842dcead85eeb2b951b437c2cd88 Author: Jia Tan Date: 2023-02-22 20:59:41 +0800 CMake: Add LZIP decoder test to list of tests. CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) commit b9f171dd00a3cc32b6d41ea8e082cf545640ec2a Author: Lasse Collin Date: 2023-02-17 20:56:49 +0200 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 2ee86d20e49985b903b78ebcfa3fa672e73e93aa Author: Lasse Collin Date: 2023-02-17 20:48:28 +0200 Build: Use only the generic symbol versioning on MicroBlaze. On MicroBlaze, GCC 12 is broken in sense that __has_attribute(__symver__) returns true but it still doesn't support the __symver__ attribute even though the platform is ELF and symbol versioning is supported if using the traditional __asm__(".symver ...") method. Avoiding the traditional method is good because it breaks LTO (-flto) builds with GCC. See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101766 For now the only extra symbols in liblzma_linux.map are the compatibility symbols with the patch that spread from RHEL/CentOS 7. These require the use of __symver__ attribute or __asm__(".symver ...") in the C code. Compatibility with the patch from CentOS 7 doesn't seem valuable on MicroBlaze so use liblzma_generic.map on MicroBlaze instead. It doesn't require anything special in the C code and thus no LTO issues either. An alternative would be to detect support for __symver__ attribute in configure.ac and CMakeLists.txt and fall back to __asm__(".symver ...") but then LTO would be silently broken on MicroBlaze. It sounds likely that MicroBlaze is a special case so let's treat it as a such because that is simpler. If a similar issue exists on some other platform too then hopefully someone will report it and this can be reconsidered. (This doesn't do the same fix in CMakeLists.txt. Perhaps it should but perhaps CMake build of liblzma doesn't matter much on MicroBlaze. The problem breaks the build so it's easy to notice and can be fixed later.) Thanks to Vincent Fazio for reporting the problem and proposing a patch (in the end that solution wasn't used): https://github.com/tukaani-project/xz/pull/32 configure.ac | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) commit d831072cceca458d94d2d5da201862f6d43a417b Author: Lasse Collin Date: 2023-02-16 21:09:00 +0200 liblzma: Very minor API doc tweaks. Use "member" to refer to struct members as that's the term used by the C standard. Use lzma_options_delta.dist and such in docs so that in Doxygen's HTML output they will link to the doc of the struct member. Clean up a few trailing white spaces too. src/liblzma/api/lzma/block.h | 6 +++--- src/liblzma/api/lzma/delta.h | 6 +++--- src/liblzma/api/lzma/index.h | 10 +++++----- src/liblzma/api/lzma/stream_flags.h | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) commit f029daea39c215fd7d5cb6b6798818b055cf5b22 Author: Jia Tan Date: 2023-02-17 00:54:33 +0800 liblzma: Adjust spacing in doc headers in bcj.h. src/liblzma/api/lzma/bcj.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) commit a5de68bac2bb7e1b9119e6cea7d761a22ea73e9c Author: Jia Tan Date: 2023-02-17 00:44:44 +0800 liblzma: Adjust documentation in bcj.h for consistent style. src/liblzma/api/lzma/bcj.h | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) commit efa498c13b883810497e0ea8a169efd6f48f5026 Author: Jia Tan Date: 2023-02-17 00:36:05 +0800 liblzma: Rename field => member in documentation. Also adjusted preset value => preset level. src/liblzma/api/lzma/base.h | 18 +++++++-------- src/liblzma/api/lzma/block.h | 44 ++++++++++++++++++------------------- src/liblzma/api/lzma/container.h | 26 +++++++++++----------- src/liblzma/api/lzma/delta.h | 12 +++++----- src/liblzma/api/lzma/index.h | 30 ++++++++++++------------- src/liblzma/api/lzma/lzma12.h | 28 +++++++++++------------ src/liblzma/api/lzma/stream_flags.h | 32 +++++++++++++-------------- 7 files changed, 95 insertions(+), 95 deletions(-) commit 718b22a6c5e3ee5de123323ea798872381f9320e Author: Lasse Collin Date: 2023-02-16 17:59:50 +0200 liblzma: Silence a warning from MSVC. It gives C4146 here since unary minus with unsigned integer is still unsigned (which is the intention here). Doing it with substraction makes it clearer and avoids the warning. Thanks to Nathan Moinvaziri for reporting this. src/liblzma/check/crc64_fast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 87c53553fa7d50f777b4edfa99f2083628f590fe Author: Jia Tan Date: 2023-02-16 21:04:54 +0800 liblzma: Improve documentation for stream_flags.h Standardizing each function to always specify parameters and return values. Also moved the parameters and return values to the end of each function description. A few small things were reworded and long sentences broken up. src/liblzma/api/lzma/stream_flags.h | 76 ++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 30 deletions(-) commit 13d99e75a543e9e5f8633cc241eae55b91a3b242 Author: Jia Tan Date: 2023-02-14 21:50:16 +0800 liblzma: Improve documentation in lzma12.h. All functions now explicitly specify parameter and return values. src/liblzma/api/lzma/lzma12.h | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) commit 43ec344c868f930e96879eb9e49212cce92a9884 Author: Jia Tan Date: 2023-01-27 22:44:06 +0800 liblzma: Improve documentation in check.h. All functions now explicitly specify parameter and return values. Also moved the note about SHA-256 functions not being exported to the top of the file. src/liblzma/api/lzma/check.h | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) commit 9c71db4e884fd49aea3d1e711036bff45ca66487 Author: Jia Tan Date: 2023-02-08 21:33:52 +0800 liblzma: Improve documentation in index.h All functions now explicitly specify parameter and return values. src/liblzma/api/lzma/index.h | 177 ++++++++++++++++++++++++++++++------------- 1 file changed, 126 insertions(+), 51 deletions(-) commit 421f2f2e160720f6009e3b6a125cafe2feaa9419 Author: Jia Tan Date: 2023-02-08 20:35:32 +0800 liblzma: Reword a comment in index.h. src/liblzma/api/lzma/index.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit b67539484981351d501b68de5e925425e50c59b1 Author: Jia Tan Date: 2023-02-08 20:30:23 +0800 liblzma: Omit lzma_index_iter's internal field from Doxygen docs. Add \private above this field and its sub-fields since it is not meant to be modified by users. src/liblzma/api/lzma/index.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) commit 0c9e4fc2ad6d88d54f299240fcc5a2ce7d695d96 Author: Jia Tan Date: 2023-01-21 21:32:03 +0800 liblzma: Fix documentation for LZMA_MEMLIMIT_ERROR. LZMA_MEMLIMIT_ERROR was missing the "<" character needed to put documentation after a member. src/liblzma/api/lzma/base.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 816fec125aa74bcef46512c73acc6d9e5a700d15 Author: Jia Tan Date: 2023-01-21 00:29:38 +0800 liblzma: Improve documentation for base.h. Standardizing each function to always specify params and return values. Also fixed a small grammar mistake. src/liblzma/api/lzma/base.h | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) commit 862dacef1a4e7e1b28d465956fa4244ed01df154 Author: Jia Tan Date: 2023-02-14 00:12:34 +0800 liblzma: Add one more missing [out] annotation in vli.h src/liblzma/api/lzma/vli.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 867b08ae4254bf55dd1f7fd502cc618231b92f75 Author: Jia Tan Date: 2023-02-14 00:08:33 +0800 liblzma: Minor improvements to vli.h. Added [out] annotations to parameters that are pointers and can have their value changed. Also added a clarification to lzma_vli_is_valid. src/liblzma/api/lzma/vli.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) commit 90d0e628ff11e5030bcc4fc000bca056adda6603 Author: Jia Tan Date: 2023-02-10 21:38:02 +0800 liblzma: Add comments for macros in delta.h. Document LZMA_DELTA_DIST_MIN and LZMA_DELTA_DIST_MAX for completeness and to avoid Doxygen warnings. src/liblzma/api/lzma/delta.h | 8 ++++++++ 1 file changed, 8 insertions(+) commit 9255fffdb13e59874bf7f95c370c410ad3a7e114 Author: Jia Tan Date: 2023-02-10 21:35:23 +0800 liblzma: Improve documentation in index_hash.h. All functions now explicitly specify parameter and return values. Also reworded the description of lzma_index_hash_init() for readability. src/liblzma/api/lzma/index_hash.h | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) commit 1dbe12b90cff79bb51923733ac0840747b4b4131 Author: Lasse Collin Date: 2023-02-07 19:07:45 +0200 xz: Improve the comment about start_time in mytime.c. start_time is relative to an arbitary point in time, it's not time of day, so using it for anything else than time differences wouldn't make sense. src/xz/mytime.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) commit 7673ef5aa80c1af7fb693360dd82f527b46c2c56 Author: Jia Tan Date: 2023-02-04 21:06:35 +0800 Build: Adjust CMake version search regex. Now, the LZMA_VERSION_MAJOR, LZMA_VERSION_MINOR, and LZMA_VERSION_PATCH macros do not need to be on consecutive lines in version.h. They can be separated by more whitespace, comments, or even other content, as long as they appear in the proper order (major, minor, patch). CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) commit b8bce89be7fb5bffe5fef4a2782ca9b2b107eaac Author: Jia Tan Date: 2023-02-04 12:01:23 +0800 xz: Add a comment clarifying the use of start_time in mytime.c. src/xz/mytime.c | 5 +++++ 1 file changed, 5 insertions(+) commit 912af91b10a18fb9bb3167247ecaaefca8248ee9 Author: Jia Tan Date: 2023-01-26 09:50:21 +0800 liblzma: Improve documentation for version.h. Specified parameter and return values for API functions and documented a few more of the macros. src/liblzma/api/lzma/version.h | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) commit 850adec171203cd22b57d016084d713f72ae5307 Author: Jia Tan Date: 2023-02-03 22:52:55 +0800 Docs: Omit SIGTSTP not handled from TODO. TODO | 4 ---- 1 file changed, 4 deletions(-) commit 2c78a83c6faec70154d9eb78022a618ed62cdcb3 Author: Jia Tan Date: 2023-02-03 00:33:32 +0800 liblzma: Fix bug in lzma_str_from_filters() not checking filters[] length. The bug is only a problem in applications that do not properly terminate the filters[] array with LZMA_VLI_UNKNOWN or have more than LZMA_FILTERS_MAX filters. This bug does not affect xz. src/liblzma/common/string_conversion.c | 7 +++++++ 1 file changed, 7 insertions(+) commit e01f01b9af1c074463b92694a16ecc16a31907c0 Author: Jia Tan Date: 2023-02-03 00:32:47 +0800 Tests: Create test_filter_str.c. Tests lzma_str_to_filters(), lzma_str_from_filters(), and lzma_str_list_filters() API functions. CMakeLists.txt | 1 + tests/Makefile.am | 2 + tests/test_filter_str.c | 593 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 596 insertions(+) commit 8dfc029e7a4ce45809c30313dc0e502f0d22be26 Author: Jia Tan Date: 2023-01-22 08:49:00 +0800 liblzma: Fix typos in comments in string_conversion.c. src/liblzma/common/string_conversion.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 54ad83c1ae2180dcc0cb2445b181dc1e9732a5d6 Author: Jia Tan Date: 2023-02-03 00:20:20 +0800 liblzma: Clarify block encoder and decoder documentation. Added a few sentences to the description for lzma_block_encoder() and lzma_block_decoder() to highlight that the Block Header must be coded before calling these functions. src/liblzma/api/lzma/block.h | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) commit f680e771b3eb2a46310fe85b3e000ac3a1a0640f Author: Jia Tan Date: 2023-02-03 00:12:24 +0800 Update lzma_block documentation for lzma_block_uncomp_encode(). src/liblzma/api/lzma/block.h | 3 +++ 1 file changed, 3 insertions(+) commit 504cf4af895fd45aad0c56eb3b49d90acd54465b Author: Jia Tan Date: 2023-02-03 00:11:37 +0800 liblzma: Minor edits to lzma_block header_size documentation. src/liblzma/api/lzma/block.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 115b720fb521f99aa832d06b2c12b7f8c6c50680 Author: Jia Tan Date: 2023-02-03 00:11:07 +0800 liblzma: Enumerate functions that read version in lzma_block. src/liblzma/api/lzma/block.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) commit 85ea0979adcf808a3830aefbe7a4ec884e542ea1 Author: Jia Tan Date: 2023-02-03 00:10:34 +0800 liblzma: Clarify comment in block.h. src/liblzma/api/lzma/block.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 1f7ab90d9ce224230a04de6b921ad6e2029023a8 Author: Jia Tan Date: 2023-02-03 00:07:23 +0800 liblzma: Improve documentation for block.h. Standardizing each function to always specify params and return values. Output pointer parameters are also marked with doxygen style [out] to make it clear. Any note sections were also moved above the parameter and return sections for consistency. src/liblzma/api/lzma/block.h | 96 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 21 deletions(-) commit c563a4bc554a96bd0b6aab3c139715b7ec8f6ca3 Author: Jia Tan Date: 2023-02-01 23:38:30 +0800 liblzma: Clarify a comment about LZMA_STR_NO_VALIDATION. The flag description for LZMA_STR_NO_VALIDATION was previously confusing about the treatment for filters than cannot be used with .xz format (lzma1) without using LZMA_STR_ALL_FILTERS. Now, it is clear that LZMA_STR_NO_VALIDATION is not a super set of LZMA_STR_ALL_FILTERS. src/liblzma/api/lzma/filter.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) commit 315c64c7e18acc59a745b68148188a73e998252b Author: Jia Tan Date: 2023-02-01 21:43:33 +0800 CI: Update .gitignore for artifacts directory in build-aux. The workflow action for our CI pipeline can only reference artifacts in the source directory, so we should ignore these files if the ci_build.sh is run locally. .gitignore | 1 + 1 file changed, 1 insertion(+) commit 2c1341f4fa06e7f487d61142aa354c433e17ec7f Author: Jia Tan Date: 2023-02-01 21:36:46 +0800 CI: Add quotes around variables in a few places. build-aux/ci_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 3a401b0e0c7a2658af7801dd0690256ef24149e0 Author: Jia Tan Date: 2023-02-01 21:36:22 +0800 CI: Upload test logs as artifacts if a test fails. .github/workflows/ci.yml | 60 ++++++++++++++++++++++++++++++++++-------------- build-aux/ci_build.sh | 31 ++++++++++++++++++++----- 2 files changed, 68 insertions(+), 23 deletions(-) commit 610dde15a88f12cc540424eb3eb3ed61f3876f74 Author: Lasse Collin Date: 2023-01-27 20:02:49 +0200 xz: Use clock_gettime() even if CLOCK_MONOTONIC isn't available. mythread.h and thus liblzma already does it. src/xz/mytime.c | 11 ++++++++--- src/xz/private.h | 3 +-- 2 files changed, 9 insertions(+), 5 deletions(-) commit 2e02877288f6576cd4595e9ac7684f867cd47d68 Author: Lasse Collin Date: 2023-01-27 19:41:19 +0200 po4a/po4a.conf: Sort the language identifiers in alphabetical order. po4a/po4a.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit ff592c616eda274215b485cf1b8d34f060c9f3be Author: Lasse Collin Date: 2023-01-26 18:29:17 +0200 xz: Add SIGTSTP handler for progress indicator time keeping. This way, if xz is stopped the elapsed time and estimated time remaining won't get confused by the amount of time spent in the stopped state. This raises SIGSTOP. It's not clear to me if this is the correct way. POSIX and glibc docs say that SIGTSTP shouldn't stop the process if it is orphaned but this commit doesn't attempt to handle that. Search for SIGTSTP in section 2.4.3: https://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html src/xz/mytime.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- src/xz/mytime.h | 6 ++++++ src/xz/private.h | 12 ++++++++++++ src/xz/signals.c | 17 ++++++++++++++++- 4 files changed, 89 insertions(+), 2 deletions(-) commit 3b1c8ac8d1d553cbb1fb22b545d2b1424c752b76 Author: Jia Tan Date: 2023-01-27 20:14:51 +0800 Translations: Add Brazilian Portuguese translation of man pages. Thanks to Rafael Fontenelle. po4a/po4a.conf | 2 +- po4a/pt_BR.po | 3677 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 3678 insertions(+), 1 deletion(-) commit a15a7552f9f67c4e402f5d2967324e0ccfd6fccc Author: Lasse Collin Date: 2023-01-26 17:51:06 +0200 Build: Avoid different quoting style in --enable-doxygen doc. configure.ac | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) commit af5a4bd5afc089d9697756dded38feafaa987ae4 Author: Lasse Collin Date: 2023-01-26 17:39:46 +0200 tuklib_physmem: Check for __has_warning before GCC version. Clang can be configured to fake a too high GCC version so this way it's more robust. src/common/tuklib_physmem.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit f35d98e20609e0be6a04ae2604bfb7cb9d5bd5e4 Author: Jia Tan Date: 2023-01-24 20:48:50 +0800 liblzma: Fix documentation in filter.h for lzma_str_to_filters() The previous documentation for lzma_str_to_filters() was technically correct, but misleading. lzma_str_to_filters() returns NULL on success, which is in practice always defined to 0. This is the same value as LZMA_OK, but lzma_str_to_filters() does not return lzma_ret so we should be more clear. src/liblzma/api/lzma/filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 2f78ecc5939b3d97ddfc2a6bd31b50108a28d0a2 Author: Lasse Collin Date: 2023-01-23 23:44:58 +0200 Revert "tuklib_common: Define __has_warning if it is not defined." This reverts commit 82e3c968bfa10e3ff13333bd9cbbadb5988d6766. Macros in the reserved namespace (_foo or __foo) shouldn't be #defined without a very good reason. Here the alternative would have been to #define tuklib_has_warning(str) to an approriate value. Also the tuklib_* files should stay namespace clean if possible. src/common/tuklib_common.h | 7 ------- 1 file changed, 7 deletions(-) commit 8366cf8738e8b7bb74c967d07bf0fd2a1878e575 Author: Lasse Collin Date: 2023-01-23 23:38:34 +0200 tuklib_physmem: Clean up the way -Wcast-function-type is silenced on Windows. __has_warning and other __has_foo macros are meant to become compiler-agnostic so it's not good to check for __clang__ with it. This also relied on tuklib_common.h for #defining __has_warning which was confusing as #defining reserved macros is generally not a good idea. src/common/tuklib_physmem.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) commit 683a3c7e2fcd922200c31078e5c9dd1348e90941 Author: Lasse Collin Date: 2023-01-24 00:05:38 +0200 xz: Flip the return value of suffix_is_set to match the documentation. Also edit style to match the existing coding style in the project. src/xz/args.c | 6 +++--- src/xz/suffix.c | 2 +- src/xz/suffix.h | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) commit cc5aa9ab138beeecaee5a1e81197591893ee9ca0 Author: Jia Tan Date: 2023-01-07 21:55:06 +0800 xz: Refactor duplicated check for custom suffix when using --format=raw src/xz/args.c | 8 ++++++++ src/xz/suffix.c | 26 ++++++++------------------ src/xz/suffix.h | 7 +++++++ 3 files changed, 23 insertions(+), 18 deletions(-) commit 9663141274e01592a281a7f2df5d7a31a1dac8bf Author: Jia Tan Date: 2023-01-20 21:53:14 +0800 liblzma: Set documentation on all reserved fields to private. This prevents the reserved fields from being part of the generated Doxygen documentation. src/liblzma/api/lzma/base.h | 17 +++++++++++++++ src/liblzma/api/lzma/block.h | 43 +++++++++++++++++++++++++++++++++++++ src/liblzma/api/lzma/container.h | 24 +++++++++++++++++++++ src/liblzma/api/lzma/delta.h | 12 +++++++++++ src/liblzma/api/lzma/index.h | 27 +++++++++++++++++++++++ src/liblzma/api/lzma/lzma12.h | 22 +++++++++++++++++++ src/liblzma/api/lzma/stream_flags.h | 28 ++++++++++++++++++++++++ 7 files changed, 173 insertions(+) commit 6327a045f34d48fc5afc58ba0d32a82c94403049 Author: Jia Tan Date: 2022-12-20 21:39:59 +0800 Doxygen: Update Doxyfile.in from 1.4.7 to 1.8.17. A few Doxygen tags were obsolete from 1.4.7. Version 1.8.17 released in 2019, so this should be compatible with resonable modern distros. The purpose of Doxygen these days is for docs on the website, so it doesn't necessarily have to work for everyone. Just when the maintainers want to update the docs. Doxyfile.in | 2523 ++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 1893 insertions(+), 630 deletions(-) commit bbf71b69ebf9d0d62a0af150a5c37d193b8159ad Author: Jia Tan Date: 2023-01-03 20:37:30 +0800 Doxygen: Make Doxygen only produce liblzma API documentation by default. Doxygen is now configurable in autotools only with --enable-doxygen=[api|all]. The default is "api", which will only generate HTML output for liblzma API functions. The LaTex documentation output was also disabled. Doxyfile.in | 18 +++++++++--------- configure.ac | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 9 deletions(-) commit 6fcf4671b6047113c583a0919fc850987a4ec5f4 Author: Jia Tan Date: 2022-12-21 23:59:43 +0800 liblzma: Highlight liblzma API headers should not be included directly. This improves the generated Doxygen HTML files to better highlight how to properly use the liblzma API header files. src/liblzma/api/lzma/base.h | 5 +++-- src/liblzma/api/lzma/bcj.h | 5 +++-- src/liblzma/api/lzma/block.h | 5 +++-- src/liblzma/api/lzma/check.h | 5 +++-- src/liblzma/api/lzma/container.h | 5 +++-- src/liblzma/api/lzma/delta.h | 5 +++-- src/liblzma/api/lzma/filter.h | 5 +++-- src/liblzma/api/lzma/hardware.h | 5 +++-- src/liblzma/api/lzma/index.h | 5 +++-- src/liblzma/api/lzma/index_hash.h | 5 +++-- src/liblzma/api/lzma/lzma12.h | 5 +++-- src/liblzma/api/lzma/stream_flags.h | 5 +++-- src/liblzma/api/lzma/version.h | 5 +++-- src/liblzma/api/lzma/vli.h | 5 +++-- 14 files changed, 42 insertions(+), 28 deletions(-) commit b43ff180fb2e372adce876bfa155fc9bcf0c3db4 Author: Jia Tan Date: 2023-01-19 20:35:09 +0800 tuklib_physmem: Silence warning from -Wcast-function-type on MinGW-w64. tuklib_physmem depends on GetProcAddress() for both MSVC and MinGW-w64 to retrieve a function address. The proper way to do this is to cast the return value to the type of function pointer retrieved. Unfortunately, this causes a cast-function-type warning, so the best solution is to simply ignore the warning. src/common/tuklib_physmem.c | 9 +++++++++ 1 file changed, 9 insertions(+) commit 82e3c968bfa10e3ff13333bd9cbbadb5988d6766 Author: Jia Tan Date: 2023-01-19 20:32:40 +0800 tuklib_common: Define __has_warning if it is not defined. clang supports the __has_warning macro to determine if the version of clang compiling the code supports a given warning. If we do not define it for other compilers, it may cause a preprocessor error. src/common/tuklib_common.h | 7 +++++++ 1 file changed, 7 insertions(+) commit b2ba1a489df451cdcd93b2334e319dd06778de19 Author: Jia Tan Date: 2023-01-18 22:11:05 +0800 CI: Reorder 32-bit build first for Linux autotool builds. The 32-bit build needs to be first so the configure cache only needs to be reset one time. The 32-bit build sets the CFLAGS env variable, so any build using that flag after will fail unless the cache is reset. .github/workflows/ci.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) commit dd1c1135741057c91e8d018be9ec4d43968b0e64 Author: Jia Tan Date: 2023-01-18 21:51:43 +0800 CI: Enable --config-cache in autotool builds. If CFLAGS are set in a build, the cache must be cleared with "make distclean", or by deleting the cache file. build-aux/ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit d3e11477053764c003eec2daa5198c747d70ff69 Author: Jia Tan Date: 2023-01-16 21:35:45 +0800 xz: Add missing comment for coder_set_compression_settings() src/xz/coder.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 123255b6ed15f4428b2aa92e4962015a5362f6bf Author: Jia Tan Date: 2023-01-16 20:55:10 +0800 xz: Do not set compression settings with raw format in list mode. Calling coder_set_compression_settings() in list mode with verbose mode on caused the filter chain and memory requirements to print. This was unnecessary since the command results in an error and not consistent with other formats like lzma and alone. src/xz/args.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 571919c47b9ff5171ede84378620ed0a9aeb98c0 Author: Jia Tan Date: 2023-01-13 20:37:06 +0800 Translations: Update the Brazilian Portuguese translation. po/pt_BR.po | 603 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 344 insertions(+), 259 deletions(-) commit 81cb02e2c22bbc036cdfaa2d2c4176f6bd60d3cf Author: Jia Tan Date: 2023-01-12 23:43:06 +0800 CI: Disable shared and nls from various jobs in autotool runners. Disabling shared library generation and linking should help speed up the runners. The shared library is still being tested in the 32 bit build and the full feature. Disabling nls is to check for any unexpected warnings or errors. .github/workflows/ci.yml | 56 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) commit 58a052198a7bcaf6e958f87fad72e69e19a2579b Author: Jia Tan Date: 2023-01-12 23:39:19 +0800 CI: Reorder the 32-bit job in the Ubuntu runner. Run the 32 bit job sooner since this is a more interesting test than some of the later jobs. .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) commit 4110a998b83459fe2bc9bc1bec30ad68afa8f797 Author: Jia Tan Date: 2023-01-12 23:09:03 +0800 CI: Allow disabling Native Language Support. build-aux/ci_build.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) commit 0dec634e705b5bf89a37c5d62d71e8511d480058 Author: Jia Tan Date: 2023-01-12 23:02:20 +0800 CI: Only run autogen.sh if it has not already run. build-aux/ci_build.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) commit 32287dc8def94df4546e903495d14c132bd54cc4 Author: Jia Tan Date: 2023-01-12 22:58:36 +0800 CI: Allow disabling shared library in autotools builds. build-aux/ci_build.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) commit 77d1ebcc99ddd82a300d1838f608150221931dcd Author: Jia Tan Date: 2023-01-12 22:44:18 +0800 CI: Improve Usage readability and add -h option. build-aux/ci_build.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) commit a8bb8358d10b059274f3cf993d9b8f490bafb268 Author: Lasse Collin Date: 2023-01-12 13:04:05 +0200 Build: Omit -Wmissing-noreturn from the default warnings. It's not that important. It can be annoying in builds that disable many features since in those cases the tests programs will correctly trigger this warning with Clang. configure.ac | 1 - 1 file changed, 1 deletion(-) commit 52dc033d0bde0d19e3912303c6c74bae559d6498 Author: Lasse Collin Date: 2023-01-12 06:05:58 +0200 xz: Use ssize_t for the to-be-ignored return value from write(fd, ptr, 1). It makes no difference here as the return value fits into an int too and it then gets ignored but this looks better. src/xz/file_io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit b1a6d180a363d57b2b1c89526ff3f0782bf863d3 Author: Lasse Collin Date: 2023-01-12 06:01:12 +0200 xz: Silence warnings from -Wsign-conversion in a 32-bit build. src/common/tuklib_mbstr_fw.c | 2 +- src/xz/list.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) commit 31c21c734b7c7d7428a3da7402a2cb7bc2587339 Author: Lasse Collin Date: 2023-01-12 05:38:48 +0200 liblzma: Silence another warning from -Wsign-conversion in a 32-bit build. It doesn't warn on a 64-bit system because truncating a ptrdiff_t (signed long) to uint32_t is diagnosed under -Wconversion by GCC and -Wshorten-64-to-32 by Clang. src/liblzma/lz/lz_encoder_mf.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) commit 37fbdfb7263522c11c7ad2685413d6295532581d Author: Lasse Collin Date: 2023-01-12 04:46:45 +0200 liblzma: Silence a warning from -Wsign-conversion in a 32-bit build. src/common/mythread.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 5ce6ddc221d0bfb57d810d845bb65fb0aac0b008 Author: Lasse Collin Date: 2023-01-12 04:17:24 +0200 Build: Make configure add more warning flags for GCC and Clang. -Wstrict-aliasing was removed from the list since it is enabled by -Wall already. A normal build is clean with these on GNU/Linux x86-64 with GCC 12.2.0 and Clang 14.0.6. configure.ac | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) commit bfc3a0a8ac16de90049c1b1ba1445a7626d0230c Author: Lasse Collin Date: 2023-01-12 04:14:18 +0200 Tests: Fix warnings from clang --Wassign-enum. Explicitly casting the integer to lzma_check silences the warning. Since such an invalid value is needed in multiple tests, a constant INVALID_LZMA_CHECK_ID was added to tests.h. The use of 0x1000 for lzma_block.check wasn't optimal as if the underlying type is a char then 0x1000 will be truncated to 0. However, in these test cases the value is ignored, thus even with such truncation the test would have passed. tests/test_block_header.c | 6 +++--- tests/test_check.c | 2 +- tests/test_stream_flags.c | 8 ++++---- tests/tests.h | 9 +++++++++ 4 files changed, 17 insertions(+), 8 deletions(-) commit 49245bb31e215ad455a1ab85e4ed6783152dc522 Author: Lasse Collin Date: 2023-01-12 03:51:07 +0200 Tests: Silence warnings from -Wsign-conversion. Note that assigning an unsigned int to lzma_check doesn't warn on GNU/Linux x86-64 since the enum type is unsigned on that platform. The enum can be signed on some other platform though so it's best to use enumeration type lzma_check in these situations. tests/test_check.c | 6 +++--- tests/test_stream_flags.c | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) commit 3f13bf6b9e8624cbe6d6e3e82d6c98a3ed1ad571 Author: Lasse Collin Date: 2023-01-12 03:19:59 +0200 liblzma: Silence warnings from clang -Wconditional-uninitialized. This is similar to 2ce4f36f179a81d0c6e182a409f363df759d1ad0. The actual initialization of the variables is done inside mythread_sync() macro. Clang doesn't seem to see that the initialization code inside the macro is always executed. src/liblzma/common/stream_decoder_mt.c | 8 +++++--- src/liblzma/common/stream_encoder_mt.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) commit 6c886cc5b3c90c6a75e6be8b1278ec2261e452a6 Author: Lasse Collin Date: 2023-01-12 03:11:40 +0200 Fix warnings from clang -Wdocumentation. src/liblzma/check/check.h | 4 ---- src/liblzma/lz/lz_encoder_mf.c | 4 ++-- src/xz/options.c | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) commit a0e7fb1c1ea658b67f30517f5d1975efd0226dba Author: Lasse Collin Date: 2023-01-12 03:04:28 +0200 Tests: test_lzip_decoder: Remove trailing white-space. tests/test_lzip_decoder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit c0f8d6782f29e219fd496dd23f6a033270509d5c Author: Lasse Collin Date: 2023-01-12 03:03:55 +0200 Tests: test_lzip_decoder: Silence warnings from -Wsign-conversion. tests/test_lzip_decoder.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) commit 62efd48a825e8f439e84c85e165d8774ddc68fd2 Author: Jia Tan Date: 2023-01-11 23:58:16 +0800 Add NEWS for 5.4.1. NEWS | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) commit d1561c47ec8cd3844a785d3741dc932f9b9c5790 Author: Jia Tan Date: 2023-01-11 22:46:48 +0800 xz: Fix warning -Wformat-nonliteral on clang in message.c. clang and gcc differ in how they handle -Wformat-nonliteral. gcc will allow a non-literal format string as long as the function takes its format arguments as a va_list. src/xz/message.c | 9 +++++++++ 1 file changed, 9 insertions(+) commit 8c0f115cc489331c48df77beca92fe378039d919 Author: Jia Tan Date: 2023-01-11 20:58:31 +0800 Tests: Fix test_filter_flags copy/paste error. tests/test_filter_flags.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 25035813d1d596fde692addc33e7f715f1fe55eb Author: Jia Tan Date: 2023-01-11 20:42:29 +0800 Tests: Fix type-limits warning in test_filter_flags. This only occurs in test_filter_flags when the BCJ filters are not configured and built. In this case, ARRAY_SIZE() returns 0 and causes a type-limits warning with the loop variable since an unsigned number will always be >= 0. tests/test_filter_flags.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) commit 0b8fa310cf56fec55663f62340e49e8e1441594f Author: Lasse Collin Date: 2023-01-10 22:14:03 +0200 liblzma: CLMUL CRC64: Work around a bug in MSVC, second attempt. This affects only 32-bit x86 builds. x86-64 is OK as is. I still cannot easily test this myself. The reporter has tested this and it passes the tests included in the CMake build and performance is good: raw CRC64 is 2-3 times faster than the C version of the slice-by-four method. (Note that liblzma doesn't include a MSVC-compatible version of the 32-bit x86 assembly code for the slice-by-four method.) Thanks to Iouri Kharon for figuring out a fix, testing, and benchmarking. src/liblzma/check/crc64_fast.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) commit 765354b50c2886fc0d294d6be3b207f7ae2ada70 Author: Jia Tan Date: 2023-01-11 01:18:50 +0800 Tests: Fix unused function warning in test_block_header. One of the global arrays of filters was only used in a test that required both encoders and decoders to be configured in the build. tests/test_block_header.c | 4 ++++ 1 file changed, 4 insertions(+) commit 7c23c05befdcc73231c0d6632a7d943dbeaea1aa Author: Jia Tan Date: 2023-01-11 01:08:03 +0800 Tests: Fix unused function warning in test_index_hash. test_index_hash does not use fill_index_hash() unless both encoders and decoders are configured in the build. tests/test_index_hash.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) commit 57464bb4ebd6c00dc8b19803f05ea55ddd0826f6 Author: Jia Tan Date: 2023-01-11 00:54:45 +0800 CI/CD: Add 32-bit build and test steps to Ubuntu autotools runner. If all goes well, Mac autotools and Linux and Mac CMake will be added later for 32-bit builds. .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) commit 923eb689a4b863b6cca8df6360d4962aae994edf Author: Jia Tan Date: 2023-01-11 00:51:01 +0800 CI/CD: Enables warnings as errors in autotool build. This will help us catch warnings and potential bugs in builds that are not often tested by us. build-aux/ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit feae5528a30c006b6e2f96a95116e20b983703fc Author: Jia Tan Date: 2023-01-11 00:48:35 +0800 CI/CD: Add -f argument to set CFLAGS in ci_build.sh. For now, the suggested option is for -m32 only, but this can be updated later if other flags are deemed useful. build-aux/ci_build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) commit cfabb62a4874c146e7d6f30445637602545bc054 Author: Lasse Collin Date: 2023-01-10 12:47:16 +0200 Revert "liblzma: CLMUL CRC64: Workaround a bug in MSVC (VS2015-2022)." This reverts commit 36edc65ab4cf10a131f239acbd423b4510ba52d5. It was reported that it wasn't a good enough fix and MSVC still produced (different kind of) bad code when building for 32-bit x86 if optimizations are enabled. Thanks to Iouri Kharon. src/liblzma/check/crc64_fast.c | 6 ------ 1 file changed, 6 deletions(-) commit 0b64215170dd3562f207ef26f794755bcd600526 Author: Lasse Collin Date: 2023-01-10 11:56:11 +0200 sysdefs.h: Don't include strings.h anymore. On some platforms src/xz/suffix.c may need for strcasecmp() but suffix.c includes the header when it needs it. Unless there is an old system that otherwise supports enough C99 to build XZ Utils but doesn't have C89/C90-compatible , there should be no need to include in sysdefs.h. src/common/sysdefs.h | 6 ------ 1 file changed, 6 deletions(-) commit ec2fc39fe4f4e6e242b3a669585049763968cdeb Author: Lasse Collin Date: 2023-01-10 11:23:41 +0200 xz: Include in suffix.c if needed for strcasecmp(). SUSv2 and POSIX.1‐2017 declare only a few functions in . Of these, strcasecmp() is used on some platforms in suffix.c. Nothing else in the project needs (at least if building on a modern system). sysdefs.h currently includes if HAVE_STRINGS_H is defined and suffix.c relied on this. Note that dos/config.h doesn't #define HAVE_STRINGS_H even though DJGPP does have strings.h. It isn't needed with DJGPP as strcasecmp() is also in in DJGPP. src/xz/suffix.c | 3 +++ 1 file changed, 3 insertions(+) commit 7049c4a76c805ad27d6cf4ee119a2ef2a7add59f Author: Lasse Collin Date: 2023-01-10 10:05:13 +0200 sysdefs.h: Fix a comment. src/common/sysdefs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 194a5fab69277d9e804a6113b5f676b8666b3a61 Author: Lasse Collin Date: 2023-01-10 10:04:06 +0200 sysdefs.h: Don't include memory.h anymore even if it were available. It quite probably was never needed, that is, any system where memory.h was required likely couldn't compile XZ Utils for other reasons anyway. XZ Utils 5.2.6 and later source packages were generated using Autoconf 2.71 which no longer defines HAVE_MEMORY_H. So the code being removed is no longer used anyway. src/common/sysdefs.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) commit 5e34774c31d1b7509b5cb77a3be9973adec59ea0 Author: Lasse Collin Date: 2023-01-10 08:29:32 +0200 CMake: Fix appending to CMAKE_RC_FLAGS. It's a string, not a list. It only worked when the variable was empty. Thanks to Iouri Kharon. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 6e652ceb18c615c578c869db300fa0756788b4e0 Author: Lasse Collin Date: 2023-01-10 00:33:14 +0200 Windows: Update INSTALL-MSVC.txt to recommend CMake over project files. windows/INSTALL-MSVC.txt | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) commit 6b117d3b1fe91eb26d533ab16a2e552f84148d47 Author: Lasse Collin Date: 2023-01-09 23:41:25 +0200 CMake: Fix windres issues again. At least on some systems, GNU windres needs --use-temp-file in addition to the \x20 hack to avoid spaces in the command line argument. Hovever, that \x20 syntax is broken with llvm-windres version 15.0.0 (results in "XZx20Utils") but luckily it works with a regular space. Thus it is best to limit the workarounds to GNU toolchain on Windows. CMakeLists.txt | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) commit 0c210ca7f489e971e94e1ddc72b0b0806e3c7935 Author: Lasse Collin Date: 2023-01-06 22:53:38 +0200 Tests: test_filter_flags: Clean up minor issues. Here are the list of the most significant issues addressed: - Avoid using internal common.h header. It's not good to copy the constants like this but common.h cannot be included for use outside of liblzma. This is the quickest thing to do that could be fixed later. - Omit the INIT_FILTER macro. Initialization should be done with just regular designated initializers. - Use start_offset = 257 for BCJ tests. It demonstrates that Filter Flags encoder and decoder don't validate the options thoroughly. 257 is valid only for the x86 filter. This is a bit silly but not a significant problem in practice because the encoder and decoder initialization functions will catch bad alignment still. Perhaps this should be fixed but it's not urgent and doesn't need to be in 5.4.x. - Various tweaks to comments such as filter id -> Filter ID tests/test_filter_flags.c | 153 +++++++++++++++++++++++----------------------- 1 file changed, 78 insertions(+), 75 deletions(-) commit 5c9fdd3bf53a9655f5eb2807d662b3af0d5e1865 Author: Jia Tan Date: 2022-12-29 23:33:33 +0800 Tests: Refactors existing filter flags tests. Converts the existing filter flags tests into tuktests. tests/test_filter_flags.c | 655 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 457 insertions(+), 198 deletions(-) commit 36edc65ab4cf10a131f239acbd423b4510ba52d5 Author: Lasse Collin Date: 2023-01-09 12:22:05 +0200 liblzma: CLMUL CRC64: Workaround a bug in MSVC (VS2015-2022). I haven't tested with MSVC myself and there doesn't seem to be information about the problem online, so I'm relying on the bug report. Thanks to Iouri Kharon for the bug report and the patch. src/liblzma/check/crc64_fast.c | 6 ++++++ 1 file changed, 6 insertions(+) commit 790a12a95a78ff82d8c6d4efe3b789851ca9470d Author: Lasse Collin Date: 2023-01-09 11:27:24 +0200 CMake: Fix a copypaste error in xzdec Windows resource file handling. It was my mistake. Thanks to Iouri Kharon for the bug report. CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 0e1545fea39c0514c7b7032a0a3592a9a33d2848 Author: Lasse Collin Date: 2023-01-08 00:32:29 +0200 Tests: tuktest.h: Support tuktest_malloc(0). It's not needed in XZ Utils at least for now. It's good to support it still because if such use is needed later, it wouldn't be caught on GNU/Linux since malloc(0) from glibc returns non-NULL. tests/tuktest.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) commit 69d5d78c6904668eb09a131da86276beec3281f8 Author: Lasse Collin Date: 2023-01-08 00:24:23 +0200 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit dd38655f80c113c9db73b9ed370dc900e1c4dc41 Author: Lasse Collin Date: 2023-01-07 21:57:11 +0200 CMake: Update cmake_minimum_required from 3.13...3.16 to 3.13...3.25. The changes listed on cmake-policies(7) for versions 3.17 to 3.25 shouldn't affect this project. CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit a890a637bee9193d5b690aefa9a59eba5b8532ae Author: Lasse Collin Date: 2023-01-07 19:50:35 +0200 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit 6e38e595dd56ac1800478cef1f6f754d0eba0d2e Author: Lasse Collin Date: 2023-01-07 19:50:03 +0200 CMake/Windows: Add resource files to xz.exe and xzdec.exe. The command line tools cannot be built with MSVC for now but they can be built with MinGW-w64. Thanks to Iouri Kharon for the bug report and the original patch. CMakeLists.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) commit 443dfebced041adc88f10d824188eeef5b5821a9 Author: Lasse Collin Date: 2023-01-07 19:48:52 +0200 CMake/Windows: Add a workaround for windres from GNU binutils. Thanks to Iouri Kharon for the bug report and the original patch. CMakeLists.txt | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) commit ceb805011747d04a915f3f39e4bed9eed151c634 Author: Lasse Collin Date: 2023-01-07 19:31:15 +0200 Build: Require that _mm_set_epi64x() is usable to enable CLMUL support. VS2013 doesn't have _mm_set_epi64x() so this way CLMUL gets disabled with VS2013. Thanks to Iouri Kharon for the bug report. CMakeLists.txt | 3 ++- configure.ac | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) commit 8d372bd94066b1a5b0570b2550f83c2868486adf Author: Jia Tan Date: 2023-01-07 21:05:15 +0800 CI/CD: Split CMake Linux and MacOS build phase to build and test. The phase split was only done for Autotools before, so should also apply to CMake. .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) commit 747c7f2b34bd498f6702c6875500a26b06201772 Author: Jia Tan Date: 2023-01-07 11:16:55 +0800 CI/CD: Reduce job runners to 4 instead of using matrix strategy. The old version used too many runners that resulted in unnecessary dependency downloads. Now, the runners are reused for the different configurations for each OS and build system. .github/workflows/ci.yml | 95 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 12 deletions(-) commit 4de35fd6b58d46fc887c78faf163f6a37b790c45 Author: Jia Tan Date: 2023-01-07 10:07:20 +0800 CI/CD: Add new -p (PHASE) argument to ci_build.sh The new PHASE argument can be build, test, or all. all is the default. This way, the CI/CD script can differentiate between the build and test phases to make it easier to track down errors when they happen. build-aux/ci_build.sh | 140 +++++++++++++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 64 deletions(-) commit 6fd39664de47801e670a16617863196bfbde4755 Merge: 78e0561d fc0c7884 Author: Jia Tan Date: 2023-01-07 00:10:50 +0800 Merge pull request #7 from tukaani-project/tuktest_index_hash Tuktest index hash commit fc0c788469159f634f09ff23c8cef6925c91da57 Author: Lasse Collin Date: 2023-01-06 17:58:48 +0200 Tests: test_index_hash: Add an assert_uint_eq(). tests/test_index_hash.c | 3 +++ 1 file changed, 3 insertions(+) commit d550304f5343b3a082da265107cd820e0d81dc71 Author: Lasse Collin Date: 2023-01-06 17:55:06 +0200 Tests: test_index_hash: Fix a memory leak. tests/test_index_hash.c | 2 ++ 1 file changed, 2 insertions(+) commit 02608f74ea1f2d2d56585711ff241c34b4ad0937 Author: Lasse Collin Date: 2023-01-06 17:53:03 +0200 Tests: test_index_hash: Don't treat pointers as booleans. tests/test_index_hash.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) commit 056766c8601a3808bea1761f6cc833197a35a3e0 Author: Lasse Collin Date: 2023-01-06 17:51:41 +0200 Tests: test_index_hash: Fix a typo in a comment. tests/test_index_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 873e684028ba9738f071c5236db7d452ed797b4c Author: Lasse Collin Date: 2023-01-06 17:44:29 +0200 Tests: test_index_hash: Avoid the variable name "index". It can trigger warnings from -Wshadow on some systems. tests/test_index_hash.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) commit d1f24c35874eeba8432d75aa77b06c50375ed937 Author: Lasse Collin Date: 2023-01-06 17:35:50 +0200 Tests: test_index_hash: Use the word "Record" instead of "entry". tests/test_index_hash.c | 102 ++++++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 51 deletions(-) commit b93f7c5cbb02b42024ac866fc0af541de3d816e2 Author: Lasse Collin Date: 2023-01-06 17:35:05 +0200 Tests: test_index_hash: Tweak comments and style. The words defined in the .xz file format specification begin with capital letter to emphasize that they have a specific meaning. tests/test_index_hash.c | 62 ++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 29 deletions(-) commit c48b24fc06d98569adb72f13c2e8e5ff30bb8036 Author: Lasse Collin Date: 2023-01-06 17:17:37 +0200 Tests: test_index_hash: Use INDEX_INDICATOR constant instead of 0. tests/test_index_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 78e0561dfebaa9d5e34558de537efcda890e0629 Author: Jia Tan Date: 2023-01-06 20:43:31 +0800 Style: Change #if !defined() to #ifndef in mythread.h. src/common/mythread.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit e834e1e934ed0af673598d8c0c34afb2af56bee0 Author: Jia Tan Date: 2023-01-06 20:35:55 +0800 Build: Add missing stream_decoder_mt.c to .vcxproj files. The line in the .vcxproj files for building with was missing in 5.4.0. Thank to Hajin Jang for reporting the issue. windows/vs2013/liblzma.vcxproj | 1 + windows/vs2013/liblzma_dll.vcxproj | 1 + windows/vs2017/liblzma.vcxproj | 1 + windows/vs2017/liblzma_dll.vcxproj | 1 + windows/vs2019/liblzma.vcxproj | 1 + windows/vs2019/liblzma_dll.vcxproj | 1 + 6 files changed, 6 insertions(+) commit 84f9687cbae972c2c342e10bf69f8ec8f70ae111 Author: Jia Tan Date: 2023-01-05 20:57:25 +0800 liblzma: Remove common.h include from common/index.h. common/index.h is needed by liblzma internally and tests. common.h will include and define many things that are not needed by the tests. Also, this prevents include order problems because common.h will redefine LZMA_API resulting in a warning. src/liblzma/common/index.c | 1 + src/liblzma/common/index.h | 9 +++++++-- src/liblzma/common/index_decoder.h | 1 + src/liblzma/common/stream_buffer_encoder.c | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) commit 7657ce1c3c4abff7560336a7b687d98e0e2bd14f Author: Lasse Collin Date: 2023-01-04 22:40:54 +0200 Update THANKS. THANKS | 1 + 1 file changed, 1 insertion(+) commit aafd67fba045ab99683971263a5a26fb2a6e8ce2 Author: Lasse Collin Date: 2023-01-04 18:40:28 +0200 Tests: Adjust style in test_compress.sh. tests/test_compress.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) commit 52380678f42364daa4510f92f6d3b18ec98c3638 Author: Jia Tan Date: 2023-01-04 23:58:58 +0800 Tests: Replace non portable shell parameter expansion The shell parameter expansion using # and ## is not supported in Solaris 10 Bourne shell (/bin/sh). Even though this is POSIX, it is not fully portable, so we should avoid it. tests/create_compress_files.c | 2 +- tests/test_compress.sh | 20 +++++++++++++------- tests/test_compress_prepared_bcj_sparc | 2 +- tests/test_compress_prepared_bcj_x86 | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) commit d0eb345bb7d148a62883ee299adec2b74a0f6f3b Author: Jia Tan Date: 2023-01-03 21:02:38 +0800 Translations: Add Korean translation of man pages. Thanks to Seong-ho Cho po4a/ko.po | 5552 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ po4a/po4a.conf | 2 +- 2 files changed, 5553 insertions(+), 1 deletion(-) commit c4145978d95ebf1690c778d354e15f7c2823d7a8 Author: Jia Tan Date: 2023-01-03 20:47:27 +0800 Translations: Update the Esperanto translation. po/eo.po | 620 ++++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 332 insertions(+), 288 deletions(-) commit 4103a2e78ac60b00c888485cd967a5fe5d1b917c Author: Lasse Collin Date: 2023-01-02 17:20:47 +0200 Bump version and soname for 5.5.0alpha. 5.5.0alpha won't be released, it's just to mark that the branch is not for stable 5.4.x. Once again there is no API/ABI stability for new features in devel versions. The major soname won't be bumped even if API/ABI of new features breaks between devel releases. src/liblzma/Makefile.am | 2 +- src/liblzma/api/lzma/version.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) commit 73c9e6d6b970ccc3d5ad61dcaa21cba050e5df0a Author: Lasse Collin Date: 2023-01-02 17:05:07 +0200 Build: Fix config.h comments. configure.ac | 2 +- m4/tuklib_progname.m4 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) commit bb740e3b117f1a3c65152d01e5755523a908ecb1 Author: Jia Tan Date: 2023-01-02 22:33:48 +0800 Build: Only define HAVE_PROGRAM_INVOCATION_NAME if it is set to 1. HAVE_DECL_PROGRAM_INVOCATION_NAME is renamed to HAVE_PROGRAM_INVOCATION_NAME. Previously, HAVE_DECL_PROGRAM_INVOCATION_NAME was always set when building with autotools. CMake would only set this when it was 1, and the dos/config.h did not define it. The new macro definition is consistent across build systems. cmake/tuklib_progname.cmake | 5 ++--- m4/tuklib_progname.m4 | 5 ++++- src/common/tuklib_progname.c | 2 +- src/common/tuklib_progname.h | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) commit 064cd385a716abc78d93a3612411a82d69ceb221 Author: Jia Tan Date: 2022-12-29 00:30:52 +0800 Adds test_index_hash to .gitignore. .gitignore | 1 + 1 file changed, 1 insertion(+) commit 3959162baec074511d83ba0fec1284c3ed724799 Author: Jia Tan Date: 2022-12-29 00:25:18 +0800 Tests: Creates test_index_hash.c Tests all API functions exported from index_hash.h. Does not have a dedicated test for lzma_index_hash_end. CMakeLists.txt | 2 + tests/Makefile.am | 3 + tests/test_index_hash.c | 379 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 384 insertions(+) commit f16e12d5e755d371247202fcccbcccd1ec16b2cf Author: Jia Tan Date: 2022-08-17 20:20:16 +0800 liblzma: Add NULL check to lzma_index_hash_append. This is for consistency with lzma_index_append. src/liblzma/common/index_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) commit 203b008eb220208981902e0db541c02d1c1c9f5e Author: Jia Tan Date: 2022-08-17 17:59:51 +0800 liblzma: Replaced hardcoded 0x0 index indicator byte with macro src/liblzma/common/index.h | 3 +++ src/liblzma/common/index_decoder.c | 2 +- src/liblzma/common/index_encoder.c | 2 +- src/liblzma/common/index_hash.c | 2 +- src/liblzma/common/stream_decoder.c | 3 ++- src/liblzma/common/stream_decoder_mt.c | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) commit dfecda875211f737d0db92dc1d3c58a3a2afb0c0 Author: Lasse Collin Date: 2022-12-30 20:10:08 +0200 Tests: test_check: Test corner cases of CLMUL CRC64. tests/test_check.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) commit ce96bb20435212fe797d6d84738fb9fd4ea13cc7 Author: Lasse Collin Date: 2022-12-30 19:36:49 +0200 Tests: Clarify a comment in test_lzip_decoder.c. tests/test_lzip_decoder.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) commit 2fcba17fc4d7eda8fc60567169cf2a0e6fcfb2f8 Author: Jia Tan Date: 2022-12-29 01:55:19 +0800 xz: Includes and conditionally in mytime.c. Previously, mytime.c depended on mythread.h for to be included. src/xz/mytime.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) commit f82294c8318a7a0990583d51ac5c7de682ad36ef Author: Jia Tan Date: 2022-12-29 01:15:27 +0800 liblzma: Includes sys/time.h conditionally in mythread Previously, was always included, even if mythread only used clock_gettime. is still needed even if clock_gettime is not used though because struct timespec is needed for mythread_condtime. src/common/mythread.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) commit 74dae7d30091e906d6a92a57952dea4354473f9b Author: Jia Tan Date: 2022-12-29 01:10:53 +0800 Build: No longer require HAVE_DECL_CLOCK_MONOTONIC to always be set. Previously, if threading was enabled HAVE_DECL_CLOCK_MONOTONIC would always be set to 0 or 1. However, this macro was needed in xz so if xz was not built with threading and HAVE_DECL_CLOCK_MONOTONIC was not defined but HAVE_CLOCK_GETTIME was, it caused a warning during build. Now, HAVE_DECL_CLOCK_MONOTONIC has been renamed to HAVE_CLOCK_MONOTONIC and will only be set if it is 1. CMakeLists.txt | 8 +++----- configure.ac | 5 ++++- src/common/mythread.h | 4 ++-- src/xz/mytime.c | 5 ++--- 4 files changed, 11 insertions(+), 11 deletions(-) commit 7339e39dc060df6eda74a2c5b69961befc3d5d24 Author: Jia Tan Date: 2022-12-28 01:14:07 +0800 Translations: Add Ukrainian translations of man pages. Thanks to Yuri Chornoivan po4a/po4a.conf | 2 +- po4a/uk.po | 3676 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 3677 insertions(+), 1 deletion(-) commit 9f05c27a58ce8cd7803079aa295e41c24665ce6e Author: Jia Tan Date: 2022-12-23 00:34:48 +0800 CI/CD: Create initial version of CI/CD workflow. The CI/CD workflow will only execute on Ubuntu and MacOS latest version. The workflow will attempt to build with autotools and CMake and execute the tests. The workflow will run for all pull requests and pushes done to the master branch. .github/workflows/ci.yml | 72 ++++++++++++++++++++++++ build-aux/ci_build.sh | 141 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+) commit 1275ebfba74230dbd028049141423c79c8b83b8f Author: Jia Tan Date: 2022-12-22 23:14:53 +0800 liblzma: Update documentation for lzma_filter_encoder. src/liblzma/common/filter_encoder.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) commit 7c9ff5f1667a16733163b75dfd4b509662c387f4 Author: Jia Tan Date: 2022-12-21 21:12:03 +0800 Tests: Adds lzip decoder tests .gitignore | 1 + tests/Makefile.am | 2 + tests/test_lzip_decoder.c | 471 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 474 insertions(+) commit 799ead162de63b8400733603d3abcd2e1977bdca Author: Jia Cheong Tan Date: 2022-12-20 22:05:21 +0800 Doxygen: Update .gitignore for generating docs for in source build. In source builds are not recommended, but we should still ignore the generated artifacts. .gitignore | 2 ++ 1 file changed, 2 insertions(+) commit 5f7ce42a16b1e86ca8408b5c670c25e2a12acc4e Author: Jia Tan Date: 2022-12-20 20:46:44 +0800 liblzma: Fix lzma_microlzma_encoder() return value. Using return_if_error on lzma_lzma_lclppb_encode was improper because return_if_error is expecting an lzma_ret value, but lzma_lzma_lclppb_encode returns a boolean. This could result in lzma_microlzma_encoder, which would be misleading for applications. src/liblzma/common/microlzma_encoder.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) commit 8ace358d65059152d9a1f43f4770170d29d35754 Author: Jia Tan Date: 2022-12-16 20:58:55 +0800 CMake: Update .gitignore for CMake artifacts from in source build. In source builds are not recommended, but we can make it easier by ignoring the generated artifacts from CMake. .gitignore | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) commit 8fd225a2c149f30aeac377e68eb5abf6b28300ad Author: Lasse Collin Date: 2022-12-16 18:30:02 +0200 liblzma: Update authors list in arm64.c. src/liblzma/simple/arm64.c | 1 + 1 file changed, 1 insertion(+) commit b69da6d4bb6bb11fc0cf066920791990d2b22a06 Author: Lasse Collin Date: 2022-12-13 20:37:17 +0200 Bump version to 5.4.0 and soname to 5.4.0. src/liblzma/Makefile.am | 2 +- src/liblzma/api/lzma/version.h | 6 +++--- src/liblzma/liblzma_generic.map | 2 +- src/liblzma/liblzma_linux.map | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/THANKS b/THANKS index 7d2d4fe82ad8..5ed0743b50f0 100644 --- a/THANKS +++ b/THANKS @@ -1,183 +1,202 @@ Thanks ====== Some people have helped more, some less, but nevertheless everyone's help has been important. :-) In alphabetical order: - Mark Adler - Kian-Meng Ang - H. Peter Anvin - Jeff Bastian - Nelson H. F. Beebe - Karl Beldan - Karl Berry - Anders F. Björklund - Emmanuel Blot - Melanie Blower - Alexander Bluhm - Martin Blumenstingl - Ben Boeckel - Jakub Bogusz - Adam Borowski - Maarten Bosmans - Lukas Braune - Benjamin Buch - Trent W. Buck - Kevin R. Bulgrien - James Buren - David Burklund - Frank Busse - Daniel Mealha Cabrita - Milo Casagrande - Marek Černocký - Tomer Chachamu - Vitaly Chikunov - Antoine Cœur + - Felix Collin - Gabi Davar - İhsan Doğan - Chris Donawa - Andrew Dudman - Markus Duft - İsmail Dönmez - Paul Eggert - Robert Elz - Gilles Espinasse - Denis Excoffier - Vincent Fazio - Michael Felt - Michael Fox - Andres Freund - Mike Frysinger - Daniel Richard G. - Tomasz Gajc - Bjarni Ingi Gislason - John Paul Adrian Glaubitz - Bill Glessner - Matthew Good - Michał Górny - Jason Gorski - Juan Manuel Guerrero - Gabriela Gutierrez - Diederik de Haas - Joachim Henke - Christian Hesse - Vincenzo Innocente - Peter Ivanov - Nicholas Jackson - Sam James - Hajin Jang - Hans Jansen - Jouk Jansen - Jun I Jin + - Christoph Junghans - Kiyoshi Kanazawa - Joona Kannisto - Per Øyvind Karlsen + - Firas Khalil Khana - Iouri Kharon - Thomas Klausner - Richard Koch - Anton Kochkov - Ville Koskinen - Sergey Kosukhin - Marcin Kowalczyk - Jan Kratochvil - Christian Kujau - Stephan Kulow - Ilya Kurdyukov - Peter Lawler - James M Leddy - Kelvin Lee - Vincent Lefevre - Hin-Tak Leung - Andraž 'ruskie' Levstik - Cary Lewis - Wim Lewis - Xin Li + - Yifeng Li - Eric Lindblad - Lorenzo De Liso - H.J. Lu - Bela Lubkin - Chenxi Mao - Gregory Margo - Julien Marrec - Ed Maste - Martin Matuška - Ivan A. Melnikov - Jim Meyering - Arkadiusz Miskiewicz - Nathan Moinvaziri - Étienne Mollier - Conley Moorhous + - Andrew Murray - Rafał Mużyło - Adrien Nader - Evan Nemerson + - Alexander Neumann - Hongbo Ni - Jonathan Nieder - Andre Noll - Peter O'Gorman - Dimitri Papadopoulos Orfanos - Daniel Packard - Filip Palian - Peter Pallinger + - Kai Pastor - Rui Paulo - Igor Pavlov - Diego Elio Pettenò - Elbert Pol - Mikko Pouru + - Frank Prochnow - Rich Prohaska - Trần Ngọc Quân - Pavel Raiskup - Ole André Vadla Ravnås - Eric S. Raymond - Robert Readman - Bernhard Reutner-Fischer - Markus Rickert - Cristian Rodríguez - Christian von Roques - Boud Roukema - Torsten Rupp - Stephen Sachs - Jukka Salmi - Agostino Sarubbo + - Vijay Sarvepalli - Alexandre Sauvé - Benno Schulenberg - Andreas Schwab + - Eli Schwartz + - Peter Seiderer - Bhargava Shastry - Dan Shechter - Stuart Shelton - Sebastian Andrzej Siewior - Ville Skyttä - Brad Smith - Bruce Stark - Pippijn van Steenhoven + - Tobias Stoeckmann - Martin Storsjö - Jonathan Stott - Dan Stromberg + - Douglas Thor - Vincent Torri - Alexey Tourbin - Paul Townsend - Mohammed Adnène Trojette + - Orange Tsai - Taiki Tsunekawa + - Mathieu Vachon - Maksym Vatsyk - Loganaden Velvindron - Patrick J. Volkerding - Martin Väth - Adam Walling - Jeffrey Walton - Christian Weisgerber - Dan Weiss - Bert Wesarg - Fredrik Wikstrom - Jim Wilcoxson - Ralf Wildenhues - Charles Wilson - Lars Wirzenius - Pilorz Wojciech - Chien Wong - Ryan Young - Andreas Zieringer +Companies: + - Google + - Sandfly Security + Also thanks to all the people who have participated in the Tukaani project. I have probably forgot to add some names to the above list. Sorry about that and thanks for your help. diff --git a/src/common/tuklib_integer.h b/src/common/tuklib_integer.h index fbd5fb2369a2..4026249e5468 100644 --- a/src/common/tuklib_integer.h +++ b/src/common/tuklib_integer.h @@ -1,948 +1,954 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_integer.h /// \brief Various integer and bit operations /// /// This file provides macros or functions to do some basic integer and bit /// operations. /// /// Native endian inline functions (XX = 16, 32, or 64): /// - Unaligned native endian reads: readXXne(ptr) /// - Unaligned native endian writes: writeXXne(ptr, num) /// - Aligned native endian reads: aligned_readXXne(ptr) /// - Aligned native endian writes: aligned_writeXXne(ptr, num) /// /// Endianness-converting integer operations (these can be macros!) /// (XX = 16, 32, or 64; Y = b or l): /// - Byte swapping: byteswapXX(num) /// - Byte order conversions to/from native (byteswaps if Y isn't /// the native endianness): convXXYe(num) /// - Unaligned reads: readXXYe(ptr) /// - Unaligned writes: writeXXYe(ptr, num) /// - Aligned reads: aligned_readXXYe(ptr) /// - Aligned writes: aligned_writeXXYe(ptr, num) /// /// Since the above can macros, the arguments should have no side effects /// because they may be evaluated more than once. /// /// Bit scan operations for non-zero 32-bit integers (inline functions): /// - Bit scan reverse (find highest non-zero bit): bsr32(num) /// - Count leading zeros: clz32(num) /// - Count trailing zeros: ctz32(num) /// - Bit scan forward (simply an alias for ctz32()): bsf32(num) /// /// The above bit scan operations return 0-31. If num is zero, /// the result is undefined. // // Authors: Lasse Collin // Joachim Henke // /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_INTEGER_H #define TUKLIB_INTEGER_H #include "tuklib_common.h" #include // Newer Intel C compilers require immintrin.h for _bit_scan_reverse() // and such functions. #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1500) # include // Only include when it is needed. GCC and Clang can both // use __builtin's, so we only need Windows instrincs when using MSVC. // GCC and Clang can set _MSC_VER on Windows, so we need to exclude these // cases explicitly. #elif defined(_MSC_VER) && !TUKLIB_GNUC_REQ(3, 4) && !defined(__clang__) # include #endif /////////////////// // Byte swapping // /////////////////// #if defined(HAVE___BUILTIN_BSWAPXX) // GCC >= 4.8 and Clang # define byteswap16(num) __builtin_bswap16(num) # define byteswap32(num) __builtin_bswap32(num) # define byteswap64(num) __builtin_bswap64(num) #elif defined(HAVE_BYTESWAP_H) // glibc, uClibc, dietlibc # include # ifdef HAVE_BSWAP_16 # define byteswap16(num) bswap_16(num) # endif # ifdef HAVE_BSWAP_32 # define byteswap32(num) bswap_32(num) # endif # ifdef HAVE_BSWAP_64 # define byteswap64(num) bswap_64(num) # endif #elif defined(HAVE_SYS_ENDIAN_H) // *BSDs and Darwin # include -# define byteswap16(num) bswap16(num) -# define byteswap32(num) bswap32(num) -# define byteswap64(num) bswap64(num) +# ifdef __OpenBSD__ +# define byteswap16(num) swap16(num) +# define byteswap32(num) swap32(num) +# define byteswap64(num) swap64(num) +# else +# define byteswap16(num) bswap16(num) +# define byteswap32(num) bswap32(num) +# define byteswap64(num) bswap64(num) +# endif #elif defined(HAVE_SYS_BYTEORDER_H) // Solaris # include # ifdef BSWAP_16 # define byteswap16(num) BSWAP_16(num) # endif # ifdef BSWAP_32 # define byteswap32(num) BSWAP_32(num) # endif # ifdef BSWAP_64 # define byteswap64(num) BSWAP_64(num) # endif # ifdef BE_16 # define conv16be(num) BE_16(num) # endif # ifdef BE_32 # define conv32be(num) BE_32(num) # endif # ifdef BE_64 # define conv64be(num) BE_64(num) # endif # ifdef LE_16 # define conv16le(num) LE_16(num) # endif # ifdef LE_32 # define conv32le(num) LE_32(num) # endif # ifdef LE_64 # define conv64le(num) LE_64(num) # endif #endif #ifndef byteswap16 # define byteswap16(n) (uint16_t)( \ (((n) & 0x00FFU) << 8) \ | (((n) & 0xFF00U) >> 8) \ ) #endif #ifndef byteswap32 # define byteswap32(n) (uint32_t)( \ (((n) & UINT32_C(0x000000FF)) << 24) \ | (((n) & UINT32_C(0x0000FF00)) << 8) \ | (((n) & UINT32_C(0x00FF0000)) >> 8) \ | (((n) & UINT32_C(0xFF000000)) >> 24) \ ) #endif #ifndef byteswap64 # define byteswap64(n) (uint64_t)( \ (((n) & UINT64_C(0x00000000000000FF)) << 56) \ | (((n) & UINT64_C(0x000000000000FF00)) << 40) \ | (((n) & UINT64_C(0x0000000000FF0000)) << 24) \ | (((n) & UINT64_C(0x00000000FF000000)) << 8) \ | (((n) & UINT64_C(0x000000FF00000000)) >> 8) \ | (((n) & UINT64_C(0x0000FF0000000000)) >> 24) \ | (((n) & UINT64_C(0x00FF000000000000)) >> 40) \ | (((n) & UINT64_C(0xFF00000000000000)) >> 56) \ ) #endif // Define conversion macros using the basic byte swapping macros. #ifdef WORDS_BIGENDIAN # ifndef conv16be # define conv16be(num) ((uint16_t)(num)) # endif # ifndef conv32be # define conv32be(num) ((uint32_t)(num)) # endif # ifndef conv64be # define conv64be(num) ((uint64_t)(num)) # endif # ifndef conv16le # define conv16le(num) byteswap16(num) # endif # ifndef conv32le # define conv32le(num) byteswap32(num) # endif # ifndef conv64le # define conv64le(num) byteswap64(num) # endif #else # ifndef conv16be # define conv16be(num) byteswap16(num) # endif # ifndef conv32be # define conv32be(num) byteswap32(num) # endif # ifndef conv64be # define conv64be(num) byteswap64(num) # endif # ifndef conv16le # define conv16le(num) ((uint16_t)(num)) # endif # ifndef conv32le # define conv32le(num) ((uint32_t)(num)) # endif # ifndef conv64le # define conv64le(num) ((uint64_t)(num)) # endif #endif //////////////////////////////// // Unaligned reads and writes // //////////////////////////////// // No-strict-align archs like x86-64 // --------------------------------- // // The traditional way of casting e.g. *(const uint16_t *)uint8_pointer // is bad even if the uint8_pointer is properly aligned because this kind // of casts break strict aliasing rules and result in undefined behavior. // With unaligned pointers it's even worse: compilers may emit vector // instructions that require aligned pointers even if non-vector // instructions work with unaligned pointers. // // Using memcpy() is the standard compliant way to do unaligned access. // Many modern compilers inline it so there is no function call overhead. // For those compilers that don't handle the memcpy() method well, the // old casting method (that violates strict aliasing) can be requested at // build time. A third method, casting to a packed struct, would also be // an option but isn't provided to keep things simpler (it's already a mess). // Hopefully this is flexible enough in practice. // // Some compilers on x86-64 like Clang >= 10 and GCC >= 5.1 detect that // // buf[0] | (buf[1] << 8) // // reads a 16-bit value and can emit a single 16-bit load and produce // identical code than with the memcpy() method. In other cases Clang and GCC // produce either the same or better code with memcpy(). For example, Clang 9 // on x86-64 can detect 32-bit load but not 16-bit load. // // MSVC uses unaligned access with the memcpy() method but emits byte-by-byte // code for "buf[0] | (buf[1] << 8)". // // Conclusion: The memcpy() method is the best choice when unaligned access // is supported. // // Strict-align archs like SPARC // ----------------------------- // // GCC versions from around 4.x to to at least 13.2.0 produce worse code // from the memcpy() method than from simple byte-by-byte shift-or code // when reading a 32-bit integer: // -// (1) It may be constructed on stack using using four 8-bit loads, +// (1) It may be constructed on stack using four 8-bit loads, // four 8-bit stores to stack, and finally one 32-bit load from stack. // // (2) Especially with -Os, an actual memcpy() call may be emitted. // // This is true on at least on ARM, ARM64, SPARC, SPARC64, MIPS64EL, and // RISC-V. Of these, ARM, ARM64, and RISC-V support unaligned access in // some processors but not all so this is relevant only in the case when // GCC assumes that unaligned is not supported or -mstrict-align or // -mno-unaligned-access is used. // // For Clang it makes little difference. ARM64 with -O2 -mstrict-align // was one the very few with a minor difference: the memcpy() version // was one instruction longer. // // Conclusion: At least in case of GCC and Clang, byte-by-byte code is // the best choice for strict-align archs to do unaligned access. // // See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111502 // // Thanks to it was easy to test different compilers. // The following is for little endian targets: /* #include #include uint32_t bytes16(const uint8_t *b) { return (uint32_t)b[0] | ((uint32_t)b[1] << 8); } uint32_t copy16(const uint8_t *b) { uint16_t v; memcpy(&v, b, sizeof(v)); return v; } uint32_t bytes32(const uint8_t *b) { return (uint32_t)b[0] | ((uint32_t)b[1] << 8) | ((uint32_t)b[2] << 16) | ((uint32_t)b[3] << 24); } uint32_t copy32(const uint8_t *b) { uint32_t v; memcpy(&v, b, sizeof(v)); return v; } void wbytes16(uint8_t *b, uint16_t v) { b[0] = (uint8_t)v; b[1] = (uint8_t)(v >> 8); } void wcopy16(uint8_t *b, uint16_t v) { memcpy(b, &v, sizeof(v)); } void wbytes32(uint8_t *b, uint32_t v) { b[0] = (uint8_t)v; b[1] = (uint8_t)(v >> 8); b[2] = (uint8_t)(v >> 16); b[3] = (uint8_t)(v >> 24); } void wcopy32(uint8_t *b, uint32_t v) { memcpy(b, &v, sizeof(v)); } */ #ifdef TUKLIB_FAST_UNALIGNED_ACCESS static inline uint16_t read16ne(const uint8_t *buf) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING return *(const uint16_t *)buf; #else uint16_t num; memcpy(&num, buf, sizeof(num)); return num; #endif } static inline uint32_t read32ne(const uint8_t *buf) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING return *(const uint32_t *)buf; #else uint32_t num; memcpy(&num, buf, sizeof(num)); return num; #endif } static inline uint64_t read64ne(const uint8_t *buf) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING return *(const uint64_t *)buf; #else uint64_t num; memcpy(&num, buf, sizeof(num)); return num; #endif } static inline void write16ne(uint8_t *buf, uint16_t num) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint16_t *)buf = num; #else memcpy(buf, &num, sizeof(num)); #endif return; } static inline void write32ne(uint8_t *buf, uint32_t num) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint32_t *)buf = num; #else memcpy(buf, &num, sizeof(num)); #endif return; } static inline void write64ne(uint8_t *buf, uint64_t num) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint64_t *)buf = num; #else memcpy(buf, &num, sizeof(num)); #endif return; } static inline uint16_t read16be(const uint8_t *buf) { uint16_t num = read16ne(buf); return conv16be(num); } static inline uint16_t read16le(const uint8_t *buf) { uint16_t num = read16ne(buf); return conv16le(num); } static inline uint32_t read32be(const uint8_t *buf) { uint32_t num = read32ne(buf); return conv32be(num); } static inline uint32_t read32le(const uint8_t *buf) { uint32_t num = read32ne(buf); return conv32le(num); } static inline uint64_t read64be(const uint8_t *buf) { uint64_t num = read64ne(buf); return conv64be(num); } static inline uint64_t read64le(const uint8_t *buf) { uint64_t num = read64ne(buf); return conv64le(num); } // NOTE: Possible byte swapping must be done in a macro to allow the compiler // to optimize byte swapping of constants when using glibc's or *BSD's // byte swapping macros. The actual write is done in an inline function // to make type checking of the buf pointer possible. #define write16be(buf, num) write16ne(buf, conv16be(num)) #define write32be(buf, num) write32ne(buf, conv32be(num)) #define write64be(buf, num) write64ne(buf, conv64be(num)) #define write16le(buf, num) write16ne(buf, conv16le(num)) #define write32le(buf, num) write32ne(buf, conv32le(num)) #define write64le(buf, num) write64ne(buf, conv64le(num)) #else #ifdef WORDS_BIGENDIAN # define read16ne read16be # define read32ne read32be # define read64ne read64be # define write16ne write16be # define write32ne write32be # define write64ne write64be #else # define read16ne read16le # define read32ne read32le # define read64ne read64le # define write16ne write16le # define write32ne write32le # define write64ne write64le #endif static inline uint16_t read16be(const uint8_t *buf) { uint16_t num = ((uint16_t)buf[0] << 8) | (uint16_t)buf[1]; return num; } static inline uint16_t read16le(const uint8_t *buf) { uint16_t num = ((uint16_t)buf[0]) | ((uint16_t)buf[1] << 8); return num; } static inline uint32_t read32be(const uint8_t *buf) { uint32_t num = (uint32_t)buf[0] << 24; num |= (uint32_t)buf[1] << 16; num |= (uint32_t)buf[2] << 8; num |= (uint32_t)buf[3]; return num; } static inline uint32_t read32le(const uint8_t *buf) { uint32_t num = (uint32_t)buf[0]; num |= (uint32_t)buf[1] << 8; num |= (uint32_t)buf[2] << 16; num |= (uint32_t)buf[3] << 24; return num; } static inline uint64_t read64be(const uint8_t *buf) { uint64_t num = (uint64_t)buf[0] << 56; num |= (uint64_t)buf[1] << 48; num |= (uint64_t)buf[2] << 40; num |= (uint64_t)buf[3] << 32; num |= (uint64_t)buf[4] << 24; num |= (uint64_t)buf[5] << 16; num |= (uint64_t)buf[6] << 8; num |= (uint64_t)buf[7]; return num; } static inline uint64_t read64le(const uint8_t *buf) { uint64_t num = (uint64_t)buf[0]; num |= (uint64_t)buf[1] << 8; num |= (uint64_t)buf[2] << 16; num |= (uint64_t)buf[3] << 24; num |= (uint64_t)buf[4] << 32; num |= (uint64_t)buf[5] << 40; num |= (uint64_t)buf[6] << 48; num |= (uint64_t)buf[7] << 56; return num; } static inline void write16be(uint8_t *buf, uint16_t num) { buf[0] = (uint8_t)(num >> 8); buf[1] = (uint8_t)num; return; } static inline void write16le(uint8_t *buf, uint16_t num) { buf[0] = (uint8_t)num; buf[1] = (uint8_t)(num >> 8); return; } static inline void write32be(uint8_t *buf, uint32_t num) { buf[0] = (uint8_t)(num >> 24); buf[1] = (uint8_t)(num >> 16); buf[2] = (uint8_t)(num >> 8); buf[3] = (uint8_t)num; return; } static inline void write32le(uint8_t *buf, uint32_t num) { buf[0] = (uint8_t)num; buf[1] = (uint8_t)(num >> 8); buf[2] = (uint8_t)(num >> 16); buf[3] = (uint8_t)(num >> 24); return; } static inline void write64be(uint8_t *buf, uint64_t num) { buf[0] = (uint8_t)(num >> 56); buf[1] = (uint8_t)(num >> 48); buf[2] = (uint8_t)(num >> 40); buf[3] = (uint8_t)(num >> 32); buf[4] = (uint8_t)(num >> 24); buf[5] = (uint8_t)(num >> 16); buf[6] = (uint8_t)(num >> 8); buf[7] = (uint8_t)num; return; } static inline void write64le(uint8_t *buf, uint64_t num) { buf[0] = (uint8_t)num; buf[1] = (uint8_t)(num >> 8); buf[2] = (uint8_t)(num >> 16); buf[3] = (uint8_t)(num >> 24); buf[4] = (uint8_t)(num >> 32); buf[5] = (uint8_t)(num >> 40); buf[6] = (uint8_t)(num >> 48); buf[7] = (uint8_t)(num >> 56); return; } #endif ////////////////////////////// // Aligned reads and writes // ////////////////////////////// // Separate functions for aligned reads and writes are provided since on // strict-align archs aligned access is much faster than unaligned access. // // Just like in the unaligned case, memcpy() is needed to avoid // strict aliasing violations. However, on archs that don't support // unaligned access the compiler cannot know that the pointers given // to memcpy() are aligned which results in slow code. As of C11 there is // no standard way to tell the compiler that we know that the address is // aligned but some compilers have language extensions to do that. With // such language extensions the memcpy() method gives excellent results. // // What to do on a strict-align system when no known language extensions // are available? Falling back to byte-by-byte access would be safe but ruin // optimizations that have been made specifically with aligned access in mind. // As a compromise, aligned reads will fall back to non-compliant type punning // but aligned writes will be byte-by-byte, that is, fast reads are preferred // over fast writes. This obviously isn't great but hopefully it's a working // compromise for now. // // __builtin_assume_aligned is support by GCC >= 4.7 and clang >= 3.6. #ifdef HAVE___BUILTIN_ASSUME_ALIGNED # define tuklib_memcpy_aligned(dest, src, size) \ memcpy(dest, __builtin_assume_aligned(src, size), size) #else # define tuklib_memcpy_aligned(dest, src, size) \ memcpy(dest, src, size) # ifndef TUKLIB_FAST_UNALIGNED_ACCESS # define TUKLIB_USE_UNSAFE_ALIGNED_READS 1 # endif #endif static inline uint16_t aligned_read16ne(const uint8_t *buf) { #if defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) \ || defined(TUKLIB_USE_UNSAFE_ALIGNED_READS) return *(const uint16_t *)buf; #else uint16_t num; tuklib_memcpy_aligned(&num, buf, sizeof(num)); return num; #endif } static inline uint32_t aligned_read32ne(const uint8_t *buf) { #if defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) \ || defined(TUKLIB_USE_UNSAFE_ALIGNED_READS) return *(const uint32_t *)buf; #else uint32_t num; tuklib_memcpy_aligned(&num, buf, sizeof(num)); return num; #endif } static inline uint64_t aligned_read64ne(const uint8_t *buf) { #if defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) \ || defined(TUKLIB_USE_UNSAFE_ALIGNED_READS) return *(const uint64_t *)buf; #else uint64_t num; tuklib_memcpy_aligned(&num, buf, sizeof(num)); return num; #endif } static inline void aligned_write16ne(uint8_t *buf, uint16_t num) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint16_t *)buf = num; #else tuklib_memcpy_aligned(buf, &num, sizeof(num)); #endif return; } static inline void aligned_write32ne(uint8_t *buf, uint32_t num) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint32_t *)buf = num; #else tuklib_memcpy_aligned(buf, &num, sizeof(num)); #endif return; } static inline void aligned_write64ne(uint8_t *buf, uint64_t num) { #ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint64_t *)buf = num; #else tuklib_memcpy_aligned(buf, &num, sizeof(num)); #endif return; } static inline uint16_t aligned_read16be(const uint8_t *buf) { uint16_t num = aligned_read16ne(buf); return conv16be(num); } static inline uint16_t aligned_read16le(const uint8_t *buf) { uint16_t num = aligned_read16ne(buf); return conv16le(num); } static inline uint32_t aligned_read32be(const uint8_t *buf) { uint32_t num = aligned_read32ne(buf); return conv32be(num); } static inline uint32_t aligned_read32le(const uint8_t *buf) { uint32_t num = aligned_read32ne(buf); return conv32le(num); } static inline uint64_t aligned_read64be(const uint8_t *buf) { uint64_t num = aligned_read64ne(buf); return conv64be(num); } static inline uint64_t aligned_read64le(const uint8_t *buf) { uint64_t num = aligned_read64ne(buf); return conv64le(num); } // These need to be macros like in the unaligned case. #define aligned_write16be(buf, num) aligned_write16ne((buf), conv16be(num)) #define aligned_write16le(buf, num) aligned_write16ne((buf), conv16le(num)) #define aligned_write32be(buf, num) aligned_write32ne((buf), conv32be(num)) #define aligned_write32le(buf, num) aligned_write32ne((buf), conv32le(num)) #define aligned_write64be(buf, num) aligned_write64ne((buf), conv64be(num)) #define aligned_write64le(buf, num) aligned_write64ne((buf), conv64le(num)) //////////////////// // Bit operations // //////////////////// static inline uint32_t bsr32(uint32_t n) { // Check for ICC first, since it tends to define __GNUC__ too. #if defined(__INTEL_COMPILER) return _bit_scan_reverse(n); #elif (TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) && UINT_MAX == UINT32_MAX // GCC >= 3.4 has __builtin_clz(), which gives good results on // multiple architectures. On x86, __builtin_clz() ^ 31U becomes // either plain BSR (so the XOR gets optimized away) or LZCNT and // XOR (if -march indicates that SSE4a instructions are supported). return (uint32_t)__builtin_clz(n) ^ 31U; #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) uint32_t i; __asm__("bsrl %1, %0" : "=r" (i) : "rm" (n)); return i; #elif defined(_MSC_VER) unsigned long i; _BitScanReverse(&i, n); return i; #else uint32_t i = 31; if ((n & 0xFFFF0000) == 0) { n <<= 16; i = 15; } if ((n & 0xFF000000) == 0) { n <<= 8; i -= 8; } if ((n & 0xF0000000) == 0) { n <<= 4; i -= 4; } if ((n & 0xC0000000) == 0) { n <<= 2; i -= 2; } if ((n & 0x80000000) == 0) --i; return i; #endif } static inline uint32_t clz32(uint32_t n) { #if defined(__INTEL_COMPILER) return _bit_scan_reverse(n) ^ 31U; #elif (TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) && UINT_MAX == UINT32_MAX return (uint32_t)__builtin_clz(n); #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) uint32_t i; __asm__("bsrl %1, %0\n\t" "xorl $31, %0" : "=r" (i) : "rm" (n)); return i; #elif defined(_MSC_VER) unsigned long i; _BitScanReverse(&i, n); return i ^ 31U; #else uint32_t i = 0; if ((n & 0xFFFF0000) == 0) { n <<= 16; i = 16; } if ((n & 0xFF000000) == 0) { n <<= 8; i += 8; } if ((n & 0xF0000000) == 0) { n <<= 4; i += 4; } if ((n & 0xC0000000) == 0) { n <<= 2; i += 2; } if ((n & 0x80000000) == 0) ++i; return i; #endif } static inline uint32_t ctz32(uint32_t n) { #if defined(__INTEL_COMPILER) return _bit_scan_forward(n); #elif (TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) && UINT_MAX >= UINT32_MAX return (uint32_t)__builtin_ctz(n); #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) uint32_t i; __asm__("bsfl %1, %0" : "=r" (i) : "rm" (n)); return i; #elif defined(_MSC_VER) unsigned long i; _BitScanForward(&i, n); return i; #else uint32_t i = 0; if ((n & 0x0000FFFF) == 0) { n >>= 16; i = 16; } if ((n & 0x000000FF) == 0) { n >>= 8; i += 8; } if ((n & 0x0000000F) == 0) { n >>= 4; i += 4; } if ((n & 0x00000003) == 0) { n >>= 2; i += 2; } if ((n & 0x00000001) == 0) ++i; return i; #endif } #define bsf32 ctz32 #endif diff --git a/src/common/w32_application.manifest b/src/common/w32_application.manifest new file mode 100644 index 000000000000..2f8750879b1f --- /dev/null +++ b/src/common/w32_application.manifest @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + true + UTF-8 + + + diff --git a/src/common/w32_application.manifest.comments.txt b/src/common/w32_application.manifest.comments.txt new file mode 100644 index 000000000000..ad0835ccb0b1 --- /dev/null +++ b/src/common/w32_application.manifest.comments.txt @@ -0,0 +1,178 @@ + +Windows application manifest for UTF-8 and long paths +===================================================== + +The .manifest file is embedded as is in the executables, thus +the comments are here in a separate file. These comments were +written in context of XZ Utils but might be useful when porting +other command line tools from POSIX environments to Windows. + + NOTE: On Cygwin and MSYS2, command line arguments and file + system access aren't tied to a Windows code page. Cygwin + and MSYS2 include a default application manifest. Replacing + it doesn't seem useful and might even be harmful if Cygwin + and MSYS2 some day change their default manifest. + + +UTF-8 code page +--------------- + +On Windows, command line applications can use main() or wmain(). +With the Windows-specific wmain(), argv contains UTF-16 code units +which is the native encoding on Windows. With main(), argv uses the +system active code page by default. It typically is a legacy code +page like Windows-1252. + + NOTE: On POSIX, argv for main() is constructed by the calling + process. On Windows, argv is constructed by a new process + itself: a program receives the command line as a single string, + and the startup code splits it into individual arguments, + including quote removal and wildcard expansion. Then main() or + wmain() is called. + +This application manifest forces the process code page to UTF-8 +when the application runs on Windows 10 version 1903 or later. +This is useful for programs that use main(): + + * UTF-8 allows such programs to access files whose names contain + characters that don't exist in the current legacy code page. + However, filenames on Windows may contain unpaired surrogates + (invalid UTF-16). Such files cannot be accesses even with the + UTF-8 code page. + + * UTF-8 avoids a security issue in command line argument handling: + If a command line contains Unicode characters (for example, + filenames) that don't exist in the current legacy code page, + the characters are converted to similar-looking characters + with best-fit mapping. Some best-fit mappings result in ASCII + characters that change the meaning of the command line, which + can be exploited with malicious filenames. For example: + + - Double quote (") breaks quoting and makes argument + injection possible. + + - Question mark (?) is a wildcard character which may + expand to one or more filenames. + + - Forward slash (/) makes a directory traversal attack + possible. This character can appear in a dangerous way + even from a wildcard expansion; a look-alike character + doesn't need to be passed directly on the command line. + + UTF-8 avoids best-fit mappings. However, it's still not + perfect. Unpaired surrogates (invalid UTF-16) on the command + line (including those from wildcard expansion) are converted + to the replacement character U+FFFD. Thus, filenames with + different unpaired surrogates appear identical when converted + to the UTF-8 code page and aren't distinguishable from + filenames that contain the actual replacement character U+FFFD. + +If different programs use different code pages, compatibility issues +are possible. For example, if one program produces a list of +filenames and another program reads it, both programs should use +the same code page because the code page affects filenames in the +char-based file system APIs. + +If building with a MinGW-w64 toolchain, it is strongly recommended +to use UCRT instead of the old MSVCRT. For example, with the UTF-8 +code page, MSVCRT doesn't convert non-ASCII characters correctly +when writing to console with printf(). With UCRT it works. + + +Long path names +--------------- + +The manifest enables support for path names longer than 259 +characters if the feature has been enabled in the Windows registry. +Omit the longPathAware element from the manifest if the application +isn't compatible with it. For example, uses of MAX_PATH might be +a sign of incompatibility. + +Documentation of the registry setting: +https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later + + +Summary of the manifest contents +-------------------------------- + +See also Microsoft's documentation: +https://learn.microsoft.com/en-us/windows/win32/sbscs/application-manifests + +assemblyIdentity (omitted) + + This is documented as mandatory but not all apps in the real world + have it, and of those that do, not all put an up-to-date version + number there. Things seem to work correctly without + so let's keep this simpler and omit it. + +compatibility + + Declare the application compatible with different Windows versions. + Without this, Windows versions newer than Vista will run the + application using Vista as the Operating System Context. + +trustInfo + + Declare the application as UAC-compliant. This avoids file system + and registry virtualization that Windows otherwise does with 32-bit + executables to make some ancient applications work. UAC-compliancy + also stops Windows from using heuristics based on the filename + (like setup.exe) to guess when elevated privileges might be + needed which would then bring up the UAC prompt. + +longPathAware + + Declare the application as long path aware. This way many file + system operations aren't limited by MAX_PATH (260 characters + including the terminating null character) if the feature has + also been enabled in the Windows registry. + +activeCodePage + + Force the process code page to UTF-8 on Windows 10 version 1903 + and later. For example: + + - main() gets the command line arguments in UTF-8 instead of + in a legacy code page. + + - File system APIs that take char-based strings use UTF-8 + instead of a legacy code page. + + - Text written to the console via stdio.h's stdout or stderr + (like calling printf()) are expected to be in UTF-8. + + +CMake notes +----------- + +As of CMake 3.30, one can add a .manifest file as a source file but +it only works with MSVC; it's ignored with MinGW-w64 toolchains. +Embedding the manifest with a resource file works with all +toolchains. However, then the default manifest needs to be +disabled with MSVC in CMakeLists.txt to avoid duplicate +manifests which would break the build. + +w32_application.manifest.rc: + + #include + CREATEPROCESS_MANIFEST_RESOURCE_ID RT_MANIFEST "w32_application.manifest" + +Or the same thing without the #include: + + 1 24 "w32_application.manifest" + +CMakeLists.txt: + + if(MSVC) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") + endif() + + add_executable(foo foo.c) + + # WIN32 isn't set on Cygwin or MSYS2, thus if(WIN32) is correct here. + if(WIN32) + target_sources(foo PRIVATE w32_application.manifest.rc) + set_source_files_properties(w32_application.manifest.rc PROPERTIES + OBJECT_DEPENDS w32_application.manifest + ) + endif() diff --git a/src/liblzma/api/lzma/container.h b/src/liblzma/api/lzma/container.h index 8e4af420389b..ee5d77e4f1af 100644 --- a/src/liblzma/api/lzma/container.h +++ b/src/liblzma/api/lzma/container.h @@ -1,995 +1,995 @@ /* SPDX-License-Identifier: 0BSD */ /** * \file lzma/container.h * \brief File formats * \note Never include this file directly. Use instead. */ /* * Author: Lasse Collin */ #ifndef LZMA_H_INTERNAL # error Never include this file directly. Use instead. #endif /************ * Encoding * ************/ /** * \brief Default compression preset * * It's not straightforward to recommend a default preset, because in some * cases keeping the resource usage relatively low is more important that * getting the maximum compression ratio. */ #define LZMA_PRESET_DEFAULT UINT32_C(6) /** * \brief Mask for preset level * * This is useful only if you need to extract the level from the preset * variable. That should be rare. */ #define LZMA_PRESET_LEVEL_MASK UINT32_C(0x1F) /* * Preset flags * * Currently only one flag is defined. */ /** * \brief Extreme compression preset * * This flag modifies the preset to make the encoding significantly slower * while improving the compression ratio only marginally. This is useful * when you don't mind spending time to get as small result as possible. * * This flag doesn't affect the memory usage requirements of the decoder (at * least not significantly). The memory usage of the encoder may be increased * a little but only at the lowest preset levels (0-3). */ #define LZMA_PRESET_EXTREME (UINT32_C(1) << 31) /** * \brief Multithreading options */ typedef struct { /** * \brief Flags * * Set this to zero if no flags are wanted. * * Encoder: No flags are currently supported. * * Decoder: Bitwise-or of zero or more of the decoder flags: * - LZMA_TELL_NO_CHECK * - LZMA_TELL_UNSUPPORTED_CHECK * - LZMA_TELL_ANY_CHECK * - LZMA_IGNORE_CHECK * - LZMA_CONCATENATED * - LZMA_FAIL_FAST */ uint32_t flags; /** * \brief Number of worker threads to use */ uint32_t threads; /** * \brief Encoder only: Maximum uncompressed size of a Block * * The encoder will start a new .xz Block every block_size bytes. * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code() * the caller may tell liblzma to start a new Block earlier. * * With LZMA2, a recommended block size is 2-4 times the LZMA2 * dictionary size. With very small dictionaries, it is recommended * to use at least 1 MiB block size for good compression ratio, even * if this is more than four times the dictionary size. Note that * these are only recommendations for typical use cases; feel free * to use other values. Just keep in mind that using a block size * less than the LZMA2 dictionary size is waste of RAM. * * Set this to 0 to let liblzma choose the block size depending * on the compression options. For LZMA2 it will be 3*dict_size * or 1 MiB, whichever is more. * * For each thread, about 3 * block_size bytes of memory will be * allocated. This may change in later liblzma versions. If so, * the memory usage will probably be reduced, not increased. */ uint64_t block_size; /** * \brief Timeout to allow lzma_code() to return early * * Multithreading can make liblzma consume input and produce * output in a very bursty way: it may first read a lot of input * to fill internal buffers, then no input or output occurs for * a while. * * In single-threaded mode, lzma_code() won't return until it has * either consumed all the input or filled the output buffer. If * this is done in multithreaded mode, it may cause a call * lzma_code() to take even tens of seconds, which isn't acceptable * in all applications. * * To avoid very long blocking times in lzma_code(), a timeout * (in milliseconds) may be set here. If lzma_code() would block * longer than this number of milliseconds, it will return with * LZMA_OK. Reasonable values are 100 ms or more. The xz command * line tool uses 300 ms. * * If long blocking times are acceptable, set timeout to a special * value of 0. This will disable the timeout mechanism and will make * lzma_code() block until all the input is consumed or the output * buffer has been filled. * * \note Even with a timeout, lzma_code() might sometimes take * a long time to return. No timing guarantees are made. */ uint32_t timeout; /** * \brief Encoder only: Compression preset * * The preset is set just like with lzma_easy_encoder(). * The preset is ignored if filters below is non-NULL. */ uint32_t preset; /** * \brief Encoder only: Filter chain (alternative to a preset) * * If this is NULL, the preset above is used. Otherwise the preset * is ignored and the filter chain specified here is used. */ const lzma_filter *filters; /** * \brief Encoder only: Integrity check type * * See check.h for available checks. The xz command line tool * defaults to LZMA_CHECK_CRC64, which is a good choice if you * are unsure. */ lzma_check check; /* * Reserved space to allow possible future extensions without * breaking the ABI. You should not touch these, because the names * of these variables may change. These are and will never be used * with the currently supported options, so it is safe to leave these * uninitialized. */ /** \private Reserved member. */ lzma_reserved_enum reserved_enum1; /** \private Reserved member. */ lzma_reserved_enum reserved_enum2; /** \private Reserved member. */ lzma_reserved_enum reserved_enum3; /** \private Reserved member. */ uint32_t reserved_int1; /** \private Reserved member. */ uint32_t reserved_int2; /** \private Reserved member. */ uint32_t reserved_int3; /** \private Reserved member. */ uint32_t reserved_int4; /** * \brief Memory usage limit to reduce the number of threads * * Encoder: Ignored. * * Decoder: * * If the number of threads has been set so high that more than * memlimit_threading bytes of memory would be needed, the number * of threads will be reduced so that the memory usage will not exceed * memlimit_threading bytes. However, if memlimit_threading cannot * be met even in single-threaded mode, then decoding will continue * in single-threaded mode and memlimit_threading may be exceeded * even by a large amount. That is, memlimit_threading will never make * lzma_code() return LZMA_MEMLIMIT_ERROR. To truly cap the memory * usage, see memlimit_stop below. * * Setting memlimit_threading to UINT64_MAX or a similar huge value * means that liblzma is allowed to keep the whole compressed file * and the whole uncompressed file in memory in addition to the memory * needed by the decompressor data structures used by each thread! * In other words, a reasonable value limit must be set here or it * will cause problems sooner or later. If you have no idea what * a reasonable value could be, try lzma_physmem() / 4 as a starting * point. Setting this limit will never prevent decompression of * a file; this will only reduce the number of threads. * * If memlimit_threading is greater than memlimit_stop, then the value * of memlimit_stop will be used for both. */ uint64_t memlimit_threading; /** * \brief Memory usage limit that should never be exceeded * * Encoder: Ignored. * * Decoder: If decompressing will need more than this amount of * memory even in the single-threaded mode, then lzma_code() will * return LZMA_MEMLIMIT_ERROR. */ uint64_t memlimit_stop; /** \private Reserved member. */ uint64_t reserved_int7; /** \private Reserved member. */ uint64_t reserved_int8; /** \private Reserved member. */ void *reserved_ptr1; /** \private Reserved member. */ void *reserved_ptr2; /** \private Reserved member. */ void *reserved_ptr3; /** \private Reserved member. */ void *reserved_ptr4; } lzma_mt; /** * \brief Calculate approximate memory usage of easy encoder * * This function is a wrapper for lzma_raw_encoder_memusage(). * * \param preset Compression preset (level and possible flags) * * \return Number of bytes of memory required for the given * preset when encoding or UINT64_MAX on error. */ extern LZMA_API(uint64_t) lzma_easy_encoder_memusage(uint32_t preset) lzma_nothrow lzma_attr_pure; /** * \brief Calculate approximate decoder memory usage of a preset * * This function is a wrapper for lzma_raw_decoder_memusage(). * * \param preset Compression preset (level and possible flags) * * \return Number of bytes of memory required to decompress a file * that was compressed using the given preset or UINT64_MAX * on error. */ extern LZMA_API(uint64_t) lzma_easy_decoder_memusage(uint32_t preset) lzma_nothrow lzma_attr_pure; /** * \brief Initialize .xz Stream encoder using a preset number * * This function is intended for those who just want to use the basic features * of liblzma (that is, most developers out there). * * If initialization fails (return value is not LZMA_OK), all the memory * allocated for *strm by liblzma is always freed. Thus, there is no need * to call lzma_end() after failed initialization. * * If initialization succeeds, use lzma_code() to do the actual encoding. * Valid values for 'action' (the second argument of lzma_code()) are * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, * there may be compression levels or flags that don't support LZMA_SYNC_FLUSH. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param preset Compression preset to use. A preset consist of level * number and zero or more flags. Usually flags aren't * used, so preset is simply a number [0, 9] which match * the options -0 ... -9 of the xz command line tool. - * Additional flags can be be set using bitwise-or with + * Additional flags can be set using bitwise-or with * the preset level number, e.g. 6 | LZMA_PRESET_EXTREME. * \param check Integrity check type to use. See check.h for available * checks. The xz command line tool defaults to * LZMA_CHECK_CRC64, which is a good choice if you are * unsure. LZMA_CHECK_CRC32 is good too as long as the * uncompressed file is not many gigabytes. * * \return Possible lzma_ret values: * - LZMA_OK: Initialization succeeded. Use lzma_code() to * encode your data. * - LZMA_MEM_ERROR: Memory allocation failed. * - LZMA_OPTIONS_ERROR: The given compression preset is not * supported by this build of liblzma. * - LZMA_UNSUPPORTED_CHECK: The given check type is not * supported by this liblzma build. * - LZMA_PROG_ERROR: One or more of the parameters have values * that will never be valid. For example, strm == NULL. */ extern LZMA_API(lzma_ret) lzma_easy_encoder( lzma_stream *strm, uint32_t preset, lzma_check check) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Single-call .xz Stream encoding using a preset number * * The maximum required output buffer size can be calculated with * lzma_stream_buffer_bound(). * * \param preset Compression preset to use. See the description * in lzma_easy_encoder(). * \param check Type of the integrity check to calculate from * uncompressed data. * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_size Size of the input buffer * \param[out] out Beginning of the output buffer * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * * \return Possible lzma_ret values: * - LZMA_OK: Encoding was successful. * - LZMA_BUF_ERROR: Not enough output buffer space. * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR * - LZMA_MEM_ERROR * - LZMA_DATA_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode( uint32_t preset, lzma_check check, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; /** * \brief Initialize .xz Stream encoder using a custom filter chain * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param filters Array of filters terminated with * .id == LZMA_VLI_UNKNOWN. See filters.h for more * information. * \param check Type of the integrity check to calculate from * uncompressed data. * * \return Possible lzma_ret values: * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, const lzma_filter *filters, lzma_check check) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Calculate approximate memory usage of multithreaded .xz encoder * * Since doing the encoding in threaded mode doesn't affect the memory * requirements of single-threaded decompressor, you can use * lzma_easy_decoder_memusage(options->preset) or * lzma_raw_decoder_memusage(options->filters) to calculate * the decompressor memory requirements. * * \param options Compression options * * \return Number of bytes of memory required for encoding with the * given options. If an error occurs, for example due to * unsupported preset or filter chain, UINT64_MAX is returned. */ extern LZMA_API(uint64_t) lzma_stream_encoder_mt_memusage( const lzma_mt *options) lzma_nothrow lzma_attr_pure; /** * \brief Initialize multithreaded .xz Stream encoder * * This provides the functionality of lzma_easy_encoder() and * lzma_stream_encoder() as a single function for multithreaded use. * * The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH, * LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be * added in the future. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param options Pointer to multithreaded compression options * * \return Possible lzma_ret values: * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( lzma_stream *strm, const lzma_mt *options) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Calculate recommended Block size for multithreaded .xz encoder * * This calculates a recommended Block size for multithreaded encoding given * a filter chain. This is used internally by lzma_stream_encoder_mt() to * determine the Block size if the block_size member is not set to the * special value of 0 in the lzma_mt options struct. * * If one wishes to change the filters between Blocks, this function is * helpful to set the block_size member of the lzma_mt struct before calling * lzma_stream_encoder_mt(). Since the block_size member represents the * maximum possible Block size for the multithreaded .xz encoder, one can * use this function to find the maximum recommended Block size based on * all planned filter chains. Otherwise, the multithreaded encoder will * base its maximum Block size on the first filter chain used (if the * block_size member is not set), which may unnecessarily limit the Block * size for a later filter chain. * * \param filters Array of filters terminated with * .id == LZMA_VLI_UNKNOWN. * * \return Recommended Block size in bytes, or UINT64_MAX if * an error occurred. */ extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) lzma_nothrow; /** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the * reason for the name of this function. The .lzma format supports only the * LZMA1 filter. There is no support for integrity checks like CRC32. * * Use this function if and only if you need to create files readable by * legacy LZMA tools such as LZMA Utils 4.32.x. Moving to the .xz format * is strongly recommended. * * The valid action values for lzma_code() are LZMA_RUN and LZMA_FINISH. * No kind of flushing is supported, because the file format doesn't make * it possible. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param options Pointer to encoder options * * \return Possible lzma_ret values: * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_alone_encoder( lzma_stream *strm, const lzma_options_lzma *options) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Calculate output buffer size for single-call Stream encoder * * When trying to compress incompressible data, the encoded size will be * slightly bigger than the input data. This function calculates how much * output buffer space is required to be sure that lzma_stream_buffer_encode() * doesn't return LZMA_BUF_ERROR. * * The calculated value is not exact, but it is guaranteed to be big enough. * The actual maximum output space required may be slightly smaller (up to * about 100 bytes). This should not be a problem in practice. * * If the calculated maximum size doesn't fit into size_t or would make the * Stream grow past LZMA_VLI_MAX (which should never happen in practice), * zero is returned to indicate the error. * * \note The limit calculated by this function applies only to * single-call encoding. Multi-call encoding may (and probably * will) have larger maximum expansion when encoding * incompressible data. Currently there is no function to * calculate the maximum expansion of multi-call encoding. * * \param uncompressed_size Size in bytes of the uncompressed * input data * * \return Maximum number of bytes needed to store the compressed data. */ extern LZMA_API(size_t) lzma_stream_buffer_bound(size_t uncompressed_size) lzma_nothrow; /** * \brief Single-call .xz Stream encoder * * \param filters Array of filters terminated with * .id == LZMA_VLI_UNKNOWN. See filters.h for more * information. * \param check Type of the integrity check to calculate from * uncompressed data. * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_size Size of the input buffer * \param[out] out Beginning of the output buffer * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * * \return Possible lzma_ret values: * - LZMA_OK: Encoding was successful. * - LZMA_BUF_ERROR: Not enough output buffer space. * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR * - LZMA_MEM_ERROR * - LZMA_DATA_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( lzma_filter *filters, lzma_check check, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief MicroLZMA encoder * * The MicroLZMA format is a raw LZMA stream whose first byte (always 0x00) * has been replaced with bitwise-negation of the LZMA properties (lc/lp/pb). * This encoding ensures that the first byte of MicroLZMA stream is never * 0x00. There is no end of payload marker and thus the uncompressed size * must be stored separately. For the best error detection the dictionary * size should be stored separately as well but alternatively one may use * the uncompressed size as the dictionary size when decoding. * * With the MicroLZMA encoder, lzma_code() behaves slightly unusually. * The action argument must be LZMA_FINISH and the return value will never be * LZMA_OK. Thus the encoding is always done with a single lzma_code() after * the initialization. The benefit of the combination of initialization * function and lzma_code() is that memory allocations can be re-used for * better performance. * * lzma_code() will try to encode as much input as is possible to fit into * the given output buffer. If not all input can be encoded, the stream will * be finished without encoding all the input. The caller must check both * input and output buffer usage after lzma_code() (total_in and total_out * in lzma_stream can be convenient). Often lzma_code() can fill the output * buffer completely if there is a lot of input, but sometimes a few bytes * may remain unused because the next LZMA symbol would require more space. * * lzma_stream.avail_out must be at least 6. Otherwise LZMA_PROG_ERROR * will be returned. * * The LZMA dictionary should be reasonably low to speed up the encoder * re-initialization. A good value is bigger than the resulting * uncompressed size of most of the output chunks. For example, if output * size is 4 KiB, dictionary size of 32 KiB or 64 KiB is good. If the * data compresses extremely well, even 128 KiB may be useful. * * The MicroLZMA format and this encoder variant were made with the EROFS * file system in mind. This format may be convenient in other embedded * uses too where many small streams are needed. XZ Embedded includes a * decoder for this format. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param options Pointer to encoder options * * \return Possible lzma_ret values: * - LZMA_STREAM_END: All good. Check the amounts of input used * and output produced. Store the amount of input used * (uncompressed size) as it needs to be known to decompress * the data. * - LZMA_OPTIONS_ERROR * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR: In addition to the generic reasons for this * error code, this may also be returned if there isn't enough * output space (6 bytes) to create a valid MicroLZMA stream. */ extern LZMA_API(lzma_ret) lzma_microlzma_encoder( lzma_stream *strm, const lzma_options_lzma *options) lzma_nothrow; /************ * Decoding * ************/ /** * This flag makes lzma_code() return LZMA_NO_CHECK if the input stream * being decoded has no integrity check. Note that when used with * lzma_auto_decoder(), all .lzma files will trigger LZMA_NO_CHECK * if LZMA_TELL_NO_CHECK is used. */ #define LZMA_TELL_NO_CHECK UINT32_C(0x01) /** * This flag makes lzma_code() return LZMA_UNSUPPORTED_CHECK if the input * stream has an integrity check, but the type of the integrity check is not * supported by this liblzma version or build. Such files can still be * decoded, but the integrity check cannot be verified. */ #define LZMA_TELL_UNSUPPORTED_CHECK UINT32_C(0x02) /** * This flag makes lzma_code() return LZMA_GET_CHECK as soon as the type * of the integrity check is known. The type can then be got with * lzma_get_check(). */ #define LZMA_TELL_ANY_CHECK UINT32_C(0x04) /** * This flag makes lzma_code() not calculate and verify the integrity check * of the compressed data in .xz files. This means that invalid integrity * check values won't be detected and LZMA_DATA_ERROR won't be returned in * such cases. * * This flag only affects the checks of the compressed data itself; the CRC32 * values in the .xz headers will still be verified normally. * * Don't use this flag unless you know what you are doing. Possible reasons * to use this flag: * * - Trying to recover data from a corrupt .xz file. * * - Speeding up decompression, which matters mostly with SHA-256 * or with files that have compressed extremely well. It's recommended * to not use this flag for this purpose unless the file integrity is * verified externally in some other way. * * Support for this flag was added in liblzma 5.1.4beta. */ #define LZMA_IGNORE_CHECK UINT32_C(0x10) /** * This flag enables decoding of concatenated files with file formats that * allow concatenating compressed files as is. From the formats currently * supported by liblzma, only the .xz and .lz formats allow concatenated * files. Concatenated files are not allowed with the legacy .lzma format. * * This flag also affects the usage of the 'action' argument for lzma_code(). * When LZMA_CONCATENATED is used, lzma_code() won't return LZMA_STREAM_END * unless LZMA_FINISH is used as 'action'. Thus, the application has to set * LZMA_FINISH in the same way as it does when encoding. * * If LZMA_CONCATENATED is not used, the decoders still accept LZMA_FINISH * as 'action' for lzma_code(), but the usage of LZMA_FINISH isn't required. */ #define LZMA_CONCATENATED UINT32_C(0x08) /** * This flag makes the threaded decoder report errors (like LZMA_DATA_ERROR) * as soon as they are detected. This saves time when the application has no * interest in a partially decompressed truncated or corrupt file. Note that * due to timing randomness, if the same truncated or corrupt input is * decompressed multiple times with this flag, a different amount of output * may be produced by different runs, and even the error code might vary. * * When using LZMA_FAIL_FAST, it is recommended to use LZMA_FINISH to tell * the decoder when no more input will be coming because it can help fast * detection and reporting of truncated files. Note that in this situation * truncated files might be diagnosed with LZMA_DATA_ERROR instead of * LZMA_OK or LZMA_BUF_ERROR! * * Without this flag the threaded decoder will provide as much output as * possible at first and then report the pending error. This default behavior * matches the single-threaded decoder and provides repeatable behavior * with truncated or corrupt input. There are a few special cases where the * behavior can still differ like memory allocation failures (LZMA_MEM_ERROR). * * Single-threaded decoders currently ignore this flag. * * Support for this flag was added in liblzma 5.3.3alpha. Note that in older * versions this flag isn't supported (LZMA_OPTIONS_ERROR) even by functions * that ignore this flag in newer liblzma versions. */ #define LZMA_FAIL_FAST UINT32_C(0x20) /** * \brief Initialize .xz Stream decoder * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. liblzma * 5.2.3 and earlier don't allow 0 here and return * LZMA_PROG_ERROR; later versions treat 0 as if 1 * had been specified. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_TELL_ANY_CHECK, LZMA_IGNORE_CHECK, * LZMA_CONCATENATED, LZMA_FAIL_FAST * * \return Possible lzma_ret values: * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_OPTIONS_ERROR: Unsupported flags * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_decoder( lzma_stream *strm, uint64_t memlimit, uint32_t flags) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Initialize multithreaded .xz Stream decoder * * The decoder can decode multiple Blocks in parallel. This requires that each * Block Header contains the Compressed Size and Uncompressed size fields * which are added by the multi-threaded encoder, see lzma_stream_encoder_mt(). * * A Stream with one Block will only utilize one thread. A Stream with multiple * Blocks but without size information in Block Headers will be processed in * single-threaded mode in the same way as done by lzma_stream_decoder(). * Concatenated Streams are processed one Stream at a time; no inter-Stream * parallelization is done. * * This function behaves like lzma_stream_decoder() when options->threads == 1 * and options->memlimit_threading <= 1. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param options Pointer to multithreaded compression options * * \return Possible lzma_ret values: * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_MEMLIMIT_ERROR: Memory usage limit was reached. * - LZMA_OPTIONS_ERROR: Unsupported flags. * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_decoder_mt( lzma_stream *strm, const lzma_mt *options) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Decode .xz, .lzma, and .lz (lzip) files with autodetection * * This decoder autodetects between the .xz, .lzma, and .lz file formats, * and calls lzma_stream_decoder(), lzma_alone_decoder(), or * lzma_lzip_decoder() once the type of the input file has been detected. * * Support for .lz was added in 5.4.0. * * If the flag LZMA_CONCATENATED is used and the input is a .lzma file: * For historical reasons concatenated .lzma files aren't supported. * If there is trailing data after one .lzma stream, lzma_code() will * return LZMA_DATA_ERROR. (lzma_alone_decoder() doesn't have such a check * as it doesn't support any decoder flags. It will return LZMA_STREAM_END * after one .lzma stream.) * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. liblzma * 5.2.3 and earlier don't allow 0 here and return * LZMA_PROG_ERROR; later versions treat 0 as if 1 * had been specified. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_TELL_ANY_CHECK, LZMA_IGNORE_CHECK, * LZMA_CONCATENATED, LZMA_FAIL_FAST * * \return Possible lzma_ret values: * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_OPTIONS_ERROR: Unsupported flags * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_auto_decoder( lzma_stream *strm, uint64_t memlimit, uint32_t flags) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Initialize .lzma decoder (legacy file format) * * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * There is no need to use LZMA_FINISH, but it's allowed because it may * simplify certain types of applications. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. liblzma * 5.2.3 and earlier don't allow 0 here and return * LZMA_PROG_ERROR; later versions treat 0 as if 1 * had been specified. * * \return Possible lzma_ret values: * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_alone_decoder( lzma_stream *strm, uint64_t memlimit) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Initialize .lz (lzip) decoder (a foreign file format) * * This decoder supports the .lz format version 0 and the unextended .lz * format version 1: * * - Files in the format version 0 were produced by lzip 1.3 and older. * Such files aren't common but may be found from file archives * as a few source packages were released in this format. People * might have old personal files in this format too. Decompression * support for the format version 0 was removed in lzip 1.18. * * - lzip 1.3 added decompression support for .lz format version 1 files. * Compression support was added in lzip 1.4. In lzip 1.6 the .lz format * version 1 was extended to support the Sync Flush marker. This extension * is not supported by liblzma. lzma_code() will return LZMA_DATA_ERROR * at the location of the Sync Flush marker. In practice files with * the Sync Flush marker are very rare and thus liblzma can decompress * almost all .lz files. * * Just like with lzma_stream_decoder() for .xz files, LZMA_CONCATENATED * should be used when decompressing normal standalone .lz files. * * The .lz format allows putting non-.lz data at the end of a file after at * least one valid .lz member. That is, one can append custom data at the end * of a .lz file and the decoder is required to ignore it. In liblzma this * is relevant only when LZMA_CONCATENATED is used. In that case lzma_code() * will return LZMA_STREAM_END and leave lzma_stream.next_in pointing to * the first byte of the non-.lz data. An exception to this is if the first * 1-3 bytes of the non-.lz data are identical to the .lz magic bytes * (0x4C, 0x5A, 0x49, 0x50; "LZIP" in US-ASCII). In such a case the 1-3 bytes * will have been ignored by lzma_code(). If one wishes to locate the non-.lz * data reliably, one must ensure that the first byte isn't 0x4C. Actually * one should ensure that none of the first four bytes of trailing data are * equal to the magic bytes because lzip >= 1.20 requires it by default. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. * \param flags Bitwise-or of flags, or zero for no flags. * All decoder flags listed above are supported * although only LZMA_CONCATENATED and (in very rare * cases) LZMA_IGNORE_CHECK are actually useful. * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * and LZMA_FAIL_FAST do nothing. LZMA_TELL_ANY_CHECK * is supported for consistency only as CRC32 is * always used in the .lz format. * * \return Possible lzma_ret values: * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_OPTIONS_ERROR: Unsupported flags * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_lzip_decoder( lzma_stream *strm, uint64_t memlimit, uint32_t flags) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief Single-call .xz Stream decoder * * \param memlimit Pointer to how much memory the decoder is allowed * to allocate. The value pointed by this pointer is * modified if and only if LZMA_MEMLIMIT_ERROR is * returned. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_IGNORE_CHECK, LZMA_CONCATENATED, * LZMA_FAIL_FAST. Note that LZMA_TELL_ANY_CHECK * is not allowed and will return LZMA_PROG_ERROR. * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_pos The next byte will be read from in[*in_pos]. * *in_pos is updated only if decoding succeeds. * \param in_size Size of the input buffer; the first byte that * won't be read is in[in_size]. * \param[out] out Beginning of the output buffer * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if decoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * * \return Possible lzma_ret values: * - LZMA_OK: Decoding was successful. * - LZMA_FORMAT_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_DATA_ERROR * - LZMA_NO_CHECK: This can be returned only if using * the LZMA_TELL_NO_CHECK flag. * - LZMA_UNSUPPORTED_CHECK: This can be returned only if using * the LZMA_TELL_UNSUPPORTED_CHECK flag. * - LZMA_MEM_ERROR * - LZMA_MEMLIMIT_ERROR: Memory usage limit was reached. * The minimum required memlimit value was stored to *memlimit. * - LZMA_BUF_ERROR: Output buffer was too small. * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode( uint64_t *memlimit, uint32_t flags, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; /** * \brief MicroLZMA decoder * * See lzma_microlzma_encoder() for more information. * * The lzma_code() usage with this decoder is completely normal. The * special behavior of lzma_code() applies to lzma_microlzma_encoder() only. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param comp_size Compressed size of the MicroLZMA stream. * The caller must somehow know this exactly. * \param uncomp_size Uncompressed size of the MicroLZMA stream. * If the exact uncompressed size isn't known, this * can be set to a value that is at most as big as * the exact uncompressed size would be, but then the * next argument uncomp_size_is_exact must be false. * \param uncomp_size_is_exact * If true, uncomp_size must be exactly correct. * This will improve error detection at the end of * the stream. If the exact uncompressed size isn't * known, this must be false. uncomp_size must still * be at most as big as the exact uncompressed size * is. Setting this to false when the exact size is * known will work but error detection at the end of * the stream will be weaker. * \param dict_size LZMA dictionary size that was used when * compressing the data. It is OK to use a bigger * value too but liblzma will then allocate more * memory than would actually be required and error * detection will be slightly worse. (Note that with * the implementation in XZ Embedded it doesn't * affect the memory usage if one specifies bigger * dictionary than actually required.) * * \return Possible lzma_ret values: * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_microlzma_decoder( lzma_stream *strm, uint64_t comp_size, uint64_t uncomp_size, lzma_bool uncomp_size_is_exact, uint32_t dict_size) lzma_nothrow; diff --git a/src/liblzma/api/lzma/version.h b/src/liblzma/api/lzma/version.h index 53526b992c95..e86c0ea4c3d1 100644 --- a/src/liblzma/api/lzma/version.h +++ b/src/liblzma/api/lzma/version.h @@ -1,134 +1,134 @@ /* SPDX-License-Identifier: 0BSD */ /** * \file lzma/version.h * \brief Version number * \note Never include this file directly. Use instead. */ /* * Author: Lasse Collin */ #ifndef LZMA_H_INTERNAL # error Never include this file directly. Use instead. #endif /** \brief Major version number of the liblzma release. */ #define LZMA_VERSION_MAJOR 5 /** \brief Minor version number of the liblzma release. */ #define LZMA_VERSION_MINOR 6 /** \brief Patch version number of the liblzma release. */ -#define LZMA_VERSION_PATCH 2 +#define LZMA_VERSION_PATCH 3 /** * \brief Version stability marker * * This will always be one of three values: * - LZMA_VERSION_STABILITY_ALPHA * - LZMA_VERSION_STABILITY_BETA * - LZMA_VERSION_STABILITY_STABLE */ #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE /** \brief Commit version number of the liblzma release */ #ifndef LZMA_VERSION_COMMIT # define LZMA_VERSION_COMMIT "" #endif /* * Map symbolic stability levels to integers. */ #define LZMA_VERSION_STABILITY_ALPHA 0 #define LZMA_VERSION_STABILITY_BETA 1 #define LZMA_VERSION_STABILITY_STABLE 2 /** * \brief Compile-time version number * * The version number is of format xyyyzzzs where * - x = major * - yyy = minor * - zzz = revision * - s indicates stability: 0 = alpha, 1 = beta, 2 = stable * * The same xyyyzzz triplet is never reused with different stability levels. * For example, if 5.1.0alpha has been released, there will never be 5.1.0beta * or 5.1.0 stable. * * \note The version number of liblzma has nothing to with * the version number of Igor Pavlov's LZMA SDK. */ #define LZMA_VERSION (LZMA_VERSION_MAJOR * UINT32_C(10000000) \ + LZMA_VERSION_MINOR * UINT32_C(10000) \ + LZMA_VERSION_PATCH * UINT32_C(10) \ + LZMA_VERSION_STABILITY) /* * Macros to construct the compile-time version string */ #if LZMA_VERSION_STABILITY == LZMA_VERSION_STABILITY_ALPHA # define LZMA_VERSION_STABILITY_STRING "alpha" #elif LZMA_VERSION_STABILITY == LZMA_VERSION_STABILITY_BETA # define LZMA_VERSION_STABILITY_STRING "beta" #elif LZMA_VERSION_STABILITY == LZMA_VERSION_STABILITY_STABLE # define LZMA_VERSION_STABILITY_STRING "" #else # error Incorrect LZMA_VERSION_STABILITY #endif #define LZMA_VERSION_STRING_C_(major, minor, patch, stability, commit) \ #major "." #minor "." #patch stability commit #define LZMA_VERSION_STRING_C(major, minor, patch, stability, commit) \ LZMA_VERSION_STRING_C_(major, minor, patch, stability, commit) /** * \brief Compile-time version as a string * * This can be for example "4.999.5alpha", "4.999.8beta", or "5.0.0" (stable * versions don't have any "stable" suffix). In future, a snapshot built * from source code repository may include an additional suffix, for example * "4.999.8beta-21-g1d92". The commit ID won't be available in numeric form * in LZMA_VERSION macro. */ #define LZMA_VERSION_STRING LZMA_VERSION_STRING_C( \ LZMA_VERSION_MAJOR, LZMA_VERSION_MINOR, \ LZMA_VERSION_PATCH, LZMA_VERSION_STABILITY_STRING, \ LZMA_VERSION_COMMIT) /* #ifndef is needed for use with windres (MinGW-w64 or Cygwin). */ #ifndef LZMA_H_INTERNAL_RC /** * \brief Run-time version number as an integer * * This allows an application to compare if it was built against the same, * older, or newer version of liblzma that is currently running. * * \return The value of LZMA_VERSION macro at the compile time of liblzma */ extern LZMA_API(uint32_t) lzma_version_number(void) lzma_nothrow lzma_attr_const; /** * \brief Run-time version as a string * * This function may be useful to display which version of liblzma an * application is currently using. * * \return Run-time version of liblzma */ extern LZMA_API(const char *) lzma_version_string(void) lzma_nothrow lzma_attr_const; #endif diff --git a/src/liblzma/check/crc32_table.c b/src/liblzma/check/crc32_table.c index c141cefe5a40..56413eec336e 100644 --- a/src/liblzma/check/crc32_table.c +++ b/src/liblzma/check/crc32_table.c @@ -1,42 +1,42 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_table.c /// \brief Precalculated CRC32 table with correct endianness // // Author: Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #include "common.h" // FIXME: Compared to crc_common.h this has to check for __x86_64__ too // so that in 32-bit builds crc32_x86.S won't break due to a missing table. #if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ && defined(__SSE4_1__) && defined(__PCLMUL__)) \ || (defined(__e2k__) && __iset__ >= 6)) # define NO_CRC32_TABLE #elif defined(HAVE_ARM64_CRC32) \ && !defined(WORDS_BIGENDIAN) \ && defined(__ARM_FEATURE_CRC32) # define NO_CRC32_TABLE #endif #if !defined(HAVE_ENCODERS) && defined(NO_CRC32_TABLE) // No table needed. Use a typedef to avoid an empty translation unit. typedef void lzma_crc32_dummy; #else // Having the declaration here silences clang -Wmissing-variable-declarations. extern const uint32_t lzma_crc32_table[8][256]; -# ifdef WORDS_BIGENDIAN -# include "crc32_table_be.h" -# else -# include "crc32_table_le.h" -# endif +# ifdef WORDS_BIGENDIAN +# include "crc32_table_be.h" +# else +# include "crc32_table_le.h" +# endif #endif diff --git a/src/liblzma/check/crc_common.h b/src/liblzma/check/crc_common.h index 63a7b5cefebf..c15d4c675c8f 100644 --- a/src/liblzma/check/crc_common.h +++ b/src/liblzma/check/crc_common.h @@ -1,137 +1,137 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file crc_common.h /// \brief Some functions and macros for CRC32 and CRC64 // // Authors: Lasse Collin // Ilya Kurdyukov // Hans Jansen // Jia Tan // /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_CRC_COMMON_H #define LZMA_CRC_COMMON_H #include "common.h" #ifdef WORDS_BIGENDIAN # define A(x) ((x) >> 24) # define B(x) (((x) >> 16) & 0xFF) # define C(x) (((x) >> 8) & 0xFF) # define D(x) ((x) & 0xFF) # define S8(x) ((x) << 8) # define S32(x) ((x) << 32) #else # define A(x) ((x) & 0xFF) # define B(x) (((x) >> 8) & 0xFF) # define C(x) (((x) >> 16) & 0xFF) # define D(x) ((x) >> 24) # define S8(x) ((x) >> 8) # define S32(x) ((x) >> 32) #endif // CRC CLMUL code needs this because accessing input buffers that aren't // aligned to the vector size will inherently trip the address sanitizer. #if lzma_has_attribute(__no_sanitize_address__) # define crc_attr_no_sanitize_address \ __attribute__((__no_sanitize_address__)) #else # define crc_attr_no_sanitize_address #endif // Keep this in sync with changes to crc32_arm64.h #if defined(_WIN32) || defined(HAVE_GETAUXVAL) \ || defined(HAVE_ELF_AUX_INFO) \ || (defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME)) # define ARM64_RUNTIME_DETECTION 1 #endif #undef CRC32_GENERIC #undef CRC64_GENERIC #undef CRC32_ARCH_OPTIMIZED #undef CRC64_ARCH_OPTIMIZED // The x86 CLMUL is used for both CRC32 and CRC64. #undef CRC_X86_CLMUL #undef CRC32_ARM64 #undef CRC64_ARM64_CLMUL #undef CRC_USE_GENERIC_FOR_SMALL_INPUTS // ARM64 CRC32 instruction is only useful for CRC32. Currently, only // little endian is supported since we were unable to test on a big // endian machine. // // NOTE: Keep this and the next check in sync with the macro // NO_CRC32_TABLE in crc32_table.c #if defined(HAVE_ARM64_CRC32) && !defined(WORDS_BIGENDIAN) -// Allow ARM64 CRC32 instruction without a runtime check if -// __ARM_FEATURE_CRC32 is defined. GCC and Clang only define this if the -// proper compiler options are used. + // Allow ARM64 CRC32 instruction without a runtime check if + // __ARM_FEATURE_CRC32 is defined. GCC and Clang only define + // this if the proper compiler options are used. # if defined(__ARM_FEATURE_CRC32) # define CRC32_ARCH_OPTIMIZED 1 # define CRC32_ARM64 1 # elif defined(ARM64_RUNTIME_DETECTION) # define CRC32_ARCH_OPTIMIZED 1 # define CRC32_ARM64 1 # define CRC32_GENERIC 1 # endif #endif #if defined(HAVE_USABLE_CLMUL) // If CLMUL is allowed unconditionally in the compiler options then the // generic version can be omitted. Note that this doesn't work with MSVC // as I don't know how to detect the features here. // // NOTE: Keep this in sync with the NO_CRC32_TABLE macro in crc32_table.c // and NO_CRC64_TABLE in crc64_table.c. # if (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__)) \ || (defined(__e2k__) && __iset__ >= 6) # define CRC32_ARCH_OPTIMIZED 1 # define CRC64_ARCH_OPTIMIZED 1 # define CRC_X86_CLMUL 1 # else # define CRC32_GENERIC 1 # define CRC64_GENERIC 1 # define CRC32_ARCH_OPTIMIZED 1 # define CRC64_ARCH_OPTIMIZED 1 # define CRC_X86_CLMUL 1 /* // The generic code is much faster with 1-8-byte inputs and // has similar performance up to 16 bytes at least in // microbenchmarks (it depends on input buffer alignment // too). If both versions are built, this #define will use // the generic version for inputs up to 16 bytes and CLMUL // for bigger inputs. It saves a little in code size since // the special cases for 0-16-byte inputs will be omitted // from the CLMUL code. # define CRC_USE_GENERIC_FOR_SMALL_INPUTS 1 */ # endif #endif // For CRC32 use the generic slice-by-eight implementation if no optimized // version is available. #if !defined(CRC32_ARCH_OPTIMIZED) && !defined(CRC32_GENERIC) # define CRC32_GENERIC 1 #endif // For CRC64 use the generic slice-by-four implementation if no optimized // version is available. #if !defined(CRC64_ARCH_OPTIMIZED) && !defined(CRC64_GENERIC) # define CRC64_GENERIC 1 #endif #endif diff --git a/src/liblzma/check/crc_x86_clmul.h b/src/liblzma/check/crc_x86_clmul.h index f1254ece18ed..50306e49a72a 100644 --- a/src/liblzma/check/crc_x86_clmul.h +++ b/src/liblzma/check/crc_x86_clmul.h @@ -1,428 +1,432 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file crc_x86_clmul.h /// \brief CRC32 and CRC64 implementations using CLMUL instructions. /// /// The CRC32 and CRC64 implementations use 32/64-bit x86 SSSE3, SSE4.1, and /// CLMUL instructions. This is compatible with Elbrus 2000 (E2K) too. /// /// They were derived from /// https://www.researchgate.net/publication/263424619_Fast_CRC_computation /// and the public domain code from https://github.com/rawrunprotected/crc /// (URLs were checked on 2023-10-14). /// /// While this file has both CRC32 and CRC64 implementations, only one /// should be built at a time to ensure that crc_simd_body() is inlined /// even with compilers with which lzma_always_inline expands to plain inline. /// The version to build is selected by defining BUILDING_CRC32_CLMUL or /// BUILDING_CRC64_CLMUL before including this file. /// /// FIXME: Builds for 32-bit x86 use the assembly .S files by default /// unless configured with --disable-assembler. Even then the lookup table /// isn't omitted in crc64_table.c since it doesn't know that assembly /// code has been disabled. // // Authors: Ilya Kurdyukov // Hans Jansen // Lasse Collin // Jia Tan // /////////////////////////////////////////////////////////////////////////////// // This file must not be included more than once. #ifdef LZMA_CRC_X86_CLMUL_H # error crc_x86_clmul.h was included twice. #endif #define LZMA_CRC_X86_CLMUL_H #include #if defined(_MSC_VER) # include #elif defined(HAVE_CPUID_H) # include #endif // EDG-based compilers (Intel's classic compiler and compiler for E2K) can // define __GNUC__ but the attribute must not be used with them. // The new Clang-based ICX needs the attribute. // // NOTE: Build systems check for this too, keep them in sync with this. #if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) # define crc_attr_target \ __attribute__((__target__("ssse3,sse4.1,pclmul"))) #else # define crc_attr_target #endif #define MASK_L(in, mask, r) r = _mm_shuffle_epi8(in, mask) #define MASK_H(in, mask, r) \ r = _mm_shuffle_epi8(in, _mm_xor_si128(mask, vsign)) #define MASK_LH(in, mask, low, high) \ MASK_L(in, mask, low); \ MASK_H(in, mask, high) crc_attr_target crc_attr_no_sanitize_address static lzma_always_inline void crc_simd_body(const uint8_t *buf, const size_t size, __m128i *v0, __m128i *v1, const __m128i vfold16, const __m128i initial_crc) { // Create a vector with 8-bit values 0 to 15. This is used to // construct control masks for _mm_blendv_epi8 and _mm_shuffle_epi8. const __m128i vramp = _mm_setr_epi32( 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c); // This is used to inverse the control mask of _mm_shuffle_epi8 // so that bytes that wouldn't be picked with the original mask // will be picked and vice versa. const __m128i vsign = _mm_set1_epi8(-0x80); // Memory addresses A to D and the distances between them: // // A B C D // [skip_start][size][skip_end] // [ size2 ] // // A and D are 16-byte aligned. B and C are 1-byte aligned. // skip_start and skip_end are 0-15 bytes. size is at least 1 byte. // // A = aligned_buf will initially point to this address. // B = The address pointed by the caller-supplied buf. // C = buf + size == aligned_buf + size2 // D = buf + size + skip_end == aligned_buf + size2 + skip_end const size_t skip_start = (size_t)((uintptr_t)buf & 15); const size_t skip_end = (size_t)((0U - (uintptr_t)(buf + size)) & 15); const __m128i *aligned_buf = (const __m128i *)( (uintptr_t)buf & ~(uintptr_t)15); // If size2 <= 16 then the whole input fits into a single 16-byte // vector. If size2 > 16 then at least two 16-byte vectors must // be processed. If size2 > 16 && size <= 16 then there is only // one 16-byte vector's worth of input but it is unaligned in memory. // // NOTE: There is no integer overflow here if the arguments // are valid. If this overflowed, buf + size would too. const size_t size2 = skip_start + size; // Masks to be used with _mm_blendv_epi8 and _mm_shuffle_epi8: // The first skip_start or skip_end bytes in the vectors will have // the high bit (0x80) set. _mm_blendv_epi8 and _mm_shuffle_epi8 // will produce zeros for these positions. (Bitwise-xor of these // masks with vsign will produce the opposite behavior.) const __m128i mask_start = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_start)); const __m128i mask_end = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_end)); // Get the first 1-16 bytes into data0. If loading less than 16 // bytes, the bytes are loaded to the high bits of the vector and // the least significant positions are filled with zeros. const __m128i data0 = _mm_blendv_epi8(_mm_load_si128(aligned_buf), _mm_setzero_si128(), mask_start); aligned_buf++; __m128i v2, v3; #ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS if (size <= 16) { // Right-shift initial_crc by 1-16 bytes based on "size" // and store the result in v1 (high bytes) and v0 (low bytes). // // NOTE: The highest 8 bytes of initial_crc are zeros so // v1 will be filled with zeros if size >= 8. The highest // 8 bytes of v1 will always become zeros. // // [ v1 ][ v0 ] // [ initial_crc ] size == 1 // [ initial_crc ] size == 2 // [ initial_crc ] size == 15 // [ initial_crc ] size == 16 (all in v0) const __m128i mask_low = _mm_add_epi8( vramp, _mm_set1_epi8((char)(size - 16))); MASK_LH(initial_crc, mask_low, *v0, *v1); if (size2 <= 16) { // There are 1-16 bytes of input and it is all // in data0. Copy the input bytes to v3. If there // are fewer than 16 bytes, the low bytes in v3 // will be filled with zeros. That is, the input // bytes are stored to the same position as // (part of) initial_crc is in v0. MASK_L(data0, mask_end, v3); } else { // There are 2-16 bytes of input but not all bytes // are in data0. const __m128i data1 = _mm_load_si128(aligned_buf); // Collect the 2-16 input bytes from data0 and data1 // to v2 and v3, and bitwise-xor them with the // low bits of initial_crc in v0. Note that the // the second xor is below this else-block as it // is shared with the other branch. MASK_H(data0, mask_end, v2); MASK_L(data1, mask_end, v3); *v0 = _mm_xor_si128(*v0, v2); } *v0 = _mm_xor_si128(*v0, v3); *v1 = _mm_alignr_epi8(*v1, *v0, 8); } else #endif { // There is more than 16 bytes of input. const __m128i data1 = _mm_load_si128(aligned_buf); const __m128i *end = (const __m128i*)( (const char *)aligned_buf - 16 + size2); aligned_buf++; MASK_LH(initial_crc, mask_start, *v0, *v1); *v0 = _mm_xor_si128(*v0, data0); *v1 = _mm_xor_si128(*v1, data1); while (aligned_buf < end) { *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( *v0, vfold16, 0x00)); *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( *v0, vfold16, 0x11)); *v1 = _mm_load_si128(aligned_buf++); } if (aligned_buf != end) { MASK_H(*v0, mask_end, v2); MASK_L(*v0, mask_end, *v0); MASK_L(*v1, mask_end, v3); *v1 = _mm_or_si128(v2, v3); } *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( *v0, vfold16, 0x00)); *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( *v0, vfold16, 0x11)); *v1 = _mm_srli_si128(*v0, 8); } } ///////////////////// // x86 CLMUL CRC32 // ///////////////////// /* // These functions were used to generate the constants // at the top of crc32_arch_optimized(). static uint64_t calc_lo(uint64_t p, uint64_t a, int n) { uint64_t b = 0; int i; for (i = 0; i < n; i++) { b = b >> 1 | (a & 1) << (n - 1); a = (a >> 1) ^ ((0 - (a & 1)) & p); } return b; } // same as ~crc(&a, sizeof(a), ~0) static uint64_t calc_hi(uint64_t p, uint64_t a, int n) { int i; for (i = 0; i < n; i++) a = (a >> 1) ^ ((0 - (a & 1)) & p); return a; } */ #ifdef BUILDING_CRC32_CLMUL crc_attr_target crc_attr_no_sanitize_address static uint32_t crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) { #ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS // The code assumes that there is at least one byte of input. if (size == 0) return crc; #endif // uint32_t poly = 0xedb88320; const int64_t p = 0x1db710640; // p << 1 const int64_t mu = 0x1f7011641; // calc_lo(p, p, 32) << 1 | 1 const int64_t k5 = 0x163cd6124; // calc_hi(p, p, 32) << 1 const int64_t k4 = 0x0ccaa009e; // calc_hi(p, p, 64) << 1 const int64_t k3 = 0x1751997d0; // calc_hi(p, p, 128) << 1 const __m128i vfold4 = _mm_set_epi64x(mu, p); const __m128i vfold8 = _mm_set_epi64x(0, k5); const __m128i vfold16 = _mm_set_epi64x(k4, k3); __m128i v0, v1, v2; - crc_simd_body(buf, size, &v0, &v1, vfold16, + crc_simd_body(buf, size, &v0, &v1, vfold16, _mm_cvtsi32_si128((int32_t)~crc)); v1 = _mm_xor_si128( _mm_clmulepi64_si128(v0, vfold16, 0x10), v1); // xxx0 v2 = _mm_shuffle_epi32(v1, 0xe7); // 0xx0 v0 = _mm_slli_epi64(v1, 32); // [0] v0 = _mm_clmulepi64_si128(v0, vfold8, 0x00); v0 = _mm_xor_si128(v0, v2); // [1] [2] v2 = _mm_clmulepi64_si128(v0, vfold4, 0x10); v2 = _mm_clmulepi64_si128(v2, vfold4, 0x00); v0 = _mm_xor_si128(v0, v2); // [2] return ~(uint32_t)_mm_extract_epi32(v0, 2); } #endif // BUILDING_CRC32_CLMUL ///////////////////// // x86 CLMUL CRC64 // ///////////////////// /* // These functions were used to generate the constants // at the top of crc64_arch_optimized(). static uint64_t calc_lo(uint64_t poly) { uint64_t a = poly; uint64_t b = 0; for (unsigned i = 0; i < 64; ++i) { b = (b >> 1) | (a << 63); a = (a >> 1) ^ (a & 1 ? poly : 0); } return b; } static uint64_t calc_hi(uint64_t poly, uint64_t a) { for (unsigned i = 0; i < 64; ++i) a = (a >> 1) ^ (a & 1 ? poly : 0); return a; } */ #ifdef BUILDING_CRC64_CLMUL // MSVC (VS2015 - VS2022) produces bad 32-bit x86 code from the CLMUL CRC // code when optimizations are enabled (release build). According to the bug // report, the ebx register is corrupted and the calculated result is wrong. // Trying to workaround the problem with "__asm mov ebx, ebx" didn't help. // The following pragma works and performance is still good. x86-64 builds // and CRC32 CLMUL aren't affected by this problem. The problem does not // happen in crc_simd_body() either (which is shared with CRC32 CLMUL anyway). // // NOTE: Another pragma after crc64_arch_optimized() restores // the optimizations. If the #if condition here is updated, // the other one must be updated too. #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ && defined(_M_IX86) # pragma optimize("g", off) #endif crc_attr_target crc_attr_no_sanitize_address static uint64_t crc64_arch_optimized(const uint8_t *buf, size_t size, uint64_t crc) { #ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS // The code assumes that there is at least one byte of input. if (size == 0) return crc; #endif // const uint64_t poly = 0xc96c5795d7870f42; // CRC polynomial const uint64_t p = 0x92d8af2baf0e1e85; // (poly << 1) | 1 const uint64_t mu = 0x9c3e466c172963d5; // (calc_lo(poly) << 1) | 1 const uint64_t k2 = 0xdabe95afc7875f40; // calc_hi(poly, 1) const uint64_t k1 = 0xe05dd497ca393ae4; // calc_hi(poly, k2) const __m128i vfold8 = _mm_set_epi64x((int64_t)p, (int64_t)mu); const __m128i vfold16 = _mm_set_epi64x((int64_t)k2, (int64_t)k1); __m128i v0, v1, v2; #if defined(__i386__) || defined(_M_IX86) - crc_simd_body(buf, size, &v0, &v1, vfold16, + crc_simd_body(buf, size, &v0, &v1, vfold16, _mm_set_epi64x(0, (int64_t)~crc)); #else // GCC and Clang would produce good code with _mm_set_epi64x // but MSVC needs _mm_cvtsi64_si128 on x86-64. - crc_simd_body(buf, size, &v0, &v1, vfold16, + crc_simd_body(buf, size, &v0, &v1, vfold16, _mm_cvtsi64_si128((int64_t)~crc)); #endif v1 = _mm_xor_si128(_mm_clmulepi64_si128(v0, vfold16, 0x10), v1); v0 = _mm_clmulepi64_si128(v1, vfold8, 0x00); v2 = _mm_clmulepi64_si128(v0, vfold8, 0x10); v0 = _mm_xor_si128(_mm_xor_si128(v1, _mm_slli_si128(v0, 8)), v2); #if defined(__i386__) || defined(_M_IX86) return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) | (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2)); #else return ~(uint64_t)_mm_extract_epi64(v0, 1); #endif } #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ && defined(_M_IX86) # pragma optimize("", on) #endif #endif // BUILDING_CRC64_CLMUL +// Even though this is an inline function, compile it only when needed. +// This way it won't appear in E2K builds at all. +#if defined(CRC32_GENERIC) || defined(CRC64_GENERIC) // Inlining this function duplicates the function body in crc32_resolve() and // crc64_resolve(), but this is acceptable because this is a tiny function. static inline bool is_arch_extension_supported(void) { int success = 1; uint32_t r[4]; // eax, ebx, ecx, edx #if defined(_MSC_VER) // This needs with MSVC. ICC has it as a built-in // on all platforms. __cpuid(r, 1); #elif defined(HAVE_CPUID_H) // Compared to just using __asm__ to run CPUID, this also checks // that CPUID is supported and saves and restores ebx as that is // needed with GCC < 5 with position-independent code (PIC). success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]); #else // Just a fallback that shouldn't be needed. __asm__("cpuid\n\t" : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3]) : "a"(1), "c"(0)); #endif // Returns true if these are supported: // CLMUL (bit 1 in ecx) // SSSE3 (bit 9 in ecx) // SSE4.1 (bit 19 in ecx) const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19); return success && (r[2] & ecx_mask) == ecx_mask; // Alternative methods that weren't used: // - ICC's _may_i_use_cpu_feature: the other methods should work too. // - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul") // - // CPUID decding is needed with MSVC anyway and older GCC. This keeps + // CPUID decoding is needed with MSVC anyway and older GCC. This keeps // the feature checks in the build system simpler too. The nice thing // about __builtin_cpu_supports would be that it generates very short // code as is it only reads a variable set at startup but a few bytes // doesn't matter here. } +#endif diff --git a/src/liblzma/check/sha256.c b/src/liblzma/check/sha256.c index bd0d2806397c..c067a3a693fa 100644 --- a/src/liblzma/check/sha256.c +++ b/src/liblzma/check/sha256.c @@ -1,189 +1,189 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file sha256.c /// \brief SHA-256 // // The C code is based on the public domain SHA-256 code found from // Crypto++ Library 5.5.1 released in 2007: https://www.cryptopp.com/ // A few minor tweaks have been made in liblzma. // // Authors: Wei Dai // Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #include "check.h" // Rotate a uint32_t. GCC can optimize this to a rotate instruction // at least on x86. static inline uint32_t rotr_32(uint32_t num, unsigned amount) { - return (num >> amount) | (num << (32 - amount)); + return (num >> amount) | (num << (32 - amount)); } #define blk0(i) (W[i] = conv32be(data[i])) #define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \ + s0(W[(i - 15) & 15])) #define Ch(x, y, z) (z ^ (x & (y ^ z))) #define Maj(x, y, z) ((x & (y ^ z)) + (y & z)) #define a(i) T[(0 - i) & 7] #define b(i) T[(1 - i) & 7] #define c(i) T[(2 - i) & 7] #define d(i) T[(3 - i) & 7] #define e(i) T[(4 - i) & 7] #define f(i) T[(5 - i) & 7] #define g(i) T[(6 - i) & 7] #define h(i) T[(7 - i) & 7] #define R(i, j, blk) \ h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] + blk; \ d(i) += h(i); \ h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) #define R0(i) R(i, 0, blk0(i)) #define R2(i) R(i, j, blk2(i)) #define S0(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 9), 11), 2) #define S1(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 14), 5), 6) #define s0(x) (rotr_32(x ^ rotr_32(x, 11), 7) ^ (x >> 3)) #define s1(x) (rotr_32(x ^ rotr_32(x, 2), 17) ^ (x >> 10)) static const uint32_t SHA256_K[64] = { 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5, 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174, 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA, 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967, 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85, 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070, 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3, 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2, }; static void transform(uint32_t state[8], const uint32_t data[16]) { uint32_t W[16]; uint32_t T[8]; // Copy state[] to working vars. memcpy(T, state, sizeof(T)); // The first 16 operations unrolled R0( 0); R0( 1); R0( 2); R0( 3); R0( 4); R0( 5); R0( 6); R0( 7); R0( 8); R0( 9); R0(10); R0(11); R0(12); R0(13); R0(14); R0(15); // The remaining 48 operations partially unrolled for (unsigned int j = 16; j < 64; j += 16) { R2( 0); R2( 1); R2( 2); R2( 3); R2( 4); R2( 5); R2( 6); R2( 7); R2( 8); R2( 9); R2(10); R2(11); R2(12); R2(13); R2(14); R2(15); } // Add the working vars back into state[]. state[0] += a(0); state[1] += b(0); state[2] += c(0); state[3] += d(0); state[4] += e(0); state[5] += f(0); state[6] += g(0); state[7] += h(0); } static void process(lzma_check_state *check) { transform(check->state.sha256.state, check->buffer.u32); return; } extern void lzma_sha256_init(lzma_check_state *check) { static const uint32_t s[8] = { 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, }; memcpy(check->state.sha256.state, s, sizeof(s)); check->state.sha256.size = 0; return; } extern void lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) { // Copy the input data into a properly aligned temporary buffer. // This way we can be called with arbitrarily sized buffers // (no need to be multiple of 64 bytes), and the code works also // on architectures that don't allow unaligned memory access. while (size > 0) { const size_t copy_start = check->state.sha256.size & 0x3F; size_t copy_size = 64 - copy_start; if (copy_size > size) copy_size = size; memcpy(check->buffer.u8 + copy_start, buf, copy_size); buf += copy_size; size -= copy_size; check->state.sha256.size += copy_size; if ((check->state.sha256.size & 0x3F) == 0) process(check); } return; } extern void lzma_sha256_finish(lzma_check_state *check) { // Add padding as described in RFC 3174 (it describes SHA-1 but // the same padding style is used for SHA-256 too). size_t pos = check->state.sha256.size & 0x3F; check->buffer.u8[pos++] = 0x80; while (pos != 64 - 8) { if (pos == 64) { process(check); pos = 0; } check->buffer.u8[pos++] = 0x00; } // Convert the message size from bytes to bits. check->state.sha256.size *= 8; check->buffer.u64[(64 - 8) / 8] = conv64be(check->state.sha256.size); process(check); for (size_t i = 0; i < 8; ++i) check->buffer.u32[i] = conv32be(check->state.sha256.state[i]); return; } diff --git a/src/liblzma/rangecoder/range_decoder.h b/src/liblzma/rangecoder/range_decoder.h index 31a58d1ffbed..a8aca9077c13 100644 --- a/src/liblzma/rangecoder/range_decoder.h +++ b/src/liblzma/rangecoder/range_decoder.h @@ -1,966 +1,966 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file range_decoder.h /// \brief Range Decoder /// // Authors: Igor Pavlov // Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_DECODER_H #define LZMA_RANGE_DECODER_H #include "range_common.h" // Choose the range decoder variants to use using a bitmask. // If no bits are set, only the basic version is used. // If more than one version is selected for the same feature, // the last one on the list below is used. // // Bitwise-or of the following enable branchless C versions: // 0x01 normal bittrees // 0x02 fixed-sized reverse bittrees // 0x04 variable-sized reverse bittrees (not faster) // 0x08 matched literal (not faster) // // GCC & Clang compatible x86-64 inline assembly: // 0x010 normal bittrees // 0x020 fixed-sized reverse bittrees // 0x040 variable-sized reverse bittrees // 0x080 matched literal // 0x100 direct bits // // The default can be overridden at build time by defining // LZMA_RANGE_DECODER_CONFIG to the desired mask. // // 2024-02-22: Feedback from benchmarks: // - Brancless C (0x003) can be better than basic on x86-64 but often it's // slightly worse on other archs. Since asm is much better on x86-64, // branchless C is not used at all. // - With x86-64 asm, there are slight differences between GCC and Clang // and different processors. Overall 0x1F0 seems to be the best choice. #ifndef LZMA_RANGE_DECODER_CONFIG # if defined(__x86_64__) && !defined(__ILP32__) \ && !defined(__NVCOMPILER) \ && (defined(__GNUC__) || defined(__clang__)) # define LZMA_RANGE_DECODER_CONFIG 0x1F0 # else # define LZMA_RANGE_DECODER_CONFIG 0 # endif #endif // Negative RC_BIT_MODEL_TOTAL but the lowest RC_MOVE_BITS are flipped. // This is useful for updating probability variables in branchless decoding: // // uint32_t decoded_bit = ...; // probability tmp = RC_BIT_MODEL_OFFSET; // tmp &= decoded_bit - 1; // prob -= (prob + tmp) >> RC_MOVE_BITS; #define RC_BIT_MODEL_OFFSET \ ((UINT32_C(1) << RC_MOVE_BITS) - 1 - RC_BIT_MODEL_TOTAL) typedef struct { uint32_t range; uint32_t code; uint32_t init_bytes_left; } lzma_range_decoder; /// Reads the first five bytes to initialize the range decoder. static inline lzma_ret rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { while (rc->init_bytes_left > 0) { if (*in_pos == in_size) return LZMA_OK; // The first byte is always 0x00. It could have been omitted // in LZMA2 but it wasn't, so one byte is wasted in every // LZMA2 chunk. if (rc->init_bytes_left == 5 && in[*in_pos] != 0x00) return LZMA_DATA_ERROR; rc->code = (rc->code << 8) | in[*in_pos]; ++*in_pos; --rc->init_bytes_left; } return LZMA_STREAM_END; } /// Makes local copies of range decoder and *in_pos variables. Doing this /// improves speed significantly. The range decoder macros expect also /// variables 'in' and 'in_size' to be defined. #define rc_to_local(range_decoder, in_pos, fast_mode_in_required) \ lzma_range_decoder rc = range_decoder; \ const uint8_t *rc_in_ptr = in + (in_pos); \ const uint8_t *rc_in_end = in + in_size; \ const uint8_t *rc_in_fast_end \ = (rc_in_end - rc_in_ptr) <= (fast_mode_in_required) \ ? rc_in_ptr \ : rc_in_end - (fast_mode_in_required); \ (void)rc_in_fast_end; /* Silence a warning with HAVE_SMALL. */ \ uint32_t rc_bound /// Evaluates to true if there is enough input remaining to use fast mode. #define rc_is_fast_allowed() (rc_in_ptr < rc_in_fast_end) /// Stores the local copes back to the range decoder structure. #define rc_from_local(range_decoder, in_pos) \ do { \ range_decoder = rc; \ in_pos = (size_t)(rc_in_ptr - in); \ } while (0) /// Resets the range decoder structure. #define rc_reset(range_decoder) \ do { \ (range_decoder).range = UINT32_MAX; \ (range_decoder).code = 0; \ (range_decoder).init_bytes_left = 5; \ } while (0) /// When decoding has been properly finished, rc.code is always zero unless /// the input stream is corrupt. So checking this can catch some corrupt /// files especially if they don't have any other integrity check. #define rc_is_finished(range_decoder) \ ((range_decoder).code == 0) // Read the next input byte if needed. #define rc_normalize() \ do { \ if (rc.range < RC_TOP_VALUE) { \ rc.range <<= RC_SHIFT_BITS; \ rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ } \ } while (0) /// If more input is needed but there is /// no more input available, "goto out" is used to jump out of the main /// decoder loop. The "_safe" macros are used in the Resumable decoder /// mode in order to save the sequence to continue decoding from that /// point later. #define rc_normalize_safe(seq) \ do { \ if (rc.range < RC_TOP_VALUE) { \ if (rc_in_ptr == rc_in_end) { \ coder->sequence = seq; \ goto out; \ } \ rc.range <<= RC_SHIFT_BITS; \ rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ } \ } while (0) /// Start decoding a bit. This must be used together with rc_update_0() /// and rc_update_1(): /// /// rc_if_0(prob) { /// rc_update_0(prob); /// // Do something /// } else { /// rc_update_1(prob); /// // Do something else /// } /// #define rc_if_0(prob) \ rc_normalize(); \ rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ if (rc.code < rc_bound) #define rc_if_0_safe(prob, seq) \ rc_normalize_safe(seq); \ rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ if (rc.code < rc_bound) /// Update the range decoder state and the used probability variable to /// match a decoded bit of 0. /// /// The x86-64 assembly uses the commented method but it seems that, /// at least on x86-64, the first version is slightly faster as C code. #define rc_update_0(prob) \ do { \ rc.range = rc_bound; \ prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \ /* prob -= ((prob) + RC_BIT_MODEL_OFFSET) >> RC_MOVE_BITS; */ \ } while (0) /// Update the range decoder state and the used probability variable to /// match a decoded bit of 1. #define rc_update_1(prob) \ do { \ rc.range -= rc_bound; \ rc.code -= rc_bound; \ prob -= (prob) >> RC_MOVE_BITS; \ } while (0) /// Decodes one bit and runs action0 or action1 depending on the decoded bit. /// This macro is used as the last step in bittree reverse decoders since /// those don't use "symbol" for anything else than indexing the probability /// arrays. #define rc_bit_last(prob, action0, action1) \ do { \ rc_if_0(prob) { \ rc_update_0(prob); \ action0; \ } else { \ rc_update_1(prob); \ action1; \ } \ } while (0) #define rc_bit_last_safe(prob, action0, action1, seq) \ do { \ rc_if_0_safe(prob, seq) { \ rc_update_0(prob); \ action0; \ } else { \ rc_update_1(prob); \ action1; \ } \ } while (0) /// Decodes one bit, updates "symbol", and runs action0 or action1 depending /// on the decoded bit. #define rc_bit(prob, action0, action1) \ rc_bit_last(prob, \ symbol <<= 1; action0, \ symbol = (symbol << 1) + 1; action1); #define rc_bit_safe(prob, action0, action1, seq) \ rc_bit_last_safe(prob, \ symbol <<= 1; action0, \ symbol = (symbol << 1) + 1; action1, \ seq); // Unroll fixed-sized bittree decoding. // // A compile-time constant in final_add can be used to get rid of the high bit // from symbol that is used for the array indexing (1U << bittree_bits). // final_add may also be used to add offset to the result (LZMA length // decoder does that). // // The reason to have final_add here is that in the asm code the addition // can be done for free: in x86-64 there is SBB instruction with -1 as // the immediate value, and final_add is combined with that value. #define rc_bittree_bit(prob) \ rc_bit(prob, , ) #define rc_bittree3(probs, final_add) \ do { \ symbol = 1; \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ symbol += (uint32_t)(final_add); \ } while (0) #define rc_bittree6(probs, final_add) \ do { \ symbol = 1; \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ symbol += (uint32_t)(final_add); \ } while (0) #define rc_bittree8(probs, final_add) \ do { \ symbol = 1; \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ rc_bittree_bit(probs[symbol]); \ symbol += (uint32_t)(final_add); \ } while (0) // Fixed-sized reverse bittree #define rc_bittree_rev4(probs) \ do { \ symbol = 0; \ rc_bit_last(probs[symbol + 1], , symbol += 1); \ rc_bit_last(probs[symbol + 2], , symbol += 2); \ rc_bit_last(probs[symbol + 4], , symbol += 4); \ rc_bit_last(probs[symbol + 8], , symbol += 8); \ } while (0) // Decode one bit from variable-sized reverse bittree. The loop is done // in the code that uses this macro. This could be changed if the assembly // version benefited from having the loop done in assembly but it didn't // seem so in early 2024. // // Also, if the loop was done here, the loop counter would likely be local // to the macro so that it wouldn't modify yet another input variable. // If a _safe version of a macro with a loop was done then a modifiable // input variable couldn't be avoided though. #define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ rc_bit(probs[symbol], \ , \ dest += value_to_add_if_1); // Matched literal #define decode_with_match_bit \ t_match_byte <<= 1; \ t_match_bit = t_match_byte & t_offset; \ t_subcoder_index = t_offset + t_match_bit + symbol; \ rc_bit(probs[t_subcoder_index], \ t_offset &= ~t_match_bit, \ t_offset &= t_match_bit) #define rc_matched_literal(probs_base_var, match_byte) \ do { \ uint32_t t_match_byte = (match_byte); \ uint32_t t_match_bit; \ uint32_t t_subcoder_index; \ uint32_t t_offset = 0x100; \ symbol = 1; \ decode_with_match_bit; \ decode_with_match_bit; \ decode_with_match_bit; \ decode_with_match_bit; \ decode_with_match_bit; \ decode_with_match_bit; \ decode_with_match_bit; \ decode_with_match_bit; \ } while (0) /// Decode a bit without using a probability. // // NOTE: GCC 13 and Clang/LLVM 16 can, at least on x86-64, optimize the bound // calculation to use an arithmetic right shift so there's no need to provide // the alternative code which, according to C99/C11/C23 6.3.1.3-p3 isn't // perfectly portable: rc_bound = (uint32_t)((int32_t)rc.code >> 31); #define rc_direct(dest, count_var) \ do { \ dest = (dest << 1) + 1; \ rc_normalize(); \ rc.range >>= 1; \ rc.code -= rc.range; \ rc_bound = UINT32_C(0) - (rc.code >> 31); \ dest += rc_bound; \ rc.code += rc.range & rc_bound; \ } while (--count_var > 0) #define rc_direct_safe(dest, count_var, seq) \ do { \ rc_normalize_safe(seq); \ rc.range >>= 1; \ rc.code -= rc.range; \ rc_bound = UINT32_C(0) - (rc.code >> 31); \ rc.code += rc.range & rc_bound; \ dest = (dest << 1) + (rc_bound + 1); \ } while (--count_var > 0) ////////////////// // Branchless C // ////////////////// /// Decode a bit using a branchless method. This reduces the number of /// mispredicted branches and thus can improve speed. #define rc_c_bit(prob, action_bit, action_neg) \ do { \ probability *p = &(prob); \ rc_normalize(); \ rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * *p; \ uint32_t rc_mask = rc.code >= rc_bound; /* rc_mask = decoded bit */ \ action_bit; /* action when rc_mask is 0 or 1 */ \ /* rc_mask becomes 0 if bit is 0 and 0xFFFFFFFF if bit is 1: */ \ rc_mask = 0U - rc_mask; \ rc.range &= rc_mask; /* If bit 0: set rc.range = 0 */ \ rc_bound ^= rc_mask; \ rc_bound -= rc_mask; /* If bit 1: rc_bound = 0U - rc_bound */ \ rc.range += rc_bound; \ rc_bound &= rc_mask; \ rc.code += rc_bound; \ action_neg; /* action when rc_mask is 0 or 0xFFFFFFFF */ \ rc_mask = ~rc_mask; /* If bit 0: all bits are set in rc_mask */ \ rc_mask &= RC_BIT_MODEL_OFFSET; \ *p -= (*p + rc_mask) >> RC_MOVE_BITS; \ } while (0) // Testing on x86-64 give an impression that only the normal bittrees and // the fixed-sized reverse bittrees are worth the branchless C code. // It should be tested on other archs for which there isn't assembly code // in this file. // Using addition in "(symbol << 1) + rc_mask" allows use of x86 LEA // or RISC-V SH1ADD instructions. Compilers might infer it from // "(symbol << 1) | rc_mask" too if they see that mask is 0 or 1 but // the use of addition doesn't require such analysis from compilers. #if LZMA_RANGE_DECODER_CONFIG & 0x01 #undef rc_bittree_bit #define rc_bittree_bit(prob) \ rc_c_bit(prob, \ symbol = (symbol << 1) + rc_mask, \ ) #endif // LZMA_RANGE_DECODER_CONFIG & 0x01 #if LZMA_RANGE_DECODER_CONFIG & 0x02 #undef rc_bittree_rev4 #define rc_bittree_rev4(probs) \ do { \ symbol = 0; \ rc_c_bit(probs[symbol + 1], symbol += rc_mask, ); \ rc_c_bit(probs[symbol + 2], symbol += rc_mask << 1, ); \ rc_c_bit(probs[symbol + 4], symbol += rc_mask << 2, ); \ rc_c_bit(probs[symbol + 8], symbol += rc_mask << 3, ); \ } while (0) #endif // LZMA_RANGE_DECODER_CONFIG & 0x02 #if LZMA_RANGE_DECODER_CONFIG & 0x04 #undef rc_bit_add_if_1 #define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ rc_c_bit(probs[symbol], \ symbol = (symbol << 1) + rc_mask, \ dest += (value_to_add_if_1) & rc_mask) #endif // LZMA_RANGE_DECODER_CONFIG & 0x04 #if LZMA_RANGE_DECODER_CONFIG & 0x08 #undef decode_with_match_bit #define decode_with_match_bit \ t_match_byte <<= 1; \ t_match_bit = t_match_byte & t_offset; \ t_subcoder_index = t_offset + t_match_bit + symbol; \ rc_c_bit(probs[t_subcoder_index], \ symbol = (symbol << 1) + rc_mask, \ t_offset &= ~t_match_bit ^ rc_mask) #endif // LZMA_RANGE_DECODER_CONFIG & 0x08 //////////// // x86-64 // //////////// #if LZMA_RANGE_DECODER_CONFIG & 0x1F0 // rc_asm_y and rc_asm_n are used as arguments to macros to control which // strings to include or omit. #define rc_asm_y(str) str #define rc_asm_n(str) // There are a few possible variations for normalization. // This is the smallest variant which is also used by LZMA SDK. // // - This has partial register write (the MOV from (%[in_ptr])). // // - INC saves one byte in code size over ADD. False dependency on // partial flags from INC shouldn't become a problem on any processor // because the instructions after normalization don't read the flags // until SUB which sets all flags. // #define rc_asm_normalize \ "cmp %[top_value], %[range]\n\t" \ "jae 1f\n\t" \ "shl %[shift_bits], %[code]\n\t" \ "mov (%[in_ptr]), %b[code]\n\t" \ "shl %[shift_bits], %[range]\n\t" \ "inc %[in_ptr]\n" \ "1:\n" // rc_asm_calc(prob) is roughly equivalent to the C version of rc_if_0(prob)... // // rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); // if (rc.code < rc_bound) // // ...but the bound is stored in "range": // // t0 = range; // range = (range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); // t0 -= range; // t1 = code; // code -= range; // // The carry flag (CF) from the last subtraction holds the negation of // the decoded bit (if CF==0 then the decoded bit is 1). // The values in t0 and t1 are needed for rc_update_0(prob) and // rc_update_1(prob). If the bit is 0, rc_update_0(prob)... // // rc.range = rc_bound; // // ...has already been done but the "code -= range" has to be reverted using // the old value stored in t1. (Also, prob needs to be updated.) // // If the bit is 1, rc_update_1(prob)... // // rc.range -= rc_bound; // rc.code -= rc_bound; // // ...is already done for "code" but the value for "range" needs to be taken // from t0. (Also, prob needs to be updated here as well.) // // The assignments from t0 and t1 can be done in a branchless manner with CMOV // after the instructions from this macro. The CF from SUB tells which moves // are needed. #define rc_asm_calc(prob) \ "mov %[range], %[t0]\n\t" \ "shr %[bit_model_total_bits], %[range]\n\t" \ "imul %[" prob "], %[range]\n\t" \ "sub %[range], %[t0]\n\t" \ "mov %[code], %[t1]\n\t" \ "sub %[range], %[code]\n\t" // Also, prob needs to be updated: The update math depends on the decoded bit. // It can be expressed in a few slightly different ways but this is fairly // convenient here: // // prob -= (prob + (bit ? 0 : RC_BIT_MODEL_OFFSET)) >> RC_MOVE_BITS; // // To do it in branchless way when the negation of the decoded bit is in CF, // both "prob" and "prob + RC_BIT_MODEL_OFFSET" are needed. Then the desired // value can be picked with CMOV. The addition can be done using LEA without // affecting CF. // // (This prob update method is a tiny bit different from LZMA SDK 23.01. // In the LZMA SDK a single register is reserved solely for a constant to // be used with CMOV when updating prob. That is fine since there are enough // free registers to do so. The method used here uses one fewer register, // which is valuable with inline assembly.) // // * * * // // In bittree decoding, each (unrolled) loop iteration decodes one bit // and needs one prob variable. To make it faster, the prob variable of // the iteration N+1 is loaded during iteration N. There are two possible // prob variables to choose from for N+1. Both are loaded from memory and // the correct one is chosen with CMOV using the same CF as is used for // other things described above. // // This preloading/prefetching requires an extra register. To avoid // useless moves from "preloaded prob register" to "current prob register", // the macros swap between the two registers for odd and even iterations. // // * * * // // Finally, the decoded bit has to be stored in "symbol". Since the negation // of the bit is in CF, this can be done with SBB: symbol -= CF - 1. That is, // if the decoded bit is 0 (CF==1) the operation is a no-op "symbol -= 0" // and when bit is 1 (CF==0) the operation is "symbol -= 0 - 1" which is // the same as "symbol += 1". // // The instructions for all things are intertwined for a few reasons: // - freeing temporary registers for new use // - not modifying CF too early // - instruction scheduling // // The first and last iterations can cheat a little. For example, // on the first iteration "symbol" is known to start from 1 so it // doesn't need to be read; it can even be immediately initialized // to 2 to prepare for the second iteration of the loop. // // * * * // // a = number of the current prob variable (0 or 1) // b = number of the next prob variable (1 or 0) // *_only = rc_asm_y or _n to include or exclude code marked with them #define rc_asm_bittree(a, b, first_only, middle_only, last_only) \ first_only( \ - "movzw 2(%[probs_base]), %[prob" #a "]\n\t" \ + "movzwl 2(%[probs_base]), %[prob" #a "]\n\t" \ "mov $2, %[symbol]\n\t" \ - "movzw 4(%[probs_base]), %[prob" #b "]\n\t" \ + "movzwl 4(%[probs_base]), %[prob" #b "]\n\t" \ ) \ middle_only( \ /* Note the scaling of 4 instead of 2: */ \ - "movzw (%[probs_base], %q[symbol], 4), %[prob" #b "]\n\t" \ + "movzwl (%[probs_base], %q[symbol], 4), %[prob" #b "]\n\t" \ ) \ last_only( \ "add %[symbol], %[symbol]\n\t" \ ) \ \ rc_asm_normalize \ rc_asm_calc("prob" #a) \ \ "cmovae %[t0], %[range]\n\t" \ \ first_only( \ - "movzw 6(%[probs_base]), %[t0]\n\t" \ + "movzwl 6(%[probs_base]), %[t0]\n\t" \ "cmovae %[t0], %[prob" #b "]\n\t" \ ) \ middle_only( \ - "movzw 2(%[probs_base], %q[symbol], 4), %[t0]\n\t" \ + "movzwl 2(%[probs_base], %q[symbol], 4), %[t0]\n\t" \ "lea (%q[symbol], %q[symbol]), %[symbol]\n\t" \ "cmovae %[t0], %[prob" #b "]\n\t" \ ) \ \ "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ "cmovb %[t1], %[code]\n\t" \ "mov %[symbol], %[t1]\n\t" \ "cmovae %[prob" #a "], %[t0]\n\t" \ \ first_only( \ "sbb $-1, %[symbol]\n\t" \ ) \ middle_only( \ "sbb $-1, %[symbol]\n\t" \ ) \ last_only( \ "sbb %[last_sbb], %[symbol]\n\t" \ ) \ \ "shr %[move_bits], %[t0]\n\t" \ "sub %[t0], %[prob" #a "]\n\t" \ /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ "mov %w[prob" #a "], (%[probs_base], %q[t1], 1)\n\t" // NOTE: The order of variables in __asm__ can affect speed and code size. #define rc_asm_bittree_n(probs_base_var, final_add, asm_str) \ do { \ uint32_t t0; \ uint32_t t1; \ uint32_t t_prob0; \ uint32_t t_prob1; \ \ __asm__( \ asm_str \ : \ [range] "+&r"(rc.range), \ [code] "+&r"(rc.code), \ [t0] "=&r"(t0), \ [t1] "=&r"(t1), \ [prob0] "=&r"(t_prob0), \ [prob1] "=&r"(t_prob1), \ [symbol] "=&r"(symbol), \ [in_ptr] "+&r"(rc_in_ptr) \ : \ [probs_base] "r"(probs_base_var), \ [last_sbb] "n"(-1 - (final_add)), \ [top_value] "n"(RC_TOP_VALUE), \ [shift_bits] "n"(RC_SHIFT_BITS), \ [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ [move_bits] "n"(RC_MOVE_BITS) \ : \ "cc", "memory"); \ } while (0) #if LZMA_RANGE_DECODER_CONFIG & 0x010 #undef rc_bittree3 #define rc_bittree3(probs_base_var, final_add) \ rc_asm_bittree_n(probs_base_var, final_add, \ rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(0, 1, rc_asm_n, rc_asm_n, rc_asm_y) \ ) #undef rc_bittree6 #define rc_bittree6(probs_base_var, final_add) \ rc_asm_bittree_n(probs_base_var, final_add, \ rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ ) #undef rc_bittree8 #define rc_bittree8(probs_base_var, final_add) \ rc_asm_bittree_n(probs_base_var, final_add, \ rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ ) #endif // LZMA_RANGE_DECODER_CONFIG & 0x010 // Fixed-sized reverse bittree // // This uses the indexing that constructs the final value in symbol directly. // add = 1, 2, 4, 8 // dcur = -, 4, 8, 16 // dnext0 = 4, 8, 16, - // dnext0 = 6, 12, 24, - #define rc_asm_bittree_rev(a, b, add, dcur, dnext0, dnext1, \ first_only, middle_only, last_only) \ first_only( \ - "movzw 2(%[probs_base]), %[prob" #a "]\n\t" \ + "movzwl 2(%[probs_base]), %[prob" #a "]\n\t" \ "xor %[symbol], %[symbol]\n\t" \ - "movzw 4(%[probs_base]), %[prob" #b "]\n\t" \ + "movzwl 4(%[probs_base]), %[prob" #b "]\n\t" \ ) \ middle_only( \ - "movzw " #dnext0 "(%[probs_base], %q[symbol], 2), " \ + "movzwl " #dnext0 "(%[probs_base], %q[symbol], 2), " \ "%[prob" #b "]\n\t" \ ) \ \ rc_asm_normalize \ rc_asm_calc("prob" #a) \ \ "cmovae %[t0], %[range]\n\t" \ \ first_only( \ - "movzw 6(%[probs_base]), %[t0]\n\t" \ + "movzwl 6(%[probs_base]), %[t0]\n\t" \ "cmovae %[t0], %[prob" #b "]\n\t" \ ) \ middle_only( \ - "movzw " #dnext1 "(%[probs_base], %q[symbol], 2), %[t0]\n\t" \ + "movzwl " #dnext1 "(%[probs_base], %q[symbol], 2), %[t0]\n\t" \ "cmovae %[t0], %[prob" #b "]\n\t" \ ) \ \ "lea " #add "(%q[symbol]), %[t0]\n\t" \ "cmovb %[t1], %[code]\n\t" \ middle_only( \ "mov %[symbol], %[t1]\n\t" \ ) \ last_only( \ "mov %[symbol], %[t1]\n\t" \ ) \ "cmovae %[t0], %[symbol]\n\t" \ "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ "cmovae %[prob" #a "], %[t0]\n\t" \ \ "shr %[move_bits], %[t0]\n\t" \ "sub %[t0], %[prob" #a "]\n\t" \ first_only( \ "mov %w[prob" #a "], 2(%[probs_base])\n\t" \ ) \ middle_only( \ "mov %w[prob" #a "], " \ #dcur "(%[probs_base], %q[t1], 2)\n\t" \ ) \ last_only( \ "mov %w[prob" #a "], " \ #dcur "(%[probs_base], %q[t1], 2)\n\t" \ ) #if LZMA_RANGE_DECODER_CONFIG & 0x020 #undef rc_bittree_rev4 #define rc_bittree_rev4(probs_base_var) \ rc_asm_bittree_n(probs_base_var, 4, \ rc_asm_bittree_rev(0, 1, 1, -, 4, 6, rc_asm_y, rc_asm_n, rc_asm_n) \ rc_asm_bittree_rev(1, 0, 2, 4, 8, 12, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree_rev(0, 1, 4, 8, 16, 24, rc_asm_n, rc_asm_y, rc_asm_n) \ rc_asm_bittree_rev(1, 0, 8, 16, -, -, rc_asm_n, rc_asm_n, rc_asm_y) \ ) #endif // LZMA_RANGE_DECODER_CONFIG & 0x020 #if LZMA_RANGE_DECODER_CONFIG & 0x040 #undef rc_bit_add_if_1 #define rc_bit_add_if_1(probs_base_var, dest_var, value_to_add_if_1) \ do { \ uint32_t t0; \ uint32_t t1; \ uint32_t t2 = (value_to_add_if_1); \ uint32_t t_prob; \ uint32_t t_index; \ \ __asm__( \ - "movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + "movzwl (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ "mov %[symbol], %[index]\n\t" \ \ "add %[dest], %[t2]\n\t" \ "add %[symbol], %[symbol]\n\t" \ \ rc_asm_normalize \ rc_asm_calc("prob") \ \ "cmovae %[t0], %[range]\n\t" \ "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ "cmovb %[t1], %[code]\n\t" \ "cmovae %[prob], %[t0]\n\t" \ \ "cmovae %[t2], %[dest]\n\t" \ "sbb $-1, %[symbol]\n\t" \ \ "sar %[move_bits], %[t0]\n\t" \ "sub %[t0], %[prob]\n\t" \ "mov %w[prob], (%[probs_base], %q[index], 2)" \ : \ [range] "+&r"(rc.range), \ [code] "+&r"(rc.code), \ [t0] "=&r"(t0), \ [t1] "=&r"(t1), \ [prob] "=&r"(t_prob), \ [index] "=&r"(t_index), \ [symbol] "+&r"(symbol), \ [t2] "+&r"(t2), \ [dest] "+&r"(dest_var), \ [in_ptr] "+&r"(rc_in_ptr) \ : \ [probs_base] "r"(probs_base_var), \ [top_value] "n"(RC_TOP_VALUE), \ [shift_bits] "n"(RC_SHIFT_BITS), \ [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ [move_bits] "n"(RC_MOVE_BITS) \ : \ "cc", "memory"); \ } while (0) #endif // LZMA_RANGE_DECODER_CONFIG & 0x040 // Literal decoding uses a normal 8-bit bittree but literal with match byte // is more complex in picking the probability variable from the correct // subtree. This doesn't use preloading/prefetching of the next prob because // there are four choices instead of two. // // FIXME? The first iteration starts with symbol = 1 so it could be optimized // by a tiny amount. #define rc_asm_matched_literal(nonlast_only) \ "add %[offset], %[symbol]\n\t" \ "and %[offset], %[match_bit]\n\t" \ "add %[match_bit], %[symbol]\n\t" \ \ - "movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + "movzwl (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ \ "add %[symbol], %[symbol]\n\t" \ \ nonlast_only( \ "xor %[match_bit], %[offset]\n\t" \ "add %[match_byte], %[match_byte]\n\t" \ ) \ \ rc_asm_normalize \ rc_asm_calc("prob") \ \ "cmovae %[t0], %[range]\n\t" \ "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ "cmovb %[t1], %[code]\n\t" \ "mov %[symbol], %[t1]\n\t" \ "cmovae %[prob], %[t0]\n\t" \ \ nonlast_only( \ "cmovae %[match_bit], %[offset]\n\t" \ "mov %[match_byte], %[match_bit]\n\t" \ ) \ \ "sbb $-1, %[symbol]\n\t" \ \ "shr %[move_bits], %[t0]\n\t" \ /* Undo symbol += match_bit + offset: */ \ "and $0x1FF, %[symbol]\n\t" \ "sub %[t0], %[prob]\n\t" \ \ /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ "mov %w[prob], (%[probs_base], %q[t1], 1)\n\t" #if LZMA_RANGE_DECODER_CONFIG & 0x080 #undef rc_matched_literal #define rc_matched_literal(probs_base_var, match_byte_value) \ do { \ uint32_t t0; \ uint32_t t1; \ uint32_t t_prob; \ uint32_t t_match_byte = (uint32_t)(match_byte_value) << 1; \ uint32_t t_match_bit = t_match_byte; \ uint32_t t_offset = 0x100; \ symbol = 1; \ \ __asm__( \ rc_asm_matched_literal(rc_asm_y) \ rc_asm_matched_literal(rc_asm_y) \ rc_asm_matched_literal(rc_asm_y) \ rc_asm_matched_literal(rc_asm_y) \ rc_asm_matched_literal(rc_asm_y) \ rc_asm_matched_literal(rc_asm_y) \ rc_asm_matched_literal(rc_asm_y) \ rc_asm_matched_literal(rc_asm_n) \ : \ [range] "+&r"(rc.range), \ [code] "+&r"(rc.code), \ [t0] "=&r"(t0), \ [t1] "=&r"(t1), \ [prob] "=&r"(t_prob), \ [match_bit] "+&r"(t_match_bit), \ [symbol] "+&r"(symbol), \ [match_byte] "+&r"(t_match_byte), \ [offset] "+&r"(t_offset), \ [in_ptr] "+&r"(rc_in_ptr) \ : \ [probs_base] "r"(probs_base_var), \ [top_value] "n"(RC_TOP_VALUE), \ [shift_bits] "n"(RC_SHIFT_BITS), \ [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ [move_bits] "n"(RC_MOVE_BITS) \ : \ "cc", "memory"); \ } while (0) #endif // LZMA_RANGE_DECODER_CONFIG & 0x080 // Doing the loop in asm instead of C seems to help a little. #if LZMA_RANGE_DECODER_CONFIG & 0x100 #undef rc_direct #define rc_direct(dest_var, count_var) \ do { \ uint32_t t0; \ uint32_t t1; \ \ __asm__( \ "2:\n\t" \ "add %[dest], %[dest]\n\t" \ "lea 1(%q[dest]), %[t1]\n\t" \ \ rc_asm_normalize \ \ "shr $1, %[range]\n\t" \ "mov %[code], %[t0]\n\t" \ "sub %[range], %[code]\n\t" \ "cmovns %[t1], %[dest]\n\t" \ "cmovs %[t0], %[code]\n\t" \ "dec %[count]\n\t" \ "jnz 2b\n\t" \ : \ [range] "+&r"(rc.range), \ [code] "+&r"(rc.code), \ [t0] "=&r"(t0), \ [t1] "=&r"(t1), \ [dest] "+&r"(dest_var), \ [count] "+&r"(count_var), \ [in_ptr] "+&r"(rc_in_ptr) \ : \ [top_value] "n"(RC_TOP_VALUE), \ [shift_bits] "n"(RC_SHIFT_BITS) \ : \ "cc", "memory"); \ } while (0) #endif // LZMA_RANGE_DECODER_CONFIG & 0x100 #endif // x86_64 #endif diff --git a/src/liblzma/simple/arm64.c b/src/liblzma/simple/arm64.c index 0a73f6c8bf2d..16c2f565f73d 100644 --- a/src/liblzma/simple/arm64.c +++ b/src/liblzma/simple/arm64.c @@ -1,136 +1,136 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file arm64.c /// \brief Filter for ARM64 binaries /// /// This converts ARM64 relative addresses in the BL and ADRP immediates /// to absolute values to increase redundancy of ARM64 code. /// /// Converting B or ADR instructions was also tested but it's not useful. /// A majority of the jumps for the B instruction are very small (+/- 0xFF). /// These are typical for loops and if-statements. Encoding them to their /// absolute address reduces redundancy since many of the small relative /// jump values are repeated, but very few of the absolute addresses are. // // Authors: Lasse Collin // Jia Tan // Igor Pavlov // /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" static size_t arm64_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { size_t i; // Clang 14.0.6 on x86-64 makes this four times bigger and 40 % slower // with auto-vectorization that is enabled by default with -O2. // Such vectorization bloat happens with -O2 when targeting ARM64 too // but performance hasn't been tested. #ifdef __clang__ # pragma clang loop vectorize(disable) #endif for (i = 0; i + 4 <= size; i += 4) { uint32_t pc = (uint32_t)(now_pos + i); uint32_t instr = read32le(buffer + i); if ((instr >> 26) == 0x25) { // BL instruction: // The full 26-bit immediate is converted. // The range is +/-128 MiB. // - // Using the full range is helps quite a lot with + // Using the full range helps quite a lot with // big executables. Smaller range would reduce false // positives in non-code sections of the input though // so this is a compromise that slightly favors big - // files. With the full range only six bits of the 32 + // files. With the full range, only six bits of the 32 // need to match to trigger a conversion. const uint32_t src = instr; instr = 0x94000000; pc >>= 2; if (!is_encoder) pc = 0U - pc; instr |= (src + pc) & 0x03FFFFFF; write32le(buffer + i, instr); } else if ((instr & 0x9F000000) == 0x90000000) { // ADRP instruction: // Only values in the range +/-512 MiB are converted. // // Using less than the full +/-4 GiB range reduces // false positives on non-code sections of the input // while being excellent for executables up to 512 MiB. // The positive effect of ADRP conversion is smaller // than that of BL but it also doesn't hurt so much in // non-code sections of input because, with +/-512 MiB // range, nine bits of 32 need to match to trigger a // conversion (two 10-bit match choices = 9 bits). const uint32_t src = ((instr >> 29) & 3) | ((instr >> 3) & 0x001FFFFC); // With the addition only one branch is needed to // check the +/- range. This is usually false when // processing ARM64 code so branch prediction will // handle it well in terms of performance. // //if ((src & 0x001E0000) != 0 // && (src & 0x001E0000) != 0x001E0000) if ((src + 0x00020000) & 0x001C0000) continue; instr &= 0x9000001F; pc >>= 12; if (!is_encoder) pc = 0U - pc; const uint32_t dest = src + pc; instr |= (dest & 3) << 29; instr |= (dest & 0x0003FFFC) << 3; instr |= (0U - (dest & 0x00020000)) & 0x00E00000; write32le(buffer + i, instr); } } return i; } static lzma_ret arm64_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, &arm64_code, 0, 4, 4, is_encoder); } #ifdef HAVE_ENCODER_ARM64 extern lzma_ret lzma_simple_arm64_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm64_coder_init(next, allocator, filters, true); } #endif #ifdef HAVE_DECODER_ARM64 extern lzma_ret lzma_simple_arm64_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm64_coder_init(next, allocator, filters, false); } #endif diff --git a/src/lzmainfo/lzmainfo.c b/src/lzmainfo/lzmainfo.c index 2550b1f1127b..d917f371c3ba 100644 --- a/src/lzmainfo/lzmainfo.c +++ b/src/lzmainfo/lzmainfo.c @@ -1,220 +1,219 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file lzmainfo.c /// \brief lzmainfo tool for compatibility with LZMA Utils // // Author: Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #include "sysdefs.h" #include #include #include "lzma.h" #include "getopt.h" #include "tuklib_gettext.h" #include "tuklib_progname.h" #include "tuklib_exit.h" #ifdef TUKLIB_DOSLIKE # include # include #endif tuklib_attr_noreturn static void help(void) { printf( _("Usage: %s [--help] [--version] [FILE]...\n" "Show information stored in the .lzma file header"), progname); printf(_( "\nWith no FILE, or when FILE is -, read standard input.\n")); printf("\n"); printf(_("Report bugs to <%s> (in English or Finnish).\n"), PACKAGE_BUGREPORT); printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); tuklib_exit(EXIT_SUCCESS, EXIT_FAILURE, true); } tuklib_attr_noreturn static void version(void) { puts("lzmainfo (" PACKAGE_NAME ") " LZMA_VERSION_STRING); tuklib_exit(EXIT_SUCCESS, EXIT_FAILURE, true); } /// Parse command line options. static void parse_args(int argc, char **argv) { enum { OPT_HELP, OPT_VERSION, }; static const struct option long_opts[] = { { "help", no_argument, NULL, OPT_HELP }, { "version", no_argument, NULL, OPT_VERSION }, { NULL, 0, NULL, 0 } }; int c; while ((c = getopt_long(argc, argv, "", long_opts, NULL)) != -1) { switch (c) { case OPT_HELP: help(); case OPT_VERSION: version(); default: exit(EXIT_FAILURE); } } return; } /// Primitive base-2 logarithm for integers static uint32_t my_log2(uint32_t n) { uint32_t e; for (e = 0; n > 1; ++e, n /= 2) ; return e; } /// Parse the .lzma header and display information about it. static bool lzmainfo(const char *name, FILE *f) { uint8_t buf[13]; const size_t size = fread(buf, 1, sizeof(buf), f); if (size != 13) { fprintf(stderr, "%s: %s: %s\n", progname, name, ferror(f) ? strerror(errno) : _("File is too small to be a .lzma file")); return true; } lzma_filter filter = { .id = LZMA_FILTER_LZMA1 }; // Parse the first five bytes. switch (lzma_properties_decode(&filter, NULL, buf, 5)) { case LZMA_OK: break; case LZMA_OPTIONS_ERROR: fprintf(stderr, "%s: %s: %s\n", progname, name, _("Not a .lzma file")); return true; case LZMA_MEM_ERROR: fprintf(stderr, "%s: %s\n", progname, strerror(ENOMEM)); exit(EXIT_FAILURE); default: fprintf(stderr, "%s: %s\n", progname, _("Internal error (bug)")); exit(EXIT_FAILURE); } // Uncompressed size uint64_t uncompressed_size = 0; for (size_t i = 0; i < 8; ++i) uncompressed_size |= (uint64_t)(buf[5 + i]) << (i * 8); // Display the results. We don't want to translate these and also // will use MB instead of MiB, because someone could be parsing // this output and we don't want to break that when people move // from LZMA Utils to XZ Utils. if (f != stdin) printf("%s\n", name); printf("Uncompressed size: "); if (uncompressed_size == UINT64_MAX) printf("Unknown"); else printf("%" PRIu64 " MB (%" PRIu64 " bytes)", - (uncompressed_size + 512 * 1024) - / (1024 * 1024), + (uncompressed_size / 1024 + 512) / 1024, uncompressed_size); lzma_options_lzma *opt = filter.options; printf("\nDictionary size: " "%" PRIu32 " MB (2^%" PRIu32 " bytes)\n" "Literal context bits (lc): %" PRIu32 "\n" "Literal pos bits (lp): %" PRIu32 "\n" "Number of pos bits (pb): %" PRIu32 "\n", - (opt->dict_size + 512 * 1024) / (1024 * 1024), + (opt->dict_size / 1024 + 512) / 1024, my_log2(opt->dict_size), opt->lc, opt->lp, opt->pb); free(opt); return false; } extern int main(int argc, char **argv) { tuklib_progname_init(argv); tuklib_gettext_init(PACKAGE, LOCALEDIR); parse_args(argc, argv); #ifdef TUKLIB_DOSLIKE setmode(fileno(stdin), O_BINARY); #endif int ret = EXIT_SUCCESS; // We print empty lines around the output only when reading from // files specified on the command line. This is due to how // LZMA Utils did it. if (optind == argc) { if (lzmainfo("(stdin)", stdin)) ret = EXIT_FAILURE; } else { printf("\n"); do { if (strcmp(argv[optind], "-") == 0) { if (lzmainfo("(stdin)", stdin)) ret = EXIT_FAILURE; } else { FILE *f = fopen(argv[optind], "r"); if (f == NULL) { ret = EXIT_FAILURE; fprintf(stderr, "%s: %s: %s\n", progname, argv[optind], strerror(errno)); continue; } if (lzmainfo(argv[optind], f)) ret = EXIT_FAILURE; printf("\n"); fclose(f); } } while (++optind < argc); } tuklib_exit(ret, EXIT_FAILURE, true); } diff --git a/src/xz/args.c b/src/xz/args.c index eba1b97dc464..b3743ceaf205 100644 --- a/src/xz/args.c +++ b/src/xz/args.c @@ -1,899 +1,898 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file args.c /// \brief Argument parsing /// /// \note Filter-specific options parsing is in options.c. // // Authors: Lasse Collin // Jia Tan // /////////////////////////////////////////////////////////////////////////////// #include "private.h" #include "getopt.h" #include bool opt_stdout = false; bool opt_force = false; bool opt_keep_original = false; bool opt_robot = false; bool opt_ignore_check = false; // We don't modify or free() this, but we need to assign it in some // non-const pointers. const char stdin_filename[] = "(stdin)"; /// Parse and set the memory usage limit for compression, decompression, /// and/or multithreaded decompression. static void parse_memlimit(const char *name, const char *name_percentage, const char *str, bool set_compress, bool set_decompress, bool set_mtdec) { bool is_percentage = false; uint64_t value; const size_t len = strlen(str); if (len > 0 && str[len - 1] == '%') { // Make a copy so that we can get rid of %. // // In the past str wasn't const and we modified it directly // but that modified argv[] and thus affected what was visible // in "ps auxf" or similar tools which was confusing. For // example, --memlimit=50% would show up as --memlimit=50 // since the percent sign was overwritten here. char *s = xstrdup(str); s[len - 1] = '\0'; is_percentage = true; value = str_to_uint64(name_percentage, s, 1, 100); free(s); } else { // On 32-bit systems, SIZE_MAX would make more sense than // UINT64_MAX. But use UINT64_MAX still so that scripts // that assume > 4 GiB values don't break. value = str_to_uint64(name, str, 0, UINT64_MAX); } hardware_memlimit_set(value, set_compress, set_decompress, set_mtdec, is_percentage); return; } static void parse_block_list(const char *str_const) { // We need a modifiable string in the for-loop. char *str_start = xstrdup(str_const); char *str = str_start; // It must be non-empty and not begin with a comma. if (str[0] == '\0' || str[0] == ',') message_fatal(_("%s: Invalid argument to --block-list"), str); // Count the number of comma-separated strings. size_t count = 1; for (size_t i = 0; str[i] != '\0'; ++i) if (str[i] == ',') ++count; // Prevent an unlikely integer overflow. if (count > SIZE_MAX / sizeof(block_list_entry) - 1) message_fatal(_("%s: Too many arguments to --block-list"), str); // Allocate memory to hold all the sizes specified. // If --block-list was specified already, its value is forgotten. free(opt_block_list); opt_block_list = xmalloc((count + 1) * sizeof(block_list_entry)); // Clear the bitmask of filter chains in use. block_list_chain_mask = 0; // Reset the largest Block size found in --block-list. block_list_largest = 0; for (size_t i = 0; i < count; ++i) { // Locate the next comma and replace it with \0. char *p = strchr(str, ','); if (p != NULL) *p = '\0'; // Use the default filter chain unless overridden. opt_block_list[i].chain_num = 0; // To specify a filter chain, the block list entry may be // prepended with "[filter-chain-number]:". The size is // still required for every block. // For instance: // --block-list=2:10MiB,1:5MiB,,8MiB,0:0 // // Translates to: // 1. Block of 10 MiB using filter chain 2 // 2. Block of 5 MiB using filter chain 1 // 3. Block of 5 MiB using filter chain 1 // 4. Block of 8 MiB using the default filter chain // 5. The last block uses the default filter chain // // The block list: // --block-list=2:MiB,1:,0 // // Is not allowed because the second block does not specify // the block size, only the filter chain. if (str[0] >= '0' && str[0] <= '9' && str[1] == ':') { if (str[2] == '\0') message_fatal(_("In --block-list, block " "size is missing after " "filter chain number '%c:'"), str[0]); const unsigned chain_num = (unsigned)(str[0] - '0'); opt_block_list[i].chain_num = chain_num; block_list_chain_mask |= 1U << chain_num; str += 2; } else { // This Block uses the default filter chain. block_list_chain_mask |= 1U << 0; } if (str[0] == '\0') { // There is no string, that is, a comma follows // another comma. Use the previous value. // // NOTE: We checked earlier that the first char // of the whole list cannot be a comma. assert(i > 0); opt_block_list[i] = opt_block_list[i - 1]; } else { opt_block_list[i].size = str_to_uint64("block-list", str, 0, UINT64_MAX); // Zero indicates no more new Blocks. if (opt_block_list[i].size == 0) { if (i + 1 != count) message_fatal(_("0 can only be used " "as the last element " "in --block-list")); opt_block_list[i].size = UINT64_MAX; } // Remember the largest Block size in the list. // // NOTE: Do this after handling the special value 0 // because when 0 is used, we don't want to reduce // the Block size of the multithreaded encoder. if (block_list_largest < opt_block_list[i].size) block_list_largest = opt_block_list[i].size; } // Be standards compliant: p + 1 is undefined behavior // if p == NULL. That occurs on the last iteration of // the loop when we won't care about the value of str // anymore anyway. That is, this is done conditionally // solely for standard conformance reasons. if (p != NULL) str = p + 1; } // Terminate the array. opt_block_list[count].size = 0; free(str_start); return; } static void parse_real(args_info *args, int argc, char **argv) { enum { OPT_FILTERS = INT_MIN, OPT_FILTERS1, OPT_FILTERS2, OPT_FILTERS3, OPT_FILTERS4, OPT_FILTERS5, OPT_FILTERS6, OPT_FILTERS7, OPT_FILTERS8, OPT_FILTERS9, OPT_FILTERS_HELP, OPT_X86, OPT_POWERPC, OPT_IA64, OPT_ARM, OPT_ARMTHUMB, OPT_ARM64, OPT_SPARC, OPT_RISCV, OPT_DELTA, OPT_LZMA1, OPT_LZMA2, OPT_SINGLE_STREAM, OPT_NO_SPARSE, OPT_FILES, OPT_FILES0, OPT_BLOCK_SIZE, OPT_BLOCK_LIST, OPT_MEM_COMPRESS, OPT_MEM_DECOMPRESS, OPT_MEM_MT_DECOMPRESS, OPT_NO_ADJUST, OPT_INFO_MEMORY, OPT_ROBOT, OPT_FLUSH_TIMEOUT, OPT_IGNORE_CHECK, }; static const char short_opts[] = "cC:defF:hHlkM:qQrS:tT:vVz0123456789"; static const struct option long_opts[] = { // Operation mode { "compress", no_argument, NULL, 'z' }, { "decompress", no_argument, NULL, 'd' }, { "uncompress", no_argument, NULL, 'd' }, { "test", no_argument, NULL, 't' }, { "list", no_argument, NULL, 'l' }, // Operation modifiers { "keep", no_argument, NULL, 'k' }, { "force", no_argument, NULL, 'f' }, { "stdout", no_argument, NULL, 'c' }, { "to-stdout", no_argument, NULL, 'c' }, { "single-stream", no_argument, NULL, OPT_SINGLE_STREAM }, { "no-sparse", no_argument, NULL, OPT_NO_SPARSE }, { "suffix", required_argument, NULL, 'S' }, - // { "recursive", no_argument, NULL, 'r' }, // TODO { "files", optional_argument, NULL, OPT_FILES }, { "files0", optional_argument, NULL, OPT_FILES0 }, // Basic compression settings { "format", required_argument, NULL, 'F' }, { "check", required_argument, NULL, 'C' }, { "ignore-check", no_argument, NULL, OPT_IGNORE_CHECK }, { "block-size", required_argument, NULL, OPT_BLOCK_SIZE }, { "block-list", required_argument, NULL, OPT_BLOCK_LIST }, { "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS }, { "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS }, { "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS }, { "memlimit", required_argument, NULL, 'M' }, { "memory", required_argument, NULL, 'M' }, // Old alias { "no-adjust", no_argument, NULL, OPT_NO_ADJUST }, { "threads", required_argument, NULL, 'T' }, { "flush-timeout", required_argument, NULL, OPT_FLUSH_TIMEOUT }, { "extreme", no_argument, NULL, 'e' }, { "fast", no_argument, NULL, '0' }, { "best", no_argument, NULL, '9' }, // Filters { "filters", optional_argument, NULL, OPT_FILTERS}, { "filters1", optional_argument, NULL, OPT_FILTERS1}, { "filters2", optional_argument, NULL, OPT_FILTERS2}, { "filters3", optional_argument, NULL, OPT_FILTERS3}, { "filters4", optional_argument, NULL, OPT_FILTERS4}, { "filters5", optional_argument, NULL, OPT_FILTERS5}, { "filters6", optional_argument, NULL, OPT_FILTERS6}, { "filters7", optional_argument, NULL, OPT_FILTERS7}, { "filters8", optional_argument, NULL, OPT_FILTERS8}, { "filters9", optional_argument, NULL, OPT_FILTERS9}, { "filters-help", optional_argument, NULL, OPT_FILTERS_HELP}, { "lzma1", optional_argument, NULL, OPT_LZMA1 }, { "lzma2", optional_argument, NULL, OPT_LZMA2 }, { "x86", optional_argument, NULL, OPT_X86 }, { "powerpc", optional_argument, NULL, OPT_POWERPC }, { "ia64", optional_argument, NULL, OPT_IA64 }, { "arm", optional_argument, NULL, OPT_ARM }, { "armthumb", optional_argument, NULL, OPT_ARMTHUMB }, { "arm64", optional_argument, NULL, OPT_ARM64 }, { "sparc", optional_argument, NULL, OPT_SPARC }, { "riscv", optional_argument, NULL, OPT_RISCV }, { "delta", optional_argument, NULL, OPT_DELTA }, // Other options { "quiet", no_argument, NULL, 'q' }, { "verbose", no_argument, NULL, 'v' }, { "no-warn", no_argument, NULL, 'Q' }, { "robot", no_argument, NULL, OPT_ROBOT }, { "info-memory", no_argument, NULL, OPT_INFO_MEMORY }, { "help", no_argument, NULL, 'h' }, { "long-help", no_argument, NULL, 'H' }, { "version", no_argument, NULL, 'V' }, { NULL, 0, NULL, 0 } }; int c; while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) { switch (c) { // Compression preset (also for decompression if --format=raw) case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': coder_set_preset((uint32_t)(c - '0')); break; // --memlimit-compress case OPT_MEM_COMPRESS: parse_memlimit("memlimit-compress", "memlimit-compress%", optarg, true, false, false); break; // --memlimit-decompress case OPT_MEM_DECOMPRESS: parse_memlimit("memlimit-decompress", "memlimit-decompress%", optarg, false, true, false); break; // --memlimit-mt-decompress case OPT_MEM_MT_DECOMPRESS: parse_memlimit("memlimit-mt-decompress", "memlimit-mt-decompress%", optarg, false, false, true); break; // --memlimit case 'M': parse_memlimit("memlimit", "memlimit%", optarg, true, true, true); break; // --suffix case 'S': suffix_set(optarg); break; case 'T': { // Since xz 5.4.0: Ignore leading '+' first. const char *s = optarg; if (optarg[0] == '+') ++s; // The max is from src/liblzma/common/common.h. uint32_t t = str_to_uint64("threads", s, 0, 16384); // If leading '+' was used then use multi-threaded // mode even if exactly one thread was specified. if (t == 1 && optarg[0] == '+') t = UINT32_MAX; hardware_threads_set(t); break; } // --version case 'V': // This doesn't return. message_version(); // --stdout case 'c': opt_stdout = true; break; // --decompress case 'd': opt_mode = MODE_DECOMPRESS; break; // --extreme case 'e': coder_set_extreme(); break; // --force case 'f': opt_force = true; break; // --info-memory case OPT_INFO_MEMORY: // This doesn't return. hardware_memlimit_show(); // --help case 'h': // This doesn't return. message_help(false); // --long-help case 'H': // This doesn't return. message_help(true); // --list case 'l': opt_mode = MODE_LIST; break; // --keep case 'k': opt_keep_original = true; break; // --quiet case 'q': message_verbosity_decrease(); break; case 'Q': set_exit_no_warn(); break; case 't': opt_mode = MODE_TEST; break; // --verbose case 'v': message_verbosity_increase(); break; // --robot case OPT_ROBOT: opt_robot = true; // This is to make sure that floating point numbers // always have a dot as decimal separator. setlocale(LC_NUMERIC, "C"); break; case 'z': opt_mode = MODE_COMPRESS; break; // --filters case OPT_FILTERS: coder_add_filters_from_str(optarg); break; // --filters1...--filters9 case OPT_FILTERS1: case OPT_FILTERS2: case OPT_FILTERS3: case OPT_FILTERS4: case OPT_FILTERS5: case OPT_FILTERS6: case OPT_FILTERS7: case OPT_FILTERS8: case OPT_FILTERS9: coder_add_block_filters(optarg, (size_t)(c - OPT_FILTERS)); break; // --filters-help case OPT_FILTERS_HELP: // This doesn't return. message_filters_help(); break; case OPT_X86: coder_add_filter(LZMA_FILTER_X86, options_bcj(optarg)); break; case OPT_POWERPC: coder_add_filter(LZMA_FILTER_POWERPC, options_bcj(optarg)); break; case OPT_IA64: coder_add_filter(LZMA_FILTER_IA64, options_bcj(optarg)); break; case OPT_ARM: coder_add_filter(LZMA_FILTER_ARM, options_bcj(optarg)); break; case OPT_ARMTHUMB: coder_add_filter(LZMA_FILTER_ARMTHUMB, options_bcj(optarg)); break; case OPT_ARM64: coder_add_filter(LZMA_FILTER_ARM64, options_bcj(optarg)); break; case OPT_SPARC: coder_add_filter(LZMA_FILTER_SPARC, options_bcj(optarg)); break; case OPT_RISCV: coder_add_filter(LZMA_FILTER_RISCV, options_bcj(optarg)); break; case OPT_DELTA: coder_add_filter(LZMA_FILTER_DELTA, options_delta(optarg)); break; case OPT_LZMA1: coder_add_filter(LZMA_FILTER_LZMA1, options_lzma(optarg)); break; case OPT_LZMA2: coder_add_filter(LZMA_FILTER_LZMA2, options_lzma(optarg)); break; // Other // --format case 'F': { // Just in case, support both "lzma" and "alone" since // the latter was used for forward compatibility in // LZMA Utils 4.32.x. static const struct { char str[8]; enum format_type format; } types[] = { { "auto", FORMAT_AUTO }, { "xz", FORMAT_XZ }, { "lzma", FORMAT_LZMA }, { "alone", FORMAT_LZMA }, #ifdef HAVE_LZIP_DECODER { "lzip", FORMAT_LZIP }, #endif { "raw", FORMAT_RAW }, }; size_t i = 0; while (strcmp(types[i].str, optarg) != 0) if (++i == ARRAY_SIZE(types)) message_fatal(_("%s: Unknown file " "format type"), optarg); opt_format = types[i].format; break; } // --check case 'C': { static const struct { char str[8]; lzma_check check; } types[] = { { "none", LZMA_CHECK_NONE }, { "crc32", LZMA_CHECK_CRC32 }, { "crc64", LZMA_CHECK_CRC64 }, { "sha256", LZMA_CHECK_SHA256 }, }; size_t i = 0; while (strcmp(types[i].str, optarg) != 0) { if (++i == ARRAY_SIZE(types)) message_fatal(_("%s: Unsupported " "integrity " "check type"), optarg); } // Use a separate check in case we are using different // liblzma than what was used to compile us. if (!lzma_check_is_supported(types[i].check)) message_fatal(_("%s: Unsupported integrity " "check type"), optarg); coder_set_check(types[i].check); break; } case OPT_IGNORE_CHECK: opt_ignore_check = true; break; case OPT_BLOCK_SIZE: opt_block_size = str_to_uint64("block-size", optarg, 0, LZMA_VLI_MAX); break; case OPT_BLOCK_LIST: { parse_block_list(optarg); break; } case OPT_SINGLE_STREAM: opt_single_stream = true; break; case OPT_NO_SPARSE: io_no_sparse(); break; case OPT_FILES: args->files_delim = '\n'; // Fall through case OPT_FILES0: if (args->files_name != NULL) message_fatal(_("Only one file can be " "specified with '--files' " "or '--files0'.")); if (optarg == NULL) { args->files_name = stdin_filename; args->files_file = stdin; } else { args->files_name = optarg; args->files_file = fopen(optarg, c == OPT_FILES ? "r" : "rb"); if (args->files_file == NULL) // TRANSLATORS: This is a translatable // string because French needs a space // before the colon ("%s : %s"). message_fatal(_("%s: %s"), optarg, strerror(errno)); } break; case OPT_NO_ADJUST: opt_auto_adjust = false; break; case OPT_FLUSH_TIMEOUT: opt_flush_timeout = str_to_uint64("flush-timeout", optarg, 0, UINT64_MAX); break; default: message_try_help(); tuklib_exit(E_ERROR, E_ERROR, false); } } return; } static void parse_environment(args_info *args, char *argv0, const char *varname) { char *env = getenv(varname); if (env == NULL) return; // We modify the string, so make a copy of it. env = xstrdup(env); // Calculate the number of arguments in env. argc stats at one // to include space for the program name. int argc = 1; bool prev_was_space = true; for (size_t i = 0; env[i] != '\0'; ++i) { // NOTE: Cast to unsigned char is needed so that correct // value gets passed to isspace(), which expects // unsigned char cast to int. Casting to int is done // automatically due to integer promotion, but we need to // force char to unsigned char manually. Otherwise 8-bit // characters would get promoted to wrong value if // char is signed. if (isspace((unsigned char)env[i])) { prev_was_space = true; } else if (prev_was_space) { prev_was_space = false; // Keep argc small enough to fit into a signed int // and to keep it usable for memory allocation. if (++argc == my_min( INT_MAX, SIZE_MAX / sizeof(char *))) message_fatal(_("The environment variable " "%s contains too many " "arguments"), varname); } } // Allocate memory to hold pointers to the arguments. Add one to get // space for the terminating NULL (if some systems happen to need it). char **argv = xmalloc(((size_t)(argc) + 1) * sizeof(char *)); argv[0] = argv0; argv[argc] = NULL; // Go through the string again. Split the arguments using '\0' // characters and add pointers to the resulting strings to argv. argc = 1; prev_was_space = true; for (size_t i = 0; env[i] != '\0'; ++i) { if (isspace((unsigned char)env[i])) { prev_was_space = true; env[i] = '\0'; } else if (prev_was_space) { prev_was_space = false; argv[argc++] = env + i; } } // Parse the argument list we got from the environment. All non-option // arguments i.e. filenames are ignored. parse_real(args, argc, argv); // Reset the state of the getopt_long() so that we can parse the // command line options too. There are two incompatible ways to // do it. #ifdef HAVE_OPTRESET // BSD optind = 1; optreset = 1; #else // GNU, Solaris optind = 0; #endif // We don't need the argument list from environment anymore. free(argv); free(env); return; } extern void args_parse(args_info *args, int argc, char **argv) { // Initialize those parts of *args that we need later. args->files_name = NULL; args->files_file = NULL; args->files_delim = '\0'; // Check how we were called. { // Remove the leading path name, if any. const char *name = strrchr(argv[0], '/'); if (name == NULL) name = argv[0]; else ++name; // NOTE: It's possible that name[0] is now '\0' if argv[0] // is weird, but it doesn't matter here. // Look for full command names instead of substrings like // "un", "cat", and "lz" to reduce possibility of false // positives when the programs have been renamed. if (strstr(name, "xzcat") != NULL) { opt_mode = MODE_DECOMPRESS; opt_stdout = true; } else if (strstr(name, "unxz") != NULL) { opt_mode = MODE_DECOMPRESS; } else if (strstr(name, "lzcat") != NULL) { opt_format = FORMAT_LZMA; opt_mode = MODE_DECOMPRESS; opt_stdout = true; } else if (strstr(name, "unlzma") != NULL) { opt_format = FORMAT_LZMA; opt_mode = MODE_DECOMPRESS; } else if (strstr(name, "lzma") != NULL) { opt_format = FORMAT_LZMA; } } // First the flags from the environment parse_environment(args, argv[0], "XZ_DEFAULTS"); parse_environment(args, argv[0], "XZ_OPT"); // Then from the command line parse_real(args, argc, argv); // If encoder or decoder support was omitted at build time, // show an error now so that the rest of the code can rely on // that whatever is in opt_mode is also supported. #ifndef HAVE_ENCODERS if (opt_mode == MODE_COMPRESS) message_fatal(_("Compression support was disabled " "at build time")); #endif #ifndef HAVE_DECODERS // Even MODE_LIST cannot work without decoder support so MODE_COMPRESS // is the only valid choice. if (opt_mode != MODE_COMPRESS) message_fatal(_("Decompression support was disabled " "at build time")); #endif #ifdef HAVE_LZIP_DECODER if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_LZIP) message_fatal(_("Compression of lzip files (.lz) " "is not supported")); #endif // Never remove the source file when the destination is not on disk. // In test mode the data is written nowhere, but setting opt_stdout // will make the rest of the code behave well. if (opt_stdout || opt_mode == MODE_TEST) { opt_keep_original = true; opt_stdout = true; } // When compressing, if no --format flag was used, or it // was --format=auto, we compress to the .xz format. if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_AUTO) opt_format = FORMAT_XZ; // Set opt_block_list to NULL if we are not compressing to the .xz // format. This option cannot be used outside of this case, and // simplifies the implementation later. if ((opt_mode != MODE_COMPRESS || opt_format != FORMAT_XZ) && opt_block_list != NULL) { message(V_WARNING, _("--block-list is ignored unless " "compressing to the .xz format")); free(opt_block_list); opt_block_list = NULL; } // If raw format is used and a custom suffix is not provided, // then only stdout mode can be used when compressing or // decompressing. if (opt_format == FORMAT_RAW && !suffix_is_set() && !opt_stdout && (opt_mode == MODE_COMPRESS || opt_mode == MODE_DECOMPRESS)) { if (args->files_name != NULL) message_fatal(_("With --format=raw, " "--suffix=.SUF is required " "unless writing to stdout")); // If all of the filenames provided are "-" (more than one // "-" could be specified) or no filenames are provided, // then we are only going to be writing to standard out. for (int i = optind; i < argc; i++) { if (strcmp(argv[i], "-") != 0) message_fatal(_("With --format=raw, " "--suffix=.SUF is required " "unless writing to stdout")); } } // Compression settings need to be validated (options themselves and // their memory usage) when compressing to any file format. It has to // be done also when uncompressing raw data, since for raw decoding // the options given on the command line are used to know what kind // of raw data we are supposed to decode. if (opt_mode == MODE_COMPRESS || (opt_format == FORMAT_RAW && opt_mode != MODE_LIST)) coder_set_compression_settings(); // If no filenames are given, use stdin. if (argv[optind] == NULL && args->files_name == NULL) { // We don't modify or free() the "-" constant. The caller // modifies this so don't make the struct itself const. static char *names_stdin[2] = { (char *)"-", NULL }; args->arg_names = names_stdin; args->arg_count = 1; } else { // We got at least one filename from the command line, or // --files or --files0 was specified. args->arg_names = argv + optind; args->arg_count = (unsigned int)(argc - optind); } return; } #ifndef NDEBUG extern void args_free(void) { free(opt_block_list); return; } #endif diff --git a/src/xz/list.c b/src/xz/list.c index ca9cf03e85b0..e4a64668c76e 100644 --- a/src/xz/list.c +++ b/src/xz/list.c @@ -1,1339 +1,1346 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file list.c /// \brief Listing information about .xz files // // Author: Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #include "private.h" #include "tuklib_integer.h" /// Information about a .xz file typedef struct { /// Combined Index of all Streams in the file lzma_index *idx; /// Total amount of Stream Padding uint64_t stream_padding; /// Highest memory usage so far uint64_t memusage_max; /// True if all Blocks so far have Compressed Size and /// Uncompressed Size fields bool all_have_sizes; /// Oldest XZ Utils version that will decompress the file uint32_t min_version; } xz_file_info; #define XZ_FILE_INFO_INIT { NULL, 0, 0, true, 50000002 } /// Information about a .xz Block typedef struct { /// Size of the Block Header uint32_t header_size; /// A few of the Block Flags as a string char flags[3]; /// Size of the Compressed Data field in the Block lzma_vli compressed_size; /// Decoder memory usage for this Block uint64_t memusage; /// The filter chain of this Block in human-readable form char *filter_chain; } block_header_info; #define BLOCK_HEADER_INFO_INIT { .filter_chain = NULL } #define block_header_info_end(bhi) free((bhi)->filter_chain) /// Strings ending in a colon. These are used for lines like /// " Foo: 123 MiB". These are grouped because translated strings /// may have different maximum string length, and we want to pad all /// strings so that the values are aligned nicely. static const char *colon_strs[] = { N_("Streams:"), N_("Blocks:"), N_("Compressed size:"), N_("Uncompressed size:"), N_("Ratio:"), N_("Check:"), N_("Stream Padding:"), N_("Memory needed:"), N_("Sizes in headers:"), // This won't be aligned because it's so long: //N_("Minimum XZ Utils version:"), N_("Number of files:"), }; /// Enum matching the above strings. enum { COLON_STR_STREAMS, COLON_STR_BLOCKS, COLON_STR_COMPRESSED_SIZE, COLON_STR_UNCOMPRESSED_SIZE, COLON_STR_RATIO, COLON_STR_CHECK, COLON_STR_STREAM_PADDING, COLON_STR_MEMORY_NEEDED, COLON_STR_SIZES_IN_HEADERS, //COLON_STR_MINIMUM_XZ_VERSION, COLON_STR_NUMBER_OF_FILES, }; /// Field widths to use with printf to pad the strings to use the same number /// of columns on a terminal. static int colon_strs_fw[ARRAY_SIZE(colon_strs)]; /// Convenience macro to get the translated string and its field width /// using a COLON_STR_foo enum. #define COLON_STR(num) colon_strs_fw[num], _(colon_strs[num]) /// Column headings static struct { /// Table column heading string const char *str; /// Number of terminal-columns to use for this table-column. /// If a translated string is longer than the initial value, /// this value will be increased in init_headings(). int columns; /// Field width to use for printf() to pad "str" to use "columns" /// number of columns on a terminal. This is calculated in /// init_headings(). int fw; } headings[] = { { N_("Stream"), 6, 0 }, { N_("Block"), 9, 0 }, { N_("Blocks"), 9, 0 }, { N_("CompOffset"), 15, 0 }, { N_("UncompOffset"), 15, 0 }, { N_("CompSize"), 15, 0 }, { N_("UncompSize"), 15, 0 }, { N_("TotalSize"), 15, 0 }, { N_("Ratio"), 5, 0 }, { N_("Check"), 10, 0 }, { N_("CheckVal"), 1, 0 }, { N_("Padding"), 7, 0 }, { N_("Header"), 5, 0 }, { N_("Flags"), 2, 0 }, { N_("MemUsage"), 7 + 4, 0 }, // +4 is for " MiB" { N_("Filters"), 1, 0 }, }; /// Enum matching the above strings. enum { HEADING_STREAM, HEADING_BLOCK, HEADING_BLOCKS, HEADING_COMPOFFSET, HEADING_UNCOMPOFFSET, HEADING_COMPSIZE, HEADING_UNCOMPSIZE, HEADING_TOTALSIZE, HEADING_RATIO, HEADING_CHECK, HEADING_CHECKVAL, HEADING_PADDING, HEADING_HEADERSIZE, HEADING_HEADERFLAGS, HEADING_MEMUSAGE, HEADING_FILTERS, }; #define HEADING_STR(num) headings[num].fw, _(headings[num].str) /// Check ID to string mapping static const char check_names[LZMA_CHECK_ID_MAX + 1][12] = { // TRANSLATORS: Indicates that there is no integrity check. // This string is used in tables. In older xz version this // string was limited to ten columns in a fixed-width font, but // nowadays there is no strict length restriction anymore. N_("None"), "CRC32", // TRANSLATORS: Indicates that integrity check name is not known, // but the Check ID is known (here 2). In older xz version these // strings were limited to ten columns in a fixed-width font, but // nowadays there is no strict length restriction anymore. N_("Unknown-2"), N_("Unknown-3"), "CRC64", N_("Unknown-5"), N_("Unknown-6"), N_("Unknown-7"), N_("Unknown-8"), N_("Unknown-9"), "SHA-256", N_("Unknown-11"), N_("Unknown-12"), N_("Unknown-13"), N_("Unknown-14"), N_("Unknown-15"), }; /// Buffer size for get_check_names(). This may be a bit ridiculous, /// but at least it's enough if some language needs many multibyte chars. #define CHECKS_STR_SIZE 1024 /// Value of the Check field as hexadecimal string. /// This is set by parse_check_value(). static char check_value[2 * LZMA_CHECK_SIZE_MAX + 1]; /// Totals that are displayed if there was more than one file. /// The "files" counter is also used in print_info_adv() to show /// the file number. static struct { uint64_t files; uint64_t streams; uint64_t blocks; uint64_t compressed_size; uint64_t uncompressed_size; uint64_t stream_padding; uint64_t memusage_max; uint32_t checks; uint32_t min_version; bool all_have_sizes; } totals = { 0, 0, 0, 0, 0, 0, 0, 0, 50000002, true }; /// Initialize colon_strs_fw[]. static void init_colon_strs(void) { // Lengths of translated strings as bytes. size_t lens[ARRAY_SIZE(colon_strs)]; // Lengths of translated strings as columns. size_t widths[ARRAY_SIZE(colon_strs)]; // Maximum number of columns needed by a translated string. size_t width_max = 0; for (unsigned i = 0; i < ARRAY_SIZE(colon_strs); ++i) { widths[i] = tuklib_mbstr_width(_(colon_strs[i]), &lens[i]); // If debugging is enabled, catch invalid strings with // an assertion. However, when not debugging, use the // byte count as the fallback width. This shouldn't // ever happen unless there is a bad string in the // translations, but in such case I guess it's better // to try to print something useful instead of failing // completely. assert(widths[i] != (size_t)-1); if (widths[i] == (size_t)-1) widths[i] = lens[i]; if (widths[i] > width_max) width_max = widths[i]; } // Calculate the field width for printf("%*s") so that the strings // will use width_max columns on a terminal. for (unsigned i = 0; i < ARRAY_SIZE(colon_strs); ++i) colon_strs_fw[i] = (int)(lens[i] + width_max - widths[i]); return; } /// Initialize headings[]. static void init_headings(void) { // Before going through the heading strings themselves, treat // the Check heading specially: Look at the widths of the various // check names and increase the width of the Check column if needed. // The width of the heading name "Check" will then be handled normally // with other heading names in the second loop in this function. for (unsigned i = 0; i < ARRAY_SIZE(check_names); ++i) { size_t len; size_t w = tuklib_mbstr_width(_(check_names[i]), &len); // Error handling like in init_colon_strs(). assert(w != (size_t)-1); if (w == (size_t)-1) w = len; // If the translated string is wider than the minimum width // set at compile time, increase the width. if ((size_t)(headings[HEADING_CHECK].columns) < w) headings[HEADING_CHECK].columns = (int)w; } for (unsigned i = 0; i < ARRAY_SIZE(headings); ++i) { size_t len; size_t w = tuklib_mbstr_width(_(headings[i].str), &len); // Error handling like in init_colon_strs(). assert(w != (size_t)-1); if (w == (size_t)-1) w = len; // If the translated string is wider than the minimum width // set at compile time, increase the width. if ((size_t)(headings[i].columns) < w) headings[i].columns = (int)w; // Calculate the field width for printf("%*s") so that // the string uses .columns number of columns on a terminal. headings[i].fw = (int)(len + (size_t)headings[i].columns - w); } return; } /// Initialize the printf field widths that are needed to get nicely aligned /// output with translated strings. static void init_field_widths(void) { init_colon_strs(); init_headings(); return; } /// Convert XZ Utils version number to a string. static const char * xz_ver_to_str(uint32_t ver) { static char buf[32]; unsigned int major = ver / 10000000U; ver -= major * 10000000U; unsigned int minor = ver / 10000U; ver -= minor * 10000U; unsigned int patch = ver / 10U; ver -= patch * 10U; const char *stability = ver == 0 ? "alpha" : ver == 1 ? "beta" : ""; snprintf(buf, sizeof(buf), "%u.%u.%u%s", major, minor, patch, stability); return buf; } /// \brief Parse the Index(es) from the given .xz file /// /// \param xfi Pointer to structure where the decoded information /// is stored. /// \param pair Input file /// /// \return On success, false is returned. On error, true is returned. /// static bool parse_indexes(xz_file_info *xfi, file_pair *pair) { if (pair->src_st.st_size <= 0) { message_error(_("%s: File is empty"), pair->src_name); return true; } if (pair->src_st.st_size < 2 * LZMA_STREAM_HEADER_SIZE) { message_error(_("%s: Too small to be a valid .xz file"), pair->src_name); return true; } io_buf buf; lzma_stream strm = LZMA_STREAM_INIT; lzma_index *idx = NULL; lzma_ret ret = lzma_file_info_decoder(&strm, &idx, hardware_memlimit_get(MODE_LIST), (uint64_t)(pair->src_st.st_size)); if (ret != LZMA_OK) { message_error(_("%s: %s"), pair->src_name, message_strm(ret)); return true; } while (true) { if (strm.avail_in == 0) { strm.next_in = buf.u8; strm.avail_in = io_read(pair, &buf, IO_BUFFER_SIZE); if (strm.avail_in == SIZE_MAX) goto error; } ret = lzma_code(&strm, LZMA_RUN); switch (ret) { case LZMA_OK: break; case LZMA_SEEK_NEEDED: // liblzma won't ask us to seek past the known size // of the input file. assert(strm.seek_pos <= (uint64_t)(pair->src_st.st_size)); if (io_seek_src(pair, strm.seek_pos)) goto error; // avail_in must be zero so that we will read new // input. strm.avail_in = 0; break; case LZMA_STREAM_END: { lzma_end(&strm); xfi->idx = idx; // Calculate xfi->stream_padding. lzma_index_iter iter; lzma_index_iter_init(&iter, xfi->idx); while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) xfi->stream_padding += iter.stream.padding; return false; } default: message_error(_("%s: %s"), pair->src_name, message_strm(ret)); // If the error was too low memory usage limit, // show also how much memory would have been needed. if (ret == LZMA_MEMLIMIT_ERROR) message_mem_needed(V_ERROR, lzma_memusage(&strm)); goto error; } } error: lzma_end(&strm); return true; } /// \brief Parse the Block Header /// /// The result is stored into *bhi. The caller takes care of initializing it. /// /// \return False on success, true on error. static bool parse_block_header(file_pair *pair, const lzma_index_iter *iter, block_header_info *bhi, xz_file_info *xfi) { #if IO_BUFFER_SIZE < LZMA_BLOCK_HEADER_SIZE_MAX # error IO_BUFFER_SIZE < LZMA_BLOCK_HEADER_SIZE_MAX #endif // Get the whole Block Header with one read, but don't read past // the end of the Block (or even its Check field). const uint32_t size = my_min(iter->block.total_size - lzma_check_size(iter->stream.flags->check), LZMA_BLOCK_HEADER_SIZE_MAX); io_buf buf; if (io_pread(pair, &buf, size, iter->block.compressed_file_offset)) return true; // Zero would mean Index Indicator and thus not a valid Block. if (buf.u8[0] == 0) goto data_error; // Initialize the block structure and decode Block Header Size. lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_block block; block.version = 0; block.check = iter->stream.flags->check; block.filters = filters; block.header_size = lzma_block_header_size_decode(buf.u8[0]); if (block.header_size > size) goto data_error; // Decode the Block Header. switch (lzma_block_header_decode(&block, NULL, buf.u8)) { case LZMA_OK: break; case LZMA_OPTIONS_ERROR: message_error(_("%s: %s"), pair->src_name, message_strm(LZMA_OPTIONS_ERROR)); return true; case LZMA_DATA_ERROR: goto data_error; default: message_bug(); } // Check the Block Flags. These must be done before calling // lzma_block_compressed_size(), because it overwrites // block.compressed_size. // // NOTE: If you add new characters here, update the minimum number of // columns in headings[HEADING_HEADERFLAGS] to match the number of // characters used here. bhi->flags[0] = block.compressed_size != LZMA_VLI_UNKNOWN ? 'c' : '-'; bhi->flags[1] = block.uncompressed_size != LZMA_VLI_UNKNOWN ? 'u' : '-'; bhi->flags[2] = '\0'; // Collect information if all Blocks have both Compressed Size // and Uncompressed Size fields. They can be useful e.g. for // multi-threaded decompression so it can be useful to know it. xfi->all_have_sizes &= block.compressed_size != LZMA_VLI_UNKNOWN && block.uncompressed_size != LZMA_VLI_UNKNOWN; // Validate or set block.compressed_size. switch (lzma_block_compressed_size(&block, iter->block.unpadded_size)) { case LZMA_OK: // Validate also block.uncompressed_size if it is present. // If it isn't present, there's no need to set it since // we aren't going to actually decompress the Block; if // we were decompressing, then we should set it so that // the Block decoder could validate the Uncompressed Size // that was stored in the Index. if (block.uncompressed_size == LZMA_VLI_UNKNOWN || block.uncompressed_size == iter->block.uncompressed_size) break; // If the above fails, the file is corrupt so // LZMA_DATA_ERROR is a good error code. // Fall through case LZMA_DATA_ERROR: // Free the memory allocated by lzma_block_header_decode(). lzma_filters_free(filters, NULL); goto data_error; default: message_bug(); } // Copy the known sizes. bhi->header_size = block.header_size; bhi->compressed_size = block.compressed_size; // Calculate the decoder memory usage and update the maximum // memory usage of this Block. bhi->memusage = lzma_raw_decoder_memusage(filters); if (xfi->memusage_max < bhi->memusage) xfi->memusage_max = bhi->memusage; // Determine the minimum XZ Utils version that supports this Block. // - RISC-V filter needs 5.6.0. // // - ARM64 filter needs 5.4.0. // // - 5.0.0 doesn't support empty LZMA2 streams and thus empty // Blocks that use LZMA2. This decoder bug was fixed in 5.0.2. if (xfi->min_version < 50060002U) { for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { if (filters[i].id == LZMA_FILTER_RISCV) { xfi->min_version = 50060002U; break; } } } if (xfi->min_version < 50040002U) { for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { if (filters[i].id == LZMA_FILTER_ARM64) { xfi->min_version = 50040002U; break; } } } if (xfi->min_version < 50000022U) { size_t i = 0; while (filters[i + 1].id != LZMA_VLI_UNKNOWN) ++i; if (filters[i].id == LZMA_FILTER_LZMA2 && iter->block.uncompressed_size == 0) xfi->min_version = 50000022U; } // Convert the filter chain to human readable form. const lzma_ret str_ret = lzma_str_from_filters( &bhi->filter_chain, filters, LZMA_STR_DECODER | LZMA_STR_GETOPT_LONG, NULL); // Free the memory allocated by lzma_block_header_decode(). lzma_filters_free(filters, NULL); // Check if the stringification succeeded. if (str_ret != LZMA_OK) { message_error(_("%s: %s"), pair->src_name, message_strm(str_ret)); return true; } return false; data_error: // Show the error message. message_error(_("%s: %s"), pair->src_name, message_strm(LZMA_DATA_ERROR)); return true; } /// \brief Parse the Check field and put it into check_value[] /// /// \return False on success, true on error. static bool parse_check_value(file_pair *pair, const lzma_index_iter *iter) { // Don't read anything from the file if there is no integrity Check. if (iter->stream.flags->check == LZMA_CHECK_NONE) { snprintf(check_value, sizeof(check_value), "---"); return false; } // Locate and read the Check field. const uint32_t size = lzma_check_size(iter->stream.flags->check); const uint64_t offset = iter->block.compressed_file_offset + iter->block.total_size - size; io_buf buf; if (io_pread(pair, &buf, size, offset)) return true; // CRC32 and CRC64 are in little endian. Guess that all the future // 32-bit and 64-bit Check values are little endian too. It shouldn't // be a too big problem if this guess is wrong. if (size == 4) snprintf(check_value, sizeof(check_value), "%08" PRIx32, conv32le(buf.u32[0])); else if (size == 8) snprintf(check_value, sizeof(check_value), "%016" PRIx64, conv64le(buf.u64[0])); else for (size_t i = 0; i < size; ++i) snprintf(check_value + i * 2, 3, "%02x", buf.u8[i]); return false; } /// \brief Parse detailed information about a Block /// /// Since this requires seek(s), listing information about all Blocks can /// be slow. /// /// \param pair Input file /// \param iter Location of the Block whose Check value should /// be printed. /// \param bhi Pointer to structure where to store the information /// about the Block Header field. /// /// \return False on success, true on error. If an error occurs, /// the error message is printed too so the caller doesn't /// need to worry about that. static bool parse_details(file_pair *pair, const lzma_index_iter *iter, block_header_info *bhi, xz_file_info *xfi) { if (parse_block_header(pair, iter, bhi, xfi)) return true; if (parse_check_value(pair, iter)) return true; return false; } /// \brief Get the compression ratio /// /// This has slightly different format than that is used in message.c. static const char * get_ratio(uint64_t compressed_size, uint64_t uncompressed_size) { if (uncompressed_size == 0) return "---"; const double ratio = (double)(compressed_size) / (double)(uncompressed_size); if (ratio > 9.999) return "---"; static char buf[16]; snprintf(buf, sizeof(buf), "%.3f", ratio); return buf; } /// \brief Get a comma-separated list of Check names /// /// The check names are translated with gettext except when in robot mode. /// /// \param buf Buffer to hold the resulting string /// \param checks Bit mask of Checks to print /// \param space_after_comma /// It's better to not use spaces in table-like listings, /// but in more verbose formats a space after a comma /// is good for readability. static void get_check_names(char buf[CHECKS_STR_SIZE], uint32_t checks, bool space_after_comma) { // If we get called when there are no Checks to print, set checks // to 1 so that we print "None". This can happen in the robot mode // when printing the totals line if there are no valid input files. if (checks == 0) checks = 1; char *pos = buf; size_t left = CHECKS_STR_SIZE; const char *sep = space_after_comma ? ", " : ","; bool comma = false; for (size_t i = 0; i <= LZMA_CHECK_ID_MAX; ++i) { if (checks & (UINT32_C(1) << i)) { my_snprintf(&pos, &left, "%s%s", comma ? sep : "", opt_robot ? check_names[i] : _(check_names[i])); comma = true; } } return; } static bool print_info_basic(const xz_file_info *xfi, file_pair *pair) { static bool headings_displayed = false; if (!headings_displayed) { headings_displayed = true; // TRANSLATORS: These are column headings. From Strms (Streams) // to Ratio, the columns are right aligned. Check and Filename // are left aligned. If you need longer words, it's OK to // use two lines here. Test with "xz -l foo.xz". puts(_("Strms Blocks Compressed Uncompressed Ratio " "Check Filename")); } char checks[CHECKS_STR_SIZE]; get_check_names(checks, lzma_index_checks(xfi->idx), false); const char *cols[7] = { uint64_to_str(lzma_index_stream_count(xfi->idx), 0), uint64_to_str(lzma_index_block_count(xfi->idx), 1), uint64_to_nicestr(lzma_index_file_size(xfi->idx), NICESTR_B, NICESTR_TIB, false, 2), uint64_to_nicestr(lzma_index_uncompressed_size(xfi->idx), NICESTR_B, NICESTR_TIB, false, 3), get_ratio(lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx)), checks, pair->src_name, }; printf("%*s %*s %*s %*s %*s %-*s %s\n", tuklib_mbstr_fw(cols[0], 5), cols[0], tuklib_mbstr_fw(cols[1], 7), cols[1], tuklib_mbstr_fw(cols[2], 11), cols[2], tuklib_mbstr_fw(cols[3], 11), cols[3], tuklib_mbstr_fw(cols[4], 5), cols[4], tuklib_mbstr_fw(cols[5], 7), cols[5], cols[6]); return false; } static void print_adv_helper(uint64_t stream_count, uint64_t block_count, uint64_t compressed_size, uint64_t uncompressed_size, uint32_t checks, uint64_t stream_padding) { char checks_str[CHECKS_STR_SIZE]; get_check_names(checks_str, checks, true); printf(" %-*s %s\n", COLON_STR(COLON_STR_STREAMS), uint64_to_str(stream_count, 0)); printf(" %-*s %s\n", COLON_STR(COLON_STR_BLOCKS), uint64_to_str(block_count, 0)); printf(" %-*s %s\n", COLON_STR(COLON_STR_COMPRESSED_SIZE), uint64_to_nicestr(compressed_size, NICESTR_B, NICESTR_TIB, true, 0)); printf(" %-*s %s\n", COLON_STR(COLON_STR_UNCOMPRESSED_SIZE), uint64_to_nicestr(uncompressed_size, NICESTR_B, NICESTR_TIB, true, 0)); printf(" %-*s %s\n", COLON_STR(COLON_STR_RATIO), get_ratio(compressed_size, uncompressed_size)); printf(" %-*s %s\n", COLON_STR(COLON_STR_CHECK), checks_str); printf(" %-*s %s\n", COLON_STR(COLON_STR_STREAM_PADDING), uint64_to_nicestr(stream_padding, NICESTR_B, NICESTR_TIB, true, 0)); return; } static bool print_info_adv(xz_file_info *xfi, file_pair *pair) { // Print the overall information. print_adv_helper(lzma_index_stream_count(xfi->idx), lzma_index_block_count(xfi->idx), lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx), lzma_index_checks(xfi->idx), xfi->stream_padding); // Size of the biggest Check. This is used to calculate the width // of the CheckVal field. The table would get insanely wide if // we always reserved space for 64-byte Check (128 chars as hex). uint32_t check_max = 0; // Print information about the Streams. // // All except Check are right aligned; Check is left aligned. // Test with "xz -lv foo.xz". printf(" %s\n %*s %*s %*s %*s %*s %*s %*s %-*s %*s\n", _(colon_strs[COLON_STR_STREAMS]), HEADING_STR(HEADING_STREAM), HEADING_STR(HEADING_BLOCKS), HEADING_STR(HEADING_COMPOFFSET), HEADING_STR(HEADING_UNCOMPOFFSET), HEADING_STR(HEADING_COMPSIZE), HEADING_STR(HEADING_UNCOMPSIZE), HEADING_STR(HEADING_RATIO), HEADING_STR(HEADING_CHECK), HEADING_STR(HEADING_PADDING)); lzma_index_iter iter; lzma_index_iter_init(&iter, xfi->idx); while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) { const char *cols1[4] = { uint64_to_str(iter.stream.number, 0), uint64_to_str(iter.stream.block_count, 1), uint64_to_str(iter.stream.compressed_offset, 2), uint64_to_str(iter.stream.uncompressed_offset, 3), }; printf(" %*s %*s %*s %*s ", tuklib_mbstr_fw(cols1[0], headings[HEADING_STREAM].columns), cols1[0], tuklib_mbstr_fw(cols1[1], headings[HEADING_BLOCKS].columns), cols1[1], tuklib_mbstr_fw(cols1[2], headings[HEADING_COMPOFFSET].columns), cols1[2], tuklib_mbstr_fw(cols1[3], headings[HEADING_UNCOMPOFFSET].columns), cols1[3]); const char *cols2[5] = { uint64_to_str(iter.stream.compressed_size, 0), uint64_to_str(iter.stream.uncompressed_size, 1), get_ratio(iter.stream.compressed_size, iter.stream.uncompressed_size), _(check_names[iter.stream.flags->check]), uint64_to_str(iter.stream.padding, 2), }; printf("%*s %*s %*s %-*s %*s\n", tuklib_mbstr_fw(cols2[0], headings[HEADING_COMPSIZE].columns), cols2[0], tuklib_mbstr_fw(cols2[1], headings[HEADING_UNCOMPSIZE].columns), cols2[1], tuklib_mbstr_fw(cols2[2], headings[HEADING_RATIO].columns), cols2[2], tuklib_mbstr_fw(cols2[3], headings[HEADING_CHECK].columns), cols2[3], tuklib_mbstr_fw(cols2[4], headings[HEADING_PADDING].columns), cols2[4]); // Update the maximum Check size. if (lzma_check_size(iter.stream.flags->check) > check_max) check_max = lzma_check_size(iter.stream.flags->check); } // Cache the verbosity level to a local variable. const bool detailed = message_verbosity_get() >= V_DEBUG; // Print information about the Blocks but only if there is // at least one Block. if (lzma_index_block_count(xfi->idx) > 0) { // Calculate the width of the CheckVal column. This can be // used as is as the field width for printf() when printing // the actual check value as it is hexadecimal. However, to // print the column heading, further calculation is needed // to handle a translated string (it's done a few lines later). assert(check_max <= LZMA_CHECK_SIZE_MAX); const int checkval_width = my_max( headings[HEADING_CHECKVAL].columns, (int)(2 * check_max)); // All except Check are right aligned; Check is left aligned. printf(" %s\n %*s %*s %*s %*s %*s %*s %*s %-*s", _(colon_strs[COLON_STR_BLOCKS]), HEADING_STR(HEADING_STREAM), HEADING_STR(HEADING_BLOCK), HEADING_STR(HEADING_COMPOFFSET), HEADING_STR(HEADING_UNCOMPOFFSET), HEADING_STR(HEADING_TOTALSIZE), HEADING_STR(HEADING_UNCOMPSIZE), HEADING_STR(HEADING_RATIO), detailed ? headings[HEADING_CHECK].fw : 1, _(headings[HEADING_CHECK].str)); if (detailed) { // CheckVal (Check value), Flags, and Filters are // left aligned. Block Header Size, CompSize, and // MemUsage are right aligned. Test with // "xz -lvv foo.xz". printf(" %-*s %*s %-*s %*s %*s %s", headings[HEADING_CHECKVAL].fw + checkval_width - headings[HEADING_CHECKVAL].columns, _(headings[HEADING_CHECKVAL].str), HEADING_STR(HEADING_HEADERSIZE), HEADING_STR(HEADING_HEADERFLAGS), HEADING_STR(HEADING_COMPSIZE), HEADING_STR(HEADING_MEMUSAGE), _(headings[HEADING_FILTERS].str)); } putchar('\n'); lzma_index_iter_init(&iter, xfi->idx); // Iterate over the Blocks. while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_BLOCK)) { // If in detailed mode, collect the information from // Block Header before starting to print the next line. block_header_info bhi = BLOCK_HEADER_INFO_INIT; if (detailed && parse_details(pair, &iter, &bhi, xfi)) return true; const char *cols1[4] = { uint64_to_str(iter.stream.number, 0), uint64_to_str( iter.block.number_in_stream, 1), uint64_to_str( iter.block.compressed_file_offset, 2), uint64_to_str( iter.block.uncompressed_file_offset, 3) }; printf(" %*s %*s %*s %*s ", tuklib_mbstr_fw(cols1[0], headings[HEADING_STREAM].columns), cols1[0], tuklib_mbstr_fw(cols1[1], headings[HEADING_BLOCK].columns), cols1[1], tuklib_mbstr_fw(cols1[2], headings[HEADING_COMPOFFSET].columns), cols1[2], tuklib_mbstr_fw(cols1[3], headings[ HEADING_UNCOMPOFFSET].columns), cols1[3]); const char *cols2[4] = { uint64_to_str(iter.block.total_size, 0), uint64_to_str(iter.block.uncompressed_size, 1), get_ratio(iter.block.total_size, iter.block.uncompressed_size), _(check_names[iter.stream.flags->check]) }; printf("%*s %*s %*s %-*s", tuklib_mbstr_fw(cols2[0], headings[HEADING_TOTALSIZE].columns), cols2[0], tuklib_mbstr_fw(cols2[1], headings[HEADING_UNCOMPSIZE].columns), cols2[1], tuklib_mbstr_fw(cols2[2], headings[HEADING_RATIO].columns), cols2[2], tuklib_mbstr_fw(cols2[3], detailed ? headings[HEADING_CHECK].columns : 1), cols2[3]); if (detailed) { const lzma_vli compressed_size = iter.block.unpadded_size - bhi.header_size - lzma_check_size( iter.stream.flags->check); const char *cols3[6] = { check_value, uint64_to_str(bhi.header_size, 0), bhi.flags, uint64_to_str(compressed_size, 1), uint64_to_str( round_up_to_mib(bhi.memusage), 2), bhi.filter_chain }; // Show MiB for memory usage, because it // is the only size which is not in bytes. printf(" %-*s %*s %-*s %*s %*s MiB %s", checkval_width, cols3[0], tuklib_mbstr_fw(cols3[1], headings[ HEADING_HEADERSIZE].columns), cols3[1], tuklib_mbstr_fw(cols3[2], headings[ HEADING_HEADERFLAGS].columns), cols3[2], tuklib_mbstr_fw(cols3[3], headings[ HEADING_COMPSIZE].columns), cols3[3], tuklib_mbstr_fw(cols3[4], headings[ HEADING_MEMUSAGE].columns - 4), cols3[4], cols3[5]); } putchar('\n'); block_header_info_end(&bhi); } } if (detailed) { printf(" %-*s %s MiB\n", COLON_STR(COLON_STR_MEMORY_NEEDED), uint64_to_str( round_up_to_mib(xfi->memusage_max), 0)); printf(" %-*s %s\n", COLON_STR(COLON_STR_SIZES_IN_HEADERS), xfi->all_have_sizes ? _("Yes") : _("No")); //printf(" %-*s %s\n", COLON_STR(COLON_STR_MINIMUM_XZ_VERSION), printf(_(" Minimum XZ Utils version: %s\n"), xz_ver_to_str(xfi->min_version)); } return false; } static bool print_info_robot(xz_file_info *xfi, file_pair *pair) { char checks[CHECKS_STR_SIZE]; get_check_names(checks, lzma_index_checks(xfi->idx), false); printf("name\t%s\n", pair->src_name); printf("file\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s\t%" PRIu64 "\n", lzma_index_stream_count(xfi->idx), lzma_index_block_count(xfi->idx), lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx), get_ratio(lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx)), checks, xfi->stream_padding); if (message_verbosity_get() >= V_VERBOSE) { lzma_index_iter iter; lzma_index_iter_init(&iter, xfi->idx); while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) printf("stream\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s\t%" PRIu64 "\n", iter.stream.number, iter.stream.block_count, iter.stream.compressed_offset, iter.stream.uncompressed_offset, iter.stream.compressed_size, iter.stream.uncompressed_size, get_ratio(iter.stream.compressed_size, iter.stream.uncompressed_size), check_names[iter.stream.flags->check], iter.stream.padding); lzma_index_iter_rewind(&iter); while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_BLOCK)) { block_header_info bhi = BLOCK_HEADER_INFO_INIT; if (message_verbosity_get() >= V_DEBUG && parse_details( pair, &iter, &bhi, xfi)) return true; printf("block\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s", iter.stream.number, iter.block.number_in_stream, iter.block.number_in_file, iter.block.compressed_file_offset, iter.block.uncompressed_file_offset, iter.block.total_size, iter.block.uncompressed_size, get_ratio(iter.block.total_size, iter.block.uncompressed_size), check_names[iter.stream.flags->check]); if (message_verbosity_get() >= V_DEBUG) printf("\t%s\t%" PRIu32 "\t%s\t%" PRIu64 "\t%" PRIu64 "\t%s", check_value, bhi.header_size, bhi.flags, bhi.compressed_size, bhi.memusage, bhi.filter_chain); putchar('\n'); block_header_info_end(&bhi); } } if (message_verbosity_get() >= V_DEBUG) printf("summary\t%" PRIu64 "\t%s\t%" PRIu32 "\n", xfi->memusage_max, xfi->all_have_sizes ? "yes" : "no", xfi->min_version); return false; } static void update_totals(const xz_file_info *xfi) { // TODO: Integer overflow checks ++totals.files; totals.streams += lzma_index_stream_count(xfi->idx); totals.blocks += lzma_index_block_count(xfi->idx); totals.compressed_size += lzma_index_file_size(xfi->idx); totals.uncompressed_size += lzma_index_uncompressed_size(xfi->idx); totals.stream_padding += xfi->stream_padding; totals.checks |= lzma_index_checks(xfi->idx); if (totals.memusage_max < xfi->memusage_max) totals.memusage_max = xfi->memusage_max; if (totals.min_version < xfi->min_version) totals.min_version = xfi->min_version; totals.all_have_sizes &= xfi->all_have_sizes; return; } static void print_totals_basic(void) { // Print a separator line. char line[80]; memset(line, '-', sizeof(line)); line[sizeof(line) - 1] = '\0'; puts(line); // Get the check names. char checks[CHECKS_STR_SIZE]; get_check_names(checks, totals.checks, false); // Print the totals except the file count, which needs // special handling. printf("%5s %7s %11s %11s %5s %-7s ", uint64_to_str(totals.streams, 0), uint64_to_str(totals.blocks, 1), uint64_to_nicestr(totals.compressed_size, NICESTR_B, NICESTR_TIB, false, 2), uint64_to_nicestr(totals.uncompressed_size, NICESTR_B, NICESTR_TIB, false, 3), get_ratio(totals.compressed_size, totals.uncompressed_size), checks); +#if defined(__sun) && (defined(__GNUC__) || defined(__clang__)) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wformat-nonliteral" +#endif // Since we print totals only when there are at least two files, // the English message will always use "%s files". But some other // languages need different forms for different plurals so we // have to translate this with ngettext(). // // TRANSLATORS: %s is an integer. Only the plural form of this // message is used (e.g. "2 files"). Test with "xz -l foo.xz bar.xz". printf(ngettext("%s file\n", "%s files\n", totals.files <= ULONG_MAX ? totals.files : (totals.files % 1000000) + 1000000), uint64_to_str(totals.files, 0)); +#if defined(__sun) && (defined(__GNUC__) || defined(__clang__)) +# pragma GCC diagnostic pop +#endif return; } static void print_totals_adv(void) { putchar('\n'); puts(_("Totals:")); printf(" %-*s %s\n", COLON_STR(COLON_STR_NUMBER_OF_FILES), uint64_to_str(totals.files, 0)); print_adv_helper(totals.streams, totals.blocks, totals.compressed_size, totals.uncompressed_size, totals.checks, totals.stream_padding); if (message_verbosity_get() >= V_DEBUG) { printf(" %-*s %s MiB\n", COLON_STR(COLON_STR_MEMORY_NEEDED), uint64_to_str( round_up_to_mib(totals.memusage_max), 0)); printf(" %-*s %s\n", COLON_STR(COLON_STR_SIZES_IN_HEADERS), totals.all_have_sizes ? _("Yes") : _("No")); //printf(" %-*s %s\n", COLON_STR(COLON_STR_MINIMUM_XZ_VERSION), printf(_(" Minimum XZ Utils version: %s\n"), xz_ver_to_str(totals.min_version)); } return; } static void print_totals_robot(void) { char checks[CHECKS_STR_SIZE]; get_check_names(checks, totals.checks, false); printf("totals\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s\t%" PRIu64 "\t%" PRIu64, totals.streams, totals.blocks, totals.compressed_size, totals.uncompressed_size, get_ratio(totals.compressed_size, totals.uncompressed_size), checks, totals.stream_padding, totals.files); if (message_verbosity_get() >= V_DEBUG) printf("\t%" PRIu64 "\t%s\t%" PRIu32, totals.memusage_max, totals.all_have_sizes ? "yes" : "no", totals.min_version); putchar('\n'); return; } extern void list_totals(void) { if (opt_robot) { // Always print totals in --robot mode. It can be convenient // in some cases and doesn't complicate usage of the // single-file case much. print_totals_robot(); } else if (totals.files > 1) { // For non-robot mode, totals are printed only if there // is more than one file. if (message_verbosity_get() <= V_WARNING) print_totals_basic(); else print_totals_adv(); } return; } extern void list_file(const char *filename) { if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO) { // The 'lzmainfo' message is printed only when --format=lzma // is used (it is implied if using "lzma" as the command // name). Thus instead of using message_fatal(), print // the messages separately and then call tuklib_exit() // like message_fatal() does. message(V_ERROR, _("--list works only on .xz files " "(--format=xz or --format=auto)")); if (opt_format == FORMAT_LZMA) message(V_ERROR, _("Try 'lzmainfo' with .lzma files.")); tuklib_exit(E_ERROR, E_ERROR, false); } message_filename(filename); if (filename == stdin_filename) { message_error(_("--list does not support reading from " "standard input")); return; } init_field_widths(); // Unset opt_stdout so that io_open_src() won't accept special files. // Set opt_force so that io_open_src() will follow symlinks. opt_stdout = false; opt_force = true; file_pair *pair = io_open_src(filename); if (pair == NULL) return; xz_file_info xfi = XZ_FILE_INFO_INIT; if (!parse_indexes(&xfi, pair)) { bool fail; // We have three main modes: // - --robot, which has submodes if --verbose is specified // once or twice // - Normal --list without --verbose // - --list with one or two --verbose if (opt_robot) fail = print_info_robot(&xfi, pair); else if (message_verbosity_get() <= V_WARNING) fail = print_info_basic(&xfi, pair); else fail = print_info_adv(&xfi, pair); // Update the totals that are displayed after all // the individual files have been listed. Don't count // broken files. if (!fail) update_totals(&xfi); lzma_index_end(xfi.idx, NULL); } io_close(pair, false); return; } diff --git a/src/xz/mytime.c b/src/xz/mytime.c index 7d9a27d58b56..c603051dcdae 100644 --- a/src/xz/mytime.c +++ b/src/xz/mytime.c @@ -1,177 +1,177 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file mytime.c /// \brief Time handling functions // // Author: Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #include "private.h" #if defined(MYTHREAD_VISTA) || defined(_MSC_VER) // Nothing #elif defined(HAVE_CLOCK_GETTIME) \ && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX)) # include #else # include #endif uint64_t opt_flush_timeout = 0; // start_time holds the time when the (de)compression was started. // It's from mytime_now() and thus only useful for calculating relative // time differences (elapsed time). start_time is initialized by calling // mytime_set_start_time() and modified by mytime_sigtstp_handler(). // // When mytime_sigtstp_handler() is used, start_time is made volatile. // I'm not sure if that is really required since access to it is guarded // by signals_block()/signals_unblock() since accessing an uint64_t isn't // atomic on all systems. But since the variable isn't accessed very // frequently making it volatile doesn't hurt. #ifdef USE_SIGTSTP_HANDLER static volatile uint64_t start_time; #else static uint64_t start_time; #endif static uint64_t next_flush; /// \brief Get the current time as milliseconds /// /// It's relative to some point but not necessarily to the UNIX Epoch. static uint64_t mytime_now(void) { #if defined(MYTHREAD_VISTA) || defined(_MSC_VER) // Since there is no SIGALRM on Windows, this function gets // called frequently when the progress indicator is in use. // Progress indicator doesn't need high-resolution time. // GetTickCount64() has very low overhead but needs at least WinVista. // // MinGW-w64 provides the POSIX functions clock_gettime() and // gettimeofday() in a manner that allow xz to run on older // than WinVista. If the threading method needs WinVista anyway, // there's no reason to avoid a WinVista API here either. return GetTickCount64(); #elif defined(HAVE_CLOCK_GETTIME) \ && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX)) // MinGW-w64: clock_gettime() is defined in winpthreads but we need // nothing else from winpthreads (unless, for some odd reason, POSIX // threading has been selected). By avoiding clock_gettime(), we // avoid the dependency on libwinpthread-1.dll or the need to link // against the static version. The downside is that the fallback // method, gettimeofday(), doesn't provide monotonic time. struct timespec tv; # ifdef HAVE_CLOCK_MONOTONIC // If CLOCK_MONOTONIC was available at compile time but for some // reason isn't at runtime, fallback to CLOCK_REALTIME which // according to POSIX is mandatory for all implementations. static clockid_t clk_id = CLOCK_MONOTONIC; while (clock_gettime(clk_id, &tv)) clk_id = CLOCK_REALTIME; # else clock_gettime(CLOCK_REALTIME, &tv); # endif return (uint64_t)tv.tv_sec * 1000 + (uint64_t)(tv.tv_nsec / 1000000); #else struct timeval tv; gettimeofday(&tv, NULL); return (uint64_t)tv.tv_sec * 1000 + (uint64_t)(tv.tv_usec / 1000); #endif } #ifdef USE_SIGTSTP_HANDLER extern void mytime_sigtstp_handler(int sig lzma_attribute((__unused__))) { // Measure how long the process stays in the stopped state and add - // that amount to start_time. This way the the progress indicator + // that amount to start_time. This way the progress indicator // won't count the stopped time as elapsed time and the estimated // remaining time won't be confused by the time spent in the // stopped state. // // FIXME? Is raising SIGSTOP the correct thing to do? POSIX.1-2017 // says that orphan processes shouldn't stop on SIGTSTP. So perhaps // the most correct thing to do could be to revert to the default // handler for SIGTSTP, unblock SIGTSTP, and then raise(SIGTSTP). // It's quite a bit more complicated than just raising SIGSTOP though. // // The difference between raising SIGTSTP vs. SIGSTOP can be seen on // the shell command line too by running "echo $?" after stopping // a process but perhaps that doesn't matter. const uint64_t t = mytime_now(); raise(SIGSTOP); start_time += mytime_now() - t; return; } #endif extern void mytime_set_start_time(void) { #ifdef USE_SIGTSTP_HANDLER // Block the signals when accessing start_time so that we cannot // end up with a garbage value. start_time is volatile but access // to it isn't atomic at least on 32-bit systems. signals_block(); #endif start_time = mytime_now(); #ifdef USE_SIGTSTP_HANDLER signals_unblock(); #endif return; } extern uint64_t mytime_get_elapsed(void) { #ifdef USE_SIGTSTP_HANDLER signals_block(); #endif const uint64_t t = mytime_now() - start_time; #ifdef USE_SIGTSTP_HANDLER signals_unblock(); #endif return t; } extern void mytime_set_flush_time(void) { next_flush = mytime_now() + opt_flush_timeout; return; } extern int mytime_get_flush_timeout(void) { if (opt_flush_timeout == 0 || opt_mode != MODE_COMPRESS) return -1; const uint64_t now = mytime_now(); if (now >= next_flush) return 0; const uint64_t remaining = next_flush - now; return remaining > INT_MAX ? INT_MAX : (int)remaining; } diff --git a/src/xzdec/xzdec.c b/src/xzdec/xzdec.c index 4d881748b2e4..a75ea42a52fb 100644 --- a/src/xzdec/xzdec.c +++ b/src/xzdec/xzdec.c @@ -1,484 +1,484 @@ // SPDX-License-Identifier: 0BSD /////////////////////////////////////////////////////////////////////////////// // /// \file xzdec.c /// \brief Simple single-threaded tool to uncompress .xz or .lzma files // // Author: Lasse Collin // /////////////////////////////////////////////////////////////////////////////// #include "sysdefs.h" #include "lzma.h" #include #include #include #ifndef _MSC_VER # include #endif #ifdef HAVE_CAP_RIGHTS_LIMIT # include #endif #ifdef HAVE_LINUX_LANDLOCK # include # include # include # ifdef LANDLOCK_ACCESS_NET_BIND_TCP # define LANDLOCK_ABI_MAX 4 # else # define LANDLOCK_ABI_MAX 3 # endif #endif #if defined(HAVE_CAP_RIGHTS_LIMIT) || defined(HAVE_PLEDGE) \ || defined(HAVE_LINUX_LANDLOCK) # define ENABLE_SANDBOX 1 #endif #include "getopt.h" #include "tuklib_progname.h" #include "tuklib_exit.h" #ifdef TUKLIB_DOSLIKE # include # include # ifdef _MSC_VER # define fileno _fileno # define setmode _setmode # endif #endif #ifdef LZMADEC # define TOOL_FORMAT "lzma" #else # define TOOL_FORMAT "xz" #endif /// Error messages are suppressed if this is zero, which is the case when /// --quiet has been given at least twice. static int display_errors = 2; lzma_attribute((__format__(__printf__, 1, 2))) static void my_errorf(const char *fmt, ...) { va_list ap; va_start(ap, fmt); if (display_errors) { fprintf(stderr, "%s: ", progname); vfprintf(stderr, fmt, ap); fprintf(stderr, "\n"); } va_end(ap); return; } tuklib_attr_noreturn static void help(void) { printf( "Usage: %s [OPTION]... [FILE]...\n" "Decompress files in the ." TOOL_FORMAT " format to standard output.\n" "\n" " -d, --decompress (ignored, only decompression is supported)\n" " -k, --keep (ignored, files are never deleted)\n" " -c, --stdout (ignored, output is always written to standard output)\n" " -q, --quiet specify *twice* to suppress errors\n" " -Q, --no-warn (ignored, the exit status 2 is never used)\n" " -h, --help display this help and exit\n" " -V, --version display the version number and exit\n" "\n" "With no FILE, or when FILE is -, read standard input.\n" "\n" "Report bugs to <" PACKAGE_BUGREPORT "> (in English or Finnish).\n" PACKAGE_NAME " home page: <" PACKAGE_URL ">\n", progname); tuklib_exit(EXIT_SUCCESS, EXIT_FAILURE, display_errors); } tuklib_attr_noreturn static void version(void) { printf(TOOL_FORMAT "dec (" PACKAGE_NAME ") " LZMA_VERSION_STRING "\n" "liblzma %s\n", lzma_version_string()); tuklib_exit(EXIT_SUCCESS, EXIT_FAILURE, display_errors); } /// Parses command line options. static void parse_options(int argc, char **argv) { - static const char short_opts[] = "cdkM:hqQV"; + static const char short_opts[] = "cdkhqQV"; static const struct option long_opts[] = { { "stdout", no_argument, NULL, 'c' }, { "to-stdout", no_argument, NULL, 'c' }, { "decompress", no_argument, NULL, 'd' }, { "uncompress", no_argument, NULL, 'd' }, { "keep", no_argument, NULL, 'k' }, { "quiet", no_argument, NULL, 'q' }, { "no-warn", no_argument, NULL, 'Q' }, { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, 'V' }, { NULL, 0, NULL, 0 } }; int c; while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) { switch (c) { case 'c': case 'd': case 'k': case 'Q': break; case 'q': if (display_errors > 0) --display_errors; break; case 'h': help(); case 'V': version(); default: exit(EXIT_FAILURE); } } return; } static void uncompress(lzma_stream *strm, FILE *file, const char *filename) { lzma_ret ret; // Initialize the decoder #ifdef LZMADEC ret = lzma_alone_decoder(strm, UINT64_MAX); #else ret = lzma_stream_decoder(strm, UINT64_MAX, LZMA_CONCATENATED); #endif // The only reasonable error here is LZMA_MEM_ERROR. if (ret != LZMA_OK) { my_errorf("%s", ret == LZMA_MEM_ERROR ? strerror(ENOMEM) : "Internal error (bug)"); exit(EXIT_FAILURE); } // Input and output buffers uint8_t in_buf[BUFSIZ]; uint8_t out_buf[BUFSIZ]; strm->avail_in = 0; strm->next_out = out_buf; strm->avail_out = BUFSIZ; lzma_action action = LZMA_RUN; while (true) { if (strm->avail_in == 0) { strm->next_in = in_buf; strm->avail_in = fread(in_buf, 1, BUFSIZ, file); if (ferror(file)) { // POSIX says that fread() sets errno if // an error occurred. ferror() doesn't // touch errno. my_errorf("%s: Error reading input file: %s", filename, strerror(errno)); exit(EXIT_FAILURE); } #ifndef LZMADEC // When using LZMA_CONCATENATED, we need to tell // liblzma when it has got all the input. if (feof(file)) action = LZMA_FINISH; #endif } ret = lzma_code(strm, action); // Write and check write error before checking decoder error. // This way as much data as possible gets written to output // even if decoder detected an error. if (strm->avail_out == 0 || ret != LZMA_OK) { const size_t write_size = BUFSIZ - strm->avail_out; if (fwrite(out_buf, 1, write_size, stdout) != write_size) { // Wouldn't be a surprise if writing to stderr // would fail too but at least try to show an // error message. my_errorf("Cannot write to standard output: " "%s", strerror(errno)); exit(EXIT_FAILURE); } strm->next_out = out_buf; strm->avail_out = BUFSIZ; } if (ret != LZMA_OK) { if (ret == LZMA_STREAM_END) { #ifdef LZMADEC // Check that there's no trailing garbage. if (strm->avail_in != 0 || fread(in_buf, 1, 1, file) != 0 || !feof(file)) ret = LZMA_DATA_ERROR; else return; #else // lzma_stream_decoder() already guarantees // that there's no trailing garbage. assert(strm->avail_in == 0); assert(action == LZMA_FINISH); assert(feof(file)); return; #endif } const char *msg; switch (ret) { case LZMA_MEM_ERROR: msg = strerror(ENOMEM); break; case LZMA_FORMAT_ERROR: msg = "File format not recognized"; break; case LZMA_OPTIONS_ERROR: // FIXME: Better message? msg = "Unsupported compression options"; break; case LZMA_DATA_ERROR: msg = "File is corrupt"; break; case LZMA_BUF_ERROR: msg = "Unexpected end of input"; break; default: msg = "Internal error (bug)"; break; } my_errorf("%s: %s", filename, msg); exit(EXIT_FAILURE); } } } #ifdef ENABLE_SANDBOX static void sandbox_enter(int src_fd) { #if defined(HAVE_CAP_RIGHTS_LIMIT) // Capsicum needs FreeBSD 10.2 or later. cap_rights_t rights; if (cap_enter()) goto error; if (cap_rights_limit(src_fd, cap_rights_init(&rights, CAP_READ))) goto error; // If not reading from stdin, remove all capabilities from it. if (src_fd != STDIN_FILENO && cap_rights_limit( STDIN_FILENO, cap_rights_clear(&rights))) goto error; if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights, CAP_WRITE))) goto error; if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights, CAP_WRITE))) goto error; #elif defined(HAVE_PLEDGE) // pledge() was introduced in OpenBSD 5.9. if (pledge("stdio", "")) goto error; (void)src_fd; #elif defined(HAVE_LINUX_LANDLOCK) int landlock_abi = syscall(SYS_landlock_create_ruleset, (void *)NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); if (landlock_abi > 0) { if (landlock_abi > LANDLOCK_ABI_MAX) landlock_abi = LANDLOCK_ABI_MAX; const struct landlock_ruleset_attr attr = { .handled_access_fs = (1ULL << (12 + my_min(3, landlock_abi))) - 1, # if LANDLOCK_ABI_MAX >= 4 .handled_access_net = landlock_abi < 4 ? 0 : (LANDLOCK_ACCESS_NET_BIND_TCP | LANDLOCK_ACCESS_NET_CONNECT_TCP), # endif }; const int ruleset_fd = syscall(SYS_landlock_create_ruleset, &attr, sizeof(attr), 0U); if (ruleset_fd < 0) goto error; // All files we need should have already been opened. Thus, // we don't need to add any rules using landlock_add_rule(2) // before activating the sandbox. if (syscall(SYS_landlock_restrict_self, ruleset_fd, 0U) != 0) goto error; } (void)src_fd; #else # error ENABLE_SANDBOX is defined but no sandboxing method was found. #endif return; error: #ifdef HAVE_CAP_RIGHTS_LIMIT // If a kernel is configured without capability mode support or // used in an emulator that does not implement the capability // system calls, then the Capsicum system calls will fail and set // errno to ENOSYS. In that case xzdec will silently run without // the sandbox. if (errno == ENOSYS) return; #endif my_errorf("Failed to enable the sandbox"); exit(EXIT_FAILURE); } #endif int main(int argc, char **argv) { #ifdef HAVE_PLEDGE // OpenBSD's pledge(2) sandbox. // Initially enable the sandbox slightly more relaxed so that // the process can still open files. This allows the sandbox to // be enabled when parsing command line arguments and decompressing // all files (the more strict sandbox only restricts the last file // that is decompressed). if (pledge("stdio rpath", "")) { my_errorf("Failed to enable the sandbox"); exit(EXIT_FAILURE); } #endif #ifdef HAVE_LINUX_LANDLOCK // Prevent the process from gaining new privileges. This must be done // before landlock_restrict_self(2) but since we will never need new // privileges, this call can be done here already. // // This is supported since Linux 3.5. Ignore the return value to // keep compatibility with old kernels. landlock_restrict_self(2) // will fail if the no_new_privs attribute isn't set, thus if prctl() // fails here the error will still be detected when it matters. (void)prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); #endif // Initialize progname which we will be used in error messages. tuklib_progname_init(argv); // Parse the command line options. parse_options(argc, argv); // The same lzma_stream is used for all files that we decode. This way // we don't need to reallocate memory for every file if they use same // compression settings. lzma_stream strm = LZMA_STREAM_INIT; // Some systems require setting stdin and stdout to binary mode. #ifdef TUKLIB_DOSLIKE setmode(fileno(stdin), O_BINARY); setmode(fileno(stdout), O_BINARY); #endif if (optind == argc) { // No filenames given, decode from stdin. #ifdef ENABLE_SANDBOX sandbox_enter(STDIN_FILENO); #endif uncompress(&strm, stdin, "(stdin)"); } else { // Loop through the filenames given on the command line. do { FILE *src_file; const char *src_name; // "-" indicates stdin. if (strcmp(argv[optind], "-") == 0) { src_file = stdin; src_name = "(stdin)"; } else { src_name = argv[optind]; src_file = fopen(src_name, "rb"); if (src_file == NULL) { my_errorf("%s: %s", src_name, strerror(errno)); exit(EXIT_FAILURE); } } #ifdef ENABLE_SANDBOX // Enable the strict sandbox for the last file. // Then the process can no longer open additional // files. The typical xzdec use case is to decompress // a single file so this way the strictest sandboxing // is used in most cases. if (optind == argc - 1) sandbox_enter(fileno(src_file)); #endif uncompress(&strm, src_file, src_name); if (src_file != stdin) (void)fclose(src_file); } while (++optind < argc); } #ifndef NDEBUG // Free the memory only when debugging. Freeing wastes some time, // but allows detecting possible memory leaks with Valgrind. lzma_end(&strm); #endif tuklib_exit(EXIT_SUCCESS, EXIT_FAILURE, display_errors); }